|
1
|
|
|
#!/usr/bin/env python |
|
2
|
|
|
# -*- coding: utf-8 -*- |
|
3
|
|
|
|
|
4
|
|
|
""" |
|
5
|
|
|
flask_jsondash.data_utils.wordcloud |
|
6
|
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
7
|
|
|
|
|
8
|
|
|
Utilities for working with wordcloud formatted data. |
|
9
|
|
|
|
|
10
|
|
|
:copyright: (c) 2016 by Chris Tabor. |
|
11
|
|
|
:license: MIT, see LICENSE for more details. |
|
12
|
|
|
""" |
|
13
|
|
|
|
|
14
|
|
|
from collections import Counter |
|
15
|
|
|
from string import punctuation |
|
16
|
|
|
import re |
|
17
|
|
|
|
|
18
|
|
|
import requests |
|
19
|
|
|
from pyquery import PyQuery as Pq |
|
20
|
|
|
|
|
21
|
|
|
# Py2/3 compat. |
|
22
|
|
|
try: |
|
23
|
|
|
_unicode = unicode |
|
24
|
|
|
except NameError: |
|
25
|
|
|
_unicode = str |
|
26
|
|
|
|
|
27
|
|
|
|
|
28
|
|
|
# NLTK stopwords |
|
29
|
|
|
stopwords = [ |
|
30
|
|
|
'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', |
|
31
|
|
|
'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', |
|
32
|
|
|
'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', |
|
33
|
|
|
'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', |
|
34
|
|
|
'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', |
|
35
|
|
|
'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', |
|
36
|
|
|
'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', |
|
37
|
|
|
'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', |
|
38
|
|
|
'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', |
|
39
|
|
|
'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', |
|
40
|
|
|
'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', |
|
41
|
|
|
'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', |
|
42
|
|
|
'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', |
|
43
|
|
|
'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', |
|
44
|
|
|
] |
|
45
|
|
|
|
|
46
|
|
|
|
|
47
|
|
|
def get_word_freq_distribution(words): |
|
48
|
|
|
"""Get the counted word frequency distribution of all words. |
|
49
|
|
|
|
|
50
|
|
|
Arg: |
|
51
|
|
|
words (list): A list of strings indicating words. |
|
52
|
|
|
|
|
53
|
|
|
Returns: |
|
54
|
|
|
collections.Counter: The Counter object with word frequencies. |
|
55
|
|
|
""" |
|
56
|
|
|
return Counter([w for w in words if w not in stopwords]) |
|
57
|
|
|
|
|
58
|
|
|
|
|
59
|
|
|
def format_4_wordcloud(words, size_multiplier=2): |
|
60
|
|
|
"""Format words in a way suitable for wordcloud plugin. |
|
61
|
|
|
|
|
62
|
|
|
Args: |
|
63
|
|
|
words (list): A list 2-tuples of format: (word-string, occurences). |
|
64
|
|
|
size_multiplier (int, optional): The size multiplier to scale |
|
65
|
|
|
word sizing. Can improve visual display of word cloud. |
|
66
|
|
|
|
|
67
|
|
|
Returns: |
|
68
|
|
|
list: A list of dicts w/ appropriate keys. |
|
69
|
|
|
""" |
|
70
|
|
|
return [ |
|
71
|
|
|
{'text': word, 'size': size * size_multiplier} |
|
72
|
|
|
for (word, size) in words if word |
|
73
|
|
|
] |
|
74
|
|
|
|
|
75
|
|
|
|
|
76
|
|
|
def url2wordcloud(url, requests_kwargs={}, |
|
77
|
|
|
exclude_punct=True, |
|
78
|
|
|
normalized=True, |
|
79
|
|
|
limit=None, |
|
80
|
|
|
size=1, |
|
81
|
|
|
min_len=None): |
|
82
|
|
|
"""Convert the text content of a urls' html to a wordcloud config. |
|
83
|
|
|
|
|
84
|
|
|
Args: |
|
85
|
|
|
url (str): The url to load. |
|
86
|
|
|
requests_kwargs (dict, optional): The kwargs to pass to the |
|
87
|
|
|
requests library. (e.g. auth, headers, mimetypes) |
|
88
|
|
|
exclude_punc (bool, optional): exclude punctuation |
|
89
|
|
|
min_length (int, optional): the minimum required length, if any |
|
90
|
|
|
limit (int, optional): the number of items to limit |
|
91
|
|
|
(by most common), if any |
|
92
|
|
|
normalized (bool, optional): normalize data by |
|
93
|
|
|
lowercasing and strippping whitespace |
|
94
|
|
|
|
|
95
|
|
|
Returns: |
|
96
|
|
|
same value as :func:`~format_4_wordcloud` |
|
97
|
|
|
""" |
|
98
|
|
|
resp = requests.get(url, **requests_kwargs) |
|
99
|
|
|
if not resp.status_code == 200: |
|
100
|
|
|
return [] |
|
101
|
|
|
resp = Pq(resp.content).find('body').text().split(' ') |
|
102
|
|
|
if exclude_punct: |
|
103
|
|
|
resp = [ |
|
104
|
|
|
re.sub(r'[^a-zA-Z0-9]+', '', w) for w |
|
105
|
|
|
in resp if w not in punctuation |
|
106
|
|
|
] |
|
107
|
|
|
if min_len is not None: |
|
108
|
|
|
resp = [w for w in resp if len(w) >= min_len] |
|
109
|
|
|
if normalized: |
|
110
|
|
|
resp = [w.lower() for w in resp] |
|
111
|
|
|
words = get_word_freq_distribution(resp) |
|
112
|
|
|
if limit is not None: |
|
113
|
|
|
words = words.most_common(limit) |
|
114
|
|
|
else: |
|
115
|
|
|
words = [(k, v) for k, v in words.items()] |
|
116
|
|
|
return format_4_wordcloud(words, size_multiplier=size) |
|
117
|
|
|
|
christabor /
flask_jsondash
