1
|
|
|
# coding: utf-8 |
2
|
|
|
import sys |
3
|
|
|
import arrow |
4
|
|
|
|
5
|
|
|
# evernote API |
6
|
|
|
from evernote.api.client import EvernoteClient |
7
|
|
|
from evernote.edam.notestore import NoteStore |
8
|
|
|
import evernote.edam.type.ttypes as Types |
9
|
|
|
from evernote.edam.error.ttypes import EDAMSystemException, EDAMUserException |
10
|
|
|
from evernote.edam.error.ttypes import EDAMErrorCode |
11
|
|
|
|
12
|
|
|
# django classes |
13
|
|
|
from django.utils.translation import ugettext as _ |
14
|
|
|
from django.conf import settings |
15
|
|
|
from django.utils.log import getLogger |
16
|
|
|
from django.core.cache import caches |
17
|
|
|
|
18
|
|
|
# django_th classes |
19
|
|
|
from django_th.services.services import ServicesMgr |
20
|
|
|
from django_th.models import UserService, ServicesActivated |
21
|
|
|
from th_evernote.models import Evernote |
22
|
|
|
from th_evernote.sanitize import sanitize |
23
|
|
|
|
24
|
|
|
|
25
|
|
|
""" |
26
|
|
|
handle process with evernote |
27
|
|
|
put the following in settings.py |
28
|
|
|
|
29
|
|
|
TH_EVERNOTE = { |
30
|
|
|
'sandbox': True, |
31
|
|
|
'consumer_key': 'abcdefghijklmnopqrstuvwxyz', |
32
|
|
|
'consumer_secret': 'abcdefghijklmnopqrstuvwxyz', |
33
|
|
|
} |
34
|
|
|
sanbox set to True to make your test and False for production purpose |
35
|
|
|
|
36
|
|
|
TH_SERVICES = ( |
37
|
|
|
... |
38
|
|
|
'th_evernote.my_evernote.ServiceEvernote', |
39
|
|
|
... |
40
|
|
|
) |
41
|
|
|
|
42
|
|
|
""" |
43
|
|
|
|
44
|
|
|
logger = getLogger('django_th.trigger_happy') |
45
|
|
|
|
46
|
|
|
cache = caches['th_evernote'] |
47
|
|
|
|
48
|
|
|
|
49
|
|
|
class ServiceEvernote(ServicesMgr): |
50
|
|
|
|
51
|
|
|
def __init__(self, token=None): |
52
|
|
|
super(ServiceEvernote, self).__init__(token) |
53
|
|
|
self.sandbox = settings.TH_EVERNOTE['sandbox'] |
54
|
|
|
self.consumer_key = settings.TH_EVERNOTE['consumer_key'] |
55
|
|
|
self.consumer_secret = settings.TH_EVERNOTE['consumer_secret'] |
56
|
|
|
self.token = token |
57
|
|
|
|
58
|
|
|
kwargs = {'consumer_key': self.consumer_key, |
59
|
|
|
'consumer_secret': self.consumer_secret, |
60
|
|
|
'sandbox': self.sandbox} |
61
|
|
|
|
62
|
|
|
if self.token: |
63
|
|
|
kwargs = {'token': token, 'sandbox': self.sandbox} |
64
|
|
|
|
65
|
|
|
self.client = EvernoteClient(**kwargs) |
66
|
|
|
|
67
|
|
|
def read_data(self, **kwargs): |
68
|
|
|
""" |
69
|
|
|
get the data from the service |
70
|
|
|
|
71
|
|
|
:param kwargs: contain keyword args : trigger_id at least |
72
|
|
|
:type kwargs: dict |
73
|
|
|
|
74
|
|
|
:rtype: list |
75
|
|
|
""" |
76
|
|
|
date_triggered = kwargs.get('date_triggered') |
77
|
|
|
trigger_id = kwargs.get('trigger_id') |
78
|
|
|
|
79
|
|
|
kwargs['model_name'] = 'Evernote' |
80
|
|
|
|
81
|
|
|
trigger = super(ServiceEvernote, self).read_data(**kwargs) |
82
|
|
|
|
83
|
|
|
data = [] |
84
|
|
|
# get the data from the last time the trigger has been started |
85
|
|
|
# the filter will use the DateTime format in standard |
86
|
|
|
new_date_triggered = arrow.get(str(date_triggered)[:-6], |
87
|
|
|
'YYYY-MM-DD HH:mm:ss') |
88
|
|
|
new_date_triggered = str(new_date_triggered).replace( |
89
|
|
|
':', '').replace('-', '').replace(' ', '') |
90
|
|
|
date_filter = "created:{} ".format(new_date_triggered[:-6]) |
91
|
|
|
|
92
|
|
|
notebook_filter = '' |
93
|
|
|
if trigger.notebook: |
94
|
|
|
notebook_filter = "notebook:{} ".format(trigger.notebook) |
95
|
|
|
tag_filter = "tag:{} ".format(trigger.tag) if trigger.tag != '' else '' |
96
|
|
|
|
97
|
|
|
complet_filter = ''.join((notebook_filter, tag_filter, date_filter)) |
98
|
|
|
|
99
|
|
|
# filter |
100
|
|
|
my_filter = NoteStore.NoteFilter() |
101
|
|
|
my_filter.words = complet_filter |
102
|
|
|
|
103
|
|
|
# result spec to tell to evernote |
104
|
|
|
# what information to include in the response |
105
|
|
|
spec = NoteStore.NotesMetadataResultSpec() |
106
|
|
|
spec.includeTitle = True |
107
|
|
|
spec.includeAttributes = True |
108
|
|
|
|
109
|
|
|
note_store = self.client.get_note_store() |
110
|
|
|
our_note_list = note_store.findNotesMetadata(self.token, |
111
|
|
|
my_filter, |
112
|
|
|
0, |
113
|
|
|
100, |
114
|
|
|
spec) |
115
|
|
|
|
116
|
|
|
for note in our_note_list.notes: |
117
|
|
|
whole_note = note_store.getNote(self.token, |
118
|
|
|
note.guid, |
119
|
|
|
True, |
120
|
|
|
True, |
121
|
|
|
False, |
122
|
|
|
False) |
123
|
|
|
content = self.cleaning_content(whole_note.content) |
124
|
|
|
data.append( |
125
|
|
|
{'title': note.title, |
126
|
|
|
'my_date': arrow.get(note.created), |
127
|
|
|
'link': whole_note.attributes.sourceURL, |
128
|
|
|
'content': content}) |
129
|
|
|
|
130
|
|
|
cache.set('th_evernote_' + str(trigger_id), data) |
131
|
|
|
|
132
|
|
|
return data |
133
|
|
|
|
134
|
|
|
def save_data(self, trigger_id, **data): |
135
|
|
|
""" |
136
|
|
|
let's save the data |
137
|
|
|
don't want to handle empty title nor content |
138
|
|
|
otherwise this will produce an Exception by |
139
|
|
|
the Evernote's API |
140
|
|
|
|
141
|
|
|
:param trigger_id: trigger ID from which to save data |
142
|
|
|
:param data: the data to check to be used and save |
143
|
|
|
:type trigger_id: int |
144
|
|
|
:type data: dict |
145
|
|
|
:return: the status of the save statement |
146
|
|
|
:rtype: boolean |
147
|
|
|
""" |
148
|
|
|
kwargs = {} |
149
|
|
|
# set the title and content of the data |
150
|
|
|
title, content = super(ServiceEvernote, self).save_data(trigger_id, |
151
|
|
|
data, |
152
|
|
|
**kwargs) |
153
|
|
|
|
154
|
|
|
if len(title): |
155
|
|
|
# get the evernote data of this trigger |
156
|
|
|
trigger = Evernote.objects.get(trigger_id=trigger_id) |
157
|
|
|
|
158
|
|
|
try: |
159
|
|
|
note_store = self.client.get_note_store() |
160
|
|
|
except EDAMSystemException as e: |
161
|
|
|
# rate limite reach have to wait 1 hour ! |
162
|
|
|
if e.errorCode == EDAMErrorCode.RATE_LIMIT_REACHED: |
163
|
|
|
sentance = "Rate limit reached {code}" |
164
|
|
|
sentance += "Retry your request in {msg} seconds" |
165
|
|
|
sentance += " - date set to cache again until" |
166
|
|
|
sentance += " limit reached" |
167
|
|
|
logger.warn(sentance.format( |
168
|
|
|
code=e.errorCode, |
169
|
|
|
msg=e.rateLimitDuration)) |
170
|
|
|
# put again in cache the data that could not be |
171
|
|
|
# published in Evernote yet |
172
|
|
|
cache.set('th_evernote_' + str(trigger_id), |
173
|
|
|
data, |
174
|
|
|
version=2) |
175
|
|
|
return True |
176
|
|
|
else: |
177
|
|
|
logger.critical(e) |
178
|
|
|
return False |
179
|
|
|
except Exception as e: |
180
|
|
|
logger.critical(e) |
181
|
|
|
return False |
182
|
|
|
|
183
|
|
|
# note object |
184
|
|
|
note = Types.Note() |
185
|
|
|
if trigger.notebook: |
186
|
|
|
# get the notebookGUID ... |
187
|
|
|
notebook_id = self.get_notebook(note_store, trigger.notebook) |
188
|
|
|
# create notebookGUID if it does not exist then return its id |
189
|
|
|
note.notebookGuid = self.set_notebook(note_store, |
190
|
|
|
trigger.notebook, |
191
|
|
|
notebook_id) |
192
|
|
|
|
193
|
|
|
if trigger.tag: |
194
|
|
|
# ... and get the tagGUID if a tag has been provided |
195
|
|
|
tag_id = self.get_tag(note_store, trigger.tag) |
196
|
|
|
if tag_id is False: |
197
|
|
|
tag_id = self.set_tag(note_store, trigger.tag, tag_id) |
198
|
|
|
# set the tag to the note if a tag has been provided |
199
|
|
|
note.tagGuids = tag_id |
200
|
|
|
|
201
|
|
|
logger.debug("notebook that will be used %s", trigger.notebook) |
202
|
|
|
|
203
|
|
|
# attribute of the note: the link to the website |
204
|
|
|
note_attribute = self.set_note_attribute(data) |
205
|
|
|
if note_attribute: |
206
|
|
|
note.attributes = note_attribute |
207
|
|
|
|
208
|
|
|
# footer of the note |
209
|
|
|
footer = self.set_note_footer(data, trigger) |
210
|
|
|
content += footer |
211
|
|
|
|
212
|
|
|
note.title = title |
213
|
|
|
note.content = self.set_evernote_header() |
214
|
|
|
note.content += self.get_sanitize_content(content) |
215
|
|
|
# create a note |
216
|
|
|
return self._create_note(note_store, note, trigger_id, data) |
217
|
|
|
|
218
|
|
|
else: |
219
|
|
|
sentence = "no title provided for trigger ID {}" |
220
|
|
|
logger.critical(sentence.format(trigger_id)) |
221
|
|
|
return False |
222
|
|
|
|
223
|
|
|
@staticmethod |
224
|
|
|
def get_notebook(note_store, my_notebook): |
225
|
|
|
""" |
226
|
|
|
get the notebook from its name |
227
|
|
|
""" |
228
|
|
|
notebook_id = 0 |
229
|
|
|
notebooks = note_store.listNotebooks() |
230
|
|
|
# get the notebookGUID ... |
231
|
|
|
for notebook in notebooks: |
232
|
|
|
if notebook.name.lower() == my_notebook.lower(): |
233
|
|
|
notebook_id = notebook.guid |
234
|
|
|
break |
235
|
|
|
return notebook_id |
236
|
|
|
|
237
|
|
|
@staticmethod |
238
|
|
|
def set_notebook(note_store, my_notebook, notebook_id): |
239
|
|
|
""" |
240
|
|
|
create a notebook |
241
|
|
|
""" |
242
|
|
|
if notebook_id == 0: |
243
|
|
|
new_notebook = Types.Notebook() |
244
|
|
|
new_notebook.name = my_notebook |
245
|
|
|
new_notebook.defaultNotebook = False |
246
|
|
|
notebook_id = note_store.createNotebook(new_notebook).guid |
247
|
|
|
|
248
|
|
|
return notebook_id |
249
|
|
|
|
250
|
|
|
@staticmethod |
251
|
|
|
def get_tag(note_store, my_tags): |
252
|
|
|
""" |
253
|
|
|
get the tags from his Evernote account |
254
|
|
|
:param note_store Evernote Instance |
255
|
|
|
:param my_tags string |
256
|
|
|
:return: array of the tag to create |
257
|
|
|
""" |
258
|
|
|
tag_id = [] |
259
|
|
|
listtags = note_store.listTags() |
260
|
|
|
# cut the string by piece of tag with comma |
261
|
|
|
if ',' in my_tags: |
262
|
|
|
for my_tag in my_tags.split(','): |
263
|
|
|
for tag in listtags: |
264
|
|
|
# remove space before and after |
265
|
|
|
# thus we keep "foo bar" |
266
|
|
|
# but not " foo bar" nor "foo bar " |
267
|
|
|
if tag.name.lower() == my_tag.lower().lstrip().rstrip(): |
268
|
|
|
tag_id.append(tag.guid) |
269
|
|
|
break |
270
|
|
|
else: |
271
|
|
|
for tag in listtags: |
272
|
|
|
if tag.name.lower() == my_tags.lower(): |
273
|
|
|
tag_id.append(tag.guid) |
274
|
|
|
break |
275
|
|
|
|
276
|
|
|
return tag_id |
277
|
|
|
|
278
|
|
|
def set_tag(self, note_store, my_tags, tag_id): |
279
|
|
|
""" |
280
|
|
|
create a tag if not exists |
281
|
|
|
:param my_tags string |
282
|
|
|
:param tag_id id of the tag(s) to create |
283
|
|
|
:return: array of the tag to create |
284
|
|
|
""" |
285
|
|
|
# tagGUID does not exist: |
286
|
|
|
# create it if a tag has been provided |
287
|
|
|
new_tag = Types.Tag() |
288
|
|
|
if ',' in my_tags: |
289
|
|
|
for my_tag in my_tags.split(','): |
290
|
|
|
new_tag.name = my_tag |
291
|
|
|
tag_id.append(self._create_tag(note_store, new_tag)) |
292
|
|
|
elif my_tags: |
293
|
|
|
new_tag.name = my_tags |
294
|
|
|
tag_id.append(self._create_tag(note_store, new_tag)) |
295
|
|
|
|
296
|
|
|
return tag_id |
297
|
|
|
|
298
|
|
|
@staticmethod |
299
|
|
|
def _create_note(note_store, note, trigger_id, data): |
300
|
|
|
""" |
301
|
|
|
create a note |
302
|
|
|
:param note_store Evernote instance |
303
|
|
|
:param note |
304
|
|
|
:param trigger_id id of the trigger |
305
|
|
|
:param data to save or to put in cache |
306
|
|
|
:type Evernote Instance |
307
|
|
|
:type note: string |
308
|
|
|
:type trigger_id: int |
309
|
|
|
:type data: dict |
310
|
|
|
:return boolean |
311
|
|
|
:rtype boolean |
312
|
|
|
""" |
313
|
|
|
# create the note ! |
314
|
|
|
try: |
315
|
|
|
created_note = note_store.createNote(note) |
316
|
|
|
sentance = str('note %s created') % created_note.guid |
317
|
|
|
logger.debug(sentance) |
318
|
|
|
return True |
319
|
|
|
except EDAMSystemException as e: |
320
|
|
|
if e.errorCode == EDAMErrorCode.RATE_LIMIT_REACHED: |
321
|
|
|
sentance = "Rate limit reached {code}" |
322
|
|
|
sentance += "Retry your request in {msg} seconds" |
323
|
|
|
logger.warn(sentance.format( |
324
|
|
|
code=e.errorCode, |
325
|
|
|
msg=e.rateLimitDuration)) |
326
|
|
|
# put again in cache the data that could not be |
327
|
|
|
# published in Evernote yet |
328
|
|
|
cache.set('th_evernote_' + str(trigger_id), |
329
|
|
|
data, |
330
|
|
|
version=2) |
331
|
|
|
return True |
332
|
|
|
else: |
333
|
|
|
logger.critical(e) |
334
|
|
|
return False |
335
|
|
|
except EDAMUserException as e: |
336
|
|
|
if e.errorCode == EDAMErrorCode.ENML_VALIDATION: |
337
|
|
|
sentance = "Data ignored due to validation" |
338
|
|
|
sentance += " error : err {code} {msg}" |
339
|
|
|
logger.warn(sentance.format( |
340
|
|
|
code=e.errorCode, |
341
|
|
|
msg=e.parameter)) |
342
|
|
|
return True |
343
|
|
|
except Exception as e: |
344
|
|
|
logger.critical(e) |
345
|
|
|
return False |
346
|
|
|
|
347
|
|
|
@staticmethod |
348
|
|
|
def _create_tag(note_store, new_tag): |
349
|
|
|
""" |
350
|
|
|
:param new_tag: create this new tag |
351
|
|
|
:return: new tag id |
352
|
|
|
""" |
353
|
|
|
try: |
354
|
|
|
return note_store.createTag(new_tag).guid |
355
|
|
|
except EDAMUserException as e: |
356
|
|
|
if e.errorCode == EDAMErrorCode.DATA_CONFLICT: |
357
|
|
|
logger.info("Evernote Data Conflict Err {0}".format(e)) |
358
|
|
|
elif e.errorCode == EDAMErrorCode.BAD_DATA_FORMAT: |
359
|
|
|
logger.critical("Evernote Err {0}".format(e)) |
360
|
|
|
|
361
|
|
|
@staticmethod |
362
|
|
|
def set_evernote_header(): |
363
|
|
|
""" |
364
|
|
|
preparing the hearder of Evernote |
365
|
|
|
""" |
366
|
|
|
prolog = '<?xml version="1.0" encoding="UTF-8"?>' |
367
|
|
|
prolog += '<!DOCTYPE en-note SYSTEM \ |
368
|
|
|
"http://xml.evernote.com/pub/enml2.dtd">\n' |
369
|
|
|
return prolog |
370
|
|
|
|
371
|
|
|
@staticmethod |
372
|
|
|
def get_sanitize_content(content): |
373
|
|
|
""" |
374
|
|
|
tidy and sanitize content |
375
|
|
|
""" |
376
|
|
|
enml = sanitize(content) |
377
|
|
|
# python 2 |
378
|
|
|
if sys.version_info.major == 2: |
379
|
|
|
return enml.encode('ascii', 'xmlcharrefreplace') |
380
|
|
|
else: |
381
|
|
|
return str(enml) |
382
|
|
|
|
383
|
|
|
@staticmethod |
384
|
|
|
def set_note_attribute(data): |
385
|
|
|
""" |
386
|
|
|
add the link of the 'source' in the note |
387
|
|
|
get a NoteAttributes object |
388
|
|
|
""" |
389
|
|
|
na = False |
390
|
|
|
if data.get('link'): |
391
|
|
|
na = Types.NoteAttributes() |
392
|
|
|
# add the url |
393
|
|
|
na.sourceURL = data['link'] |
394
|
|
|
# add the object to the note |
395
|
|
|
return na |
396
|
|
|
|
397
|
|
|
@staticmethod |
398
|
|
|
def set_note_footer(data, trigger): |
399
|
|
|
""" |
400
|
|
|
handle the footer of the note |
401
|
|
|
""" |
402
|
|
|
footer = '' |
403
|
|
|
if data.get('link'): |
404
|
|
|
provided_by = _('Provided by') |
405
|
|
|
provided_from = _('from') |
406
|
|
|
footer_from = "<br/><br/>{} <em>{}</em> {} <a href='{}'>{}</a>" |
407
|
|
|
|
408
|
|
|
footer = footer_from.format( |
409
|
|
|
provided_by, trigger.trigger.description, provided_from, |
410
|
|
|
data.get('link'), data.get('link')) |
411
|
|
|
|
412
|
|
|
return footer |
413
|
|
|
|
414
|
|
|
def get_evernote_client(self, token=None): |
415
|
|
|
""" |
416
|
|
|
get the token from evernote |
417
|
|
|
""" |
418
|
|
|
if token: |
419
|
|
|
return EvernoteClient( |
420
|
|
|
token=token, |
421
|
|
|
sandbox=self.sandbox) |
422
|
|
|
else: |
423
|
|
|
return EvernoteClient( |
424
|
|
|
consumer_key=self.consumer_key, |
425
|
|
|
consumer_secret=self.consumer_secret, |
426
|
|
|
sandbox=self.sandbox) |
427
|
|
|
|
428
|
|
|
def auth(self, request): |
429
|
|
|
""" |
430
|
|
|
let's auth the user to the Service |
431
|
|
|
""" |
432
|
|
|
client = self.get_evernote_client() |
433
|
|
View Code Duplication |
request_token = client.get_request_token( |
|
|
|
|
434
|
|
|
self.callback_url(request, 'evernote')) |
435
|
|
|
|
436
|
|
|
# Save the request token information for later |
437
|
|
|
request.session['oauth_token'] = request_token['oauth_token'] |
438
|
|
|
request.session['oauth_token_secret'] = request_token[ |
439
|
|
|
'oauth_token_secret'] |
440
|
|
|
|
441
|
|
|
# Redirect the user to the Evernote authorization URL |
442
|
|
|
# return the URL string which will be used by redirect() |
443
|
|
|
# from the calling func |
444
|
|
|
return client.get_authorize_url(request_token) |
445
|
|
|
|
446
|
|
|
def callback(self, request, **kwargs): |
447
|
|
|
""" |
448
|
|
|
Called from the Service when the user accept to activate it |
449
|
|
|
""" |
450
|
|
|
try: |
451
|
|
|
client = self.get_evernote_client() |
452
|
|
|
# finally we save the user auth token |
453
|
|
|
# As we already stored the object ServicesActivated |
454
|
|
|
# from the UserServiceCreateView now we update the same |
455
|
|
|
# object to the database so : |
456
|
|
|
# 1) we get the previous objet |
457
|
|
|
us = UserService.objects.get( |
458
|
|
|
user=request.user, |
459
|
|
|
name=ServicesActivated.objects.get(name='ServiceEvernote')) |
460
|
|
|
# 2) then get the token |
461
|
|
|
us.token = client.get_access_token( |
462
|
|
|
request.session['oauth_token'], |
463
|
|
|
request.session['oauth_token_secret'], |
464
|
|
|
request.GET.get('oauth_verifier', '') |
465
|
|
|
) |
466
|
|
|
# 3) and save everything |
467
|
|
|
us.save() |
468
|
|
|
except KeyError: |
469
|
|
|
return '/' |
470
|
|
|
|
471
|
|
|
return 'evernote/callback.html' |
472
|
|
|
|
473
|
|
|
@staticmethod |
474
|
|
|
def cleaning_content(data): |
475
|
|
|
|
476
|
|
|
data = data.replace('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">\n<en-note>', '') |
477
|
|
|
data = data.replace('</en-note>', '') |
478
|
|
|
|
479
|
|
|
return data |
480
|
|
|
|