ServiceEvernote._cleaning_content() - Code Metrics - Inspection of "improvment of evernote and github" - foxmask/django-th - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( b9effa...31e28e )

by Fox

created 2016-06-14 20:44 UTC

ServiceEvernote._cleaning_content() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
dl	0
loc	10
rs	9.4285
c	0
b	0
f	0

# coding: utf-8
import arrow

# evernote API
from evernote.api.client import EvernoteClient
from evernote.edam.notestore import NoteStore
import evernote.edam.type.ttypes as Types
from evernote.edam.error.ttypes import EDAMSystemException
from evernote.edam.error.ttypes import EDAMErrorCode

# django classes
from django.conf import settings
from django.utils.log import getLogger
from django.core.cache import caches

# django_th classes
from django_th.services.services import ServicesMgr
from django_th.models import UserService, ServicesActivated
from th_evernote.models import Evernote
from th_evernote.evernote_mgr import create_note, set_notebook, get_notebook, \
    set_header, set_tag, get_tag, set_note_attribute, set_note_footer
from th_evernote.sanitize import sanitize

"""
    handle process with evernote
    put the following in settings.py

    TH_EVERNOTE = {
        'sandbox': True,
        'consumer_key': 'abcdefghijklmnopqrstuvwxyz',
        'consumer_secret': 'abcdefghijklmnopqrstuvwxyz',
    }
    sanbox set to True to make your test and False for production purpose

    TH_SERVICES = (
        ...
        'th_evernote.my_evernote.ServiceEvernote',
        ...
    )
"""

logger = getLogger('django_th.trigger_happy')

cache = caches['th_evernote']


class ServiceEvernote(ServicesMgr):

    def __init__(self, token=None, **kwargs):
        super(ServiceEvernote, self).__init__(token, **kwargs)
        self.sandbox = settings.TH_EVERNOTE['sandbox']
        self.consumer_key = settings.TH_EVERNOTE['consumer_key']
        self.consumer_secret = settings.TH_EVERNOTE['consumer_secret']
        self.token = token
        self.service = 'ServiceEvernote'
        self.oauth = 'oauth1'

        kwargs = {'consumer_key': self.consumer_key,
                  'consumer_secret': self.consumer_secret,
                  'sandbox': self.sandbox}

        if self.token:
            kwargs = {'token': token, 'sandbox': self.sandbox}

        self.client = EvernoteClient(**kwargs)

    def read_data(self, **kwargs):
        """
            get the data from the service

            :param kwargs: contain keyword args : trigger_id at least
            :type kwargs: dict

            :rtype: list
        """
        date_triggered = kwargs.get('date_triggered')
        trigger_id = kwargs.get('trigger_id')

        kwargs['model_name'] = 'Evernote'

        trigger = super(ServiceEvernote, self).read_data(**kwargs)

        data = []
        new_date_triggered = arrow.get(str(date_triggered)[:-6],
                                       'YYYY-MM-DD HH:mm:ss')
        new_date_triggered = str(new_date_triggered).replace(
            ':', '').replace('-', '').replace(' ', '')
        date_filter = "created:{} ".format(new_date_triggered[:-6])

        notebook_filter = ''
        if trigger.notebook:
            notebook_filter = "notebook:{} ".format(trigger.notebook)
        tag_filter = "tag:{} ".format(trigger.tag) if trigger.tag != '' else ''

        complet_filter = ''.join((notebook_filter, tag_filter, date_filter))

        # filter
        my_filter = NoteStore.NoteFilter()
        my_filter.words = complet_filter

        spec = NoteStore.NotesMetadataResultSpec()
        spec.includeTitle = True
        spec.includeAttributes = True

        note_store = self.client.get_note_store()
        our_note_list = note_store.findNotesMetadata(self.token,
                                                     my_filter,
                                                     0,
                                                     100,
                                                     spec)

        for note in our_note_list.notes:
            whole_note = note_store.getNote(self.token,
                                            note.guid,
                                            True,
                                            True,
                                            False,
                                            False)
            content = self._cleaning_content(whole_note.content)
            data.append(
                {'title': note.title,
                 'my_date': arrow.get(note.created),
                 'link': whole_note.attributes.sourceURL,
                 'content': content})

        cache.set('th_evernote_' + str(trigger_id), data)

        return data

    def save_data(self, trigger_id, **data):
        """
            let's save the data
            don't want to handle empty title nor content
            otherwise this will produce an Exception by
            the Evernote's API

            :param trigger_id: trigger ID from which to save data
            :param data: the data to check to be used and save
            :type trigger_id: int
            :type data:  dict
            :return: the status of the save statement
            :rtype: boolean
        """
        kwargs = {}
        # set the title and content of the data
        title, content = super(ServiceEvernote, self).save_data(trigger_id,
                                                                data,
                                                                **kwargs)

        if len(title):
            # get the evernote data of this trigger
            trigger = Evernote.objects.get(trigger_id=trigger_id)
            # initialize notestore process
            note_store = self._notestore(trigger_id, data)
            if note_store is False:
                return False
            # note object
            note = self._notebook(trigger, note_store)
            # its attributes
            note = self._attributes(note, data)
            # its footer
            content = self._footer(trigger, data, content)
            # its title
            note.title = title
            # its content
            note = self._content(note, content)
            # create a note
            return create_note(note_store, note, trigger_id, data)

        else:
            logger.critical("no title provided "
                            "for trigger ID {}".format(trigger_id))
            return False

    def _notestore(self, trigger_id, data):
        try:
            note_store = self.client.get_note_store()
            return note_store
        except EDAMSystemException as e:
            # rate limit reach have to wait 1 hour !
            if e.errorCode == EDAMErrorCode.RATE_LIMIT_REACHED:
                logger.warn("Rate limit reached {code}\n"
                            "Retry your request in {msg} seconds\n"
                            "Data set to cache again until"
                            " limit reached".format(code=e.errorCode,
                                                    msg=e.rateLimitDuration)
                            )
                cache.set('th_evernote_' + str(trigger_id),
                          data,
                          version=2)
                return True
            else:
                logger.critical(e)
                return False
        except Exception as e:
            logger.critical(e)
            return False

    @staticmethod
    def _notebook(trigger, note_store):
        """
        :param trigger: trigger object
        :param note_store: note_store object
        :return: note object
        """
        note = Types.Note()
        if trigger.notebook:
            # get the notebookGUID ...
            notebook_id = get_notebook(note_store, trigger.notebook)
            # create notebookGUID if it does not exist then return its id
            note.notebookGuid = set_notebook(note_store,
                                             trigger.notebook,
                                             notebook_id)

            if trigger.tag:
                # ... and get the tagGUID if a tag has been provided
                tag_id = get_tag(note_store, trigger.tag)
                if tag_id is False:
                    tag_id = set_tag(note_store, trigger.tag, tag_id)
                    # set the tag to the note if a tag has been provided
                    note.tagGuids = tag_id

            logger.debug("notebook that will be used %s", trigger.notebook)
        return note

    @staticmethod
    def _attributes(note, data):
        """
        attribute of the note
        :param note: note object
        :param data:
        :return:
        """
        # attribute of the note: the link to the website
        note_attribute = set_note_attribute(data)
        if note_attribute:
            note.attributes = note_attribute
        return note

    @staticmethod
    def _footer(trigger, data, content):
        """
        footer of the note
        :param trigger: trigger object
        :param data: data to be used
        :param content: add the footer of the note to the content
        :return: content string
        """
        # footer of the note
        footer = set_note_footer(data, trigger)
        content += footer
        return content

    @staticmethod
    def _content(note, content):
        """
        content of the note
        :param note: note object
        :param content: content string to make the main body of the note
        :return:
        """
        note.content = set_header()
        note.content += sanitize(content)
        return note

    def get_evernote_client(self, token=None):
        """
            get the token from evernote
        """
        if token:
            return EvernoteClient(
                token=token,
                sandbox=self.sandbox)
        else:
            return EvernoteClient(
                consumer_key=self.consumer_key,
                consumer_secret=self.consumer_secret,
                sandbox=self.sandbox)

    def auth(self, request):
        """
            let's auth the user to the Service
        """
        client = self.get_evernote_client()
        request_token = client.get_request_token(
            self.callback_url(request))

        # Save the request token information for later
        request.session['oauth_token'] = request_token['oauth_token']
        request.session['oauth_token_secret'] = request_token[
            'oauth_token_secret']

        # Redirect the user to the Evernote authorization URL
        # return the URL string which will be used by redirect()
        # from the calling func
        return client.get_authorize_url(request_token)

    def callback(self, request, **kwargs):
        """
            Called from the Service when the user accept to activate it
        """
        try:
            client = self.get_evernote_client()
            # finally we save the user auth token
            # As we already stored the object ServicesActivated
            # from the UserServiceCreateView now we update the same
            # object to the database so :
            # 1) we get the previous object
            us = UserService.objects.get(
                user=request.user,
                name=ServicesActivated.objects.get(name='ServiceEvernote'))
            # 2) then get the token
            us.token = client.get_access_token(
                request.session['oauth_token'],
                request.session['oauth_token_secret'],
                request.GET.get('oauth_verifier', '')
            )
            # 3) and save everything
            us.save()
        except KeyError:
            return '/'

        return 'evernote/callback.html'

    @staticmethod
    def _cleaning_content(data):

        data = data.replace(
            '<?xml version="1.0" encoding="UTF-8"?>\n<'
            '!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">'
            '\n<en-note>', '')
        data = data.replace('</en-note>', '')

        return data


1			# coding: utf-8
2			import arrow
3
4			# evernote API
5			from evernote.api.client import EvernoteClient
6			from evernote.edam.notestore import NoteStore
7			import evernote.edam.type.ttypes as Types
8			from evernote.edam.error.ttypes import EDAMSystemException
9			from evernote.edam.error.ttypes import EDAMErrorCode
10
11			# django classes
12			from django.conf import settings
13			from django.utils.log import getLogger
14			from django.core.cache import caches
15
16			# django_th classes
17			from django_th.services.services import ServicesMgr
18			from django_th.models import UserService, ServicesActivated
19			from th_evernote.models import Evernote
20			from th_evernote.evernote_mgr import create_note, set_notebook, get_notebook, \
21			set_header, set_tag, get_tag, set_note_attribute, set_note_footer
22			from th_evernote.sanitize import sanitize
23
24			"""
25			handle process with evernote
26			put the following in settings.py
27
28			TH_EVERNOTE = {
29			'sandbox': True,
30			'consumer_key': 'abcdefghijklmnopqrstuvwxyz',
31			'consumer_secret': 'abcdefghijklmnopqrstuvwxyz',
32			}
33			sanbox set to True to make your test and False for production purpose
34
35			TH_SERVICES = (
36			...
37			'th_evernote.my_evernote.ServiceEvernote',
38			...
39			)
40			"""
41
42			logger = getLogger('django_th.trigger_happy')
43
44			cache = caches['th_evernote']
45
46
47			class ServiceEvernote(ServicesMgr):
48
49			def __init__(self, token=None, **kwargs):
50			super(ServiceEvernote, self).__init__(token, **kwargs)
51			self.sandbox = settings.TH_EVERNOTE['sandbox']
52			self.consumer_key = settings.TH_EVERNOTE['consumer_key']
53			self.consumer_secret = settings.TH_EVERNOTE['consumer_secret']
54			self.token = token
55			self.service = 'ServiceEvernote'
56			self.oauth = 'oauth1'
57
58			kwargs = {'consumer_key': self.consumer_key,
59			'consumer_secret': self.consumer_secret,
60			'sandbox': self.sandbox}
61
62			if self.token:
63			kwargs = {'token': token, 'sandbox': self.sandbox}
64
65			self.client = EvernoteClient(**kwargs)
66
67			def read_data(self, **kwargs):
68			"""
69			get the data from the service
70
71			:param kwargs: contain keyword args : trigger_id at least
72			:type kwargs: dict
73
74			:rtype: list
75			"""
76			date_triggered = kwargs.get('date_triggered')
77			trigger_id = kwargs.get('trigger_id')
78
79			kwargs['model_name'] = 'Evernote'
80
81			trigger = super(ServiceEvernote, self).read_data(**kwargs)
82
83			data = []
84			new_date_triggered = arrow.get(str(date_triggered)[:-6],
85			'YYYY-MM-DD HH:mm:ss')
86			new_date_triggered = str(new_date_triggered).replace(
87			':', '').replace('-', '').replace(' ', '')
88			date_filter = "created:{} ".format(new_date_triggered[:-6])
89
90			notebook_filter = ''
91			if trigger.notebook:
92			notebook_filter = "notebook:{} ".format(trigger.notebook)
93			tag_filter = "tag:{} ".format(trigger.tag) if trigger.tag != '' else ''
94
95			complet_filter = ''.join((notebook_filter, tag_filter, date_filter))
96
97			# filter
98			my_filter = NoteStore.NoteFilter()
99			my_filter.words = complet_filter
100
101			spec = NoteStore.NotesMetadataResultSpec()
102			spec.includeTitle = True
103			spec.includeAttributes = True
104
105			note_store = self.client.get_note_store()
106			our_note_list = note_store.findNotesMetadata(self.token,
107			my_filter,
108			0,
109			100,
110			spec)
111
112			for note in our_note_list.notes:
113			whole_note = note_store.getNote(self.token,
114			note.guid,
115			True,
116			True,
117			False,
118			False)
119			content = self._cleaning_content(whole_note.content)
120			data.append(
121			{'title': note.title,
122			'my_date': arrow.get(note.created),
123			'link': whole_note.attributes.sourceURL,
124			'content': content})
125
126			cache.set('th_evernote_' + str(trigger_id), data)
127
128			return data
129
130			def save_data(self, trigger_id, **data):
131			"""
132			let's save the data
133			don't want to handle empty title nor content
134			otherwise this will produce an Exception by
135			the Evernote's API
136
137			:param trigger_id: trigger ID from which to save data
138			:param data: the data to check to be used and save
139			:type trigger_id: int
140			:type data: dict
141			:return: the status of the save statement
142			:rtype: boolean
143			"""
144			kwargs = {}
145			# set the title and content of the data
146			title, content = super(ServiceEvernote, self).save_data(trigger_id,
147			data,
148			**kwargs)
149
150			if len(title):
151			# get the evernote data of this trigger
152			trigger = Evernote.objects.get(trigger_id=trigger_id)
153			# initialize notestore process
154			note_store = self._notestore(trigger_id, data)
155			if note_store is False:
156			return False
157			# note object
158			note = self._notebook(trigger, note_store)
159			# its attributes
160			note = self._attributes(note, data)
161			# its footer
162			content = self._footer(trigger, data, content)
163			# its title
164			note.title = title
165			# its content
166			note = self._content(note, content)
167			# create a note
168			return create_note(note_store, note, trigger_id, data)
169
170			else:
171			logger.critical("no title provided "
172			"for trigger ID {}".format(trigger_id))
173			return False
174
175			def _notestore(self, trigger_id, data):
176			try:
177			note_store = self.client.get_note_store()
178			return note_store
179			except EDAMSystemException as e:
180			# rate limit reach have to wait 1 hour !
181			if e.errorCode == EDAMErrorCode.RATE_LIMIT_REACHED:
182			logger.warn("Rate limit reached {code}\n"
183			"Retry your request in {msg} seconds\n"
184			"Data set to cache again until"
185			" limit reached".format(code=e.errorCode,
186			msg=e.rateLimitDuration)
187			)
188			cache.set('th_evernote_' + str(trigger_id),
189			data,
190			version=2)
191			return True
192			else:
193			logger.critical(e)
194			return False
195			except Exception as e:
196			logger.critical(e)
197			return False
198
199			@staticmethod
200			def _notebook(trigger, note_store):
201			"""
202			:param trigger: trigger object
203			:param note_store: note_store object
204			:return: note object
205			"""
206			note = Types.Note()
207			if trigger.notebook:
208			# get the notebookGUID ...
209			notebook_id = get_notebook(note_store, trigger.notebook)
210			# create notebookGUID if it does not exist then return its id
211			note.notebookGuid = set_notebook(note_store,
212			trigger.notebook,
213			notebook_id)
214
215			if trigger.tag:
216			# ... and get the tagGUID if a tag has been provided
217			tag_id = get_tag(note_store, trigger.tag)
218			if tag_id is False:
219			tag_id = set_tag(note_store, trigger.tag, tag_id)
220			# set the tag to the note if a tag has been provided
221			note.tagGuids = tag_id
222
223			logger.debug("notebook that will be used %s", trigger.notebook)
224			return note
225
226			@staticmethod
227			def _attributes(note, data):
228			"""
229			attribute of the note
230			:param note: note object
231			:param data:
232			:return:
233			"""
234			# attribute of the note: the link to the website
235			note_attribute = set_note_attribute(data)
236			if note_attribute:
237			note.attributes = note_attribute
238			return note
239
240			@staticmethod
241			def _footer(trigger, data, content):
242			"""
243			footer of the note
244			:param trigger: trigger object
245			:param data: data to be used
246			:param content: add the footer of the note to the content
247			:return: content string
248			"""
249			# footer of the note
250			footer = set_note_footer(data, trigger)
251			content += footer
252			return content
253
254			@staticmethod
255			def _content(note, content):
256			"""
257			content of the note
258			:param note: note object
259			:param content: content string to make the main body of the note
260			:return:
261			"""
262			note.content = set_header()
263			note.content += sanitize(content)
264			return note
265
266			def get_evernote_client(self, token=None):
267			"""
268			get the token from evernote
269			"""
270			if token:
271			return EvernoteClient(
272			token=token,
273			sandbox=self.sandbox)
274			else:
275			return EvernoteClient(
276			consumer_key=self.consumer_key,
277			consumer_secret=self.consumer_secret,
278			sandbox=self.sandbox)
279
280			def auth(self, request):
281			"""
282			let's auth the user to the Service
283			"""
284			client = self.get_evernote_client()
285			request_token = client.get_request_token(
286			self.callback_url(request))
287
288			# Save the request token information for later
289			request.session['oauth_token'] = request_token['oauth_token']
290			request.session['oauth_token_secret'] = request_token[
291			'oauth_token_secret']
292
293			# Redirect the user to the Evernote authorization URL
294			# return the URL string which will be used by redirect()
295			# from the calling func
296			return client.get_authorize_url(request_token)
297
298			def callback(self, request, **kwargs):
299			"""
300			Called from the Service when the user accept to activate it
301			"""
302			try:
303			client = self.get_evernote_client()
304			# finally we save the user auth token
305			# As we already stored the object ServicesActivated
306			# from the UserServiceCreateView now we update the same
307			# object to the database so :
308			# 1) we get the previous object
309			us = UserService.objects.get(
310			user=request.user,
311			name=ServicesActivated.objects.get(name='ServiceEvernote'))
312			# 2) then get the token
313			us.token = client.get_access_token(
314			request.session['oauth_token'],
315			request.session['oauth_token_secret'],
316			request.GET.get('oauth_verifier', '')
317			)
318			# 3) and save everything
319			us.save()
320			except KeyError:
321			return '/'
322
323			return 'evernote/callback.html'
324
325			@staticmethod
326			def _cleaning_content(data):
327
328			data = data.replace(
329			'<?xml version="1.0" encoding="UTF-8"?>\n<'
330			'!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">'
331			'\n<en-note>', '')
332			data = data.replace('</en-note>', '')
333
334			return data
335

foxmask / django-th

Push — master ( b9effa...31e28e )

ServiceEvernote._cleaning_content() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like