check_and_update_feed() - Code Metrics - Inspection of "Enhances feed entry printing in debug mode." - dupgit/versions - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 0787d3...710b5a )

by dup

created 2017-06-28 20:41 UTC

check_and_update_feed() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	7
Bugs	0	Features	0

Metric	Value
cc	3
c	7
b	0
f	0
dl	0
loc	23
rs	9.0856

#!/usr/bin/env python
# -*- coding: utf8 -*-
#
#  versions.py : checks releases and versions of programs through RSS
#                or Atom feeds and tells you
#
#  (C) Copyright 2016 Olivier Delhomme
#  e-mail : [email protected]
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2, or (at your option)
#  any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
import codecs
import feedparser
import yaml
import argparse
import os
import errno
import time
import doctest


__author__ = "Olivier Delhomme <[email protected]>"
__date__ = "16.04.2017"
__version__ = "0.0.4"

"""
This program checks projects versions throught RSS and Atom feeds and
should only print those with new release version.

It implements checking for projects in github.com and freshcode.club.
Projects must be added to a YAML file (named by default
~/.config/versions/versions.yaml). One can use --file=FILENAME option
to specify an alternative YAML file.
github projects must be listed under a "github.com:" section and
freshcode ones must be listed under a "freshcode.club:" section.

Versions uses and produces text files. Those files are cache files
written into ~/.local/versions directory. "*.cache" are cache files
containing the project list and their associated version (the latest).
"*.feed" are information feed cache files containing on only one line
the latest parsed post of the feed.
"""



class Conf:
    """
    Class to store configuration of the program
    """

    config_dir = ''
    local_dir = ''
    config_filename = ''
    description = {}
    options = None

    def __init__(self):
        """
        Inits the class
        """
        self.config_dir = os.path.expanduser("~/.config/versions")
        self.local_dir = os.path.expanduser("~/.local/versions")
        self.config_filename = ''  # At this stage we do not know if a filename has been set on the command line
        self.description = {}
        self.options = None

        # Make sure that the directories exists
        make_directories(self.config_dir)
        make_directories(self.local_dir)

        self._get_command_line_arguments()

    # End of init() function


    def load_yaml_from_config_file(self, filename):
        """
        Loads definitions from the YAML config file filename
        """

        config_file = codecs.open(filename, 'r', encoding='utf-8')

        self.description = yaml.safe_load(config_file)

        config_file.close()

    # End of load_yaml_from_config_file() function


    def _get_command_line_arguments(self):
        """
        Defines and gets all the arguments for the command line using
        argparse module. This function is called in the __init__ function
        of this class.
        """
        str_version = 'versions.py - %s' % __version__

        parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds', version=str_version)

        parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='versions.yaml')
        parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False)
        parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False)

        self.options = parser.parse_args()
        self.config_filename = os.path.join(self.config_dir, self.options.filename)

    # End of get_command_line_arguments() function
# End of Conf class


class FileCache:
    """
    This class should help in managing cache files
    """

    cache_filename = ''
    cache_dict = {}  # Dictionnary of projects and their associated version

    def __init__(self, local_dir, filename):
        """
        Inits the class. 'local_dir' must be a directory where we want to
        store the cache file named 'filename'
        """

        self.cache_filename = os.path.join(local_dir, filename)
        self.cache_dict = {}
        self._read_cache_file()

    # End of __init__() function


    def _return_project_and_version_from_line(self, line):
        """
        Splits the line into a project and a version if possible (the line
        must contain a whitespace.
        """

        line = line.strip()

        if line.count(' ') > 0:
            (project, version) = line.split(' ', 1)

        elif line != '':
            project = line
            version = ''

        return (project, version)

    # End of _return_project_and_version_from_line() function


    def _read_cache_file(self):
        """
        Reads the cache file and puts it into a dictionnary of project with
        their associated version
        """

        if os.path.isfile(self.cache_filename):
            cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')

            for line in cache_file:
                (project, version) = self._return_project_and_version_from_line(line)
                self.cache_dict[project] = version

            cache_file.close()

    # End of _read_cache_file() function


    def write_cache_file(self):
        """
        Owerwrites dictionnary cache to the cache file
        """

        cache_file = open_and_truncate_file(self.cache_filename)

        for (project, version) in self.cache_dict.iteritems():
            cache_file.write('%s %s\n' % (project, version))

        cache_file.close()

    # End of write_cache_file() function


    def update_cache_dict(self, project, version, debug):
        """
        Updates cache dictionnary if needed
        """

        try:
            version_cache = self.cache_dict[project]
            print_debug(debug, '\t\tIn cache: {}'.format(version_cache))

            if version != version_cache:
                print('%s %s' % (project, version))
                self.cache_dict[project] = version

        except KeyError:
            print('%s %s' % (project, version))
            self.cache_dict[project] = version

    # End of update_cache_dict() function


    def print_cache_dict(self, sitename):
        """
        Pretty prints the cache dictionary as it is recorded in the files.
        """

        print('%s:' % sitename)

        # Gets project and version tuple sorted by project lowered while sorting
        for project, version in sorted(self.cache_dict.iteritems(), key=lambda proj: proj[0].lower()):
            print('\t%s %s' % (project, version))

        print('')

    # End of print_cache_dict() function
# End of FileCache class


class FeedCache:

    cache_filename = ''
    year = 2016
    month = 05
    day = 1
    hour = 0
    minute = 0
    date_minutes = 0


    def __init__(self, local_dir, filename):
        """
        Inits the class. 'local_dir' must be a directory where we want to
        store the cache file named 'filename'
        """

        self.cache_filename = os.path.join(local_dir, filename)
        self.read_cache_feed()

    # End of __init__() function


    def read_cache_feed(self):
        """
        Reads the cache file which should only contain one date on the
        first line
        """

        if os.path.isfile(self.cache_filename):
            cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
            (self.year, self.month, self.day, self.hour, self.minute) = cache_file.readline().strip().split(' ', 4)
            self.date_minutes = self._calculate_minutes(int(self.year), int(self.month), int(self.day), int(self.hour), int(self.minute))
            cache_file.close()

    # End of read_cache_feed() function


    def write_cache_feed(self):
        """
        Overwrites the cache file with values stored in this class
        """
        cache_file = open_and_truncate_file(self.cache_filename)

        cache_file.write('%s %s %s %s %s' % (self.year, self.month, self.day, self.hour, self.minute))

        cache_file.close()

    # End of write_cache_feed() function


    def update_cache_feed(self, date):
        """
        Updates the values stored in the class with the date which should
        be a time.struct_time
        """

        self.year = date.tm_year
        self.month = date.tm_mon
        self.day = date.tm_mday
        self.hour = date.tm_hour
        self.minute = date.tm_min
        self.date_minutes = self._calculate_minutes_from_date(date)

    # End of update_cache_feed() function


    def _calculate_minutes(self, year, mon, day, hour, mins):
        """
        Calculate a number of minutes with all parameters and returns
        this.
        >>> fc = FeedCache('localdir','filename')
        >>> fc._calculate_minutes(2016, 5, 1, 0, 0)
        1059827040
        """

        minutes = (year * 365 * 24 * 60) + \
                  (mon * 30 * 24 * 60) + \
                  (day * 24 * 60) + \
                  (hour * 60) + \
                  (mins)

        return minutes

    # End of _calculate_minutes() function


    def _calculate_minutes_from_date(self, date):
        """
        Transforms a date in a number of minutes to ease comparisons
        and returns this number of minutes
        """

        return self._calculate_minutes(date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min)

    # End of _calculate_minutes() function


    def is_newer(self, date):
        """
        Tells wether "date" is newer than the one in the cache (returns True
        or not (returns False)
        """

        minutes = self._calculate_minutes_from_date(date)

        if minutes > self.date_minutes:
            return True

        else:
            return False

    # End of is_newer() function
# End of FeedCache class


######## Below are some utility functions used by classes above ########

def make_directories(path):
    """
    Makes all directories in path if possible. It is not an error if
    path already exists.
    """

    try:
        os.makedirs(path)

    except OSError as exc:

        if exc.errno != errno.EEXIST or os.path.isdir(path) != True:
            raise

# End of make_directories() function


def open_and_truncate_file(filename):
    """
    Opens filename for writing truncating it to a zero length file
    and returns a python file object.
    """

    cache_file = codecs.open(filename, 'w', encoding='utf-8')
    cache_file.truncate(0)
    cache_file.flush()

    return cache_file

# End of open_and_truncate_file() function
####################### End of utility functions #######################


def get_latest_github_release(program, debug):
    """
    Gets the latest release of a program on github. program must be a
    string of the form user/repository.
    """

    version = ''
    url = 'https://github.com/' + program + '/releases.atom'
    feed = feedparser.parse(url)

    if len(feed.entries) > 0:
        version = feed.entries[0].title

    print_debug(debug, '\tProject {}: {}'.format(program, version))

    return version

# End of get_latest_github_release() function


def check_versions_for_github_projects(github_project_list, local_dir, debug):
    """
    Checks project's versions on github if any are defined in the yaml
    file under the github.com tag.
    """

    github_cache = FileCache(local_dir, 'github.cache')

    for project in github_project_list:
        version = get_latest_github_release(project, debug)
        if version != '':
            github_cache.update_cache_dict(project, version, debug)

    github_cache.write_cache_file()

# End of check_versions_for_github_projects() function


def make_list_of_newer_feeds(feed, feed_info, debug):
    """
    Compares feed entries and keep those that are newer than the latest
    check we've done and inserting the newer ones in reverse order in
    a list to be returned
    """

    feed_list = []

    # inserting into a list in reverse order to keep the most recent
    # version in case of multiple release of the same project in the
    # feeds
    for a_feed in feed.entries:
        (project, version) = a_feed.title.strip().split(' ', 1)
        print_debug(debug, u'\tFeed entry ({0}): project: {1:16} version: {2}'.format(time.strftime('%x %X', a_feed.published_parsed), project, version))
        if feed_info.is_newer(a_feed.published_parsed):
            feed_list.insert(0, a_feed)

    return feed_list

# End of make_list_of_newer_feeds() function


def lower_list_of_strings(project_list):
    """
    Lowers every string in the list to ease sorting and comparisons
    """

    project_list_low = [project.lower() for project in project_list]

    return project_list_low

# End of lower_list_of_strings() function


def check_and_update_feed(feed_list, project_list, cache, debug):
    """
    Checks every feed entry in the list against project list cache and
    then updates the dictionnary then writes the cache file to the disk.
     - feed_list    is a list of feed (from feedparser module)
     - project_list is the list of project as read from the yaml
                    configuration file
     - cache is an initialized instance of FileCache
    """

    # Lowers the list before searching in it
    project_list_low = lower_list_of_strings(project_list)

    # Checking every feed entry that are newer than the last check
    # and updates the dictionnary accordingly
    for entry in feed_list:
        (project, version) = entry.title.strip().split(' ', 1)
        print_debug(debug, '\tChecking {}: {}'.format(project, version))

        if project.lower() in project_list_low:
            cache.update_cache_dict(project, version, debug)

    cache.write_cache_file()

# End of check_and_update_feed()


def check_versions_for_freshcode(freshcode_project_list, local_dir, debug):
    """
    Checks projects with freshcode's web site's RSS
    """

    freshcode_cache = FileCache(local_dir, 'freshcode.cache')

    url = 'http://freshcode.club/projects.rss'
    feed = feedparser.parse(url)

    feed_info = FeedCache(local_dir, 'freshcode.feed')
    feed_info.read_cache_feed()

    length = len(feed.entries)

    if length > 0:
        print_debug(debug, '\tFound {} entries'.format(length))

        feed_list = make_list_of_newer_feeds(feed, feed_info, debug)
        length = len(feed_list)
        print_debug(debug, '\tFound {} new entries (relative to {})'.format(length, feed_info.date_minutes))

        check_and_update_feed(feed_list, freshcode_project_list, freshcode_cache, debug)

        # Updating feed_info with the latest parsed feed entry date
        feed_info.update_cache_feed(feed.entries[0].published_parsed)

    else:
        print_debug(debug, 'No entries found in feed')

    feed_info.write_cache_feed()

# End of check_versions_for_freshcode() function


def print_versions_from_cache(local_dir, debug):
    """
    Prints all projects and their associated data from the cache
    """

    github_cache = FileCache(local_dir, 'github.cache')
    freshcode_cache = FileCache(local_dir, 'freshcode.cache')

    github_cache.print_cache_dict('Github')
    freshcode_cache.print_cache_dict('Freshcode')

# End of print_versions_from_cache()


def print_cache_or_check_versions(versions_conf):
    """
    Decide to pretty print projects and their associated version that
    are already in the cache or to check versions of that projects upon
    selections made at the command line
    """

    debug = versions_conf.options.debug
    print_debug(debug, 'Loading yaml config file')
    versions_conf.load_yaml_from_config_file(versions_conf.config_filename)

    if versions_conf.options.list_cache == True:
        # Pretty prints all caches.
        print_versions_from_cache(versions_conf.local_dir, debug)

    else:
        # Checks projects from github
        print_debug(debug, 'Checking github prolects')
        #check_versions_for_github_projects(versions_conf.description['github.com'], versions_conf.local_dir, debug)

        # Checks projects from freshcode.club
        print_debug(debug, 'Checking freshcode updates')
        check_versions_for_freshcode(versions_conf.description['freshcode.club'], versions_conf.local_dir, debug)

# End of print_list_or_check_versions() function.


def main():
    """
    This is the where the program begins
    """

    versions_conf = Conf()  # Configuration options

    if versions_conf.options.debug:
        doctest.testmod(verbose=True)

    if os.path.isfile(versions_conf.config_filename):
        print_cache_or_check_versions(versions_conf)

    else:
        print('Error: file %s does not exist' % config_filename)

# End of main() function


def print_debug(debug, message):
    """
    Prints 'message' if debug mode is True
    """

    if debug:
        print(message)

# End of print_debug() function


if __name__=="__main__" :
    main()


1			#!/usr/bin/env python
2			# -- coding: utf8 --
3			#
4			# versions.py : checks releases and versions of programs through RSS
5			# or Atom feeds and tells you
6			#
7			# (C) Copyright 2016 Olivier Delhomme
8			# e-mail : [email protected]
9			#
10			# This program is free software; you can redistribute it and/or modify
11			# it under the terms of the GNU General Public License as published by
12			# the Free Software Foundation; either version 2, or (at your option)
13			# any later version.
14			#
15			# This program is distributed in the hope that it will be useful,
16			# but WITHOUT ANY WARRANTY; without even the implied warranty of
17			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18			# GNU General Public License for more details.
19			#
20			# You should have received a copy of the GNU General Public License
21			# along with this program; if not, write to the Free Software Foundation,
22			# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23			#
24			import codecs
25			import feedparser
26			import yaml
27			import argparse
28			import os
29			import errno
30			import time
31			import doctest
32
33
34			__author__ = "Olivier Delhomme <[email protected]>"
35			__date__ = "16.04.2017"
36			__version__ = "0.0.4"
37
38			"""
39			This program checks projects versions throught RSS and Atom feeds and
40			should only print those with new release version.
41
42			It implements checking for projects in github.com and freshcode.club.
43			Projects must be added to a YAML file (named by default
44			~/.config/versions/versions.yaml). One can use --file=FILENAME option
45			to specify an alternative YAML file.
46			github projects must be listed under a "github.com:" section and
47			freshcode ones must be listed under a "freshcode.club:" section.
48
49			Versions uses and produces text files. Those files are cache files
50			written into ~/.local/versions directory. "*.cache" are cache files
51			containing the project list and their associated version (the latest).
52			"*.feed" are information feed cache files containing on only one line
53			the latest parsed post of the feed.
54			"""
55
56
57
58			class Conf:
59			"""
60			Class to store configuration of the program
61			"""
62
63			config_dir = ''
64			local_dir = ''
65			config_filename = ''
66			description = {}
67			options = None
68
69			def __init__(self):
70			"""
71			Inits the class
72			"""
73			self.config_dir = os.path.expanduser("~/.config/versions")
74			self.local_dir = os.path.expanduser("~/.local/versions")
75			self.config_filename = '' # At this stage we do not know if a filename has been set on the command line
76			self.description = {}
77			self.options = None
78
79			# Make sure that the directories exists
80			make_directories(self.config_dir)
81			make_directories(self.local_dir)
82
83			self._get_command_line_arguments()
84
85			# End of init() function
86
87
88			def load_yaml_from_config_file(self, filename):
89			"""
90			Loads definitions from the YAML config file filename
91			"""
92
93			config_file = codecs.open(filename, 'r', encoding='utf-8')
94
95			self.description = yaml.safe_load(config_file)
96
97			config_file.close()
98
99			# End of load_yaml_from_config_file() function
100
101
102			def _get_command_line_arguments(self):
103			"""
104			Defines and gets all the arguments for the command line using
105			argparse module. This function is called in the __init__ function
106			of this class.
107			"""
108			str_version = 'versions.py - %s' % __version__
109
110			parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds', version=str_version)
111
112			parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='versions.yaml')
113			parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False)
114			parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False)
115
116			self.options = parser.parse_args()
117			self.config_filename = os.path.join(self.config_dir, self.options.filename)
118
119			# End of get_command_line_arguments() function
120			# End of Conf class
121
122
123			class FileCache:
124			"""
125			This class should help in managing cache files
126			"""
127
128			cache_filename = ''
129			cache_dict = {} # Dictionnary of projects and their associated version
130
131			def __init__(self, local_dir, filename):
132			"""
133			Inits the class. 'local_dir' must be a directory where we want to
134			store the cache file named 'filename'
135			"""
136
137			self.cache_filename = os.path.join(local_dir, filename)
138			self.cache_dict = {}
139			self._read_cache_file()
140
141			# End of __init__() function
142
143
144			def _return_project_and_version_from_line(self, line):
145			"""
146			Splits the line into a project and a version if possible (the line
147			must contain a whitespace.
148			"""
149
150			line = line.strip()
151
152			if line.count(' ') > 0:
153			(project, version) = line.split(' ', 1)
154
155			elif line != '':
156			project = line
157			version = ''
158
159			return (project, version)
160
161			# End of _return_project_and_version_from_line() function
162
163
164			def _read_cache_file(self):
165			"""
166			Reads the cache file and puts it into a dictionnary of project with
167			their associated version
168			"""
169
170			if os.path.isfile(self.cache_filename):
171			cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
172
173			for line in cache_file:
174			(project, version) = self._return_project_and_version_from_line(line)
175			self.cache_dict[project] = version
176
177			cache_file.close()
178
179			# End of _read_cache_file() function
180
181
182			def write_cache_file(self):
183			"""
184			Owerwrites dictionnary cache to the cache file
185			"""
186
187			cache_file = open_and_truncate_file(self.cache_filename)
188
189			for (project, version) in self.cache_dict.iteritems():
190			cache_file.write('%s %s\n' % (project, version))
191
192			cache_file.close()
193
194			# End of write_cache_file() function
195
196
197			def update_cache_dict(self, project, version, debug):
198			"""
199			Updates cache dictionnary if needed
200			"""
201
202			try:
203			version_cache = self.cache_dict[project]
204			print_debug(debug, '\t\tIn cache: {}'.format(version_cache))
205
206			if version != version_cache:
207			print('%s %s' % (project, version))
208			self.cache_dict[project] = version
209
210			except KeyError:
211			print('%s %s' % (project, version))
212			self.cache_dict[project] = version
213
214			# End of update_cache_dict() function
215
216
217			def print_cache_dict(self, sitename):
218			"""
219			Pretty prints the cache dictionary as it is recorded in the files.
220			"""
221
222			print('%s:' % sitename)
223
224			# Gets project and version tuple sorted by project lowered while sorting
225			for project, version in sorted(self.cache_dict.iteritems(), key=lambda proj: proj[0].lower()):
226			print('\t%s %s' % (project, version))
227
228			print('')
229
230			# End of print_cache_dict() function
231			# End of FileCache class
232
233
234			class FeedCache:
235
236			cache_filename = ''
237			year = 2016
238			month = 05
239			day = 1
240			hour = 0
241			minute = 0
242			date_minutes = 0
243
244
245			def __init__(self, local_dir, filename):
246			"""
247			Inits the class. 'local_dir' must be a directory where we want to
248			store the cache file named 'filename'
249			"""
250
251			self.cache_filename = os.path.join(local_dir, filename)
252			self.read_cache_feed()
253
254			# End of __init__() function
255
256
257			def read_cache_feed(self):
258			"""
259			Reads the cache file which should only contain one date on the
260			first line
261			"""
262
263			if os.path.isfile(self.cache_filename):
264			cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
265			(self.year, self.month, self.day, self.hour, self.minute) = cache_file.readline().strip().split(' ', 4)
266			self.date_minutes = self._calculate_minutes(int(self.year), int(self.month), int(self.day), int(self.hour), int(self.minute))
267			cache_file.close()
268
269			# End of read_cache_feed() function
270
271
272			def write_cache_feed(self):
273			"""
274			Overwrites the cache file with values stored in this class
275			"""
276			cache_file = open_and_truncate_file(self.cache_filename)
277
278			cache_file.write('%s %s %s %s %s' % (self.year, self.month, self.day, self.hour, self.minute))
279
280			cache_file.close()
281
282			# End of write_cache_feed() function
283
284
285			def update_cache_feed(self, date):
286			"""
287			Updates the values stored in the class with the date which should
288			be a time.struct_time
289			"""
290
291			self.year = date.tm_year
292			self.month = date.tm_mon
293			self.day = date.tm_mday
294			self.hour = date.tm_hour
295			self.minute = date.tm_min
296			self.date_minutes = self._calculate_minutes_from_date(date)
297
298			# End of update_cache_feed() function
299
300
301			def _calculate_minutes(self, year, mon, day, hour, mins):
302			"""
303			Calculate a number of minutes with all parameters and returns
304			this.
305			>>> fc = FeedCache('localdir','filename')
306			>>> fc._calculate_minutes(2016, 5, 1, 0, 0)
307			1059827040
308			"""
309
310			minutes = (year * 365 * 24 * 60) + \
311			(mon * 30 * 24 * 60) + \
312			(day * 24 * 60) + \
313			(hour * 60) + \
314			(mins)
315
316			return minutes
317
318			# End of _calculate_minutes() function
319
320
321			def _calculate_minutes_from_date(self, date):
322			"""
323			Transforms a date in a number of minutes to ease comparisons
324			and returns this number of minutes
325			"""
326
327			return self._calculate_minutes(date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min)
328
329			# End of _calculate_minutes() function
330
331
332			def is_newer(self, date):
333			"""
334			Tells wether "date" is newer than the one in the cache (returns True
335			or not (returns False)
336			"""
337
338			minutes = self._calculate_minutes_from_date(date)
339
340			if minutes > self.date_minutes:
341			return True
342
343			else:
344			return False
345
346			# End of is_newer() function
347			# End of FeedCache class
348
349
350			######## Below are some utility functions used by classes above ########
351
352			def make_directories(path):
353			"""
354			Makes all directories in path if possible. It is not an error if
355			path already exists.
356			"""
357
358			try:
359			os.makedirs(path)
360
361			except OSError as exc:
362
363			if exc.errno != errno.EEXIST or os.path.isdir(path) != True:
364			raise
365
366			# End of make_directories() function
367
368
369			def open_and_truncate_file(filename):
370			"""
371			Opens filename for writing truncating it to a zero length file
372			and returns a python file object.
373			"""
374
375			cache_file = codecs.open(filename, 'w', encoding='utf-8')
376			cache_file.truncate(0)
377			cache_file.flush()
378
379			return cache_file
380
381			# End of open_and_truncate_file() function
382			####################### End of utility functions #######################
383
384
385			def get_latest_github_release(program, debug):
386			"""
387			Gets the latest release of a program on github. program must be a
388			string of the form user/repository.
389			"""
390
391			version = ''
392			url = 'https://github.com/' + program + '/releases.atom'
393			feed = feedparser.parse(url)
394
395			if len(feed.entries) > 0:
396			version = feed.entries[0].title
397
398			print_debug(debug, '\tProject {}: {}'.format(program, version))
399
400			return version
401
402			# End of get_latest_github_release() function
403
404
405			def check_versions_for_github_projects(github_project_list, local_dir, debug):
406			"""
407			Checks project's versions on github if any are defined in the yaml
408			file under the github.com tag.
409			"""
410
411			github_cache = FileCache(local_dir, 'github.cache')
412
413			for project in github_project_list:
414			version = get_latest_github_release(project, debug)
415			if version != '':
416			github_cache.update_cache_dict(project, version, debug)
417
418			github_cache.write_cache_file()
419
420			# End of check_versions_for_github_projects() function
421
422
423			def make_list_of_newer_feeds(feed, feed_info, debug):
424			"""
425			Compares feed entries and keep those that are newer than the latest
426			check we've done and inserting the newer ones in reverse order in
427			a list to be returned
428			"""
429
430			feed_list = []
431
432			# inserting into a list in reverse order to keep the most recent
433			# version in case of multiple release of the same project in the
434			# feeds
435			for a_feed in feed.entries:
436			(project, version) = a_feed.title.strip().split(' ', 1)
437			print_debug(debug, u'\tFeed entry ({0}): project: {1:16} version: {2}'.format(time.strftime('%x %X', a_feed.published_parsed), project, version))
438			if feed_info.is_newer(a_feed.published_parsed):
439			feed_list.insert(0, a_feed)
440
441			return feed_list
442
443			# End of make_list_of_newer_feeds() function
444
445
446			def lower_list_of_strings(project_list):
447			"""
448			Lowers every string in the list to ease sorting and comparisons
449			"""
450
451			project_list_low = [project.lower() for project in project_list]
452
453			return project_list_low
454
455			# End of lower_list_of_strings() function
456
457
458			def check_and_update_feed(feed_list, project_list, cache, debug):
459			"""
460			Checks every feed entry in the list against project list cache and
461			then updates the dictionnary then writes the cache file to the disk.
462			- feed_list is a list of feed (from feedparser module)
463			- project_list is the list of project as read from the yaml
464			configuration file
465			- cache is an initialized instance of FileCache
466			"""
467
468			# Lowers the list before searching in it
469			project_list_low = lower_list_of_strings(project_list)
470
471			# Checking every feed entry that are newer than the last check
472			# and updates the dictionnary accordingly
473			for entry in feed_list:
474			(project, version) = entry.title.strip().split(' ', 1)
475			print_debug(debug, '\tChecking {}: {}'.format(project, version))
476
477			if project.lower() in project_list_low:
478			cache.update_cache_dict(project, version, debug)
479
480			cache.write_cache_file()
481
482			# End of check_and_update_feed()
483
484
485			def check_versions_for_freshcode(freshcode_project_list, local_dir, debug):
486			"""
487			Checks projects with freshcode's web site's RSS
488			"""
489
490			freshcode_cache = FileCache(local_dir, 'freshcode.cache')
491
492			url = 'http://freshcode.club/projects.rss'
493			feed = feedparser.parse(url)
494
495			feed_info = FeedCache(local_dir, 'freshcode.feed')
496			feed_info.read_cache_feed()
497
498			length = len(feed.entries)
499
500			if length > 0:
501			print_debug(debug, '\tFound {} entries'.format(length))
502
503			feed_list = make_list_of_newer_feeds(feed, feed_info, debug)
504			length = len(feed_list)
505			print_debug(debug, '\tFound {} new entries (relative to {})'.format(length, feed_info.date_minutes))
506
507			check_and_update_feed(feed_list, freshcode_project_list, freshcode_cache, debug)
508
509			# Updating feed_info with the latest parsed feed entry date
510			feed_info.update_cache_feed(feed.entries[0].published_parsed)
511
512			else:
513			print_debug(debug, 'No entries found in feed')
514
515			feed_info.write_cache_feed()
516
517			# End of check_versions_for_freshcode() function
518
519
520			def print_versions_from_cache(local_dir, debug):
521			"""
522			Prints all projects and their associated data from the cache
523			"""
524
525			github_cache = FileCache(local_dir, 'github.cache')
526			freshcode_cache = FileCache(local_dir, 'freshcode.cache')
527
528			github_cache.print_cache_dict('Github')
529			freshcode_cache.print_cache_dict('Freshcode')
530
531			# End of print_versions_from_cache()
532
533
534			def print_cache_or_check_versions(versions_conf):
535			"""
536			Decide to pretty print projects and their associated version that
537			are already in the cache or to check versions of that projects upon
538			selections made at the command line
539			"""
540
541			debug = versions_conf.options.debug
542			print_debug(debug, 'Loading yaml config file')
543			versions_conf.load_yaml_from_config_file(versions_conf.config_filename)
544
545			if versions_conf.options.list_cache == True:
546			# Pretty prints all caches.
547			print_versions_from_cache(versions_conf.local_dir, debug)
548
549			else:
550			# Checks projects from github
551			print_debug(debug, 'Checking github prolects')
552			#check_versions_for_github_projects(versions_conf.description['github.com'], versions_conf.local_dir, debug)
553
554			# Checks projects from freshcode.club
555			print_debug(debug, 'Checking freshcode updates')
556			check_versions_for_freshcode(versions_conf.description['freshcode.club'], versions_conf.local_dir, debug)
557
558			# End of print_list_or_check_versions() function.
559
560
561			def main():
562			"""
563			This is the where the program begins
564			"""
565
566			versions_conf = Conf() # Configuration options
567
568			if versions_conf.options.debug:
569			doctest.testmod(verbose=True)
570
571			if os.path.isfile(versions_conf.config_filename):
572			print_cache_or_check_versions(versions_conf)
573
574			else:
575			print('Error: file %s does not exist' % config_filename)
576
577			# End of main() function
578
579
580			def print_debug(debug, message):
581			"""
582			Prints 'message' if debug mode is True
583			"""
584
585			if debug:
586			print(message)
587
588			# End of print_debug() function
589
590
591			if __name__=="__main__" :
592			main()
593

dupgit / versions

GitHub Access Token became invalid

Push — master ( 0787d3...710b5a )

check_and_update_feed() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like