check_and_update_feed() - Code Metrics - Inspection of "Corrects a bug that prevented updates from freshco..." - dupgit/versions - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( f4450d...afed55 )

by dup

created 2017-06-26 21:09 UTC

check_and_update_feed() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	6
Bugs	0	Features	0

Metric	Value
cc	3
c	6
b	0
f	0
dl	0
loc	24
rs	8.9713

#!/usr/bin/env python
# -*- coding: utf8 -*-
#
#  versions.py : checks releases and versions of programs through RSS
#                or Atom feeds and tells you
#
#  (C) Copyright 2016 Olivier Delhomme
#  e-mail : [email protected]
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2, or (at your option)
#  any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
import codecs
import feedparser
import yaml
import argparse
import os
import errno
import time


__author__ = "Olivier Delhomme <[email protected]>"
__date__ = "16.04.2017"
__version__ = "0.0.4"

"""
This program checks projects versions throught RSS and Atom feeds and
should only print those with new release version.

It implements checking for projects in github.com and freshcode.club.
Projects must be added to a YAML file (named by default
~/.config/versions/versions.yaml). One can use --file=FILENAME option
to specify an alternative YAML file.
github projects must be listed under a "github.com:" section and
freshcode ones must be listed under a "freshcode.club:" section.

Versions uses and produces text files. Those files are cache files
written into ~/.local/versions directory. "*.cache" are cache files
containing the project list and their associated version (the latest).
"*.feed" are information feed cache files containing on only one line
the latest parsed post of the feed.
"""



class Conf:
    """
    Class to store configuration of the program
    """

    config_dir = ''
    local_dir = ''
    config_filename = ''
    description = {}
    options = None

    def __init__(self):
        """
        Inits the class
        """
        self.config_dir = os.path.expanduser("~/.config/versions")
        self.local_dir = os.path.expanduser("~/.local/versions")
        self.config_filename = ''  # At this stage we do not know if a filename has been set on the command line
        self.description = {}
        self.options = None

        # Make sure that the directories exists
        make_directories(self.config_dir)
        make_directories(self.local_dir)

        self._get_command_line_arguments()

    # End of init() function


    def load_yaml_from_config_file(self, filename):
        """
        Loads definitions from the YAML config file filename
        """

        config_file = codecs.open(filename, 'r', encoding='utf-8')

        self.description = yaml.safe_load(config_file)

        config_file.close()

    # End of load_yaml_from_config_file() function


    def _get_command_line_arguments(self):
        """
        Defines and gets all the arguments for the command line using
        argparse module. This function is called in the __init__ function
        of this class.
        """
        str_version = 'versions.py - %s' % __version__

        parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds', version=str_version)

        parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='versions.yaml')
        parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False)
        parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False)

        self.options = parser.parse_args()
        self.config_filename = os.path.join(self.config_dir, self.options.filename)

    # End of get_command_line_arguments() function
# End of Conf class


class FileCache:
    """
    This class should help in managing cache files
    """

    cache_filename = ''
    cache_dict = {}  # Dictionnary of projects and their associated version

    def __init__(self, local_dir, filename):
        """
        Inits the class. 'local_dir' must be a directory where we want to
        store the cache file named 'filename'
        """

        self.cache_filename = os.path.join(local_dir, filename)
        self.cache_dict = {}
        self._read_cache_file()

    # End of __init__() function


    def _return_project_and_version_from_line(self, line):
        """
        Splits the line into a project and a version if possible (the line
        must contain a whitespace.
        """

        line = line.strip()

        if line.count(' ') > 0:
            (project, version) = line.split(' ', 1)

        elif line != '':
            project = line
            version = ''

        return (project, version)

    # End of _return_project_and_version_from_line() function


    def _read_cache_file(self):
        """
        Reads the cache file and puts it into a dictionnary of project with
        their associated version
        """

        if os.path.isfile(self.cache_filename):
            cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')

            for line in cache_file:
                (project, version) = self._return_project_and_version_from_line(line)
                self.cache_dict[project] = version

            cache_file.close()

    # End of _read_cache_file() function


    def write_cache_file(self):
        """
        Owerwrites dictionnary cache to the cache file
        """

        cache_file = open_and_truncate_file(self.cache_filename)

        for (project, version) in self.cache_dict.iteritems():
            cache_file.write('%s %s\n' % (project, version))

        cache_file.close()

    # End of write_cache_file() function


    def update_cache_dict(self, project, version, debug):
        """
        Updates cache dictionnary if needed
        """

        try:
            version_cache = self.cache_dict[project]
            debug_message = '\t\tIn cache: ' + version_cache
            print_debug(debug_message, debug)

            if version != version_cache:
                print('%s %s' % (project, version))
                self.cache_dict[project] = version

        except KeyError:
            print('%s %s' % (project, version))
            self.cache_dict[project] = version

    # End of update_cache_dict() function


    def print_cache_dict(self, sitename):
        """
        Pretty prints the cache dictionary as it is recorded in the files.
        """

        print('%s:' % sitename)

        # Gets project and version tuple sorted by project lowered while sorting
        for project, version in sorted(self.cache_dict.iteritems(), key=lambda proj: proj[0].lower()):
            print('\t%s %s' % (project, version))

        print('')

    # End of print_cache_dict() function
# End of FileCache class


class FeedCache:

    cache_filename = ''
    year = 2016
    month = 05
    day = 1
    hour = 0
    minute = 0
    date_minutes = 0


    def __init__(self, local_dir, filename):
        """
        Inits the class. 'local_dir' must be a directory where we want to
        store the cache file named 'filename'
        """

        self.cache_filename = os.path.join(local_dir, filename)
        self.read_cache_feed()

    # End of __init__() function


    def read_cache_feed(self):
        """
        Reads the cache file which should only contain one date on the
        first line
        """

        if os.path.isfile(self.cache_filename):
            cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
            (self.year, self.month, self.day, self.hour, self.minute) = cache_file.readline().strip().split(' ', 4)
            self.date_minutes = self._calculate_minutes(int(self.year), int(self.month), int(self.day), int(self.hour), int(self.minute))
            cache_file.close()

    # End of read_cache_feed() function


    def write_cache_feed(self):
        """
        Overwrites the cache file with values stored in this class
        """
        cache_file = open_and_truncate_file(self.cache_filename)

        cache_file.write('%s %s %s %s %s' % (self.year, self.month, self.day, self.hour, self.minute))

        cache_file.close()

    # End of write_cache_feed() function


    def update_cache_feed(self, date):
        """
        Updates the values stored in the class with the date which should
        be a time.struct_time
        """

        self.year = date.tm_year
        self.month = date.tm_mon
        self.day = date.tm_mday
        self.hour = date.tm_hour
        self.minute = date.tm_min
        self.date_minutes = self._calculate_minutes_from_date(date)

    # End of update_cache_feed() function


    def _calculate_minutes(self, year, mon, day, hour, mins):
        """
        Calculate a number of minutes with all parameters and returns
        this.
        >>> fc = FeedCache('localdir','filename')
        >>> fc._calculate_minutes(2016, 5, 1, 0, 0)
        1059827040
        """

        minutes = (year * 365 * 24 * 60) + \
                  (mon * 30 * 24 * 60) + \
                  (day * 24 * 60) + \
                  (hour * 60) + \
                  (mins)

        return minutes

    # End of _calculate_minutes() function


    def _calculate_minutes_from_date(self, date):
        """
        Transforms a date in a number of minutes to ease comparisons
        and returns this number of minutes
        """

        return self._calculate_minutes(date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min)

    # End of _calculate_minutes() function


    def is_newer(self, date):
        """
        Tells wether "date" is newer than the one in the cache (returns True
        or not (returns False)
        """

        minutes = self._calculate_minutes_from_date(date)

        if minutes > self.date_minutes:
            return True

        else:
            return False

    # End of is_newer() function
# End of FeedCache class


######## Below are some utility functions used by classes above ########

def make_directories(path):
    """
    Makes all directories in path if possible. It is not an error if
    path already exists.
    """

    try:
        os.makedirs(path)

    except OSError as exc:

        if exc.errno != errno.EEXIST or os.path.isdir(path) != True:
            raise

# End of make_directories() function


def open_and_truncate_file(filename):
    """
    Opens filename for writing truncating it to a zero length file
    and returns a python file object.
    """

    cache_file = codecs.open(filename, 'w', encoding='utf-8')
    cache_file.truncate(0)
    cache_file.flush()

    return cache_file

# End of open_and_truncate_file() function
####################### End of utility functions #######################


def get_latest_github_release(program, debug):
    """
    Gets the latest release of a program on github. program must be a
    string of the form user/repository.
    """

    version = ''
    url = 'https://github.com/' + program + '/releases.atom'
    feed = feedparser.parse(url)

    if len(feed.entries) > 0:
        version = feed.entries[0].title

    debug_message = '\tProject ' + program + ': ' + version
    print_debug(debug_message, debug)

    return version

# End of get_latest_github_release() function


def check_versions_for_github_projects(github_project_list, local_dir, debug):
    """
    Checks project's versions on github if any are defined in the yaml
    file under the github.com tag.
    """

    github_cache = FileCache(local_dir, 'github.cache')

    for project in github_project_list:
        version = get_latest_github_release(project, debug)
        if version != '':
            github_cache.update_cache_dict(project, version, debug)

    github_cache.write_cache_file()

# End of check_versions_for_github_projects() function


def make_list_of_newer_feeds(feed, feed_info, debug):
    """
    Compares feed entries and keep those that are newer than the latest
    check we've done and inserting the newer ones in reverse order in
    a list to be returned
    """

    feed_list = []

    # inserting into a list in reverse order to keep the most recent
    # version in case of multiple release of the same project in the
    # feeds
    for a_feed in feed.entries:
        debug_message = '\tEntry: %s - %s' % (a_feed.title.strip().split(' ', 1), time.strftime('%x %X', a_feed.published_parsed))
        print_debug(debug_message, debug)
        if feed_info.is_newer(a_feed.published_parsed):
            feed_list.insert(0, a_feed)

    return feed_list

# End of make_list_of_newer_feeds() function


def lower_list_of_strings(project_list):
    """
    Lowers every string in the list to ease sorting and comparisons
    """

    project_list_low = [project.lower() for project in project_list]

    return project_list_low

# End of lower_list_of_strings() function


def check_and_update_feed(feed_list, project_list, cache, debug):
    """
    Checks every feed entry in the list against project list cache and
    then updates the dictionnary then writes the cache file to the disk.
     - feed_list    is a list of feed (from feedparser module)
     - project_list is the list of project as read from the yaml
                    configuration file
     - cache is an initialized instance of FileCache
    """

    # Lowers the list before searching in it
    project_list_low = lower_list_of_strings(project_list)

    # Checking every feed entry that are newer than the last check
    # and updates the dictionnary accordingly
    for entry in feed_list:
        (project, version) = entry.title.strip().split(' ', 1)
        debug_message = '\tChecking %s: %s' % (project, version)
        print_debug(debug_message, debug)

        if project.lower() in project_list_low:
            cache.update_cache_dict(project, version, debug)

    cache.write_cache_file()

# End of check_and_update_feed()


def check_versions_for_freshcode(freshcode_project_list, local_dir, debug):
    """
    Checks projects with freshcode's web site's RSS
    """

    freshcode_cache = FileCache(local_dir, 'freshcode.cache')

    url = 'http://freshcode.club/projects.rss'
    feed = feedparser.parse(url)

    feed_info = FeedCache(local_dir, 'freshcode.feed')
    feed_info.read_cache_feed()

    length = len(feed.entries)

    if length > 0:
        debug_message = '\tFound %d entries' % length 
        print_debug(debug_message, debug)

        feed_list = make_list_of_newer_feeds(feed, feed_info, debug)
        length = len(feed_list)
        debug_message = '\tFound %d new entries (relative to %s)' % (length, feed_info.date_minutes)
        print_debug(debug_message, debug)

        check_and_update_feed(feed_list, freshcode_project_list, freshcode_cache, debug)

        # Updating feed_info with the latest parsed feed entry date
        feed_info.update_cache_feed(feed.entries[0].published_parsed)

    else:
        print_debug('No entries found in feed')

    feed_info.write_cache_feed()

# End of check_versions_for_freshcode() function


def print_versions_from_cache(local_dir, debug):
    """
    Prints all projects and their associated data from the cache
    """

    github_cache = FileCache(local_dir, 'github.cache')
    freshcode_cache = FileCache(local_dir, 'freshcode.cache')

    github_cache.print_cache_dict('Github')
    freshcode_cache.print_cache_dict('Freshcode')

# End of print_versions_from_cache()


def print_cache_or_check_versions(versions_conf):
    """
    Decide to pretty print projects and their associated version that
    are already in the cache or to check versions of that projects upon
    selections made at the command line
    """

    debug = versions_conf.options.debug
    print_debug('Loading yaml config file', debug)
    versions_conf.load_yaml_from_config_file(versions_conf.config_filename)

    if versions_conf.options.list_cache == True:
        # Pretty prints all caches.
        print_versions_from_cache(versions_conf.local_dir, debug)

    else:
        # Checks projects from github
        print_debug('Checking github prolects', debug)
        #check_versions_for_github_projects(versions_conf.description['github.com'], versions_conf.local_dir, debug)

        # Checks projects from freshcode.club
        print_debug('Checking freshcode updates', debug)
        check_versions_for_freshcode(versions_conf.description['freshcode.club'], versions_conf.local_dir, debug)

# End of print_list_or_check_versions() function.


def main():
    """
    This is the where the program begins
    """

    versions_conf = Conf()  # Configuration options

    if os.path.isfile(versions_conf.config_filename):
        print_cache_or_check_versions(versions_conf)

    else:
        print('Error: file %s does not exist' % config_filename)

# End of main() function


def print_debug(message, debug):
    """
    Prints 'message' if debug mode is True
    """

    if debug:
        print('%s' % message)

# End of print_debug() function


if __name__=="__main__" :
    import doctest
    doctest.testmod()
    main()


1			#!/usr/bin/env python
2			# -- coding: utf8 --
3			#
4			# versions.py : checks releases and versions of programs through RSS
5			# or Atom feeds and tells you
6			#
7			# (C) Copyright 2016 Olivier Delhomme
8			# e-mail : [email protected]
9			#
10			# This program is free software; you can redistribute it and/or modify
11			# it under the terms of the GNU General Public License as published by
12			# the Free Software Foundation; either version 2, or (at your option)
13			# any later version.
14			#
15			# This program is distributed in the hope that it will be useful,
16			# but WITHOUT ANY WARRANTY; without even the implied warranty of
17			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18			# GNU General Public License for more details.
19			#
20			# You should have received a copy of the GNU General Public License
21			# along with this program; if not, write to the Free Software Foundation,
22			# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23			#
24			import codecs
25			import feedparser
26			import yaml
27			import argparse
28			import os
29			import errno
30			import time
31
32
33			__author__ = "Olivier Delhomme <[email protected]>"
34			__date__ = "16.04.2017"
35			__version__ = "0.0.4"
36
37			"""
38			This program checks projects versions throught RSS and Atom feeds and
39			should only print those with new release version.
40
41			It implements checking for projects in github.com and freshcode.club.
42			Projects must be added to a YAML file (named by default
43			~/.config/versions/versions.yaml). One can use --file=FILENAME option
44			to specify an alternative YAML file.
45			github projects must be listed under a "github.com:" section and
46			freshcode ones must be listed under a "freshcode.club:" section.
47
48			Versions uses and produces text files. Those files are cache files
49			written into ~/.local/versions directory. "*.cache" are cache files
50			containing the project list and their associated version (the latest).
51			"*.feed" are information feed cache files containing on only one line
52			the latest parsed post of the feed.
53			"""
54
55
56
57			class Conf:
58			"""
59			Class to store configuration of the program
60			"""
61
62			config_dir = ''
63			local_dir = ''
64			config_filename = ''
65			description = {}
66			options = None
67
68			def __init__(self):
69			"""
70			Inits the class
71			"""
72			self.config_dir = os.path.expanduser("~/.config/versions")
73			self.local_dir = os.path.expanduser("~/.local/versions")
74			self.config_filename = '' # At this stage we do not know if a filename has been set on the command line
75			self.description = {}
76			self.options = None
77
78			# Make sure that the directories exists
79			make_directories(self.config_dir)
80			make_directories(self.local_dir)
81
82			self._get_command_line_arguments()
83
84			# End of init() function
85
86
87			def load_yaml_from_config_file(self, filename):
88			"""
89			Loads definitions from the YAML config file filename
90			"""
91
92			config_file = codecs.open(filename, 'r', encoding='utf-8')
93
94			self.description = yaml.safe_load(config_file)
95
96			config_file.close()
97
98			# End of load_yaml_from_config_file() function
99
100
101			def _get_command_line_arguments(self):
102			"""
103			Defines and gets all the arguments for the command line using
104			argparse module. This function is called in the __init__ function
105			of this class.
106			"""
107			str_version = 'versions.py - %s' % __version__
108
109			parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds', version=str_version)
110
111			parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='versions.yaml')
112			parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False)
113			parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False)
114
115			self.options = parser.parse_args()
116			self.config_filename = os.path.join(self.config_dir, self.options.filename)
117
118			# End of get_command_line_arguments() function
119			# End of Conf class
120
121
122			class FileCache:
123			"""
124			This class should help in managing cache files
125			"""
126
127			cache_filename = ''
128			cache_dict = {} # Dictionnary of projects and their associated version
129
130			def __init__(self, local_dir, filename):
131			"""
132			Inits the class. 'local_dir' must be a directory where we want to
133			store the cache file named 'filename'
134			"""
135
136			self.cache_filename = os.path.join(local_dir, filename)
137			self.cache_dict = {}
138			self._read_cache_file()
139
140			# End of __init__() function
141
142
143			def _return_project_and_version_from_line(self, line):
144			"""
145			Splits the line into a project and a version if possible (the line
146			must contain a whitespace.
147			"""
148
149			line = line.strip()
150
151			if line.count(' ') > 0:
152			(project, version) = line.split(' ', 1)
153
154			elif line != '':
155			project = line
156			version = ''
157
158			return (project, version)
159
160			# End of _return_project_and_version_from_line() function
161
162
163			def _read_cache_file(self):
164			"""
165			Reads the cache file and puts it into a dictionnary of project with
166			their associated version
167			"""
168
169			if os.path.isfile(self.cache_filename):
170			cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
171
172			for line in cache_file:
173			(project, version) = self._return_project_and_version_from_line(line)
174			self.cache_dict[project] = version
175
176			cache_file.close()
177
178			# End of _read_cache_file() function
179
180
181			def write_cache_file(self):
182			"""
183			Owerwrites dictionnary cache to the cache file
184			"""
185
186			cache_file = open_and_truncate_file(self.cache_filename)
187
188			for (project, version) in self.cache_dict.iteritems():
189			cache_file.write('%s %s\n' % (project, version))
190
191			cache_file.close()
192
193			# End of write_cache_file() function
194
195
196			def update_cache_dict(self, project, version, debug):
197			"""
198			Updates cache dictionnary if needed
199			"""
200
201			try:
202			version_cache = self.cache_dict[project]
203			debug_message = '\t\tIn cache: ' + version_cache
204			print_debug(debug_message, debug)
205
206			if version != version_cache:
207			print('%s %s' % (project, version))
208			self.cache_dict[project] = version
209
210			except KeyError:
211			print('%s %s' % (project, version))
212			self.cache_dict[project] = version
213
214			# End of update_cache_dict() function
215
216
217			def print_cache_dict(self, sitename):
218			"""
219			Pretty prints the cache dictionary as it is recorded in the files.
220			"""
221
222			print('%s:' % sitename)
223
224			# Gets project and version tuple sorted by project lowered while sorting
225			for project, version in sorted(self.cache_dict.iteritems(), key=lambda proj: proj[0].lower()):
226			print('\t%s %s' % (project, version))
227
228			print('')
229
230			# End of print_cache_dict() function
231			# End of FileCache class
232
233
234			class FeedCache:
235
236			cache_filename = ''
237			year = 2016
238			month = 05
239			day = 1
240			hour = 0
241			minute = 0
242			date_minutes = 0
243
244
245			def __init__(self, local_dir, filename):
246			"""
247			Inits the class. 'local_dir' must be a directory where we want to
248			store the cache file named 'filename'
249			"""
250
251			self.cache_filename = os.path.join(local_dir, filename)
252			self.read_cache_feed()
253
254			# End of __init__() function
255
256
257			def read_cache_feed(self):
258			"""
259			Reads the cache file which should only contain one date on the
260			first line
261			"""
262
263			if os.path.isfile(self.cache_filename):
264			cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
265			(self.year, self.month, self.day, self.hour, self.minute) = cache_file.readline().strip().split(' ', 4)
266			self.date_minutes = self._calculate_minutes(int(self.year), int(self.month), int(self.day), int(self.hour), int(self.minute))
267			cache_file.close()
268
269			# End of read_cache_feed() function
270
271
272			def write_cache_feed(self):
273			"""
274			Overwrites the cache file with values stored in this class
275			"""
276			cache_file = open_and_truncate_file(self.cache_filename)
277
278			cache_file.write('%s %s %s %s %s' % (self.year, self.month, self.day, self.hour, self.minute))
279
280			cache_file.close()
281
282			# End of write_cache_feed() function
283
284
285			def update_cache_feed(self, date):
286			"""
287			Updates the values stored in the class with the date which should
288			be a time.struct_time
289			"""
290
291			self.year = date.tm_year
292			self.month = date.tm_mon
293			self.day = date.tm_mday
294			self.hour = date.tm_hour
295			self.minute = date.tm_min
296			self.date_minutes = self._calculate_minutes_from_date(date)
297
298			# End of update_cache_feed() function
299
300
301			def _calculate_minutes(self, year, mon, day, hour, mins):
302			"""
303			Calculate a number of minutes with all parameters and returns
304			this.
305			>>> fc = FeedCache('localdir','filename')
306			>>> fc._calculate_minutes(2016, 5, 1, 0, 0)
307			1059827040
308			"""
309
310			minutes = (year * 365 * 24 * 60) + \
311			(mon * 30 * 24 * 60) + \
312			(day * 24 * 60) + \
313			(hour * 60) + \
314			(mins)
315
316			return minutes
317
318			# End of _calculate_minutes() function
319
320
321			def _calculate_minutes_from_date(self, date):
322			"""
323			Transforms a date in a number of minutes to ease comparisons
324			and returns this number of minutes
325			"""
326
327			return self._calculate_minutes(date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min)
328
329			# End of _calculate_minutes() function
330
331
332			def is_newer(self, date):
333			"""
334			Tells wether "date" is newer than the one in the cache (returns True
335			or not (returns False)
336			"""
337
338			minutes = self._calculate_minutes_from_date(date)
339
340			if minutes > self.date_minutes:
341			return True
342
343			else:
344			return False
345
346			# End of is_newer() function
347			# End of FeedCache class
348
349
350			######## Below are some utility functions used by classes above ########
351
352			def make_directories(path):
353			"""
354			Makes all directories in path if possible. It is not an error if
355			path already exists.
356			"""
357
358			try:
359			os.makedirs(path)
360
361			except OSError as exc:
362
363			if exc.errno != errno.EEXIST or os.path.isdir(path) != True:
364			raise
365
366			# End of make_directories() function
367
368
369			def open_and_truncate_file(filename):
370			"""
371			Opens filename for writing truncating it to a zero length file
372			and returns a python file object.
373			"""
374
375			cache_file = codecs.open(filename, 'w', encoding='utf-8')
376			cache_file.truncate(0)
377			cache_file.flush()
378
379			return cache_file
380
381			# End of open_and_truncate_file() function
382			####################### End of utility functions #######################
383
384
385			def get_latest_github_release(program, debug):
386			"""
387			Gets the latest release of a program on github. program must be a
388			string of the form user/repository.
389			"""
390
391			version = ''
392			url = 'https://github.com/' + program + '/releases.atom'
393			feed = feedparser.parse(url)
394
395			if len(feed.entries) > 0:
396			version = feed.entries[0].title
397
398			debug_message = '\tProject ' + program + ': ' + version
399			print_debug(debug_message, debug)
400
401			return version
402
403			# End of get_latest_github_release() function
404
405
406			def check_versions_for_github_projects(github_project_list, local_dir, debug):
407			"""
408			Checks project's versions on github if any are defined in the yaml
409			file under the github.com tag.
410			"""
411
412			github_cache = FileCache(local_dir, 'github.cache')
413
414			for project in github_project_list:
415			version = get_latest_github_release(project, debug)
416			if version != '':
417			github_cache.update_cache_dict(project, version, debug)
418
419			github_cache.write_cache_file()
420
421			# End of check_versions_for_github_projects() function
422
423
424			def make_list_of_newer_feeds(feed, feed_info, debug):
425			"""
426			Compares feed entries and keep those that are newer than the latest
427			check we've done and inserting the newer ones in reverse order in
428			a list to be returned
429			"""
430
431			feed_list = []
432
433			# inserting into a list in reverse order to keep the most recent
434			# version in case of multiple release of the same project in the
435			# feeds
436			for a_feed in feed.entries:
437			debug_message = '\tEntry: %s - %s' % (a_feed.title.strip().split(' ', 1), time.strftime('%x %X', a_feed.published_parsed))
438			print_debug(debug_message, debug)
439			if feed_info.is_newer(a_feed.published_parsed):
440			feed_list.insert(0, a_feed)
441
442			return feed_list
443
444			# End of make_list_of_newer_feeds() function
445
446
447			def lower_list_of_strings(project_list):
448			"""
449			Lowers every string in the list to ease sorting and comparisons
450			"""
451
452			project_list_low = [project.lower() for project in project_list]
453
454			return project_list_low
455
456			# End of lower_list_of_strings() function
457
458
459			def check_and_update_feed(feed_list, project_list, cache, debug):
460			"""
461			Checks every feed entry in the list against project list cache and
462			then updates the dictionnary then writes the cache file to the disk.
463			- feed_list is a list of feed (from feedparser module)
464			- project_list is the list of project as read from the yaml
465			configuration file
466			- cache is an initialized instance of FileCache
467			"""
468
469			# Lowers the list before searching in it
470			project_list_low = lower_list_of_strings(project_list)
471
472			# Checking every feed entry that are newer than the last check
473			# and updates the dictionnary accordingly
474			for entry in feed_list:
475			(project, version) = entry.title.strip().split(' ', 1)
476			debug_message = '\tChecking %s: %s' % (project, version)
477			print_debug(debug_message, debug)
478
479			if project.lower() in project_list_low:
480			cache.update_cache_dict(project, version, debug)
481
482			cache.write_cache_file()
483
484			# End of check_and_update_feed()
485
486
487			def check_versions_for_freshcode(freshcode_project_list, local_dir, debug):
488			"""
489			Checks projects with freshcode's web site's RSS
490			"""
491
492			freshcode_cache = FileCache(local_dir, 'freshcode.cache')
493
494			url = 'http://freshcode.club/projects.rss'
495			feed = feedparser.parse(url)
496
497			feed_info = FeedCache(local_dir, 'freshcode.feed')
498			feed_info.read_cache_feed()
499
500			length = len(feed.entries)
501
502			if length > 0:
503			debug_message = '\tFound %d entries' % length
504			print_debug(debug_message, debug)
505
506			feed_list = make_list_of_newer_feeds(feed, feed_info, debug)
507			length = len(feed_list)
508			debug_message = '\tFound %d new entries (relative to %s)' % (length, feed_info.date_minutes)
509			print_debug(debug_message, debug)
510
511			check_and_update_feed(feed_list, freshcode_project_list, freshcode_cache, debug)
512
513			# Updating feed_info with the latest parsed feed entry date
514			feed_info.update_cache_feed(feed.entries[0].published_parsed)
515
516			else:
517			print_debug('No entries found in feed')
518
519			feed_info.write_cache_feed()
520
521			# End of check_versions_for_freshcode() function
522
523
524			def print_versions_from_cache(local_dir, debug):
525			"""
526			Prints all projects and their associated data from the cache
527			"""
528
529			github_cache = FileCache(local_dir, 'github.cache')
530			freshcode_cache = FileCache(local_dir, 'freshcode.cache')
531
532			github_cache.print_cache_dict('Github')
533			freshcode_cache.print_cache_dict('Freshcode')
534
535			# End of print_versions_from_cache()
536
537
538			def print_cache_or_check_versions(versions_conf):
539			"""
540			Decide to pretty print projects and their associated version that
541			are already in the cache or to check versions of that projects upon
542			selections made at the command line
543			"""
544
545			debug = versions_conf.options.debug
546			print_debug('Loading yaml config file', debug)
547			versions_conf.load_yaml_from_config_file(versions_conf.config_filename)
548
549			if versions_conf.options.list_cache == True:
550			# Pretty prints all caches.
551			print_versions_from_cache(versions_conf.local_dir, debug)
552
553			else:
554			# Checks projects from github
555			print_debug('Checking github prolects', debug)
556			#check_versions_for_github_projects(versions_conf.description['github.com'], versions_conf.local_dir, debug)
557
558			# Checks projects from freshcode.club
559			print_debug('Checking freshcode updates', debug)
560			check_versions_for_freshcode(versions_conf.description['freshcode.club'], versions_conf.local_dir, debug)
561
562			# End of print_list_or_check_versions() function.
563
564
565			def main():
566			"""
567			This is the where the program begins
568			"""
569
570			versions_conf = Conf() # Configuration options
571
572			if os.path.isfile(versions_conf.config_filename):
573			print_cache_or_check_versions(versions_conf)
574
575			else:
576			print('Error: file %s does not exist' % config_filename)
577
578			# End of main() function
579
580
581			def print_debug(message, debug):
582			"""
583			Prints 'message' if debug mode is True
584			"""
585
586			if debug:
587			print('%s' % message)
588
589			# End of print_debug() function
590
591
592			if __name__=="__main__" :
593			import doctest
594			doctest.testmod()
595			main()
596

dupgit / versions

GitHub Access Token became invalid

Push — master ( f4450d...afed55 )

check_and_update_feed() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like