get_latest_release_by_title() - Code Metrics - Inspection of "New release." - dupgit/versions - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 329397...e64093 )

by dup

created 2017-10-12 19:58 UTC

get_latest_release_by_title() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	8
CRAP Score	2

Importance

Changes

Metric	Value
cc	2
c	0
b	0
f	0
dl	0
loc	16
ccs	8
cts	8
cp	1
crap	2
rs	9.4285

#!/usr/bin/env python
# -*- coding: utf8 -*-
#
#  versions.py : checks releases and versions of programs through RSS
#                or Atom feeds and tells you
#
#  (C) Copyright 2016 Olivier Delhomme
#  e-mail : [email protected]
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2, or (at your option)
#  any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
import codecs
import feedparser
import yaml
import argparse
import os
import errno
import time
import doctest


__author__ = "Olivier Delhomme <[email protected]>"
__date__ = "12.10.2017"
__version__ = "0.0.5"

"""
This program checks projects versions throught RSS and Atom feeds and
should only print those with new release version.

It implements checking for projects in github.com and freshcode.club.
Projects must be added to a YAML file (named by default
~/.config/versions/versions.yaml). One can use --file=FILENAME option
to specify an alternative YAML file.
github projects must be listed under a "github.com:" section and
freshcode ones must be listed under a "freshcode.club:" section.

Versions uses and produces text files. Those files are cache files
written into ~/.local/versions directory. "*.cache" are cache files
containing the project list and their associated version (the latest).
"*.feed" are information feed cache files containing on only one line
the latest parsed post of the feed.
"""



class Conf:
    """
    Class to store configuration of the program
    """

    config_dir = ''
    local_dir = ''
    config_filename = ''
    description = {}
    options = None

    def __init__(self):
        """
        Inits the class
        """
        self.config_dir = os.path.expanduser("~/.config/versions")
        self.local_dir = os.path.expanduser("~/.local/versions")
        self.config_filename = ''  # At this stage we do not know if a filename has been set on the command line
        self.description = {}
        self.options = None

        # Make sure that the directories exists
        make_directories(self.config_dir)
        make_directories(self.local_dir)

        self._get_command_line_arguments()

    # End of init() function


    def load_yaml_from_config_file(self, filename):
        """
        Loads definitions from the YAML config file filename
        """

        config_file = codecs.open(filename, 'r', encoding='utf-8')

        self.description = yaml.safe_load(config_file)

        config_file.close()

    # End of load_yaml_from_config_file() function


    def _get_command_line_arguments(self):
        """
        Defines and gets all the arguments for the command line using
        argparse module. This function is called in the __init__ function
        of this class.
        """
        str_version = 'versions.py - %s' % __version__

        parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds', version=str_version)

        parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='')
        parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False)
        parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False)

        self.options = parser.parse_args()

        if self.options.filename != '':
            self.config_filename = self.options.filename
        else:
            self.config_filename = os.path.join(self.config_dir, 'versions.yaml')

    # End of get_command_line_arguments() function
# End of Conf class


class FileCache:
    """
    This class should help in managing cache files
    """

    cache_filename = ''
    cache_dict = {}  # Dictionnary of projects and their associated version

    def __init__(self, local_dir, filename):
        """
        Inits the class. 'local_dir' must be a directory where we want to
        store the cache file named 'filename'
        """

        self.cache_filename = os.path.join(local_dir, filename)
        self.cache_dict = {}
        self._read_cache_file()

    # End of __init__() function


    def _return_project_and_version_from_line(self, line):
        """
        Splits the line into a project and a version if possible (the line
        must contain a whitespace.
        """

        line = line.strip()

        if line.count(' ') > 0:
            (project, version) = line.split(' ', 1)

        elif line != '':
            project = line
            version = ''

        return (project, version)

    # End of _return_project_and_version_from_line() function


    def _read_cache_file(self):
        """
        Reads the cache file and puts it into a dictionnary of project with
        their associated version
        """

        if os.path.isfile(self.cache_filename):
            cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')

            for line in cache_file:
                (project, version) = self._return_project_and_version_from_line(line)
                self.cache_dict[project] = version

            cache_file.close()

    # End of _read_cache_file() function


    def write_cache_file(self):
        """
        Owerwrites dictionnary cache to the cache file
        """

        cache_file = open_and_truncate_file(self.cache_filename)

        for (project, version) in self.cache_dict.iteritems():
            cache_file.write('%s %s\n' % (project, version))

        cache_file.close()

    # End of write_cache_file() function


    def update_cache_dict(self, project, version, debug):
        """
        Updates cache dictionnary if needed
        """

        try:
            version_cache = self.cache_dict[project]
            print_debug(debug, u'\t\tIn cache: {}'.format(version_cache))

            if version != version_cache:
                print('%s %s' % (project, version))
                self.cache_dict[project] = version

        except KeyError:
            print('%s %s' % (project, version))
            self.cache_dict[project] = version

    # End of update_cache_dict() function


    def print_cache_dict(self, sitename):
        """
        Pretty prints the cache dictionary as it is recorded in the files.
        """

        print('%s:' % sitename)

        # Gets project and version tuple sorted by project lowered while sorting
        for project, version in sorted(self.cache_dict.iteritems(), key=lambda proj: proj[0].lower()):
            print('\t%s %s' % (project, version))

        print('')

    # End of print_cache_dict() function
# End of FileCache class


class FeedCache:

    cache_filename = ''
    year = 2016
    month = 05
    day = 1
    hour = 0
    minute = 0
    date_minutes = 0


    def __init__(self, local_dir, filename):
        """
        Inits the class. 'local_dir' must be a directory where we want to
        store the cache file named 'filename'
        """

        self.cache_filename = os.path.join(local_dir, filename)
        self.read_cache_feed()

    # End of __init__() function


    def read_cache_feed(self):
        """
        Reads the cache file which should only contain one date on the
        first line
        """

        if os.path.isfile(self.cache_filename):
            cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
            (self.year, self.month, self.day, self.hour, self.minute) = cache_file.readline().strip().split(' ', 4)
            self.date_minutes = self._calculate_minutes(int(self.year), int(self.month), int(self.day), int(self.hour), int(self.minute))
            cache_file.close()

    # End of read_cache_feed() function


    def write_cache_feed(self):
        """
        Overwrites the cache file with values stored in this class
        """
        cache_file = open_and_truncate_file(self.cache_filename)

        cache_file.write('%s %s %s %s %s' % (self.year, self.month, self.day, self.hour, self.minute))

        cache_file.close()

    # End of write_cache_feed() function


    def update_cache_feed(self, date):
        """
        Updates the values stored in the class with the date which should
        be a time.struct_time
        """

        self.year = date.tm_year
        self.month = date.tm_mon
        self.day = date.tm_mday
        self.hour = date.tm_hour
        self.minute = date.tm_min
        self.date_minutes = self._calculate_minutes_from_date(date)

    # End of update_cache_feed() function


    def _calculate_minutes(self, year, mon, day, hour, mins):
        """
        Calculate a number of minutes with all parameters and returns
        this.
        >>> fc = FeedCache('localdir','filename')
        >>> fc._calculate_minutes(2016, 5, 1, 0, 0)
        1059827040
        """

        minutes = (year * 365 * 24 * 60) + \
                  (mon * 30 * 24 * 60) + \
                  (day * 24 * 60) + \
                  (hour * 60) + \
                  (mins)

        return minutes

    # End of _calculate_minutes() function


    def _calculate_minutes_from_date(self, date):
        """
        Transforms a date in a number of minutes to ease comparisons
        and returns this number of minutes
        """

        return self._calculate_minutes(date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min)

    # End of _calculate_minutes() function


    def is_newer(self, date):
        """
        Tells wether "date" is newer than the one in the cache (returns True
        or not (returns False)
        """

        minutes = self._calculate_minutes_from_date(date)

        if minutes > self.date_minutes:
            return True

        else:
            return False

    # End of is_newer() function
# End of FeedCache class


######## Below are some utility functions used by classes above ########

def make_directories(path):
    """
    Makes all directories in path if possible. It is not an error if
    path already exists.
    """

    try:
        os.makedirs(path)

    except OSError as exc:

        if exc.errno != errno.EEXIST or os.path.isdir(path) is not True:
            raise

# End of make_directories() function


def open_and_truncate_file(filename):
    """
    Opens filename for writing truncating it to a zero length file
    and returns a python file object.
    """

    cache_file = codecs.open(filename, 'w', encoding='utf-8')
    cache_file.truncate(0)
    cache_file.flush()

    return cache_file

# End of open_and_truncate_file() function
####################### End of utility functions #######################


def get_latest_release_by_title(program, debug, feed_url):
    """
    Gets the latest release of a program on github. program must be a
    string of the form user/repository.
    """

    version = ''
    url = feed_url.format(program)     
    feed = feedparser.parse(url)

    if len(feed.entries) > 0:
        version = feed.entries[0].title

    print_debug(debug, u'\tProject {}: {}'.format(program, version))

    return version

# End of get_latest_github_release() function


def check_versions_feeds_by_projects(github_project_list, local_dir, debug, feed_url):
    """
    Checks project's versions on github if any are defined in the yaml
    file under the github.com tag.
    """

    github_cache = FileCache(local_dir, 'github.cache')

    for project in github_project_list:
        version = get_latest_release_by_title(project, debug, feed_url)
        if version != '':
            github_cache.update_cache_dict(project, version, debug)

    github_cache.write_cache_file()

# End of check_versions_for_github_projects() function


def make_list_of_newer_feeds(feed, feed_info, debug):
    """
    Compares feed entries and keep those that are newer than the latest
    check we've done and inserting the newer ones in reverse order in
    a list to be returned
    """

    feed_list = []

    # inserting into a list in reverse order to keep the most recent
    # version in case of multiple release of the same project in the
    # feeds
    for a_feed in feed.entries:
        (project, version) = a_feed.title.strip().split(' ', 1)
        print_debug(debug, u'\tFeed entry ({0}): project: {1:16} version: {2}'.format(time.strftime('%x %X', a_feed.published_parsed), project, version))
        if feed_info.is_newer(a_feed.published_parsed):
            feed_list.insert(0, a_feed)

    return feed_list

# End of make_list_of_newer_feeds() function


def lower_list_of_strings(project_list):
    """
    Lowers every string in the list to ease sorting and comparisons
    """

    project_list_low = [project.lower() for project in project_list]

    return project_list_low

# End of lower_list_of_strings() function


def check_and_update_feed(feed_list, project_list, cache, debug):
    """
    Checks every feed entry in the list against project list cache and
    then updates the dictionnary then writes the cache file to the disk.
     - feed_list    is a list of feed (from feedparser module)
     - project_list is the list of project as read from the yaml
                    configuration file
     - cache is an initialized instance of FileCache
    """

    # Lowers the list before searching in it
    project_list_low = lower_list_of_strings(project_list)

    # Checking every feed entry that are newer than the last check
    # and updates the dictionnary accordingly
    for entry in feed_list:
        (project, version) = entry.title.strip().split(' ', 1)
        print_debug(debug, u'\tChecking {0:16}: {1}'.format(project, version))

        if project.lower() in project_list_low:
            cache.update_cache_dict(project, version, debug)

    cache.write_cache_file()

# End of check_and_update_feed()


def check_versions_for_freshcode(freshcode_project_list, local_dir, debug):
    """
    Checks projects with freshcode's web site's RSS
    """

    freshcode_cache = FileCache(local_dir, 'freshcode.cache')

    url = 'http://freshcode.club/projects.rss'
    feed = feedparser.parse(url)

    feed_info = FeedCache(local_dir, 'freshcode.feed')
    feed_info.read_cache_feed()

    length = len(feed.entries)

    if length > 0:
        print_debug(debug, u'\tFound {} entries'.format(length))

        feed_list = make_list_of_newer_feeds(feed, feed_info, debug)
        print_debug(debug, u'\tFound {} new entries (relative to {})'.format(len(feed_list), feed_info.date_minutes))

        check_and_update_feed(feed_list, freshcode_project_list, freshcode_cache, debug)

        # Updating feed_info with the latest parsed feed entry date
        feed_info.update_cache_feed(feed.entries[0].published_parsed)

    else:
        print_debug(debug, u'No entries found in feed')

    feed_info.write_cache_feed()

# End of check_versions_for_freshcode() function


def print_versions_from_cache(local_dir, debug):
    """
    Prints all projects and their associated data from the cache
    """

    github_cache = FileCache(local_dir, 'github.cache')
    freshcode_cache = FileCache(local_dir, 'freshcode.cache')

    github_cache.print_cache_dict('Github')
    freshcode_cache.print_cache_dict('Freshcode')

# End of print_versions_from_cache()


def print_cache_or_check_versions(versions_conf):
    """
    Decide to pretty print projects and their associated version that
    are already in the cache or to check versions of that projects upon
    selections made at the command line
    """

    debug = versions_conf.options.debug
    print_debug(debug, u'Loading yaml config file')
    versions_conf.load_yaml_from_config_file(versions_conf.config_filename)

    if versions_conf.options.list_cache is True:
        # Pretty prints all caches.
        print_versions_from_cache(versions_conf.local_dir, debug)

    else:
        # Checks projects from sourceforge
        print_debug(debug, u'Checking sourceforge prolects')
        check_versions_feeds_by_projects(versions_conf.description['sourceforge.net'], versions_conf.local_dir, debug, 'https://sourceforge.net/projects/{}/rss?path=/')

        # Checks projects from github
        print_debug(debug, u'Checking github prolects')
        check_versions_feeds_by_projects(versions_conf.description['github.com'], versions_conf.local_dir, debug, 'https://github.com/{}/releases.atom')

        # Checks projects from freshcode.club
        print_debug(debug, u'Checking freshcode updates')
        check_versions_for_freshcode(versions_conf.description['freshcode.club'], versions_conf.local_dir, debug)

# End of print_list_or_check_versions() function.


def main():
    """
    This is the where the program begins
    """

    versions_conf = Conf()  # Configuration options

    if versions_conf.options.debug:
        doctest.testmod(verbose=True)

    if os.path.isfile(versions_conf.config_filename):
        print_cache_or_check_versions(versions_conf)

    else:
        print('Error: file %s does not exist' % versions_conf.config_filename)

# End of main() function


def print_debug(debug, message):
    """
    Prints 'message' if debug mode is True
    """

    if debug:
        print(message)

# End of print_debug() function


if __name__ == "__main__":
    main()


1		#!/usr/bin/env python
2		# -- coding: utf8 --
3		#
4		# versions.py : checks releases and versions of programs through RSS
5		# or Atom feeds and tells you
6		#
7		# (C) Copyright 2016 Olivier Delhomme
8		# e-mail : [email protected]
9		#
10		# This program is free software; you can redistribute it and/or modify
11		# it under the terms of the GNU General Public License as published by
12		# the Free Software Foundation; either version 2, or (at your option)
13		# any later version.
14		#
15		# This program is distributed in the hope that it will be useful,
16		# but WITHOUT ANY WARRANTY; without even the implied warranty of
17		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18		# GNU General Public License for more details.
19		#
20		# You should have received a copy of the GNU General Public License
21		# along with this program; if not, write to the Free Software Foundation,
22		# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23		#
24	1	import codecs
25	1	import feedparser
26	1	import yaml
27	1	import argparse
28	1	import os
29	1	import errno
30	1	import time
31	1	import doctest
32
33
34	1	__author__ = "Olivier Delhomme <[email protected]>"
35	1	__date__ = "12.10.2017"
36	1	__version__ = "0.0.5"
37
38		"""
39		This program checks projects versions throught RSS and Atom feeds and
40		should only print those with new release version.
41
42		It implements checking for projects in github.com and freshcode.club.
43		Projects must be added to a YAML file (named by default
44		~/.config/versions/versions.yaml). One can use --file=FILENAME option
45		to specify an alternative YAML file.
46		github projects must be listed under a "github.com:" section and
47		freshcode ones must be listed under a "freshcode.club:" section.
48
49		Versions uses and produces text files. Those files are cache files
50		written into ~/.local/versions directory. "*.cache" are cache files
51		containing the project list and their associated version (the latest).
52		"*.feed" are information feed cache files containing on only one line
53		the latest parsed post of the feed.
54		"""
55
56
57
58	1	class Conf:
59		"""
60		Class to store configuration of the program
61		"""
62
63	1	config_dir = ''
64	1	local_dir = ''
65	1	config_filename = ''
66	1	description = {}
67	1	options = None
68
69	1	def __init__(self):
70		"""
71		Inits the class
72		"""
73	1	self.config_dir = os.path.expanduser("~/.config/versions")
74	1	self.local_dir = os.path.expanduser("~/.local/versions")
75	1	self.config_filename = '' # At this stage we do not know if a filename has been set on the command line
76	1	self.description = {}
77	1	self.options = None
78
79		# Make sure that the directories exists
80	1	make_directories(self.config_dir)
81	1	make_directories(self.local_dir)
82
83	1	self._get_command_line_arguments()
84
85		# End of init() function
86
87
88	1	def load_yaml_from_config_file(self, filename):
89		"""
90		Loads definitions from the YAML config file filename
91		"""
92
93	1	config_file = codecs.open(filename, 'r', encoding='utf-8')
94
95	1	self.description = yaml.safe_load(config_file)
96
97	1	config_file.close()
98
99		# End of load_yaml_from_config_file() function
100
101
102	1	def _get_command_line_arguments(self):
103		"""
104		Defines and gets all the arguments for the command line using
105		argparse module. This function is called in the __init__ function
106		of this class.
107		"""
108	1	str_version = 'versions.py - %s' % __version__
109
110	1	parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds', version=str_version)
111
112	1	parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='')
113	1	parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False)
114	1	parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False)
115
116	1	self.options = parser.parse_args()
117
118	1	if self.options.filename != '':
119	1	self.config_filename = self.options.filename
120		else:
121		self.config_filename = os.path.join(self.config_dir, 'versions.yaml')
122
123		# End of get_command_line_arguments() function
124		# End of Conf class
125
126
127	1	class FileCache:
128		"""
129		This class should help in managing cache files
130		"""
131
132	1	cache_filename = ''
133	1	cache_dict = {} # Dictionnary of projects and their associated version
134
135	1	def __init__(self, local_dir, filename):
136		"""
137		Inits the class. 'local_dir' must be a directory where we want to
138		store the cache file named 'filename'
139		"""
140
141	1	self.cache_filename = os.path.join(local_dir, filename)
142	1	self.cache_dict = {}
143	1	self._read_cache_file()
144
145		# End of __init__() function
146
147
148	1	def _return_project_and_version_from_line(self, line):
149		"""
150		Splits the line into a project and a version if possible (the line
151		must contain a whitespace.
152		"""
153
154	1	line = line.strip()
155
156	1	if line.count(' ') > 0:
157	1	(project, version) = line.split(' ', 1)
158
159		elif line != '':
160		project = line
161		version = ''
162
163	1	return (project, version)
164
165		# End of _return_project_and_version_from_line() function
166
167
168	1	def _read_cache_file(self):
169		"""
170		Reads the cache file and puts it into a dictionnary of project with
171		their associated version
172		"""
173
174	1	if os.path.isfile(self.cache_filename):
175	1	cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
176
177	1	for line in cache_file:
178	1	(project, version) = self._return_project_and_version_from_line(line)
179	1	self.cache_dict[project] = version
180
181	1	cache_file.close()
182
183		# End of _read_cache_file() function
184
185
186	1	def write_cache_file(self):
187		"""
188		Owerwrites dictionnary cache to the cache file
189		"""
190
191	1	cache_file = open_and_truncate_file(self.cache_filename)
192
193	1	for (project, version) in self.cache_dict.iteritems():
194	1	cache_file.write('%s %s\n' % (project, version))
195
196	1	cache_file.close()
197
198		# End of write_cache_file() function
199
200
201	1	def update_cache_dict(self, project, version, debug):
202		"""
203		Updates cache dictionnary if needed
204		"""
205
206	1	try:
207	1	version_cache = self.cache_dict[project]
208		print_debug(debug, u'\t\tIn cache: {}'.format(version_cache))
209
210		if version != version_cache:
211		print('%s %s' % (project, version))
212		self.cache_dict[project] = version
213
214	1	except KeyError:
215	1	print('%s %s' % (project, version))
216	1	self.cache_dict[project] = version
217
218		# End of update_cache_dict() function
219
220
221	1	def print_cache_dict(self, sitename):
222		"""
223		Pretty prints the cache dictionary as it is recorded in the files.
224		"""
225
226		print('%s:' % sitename)
227
228		# Gets project and version tuple sorted by project lowered while sorting
229		for project, version in sorted(self.cache_dict.iteritems(), key=lambda proj: proj[0].lower()):
230		print('\t%s %s' % (project, version))
231
232		print('')
233
234		# End of print_cache_dict() function
235		# End of FileCache class
236
237
238	1	class FeedCache:
239
240	1	cache_filename = ''
241	1	year = 2016
242	1	month = 05
243	1	day = 1
244	1	hour = 0
245	1	minute = 0
246	1	date_minutes = 0
247
248
249	1	def __init__(self, local_dir, filename):
250		"""
251		Inits the class. 'local_dir' must be a directory where we want to
252		store the cache file named 'filename'
253		"""
254
255	1	self.cache_filename = os.path.join(local_dir, filename)
256	1	self.read_cache_feed()
257
258		# End of __init__() function
259
260
261	1	def read_cache_feed(self):
262		"""
263		Reads the cache file which should only contain one date on the
264		first line
265		"""
266
267	1	if os.path.isfile(self.cache_filename):
268		cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
269		(self.year, self.month, self.day, self.hour, self.minute) = cache_file.readline().strip().split(' ', 4)
270		self.date_minutes = self._calculate_minutes(int(self.year), int(self.month), int(self.day), int(self.hour), int(self.minute))
271		cache_file.close()
272
273		# End of read_cache_feed() function
274
275
276	1	def write_cache_feed(self):
277		"""
278		Overwrites the cache file with values stored in this class
279		"""
280	1	cache_file = open_and_truncate_file(self.cache_filename)
281
282	1	cache_file.write('%s %s %s %s %s' % (self.year, self.month, self.day, self.hour, self.minute))
283
284	1	cache_file.close()
285
286		# End of write_cache_feed() function
287
288
289	1	def update_cache_feed(self, date):
290		"""
291		Updates the values stored in the class with the date which should
292		be a time.struct_time
293		"""
294
295	1	self.year = date.tm_year
296	1	self.month = date.tm_mon
297	1	self.day = date.tm_mday
298	1	self.hour = date.tm_hour
299	1	self.minute = date.tm_min
300	1	self.date_minutes = self._calculate_minutes_from_date(date)
301
302		# End of update_cache_feed() function
303
304
305	1	def _calculate_minutes(self, year, mon, day, hour, mins):
306		"""
307		Calculate a number of minutes with all parameters and returns
308		this.
309		>>> fc = FeedCache('localdir','filename')
310		>>> fc._calculate_minutes(2016, 5, 1, 0, 0)
311		1059827040
312		"""
313
314	1	minutes = (year * 365 * 24 * 60) + \
315		(mon * 30 * 24 * 60) + \
316		(day * 24 * 60) + \
317		(hour * 60) + \
318		(mins)
319
320	1	return minutes
321
322		# End of _calculate_minutes() function
323
324
325	1	def _calculate_minutes_from_date(self, date):
326		"""
327		Transforms a date in a number of minutes to ease comparisons
328		and returns this number of minutes
329		"""
330
331	1	return self._calculate_minutes(date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min)
332
333		# End of _calculate_minutes() function
334
335
336	1	def is_newer(self, date):
337		"""
338		Tells wether "date" is newer than the one in the cache (returns True
339		or not (returns False)
340		"""
341
342	1	minutes = self._calculate_minutes_from_date(date)
343
344	1	if minutes > self.date_minutes:
345	1	return True
346
347		else:
348		return False
349
350		# End of is_newer() function
351		# End of FeedCache class
352
353
354		######## Below are some utility functions used by classes above ########
355
356	1	def make_directories(path):
357		"""
358		Makes all directories in path if possible. It is not an error if
359		path already exists.
360		"""
361
362	1	try:
363	1	os.makedirs(path)
364
365		except OSError as exc:
366
367		if exc.errno != errno.EEXIST or os.path.isdir(path) is not True:
368		raise
369
370		# End of make_directories() function
371
372
373	1	def open_and_truncate_file(filename):
374		"""
375		Opens filename for writing truncating it to a zero length file
376		and returns a python file object.
377		"""
378
379	1	cache_file = codecs.open(filename, 'w', encoding='utf-8')
380	1	cache_file.truncate(0)
381	1	cache_file.flush()
382
383	1	return cache_file
384
385		# End of open_and_truncate_file() function
386		####################### End of utility functions #######################
387
388
389	1	def get_latest_release_by_title(program, debug, feed_url):
390		"""
391		Gets the latest release of a program on github. program must be a
392		string of the form user/repository.
393		"""
394
395	1	version = ''
396	1	url = feed_url.format(program)
397	1	feed = feedparser.parse(url)
398
399	1	if len(feed.entries) > 0:
400	1	version = feed.entries[0].title
401
402	1	print_debug(debug, u'\tProject {}: {}'.format(program, version))
403
404	1	return version
405
406		# End of get_latest_github_release() function
407
408
409	1	def check_versions_feeds_by_projects(github_project_list, local_dir, debug, feed_url):
410		"""
411		Checks project's versions on github if any are defined in the yaml
412		file under the github.com tag.
413		"""
414
415	1	github_cache = FileCache(local_dir, 'github.cache')
416
417	1	for project in github_project_list:
418	1	version = get_latest_release_by_title(project, debug, feed_url)
419	1	if version != '':
420	1	github_cache.update_cache_dict(project, version, debug)
421
422	1	github_cache.write_cache_file()
423
424		# End of check_versions_for_github_projects() function
425
426
427	1	def make_list_of_newer_feeds(feed, feed_info, debug):
428		"""
429		Compares feed entries and keep those that are newer than the latest
430		check we've done and inserting the newer ones in reverse order in
431		a list to be returned
432		"""
433
434	1	feed_list = []
435
436		# inserting into a list in reverse order to keep the most recent
437		# version in case of multiple release of the same project in the
438		# feeds
439	1	for a_feed in feed.entries:
440	1	(project, version) = a_feed.title.strip().split(' ', 1)
441	1	print_debug(debug, u'\tFeed entry ({0}): project: {1:16} version: {2}'.format(time.strftime('%x %X', a_feed.published_parsed), project, version))
442	1	if feed_info.is_newer(a_feed.published_parsed):
443	1	feed_list.insert(0, a_feed)
444
445	1	return feed_list
446
447		# End of make_list_of_newer_feeds() function
448
449
450	1	def lower_list_of_strings(project_list):
451		"""
452		Lowers every string in the list to ease sorting and comparisons
453		"""
454
455	1	project_list_low = [project.lower() for project in project_list]
456
457	1	return project_list_low
458
459		# End of lower_list_of_strings() function
460
461
462	1	def check_and_update_feed(feed_list, project_list, cache, debug):
463		"""
464		Checks every feed entry in the list against project list cache and
465		then updates the dictionnary then writes the cache file to the disk.
466		- feed_list is a list of feed (from feedparser module)
467		- project_list is the list of project as read from the yaml
468		configuration file
469		- cache is an initialized instance of FileCache
470		"""
471
472		# Lowers the list before searching in it
473	1	project_list_low = lower_list_of_strings(project_list)
474
475		# Checking every feed entry that are newer than the last check
476		# and updates the dictionnary accordingly
477	1	for entry in feed_list:
478	1	(project, version) = entry.title.strip().split(' ', 1)
479	1	print_debug(debug, u'\tChecking {0:16}: {1}'.format(project, version))
480
481	1	if project.lower() in project_list_low:
482	1	cache.update_cache_dict(project, version, debug)
483
484	1	cache.write_cache_file()
485
486		# End of check_and_update_feed()
487
488
489	1	def check_versions_for_freshcode(freshcode_project_list, local_dir, debug):
490		"""
491		Checks projects with freshcode's web site's RSS
492		"""
493
494	1	freshcode_cache = FileCache(local_dir, 'freshcode.cache')
495
496	1	url = 'http://freshcode.club/projects.rss'
497	1	feed = feedparser.parse(url)
498
499	1	feed_info = FeedCache(local_dir, 'freshcode.feed')
500	1	feed_info.read_cache_feed()
501
502	1	length = len(feed.entries)
503
504	1	if length > 0:
505	1	print_debug(debug, u'\tFound {} entries'.format(length))
506
507	1	feed_list = make_list_of_newer_feeds(feed, feed_info, debug)
508	1	print_debug(debug, u'\tFound {} new entries (relative to {})'.format(len(feed_list), feed_info.date_minutes))
509
510	1	check_and_update_feed(feed_list, freshcode_project_list, freshcode_cache, debug)
511
512		# Updating feed_info with the latest parsed feed entry date
513	1	feed_info.update_cache_feed(feed.entries[0].published_parsed)
514
515		else:
516		print_debug(debug, u'No entries found in feed')
517
518	1	feed_info.write_cache_feed()
519
520		# End of check_versions_for_freshcode() function
521
522
523	1	def print_versions_from_cache(local_dir, debug):
524		"""
525		Prints all projects and their associated data from the cache
526		"""
527
528		github_cache = FileCache(local_dir, 'github.cache')
529		freshcode_cache = FileCache(local_dir, 'freshcode.cache')
530
531		github_cache.print_cache_dict('Github')
532		freshcode_cache.print_cache_dict('Freshcode')
533
534		# End of print_versions_from_cache()
535
536
537	1	def print_cache_or_check_versions(versions_conf):
538		"""
539		Decide to pretty print projects and their associated version that
540		are already in the cache or to check versions of that projects upon
541		selections made at the command line
542		"""
543
544	1	debug = versions_conf.options.debug
545	1	print_debug(debug, u'Loading yaml config file')
546	1	versions_conf.load_yaml_from_config_file(versions_conf.config_filename)
547
548	1	if versions_conf.options.list_cache is True:
549		# Pretty prints all caches.
550		print_versions_from_cache(versions_conf.local_dir, debug)
551
552		else:
553		# Checks projects from sourceforge
554	1	print_debug(debug, u'Checking sourceforge prolects')
555	1	check_versions_feeds_by_projects(versions_conf.description['sourceforge.net'], versions_conf.local_dir, debug, 'https://sourceforge.net/projects/{}/rss?path=/')
556
557		# Checks projects from github
558	1	print_debug(debug, u'Checking github prolects')
559	1	check_versions_feeds_by_projects(versions_conf.description['github.com'], versions_conf.local_dir, debug, 'https://github.com/{}/releases.atom')
560
561		# Checks projects from freshcode.club
562	1	print_debug(debug, u'Checking freshcode updates')
563	1	check_versions_for_freshcode(versions_conf.description['freshcode.club'], versions_conf.local_dir, debug)
564
565		# End of print_list_or_check_versions() function.
566
567
568	1	def main():
569		"""
570		This is the where the program begins
571		"""
572
573	1	versions_conf = Conf() # Configuration options
574
575	1	if versions_conf.options.debug:
576	1	doctest.testmod(verbose=True)
577
578	1	if os.path.isfile(versions_conf.config_filename):
579	1	print_cache_or_check_versions(versions_conf)
580
581		else:
582		print('Error: file %s does not exist' % versions_conf.config_filename)
583
584		# End of main() function
585
586
587	1	def print_debug(debug, message):
588		"""
589		Prints 'message' if debug mode is True
590		"""
591
592	1	if debug:
593	1	print(message)
594
595		# End of print_debug() function
596
597
598	1	if __name__ == "__main__":
599		main()
600

dupgit / versions

GitHub Access Token became invalid

Push — master ( 329397...e64093 )

get_latest_release_by_title() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like