check_versions_feeds_by_projects() - Code Metrics - Inspection of "Adds 'url', 'type' and 'projects' arguments to 'li..." - dupgit/versions - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 5562bc...7165b1 )

by dup

created 2017-10-21 19:09 UTC

check_versions_feeds_by_projects() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	7
CRAP Score	3

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
cc	3
c	2
b	0
f	0
dl	0
loc	14
ccs	7
cts	7
cp	1
crap	3
rs	9.4285

#!/usr/bin/env python
# -*- coding: utf8 -*-
#
#  versions.py : checks releases and versions of programs through RSS
#                or Atom feeds and tells you
#
#  (C) Copyright 2016 Olivier Delhomme
#  e-mail : [email protected]
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2, or (at your option)
#  any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
import codecs
import feedparser
import yaml
import argparse
import os
import errno
import time
import doctest


__author__ = "Olivier Delhomme <[email protected]>"
__date__ = "20.10.2017"
__version__ = "1.0.0"

"""
This program checks projects versions throught RSS and Atom feeds and
should only print those with new release version.

It implements checking for projects in github.com and freshcode.club.
Projects must be added to a YAML file (named by default
~/.config/versions/versions.yaml). One can use --file=FILENAME option
to specify an alternative YAML file.
github projects must be listed under a "github.com:" section and
freshcode ones must be listed under a "freshcode.club:" section.

Versions uses and produces text files. Those files are cache files
written into ~/.local/versions directory. "*.cache" are cache files
containing the project list and their associated version (the latest).
"*.feed" are information feed cache files containing on only one line
the latest parsed post of the feed.
"""


class Conf:
    """
    Class to store configuration of the program
    """

    config_dir = ''
    local_dir = ''
    config_filename = ''
    description = {}
    options = None

    def __init__(self):
        """
        Inits the class
        """
        self.config_dir = os.path.expanduser("~/.config/versions")
        self.local_dir = os.path.expanduser("~/.local/versions")
        self.config_filename = ''  # At this stage we do not know if a filename has been set on the command line
        self.description = {}
        self.options = None

        # Make sure that the directories exists
        make_directories(self.config_dir)
        make_directories(self.local_dir)

        self._get_command_line_arguments()

    # End of init() function


    def load_yaml_from_config_file(self, filename):
        """
        Loads definitions from the YAML config file filename
        """

        config_file = codecs.open(filename, 'r', encoding='utf-8')

        self.description = yaml.safe_load(config_file)

        config_file.close()

    # End of load_yaml_from_config_file() function


    def _get_command_line_arguments(self):
        """
        Defines and gets all the arguments for the command line using
        argparse module. This function is called in the __init__ function
        of this class.
        """
        str_version = 'versions.py - %s' % __version__

        parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds', version=str_version)

        parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='')
        parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False)
        parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False)

        self.options = parser.parse_args()

        if self.options.filename != '':
            self.config_filename = self.options.filename
        else:
            self.config_filename = os.path.join(self.config_dir, 'versions.yaml')

    # End of get_command_line_arguments() function

    def extract_site_definition(self, site_name):
        """
        extracts whole site definition
        """

        if site_name in self.description:
            return self.description[site_name]
        else:
            return dict()

    # End of extract_site_definition()


    def extract_project_list_from_site_def(self, site_name):
        """
        Extracts a project list from a site by project definition
        """

        site_definition = self.extract_site_definition(site_name)

        if 'projects' in site_definition:
            project_list = site_definition['projects']
        else:
            project_list = []

        return project_list

    # End of extract_project_list_from_site_def() function


    def extract_project_url(self, site_name):
        """
        Extracts the url definition where to check project version.
        """

        site_definition = self.extract_site_definition(site_name)

        if 'url' in site_definition:
            project_url = site_definition['url']
        else:
            project_url = ''

        return project_url

    # End of extract_project_url() function


    def is_site_of_type(self, site_name, type):
        """
        Returns True if site_name is of type 'type'
        """

        site_definition = self.extract_site_definition(site_name)
        if 'type' in site_definition:
            return (site_definition['type'] == type)
        else:
            return False

    # End of is_site_of_type() function


    def extract_site_list(self, type):
        """
        Extracts all sites from a specific type (byproject or list)
        """

        all_site_list = list(self.description.keys())
        site_list = []
        for site_name in all_site_list:
            if self.is_site_of_type(site_name, type):
                site_list.insert(0, site_name)
       
        return site_list

    # End of extract_site_list() function


    def get_site_cache_liste_name(self):
        """
        Formats list of cache filenames for all sites
        """

        all_site_list = list(self.description.keys())
        cache_list = []
        for site_name in all_site_list:
            site_cache = u'{}.cache'.format(site_name)
            cache_list.insert(0, site_cache)

        return cache_list

    # End of get_site_cache_liste_name() function

# End of Conf class


class FileCache:
    """
    This class should help in managing cache files
    """

    cache_filename = ''
    cache_dict = {}  # Dictionnary of projects and their associated version

    def __init__(self, local_dir, filename):
        """
        Inits the class. 'local_dir' must be a directory where we want to
        store the cache file named 'filename'
        """

        self.cache_filename = os.path.join(local_dir, filename)
        self.cache_dict = {}
        self._read_cache_file()

    # End of __init__() function


    def _return_project_and_version_from_line(self, line):
        """
        Splits the line into a project and a version if possible (the line
        must contain a whitespace.
        """

        line = line.strip()

        if line.count(' ') > 0:
            (project, version) = line.split(' ', 1)

        elif line != '':
            project = line
            version = ''

        return (project, version)

    # End of _return_project_and_version_from_line() function


    def _read_cache_file(self):
        """
        Reads the cache file and puts it into a dictionnary of project with
        their associated version
        """

        if os.path.isfile(self.cache_filename):
            cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')

            for line in cache_file:
                (project, version) = self._return_project_and_version_from_line(line)
                self.cache_dict[project] = version

            cache_file.close()

    # End of _read_cache_file() function


    def write_cache_file(self):
        """
        Owerwrites dictionnary cache to the cache file
        """

        cache_file = open_and_truncate_file(self.cache_filename)

        for (project, version) in self.cache_dict.iteritems():
            cache_file.write('%s %s\n' % (project, version))

        cache_file.close()

    # End of write_cache_file() function


    def update_cache_dict(self, project, version, debug):
        """
        Updates cache dictionnary if needed
        """

        try:
            version_cache = self.cache_dict[project]
            print_debug(debug, u'\t\tIn cache: {}'.format(version_cache))

            if version != version_cache:
                print('%s %s' % (project, version))
                self.cache_dict[project] = version

        except KeyError:
            print('%s %s' % (project, version))
            self.cache_dict[project] = version

    # End of update_cache_dict() function


    def print_cache_dict(self, sitename):
        """
        Pretty prints the cache dictionary as it is recorded in the files.
        """

        print('%s:' % sitename)

        # Gets project and version tuple sorted by project lowered while sorting
        for project, version in sorted(self.cache_dict.iteritems(), key=lambda proj: proj[0].lower()):
            print('\t%s %s' % (project, version))

        print('')

    # End of print_cache_dict() function
# End of FileCache class


class FeedCache:

    cache_filename = ''
    year = 2016
    month = 05
    day = 1
    hour = 0
    minute = 0
    date_minutes = 0


    def __init__(self, local_dir, filename):
        """
        Inits the class. 'local_dir' must be a directory where we want to
        store the cache file named 'filename'
        """

        self.cache_filename = os.path.join(local_dir, filename)
        self.read_cache_feed()

    # End of __init__() function


    def read_cache_feed(self):
        """
        Reads the cache file which should only contain one date on the
        first line
        """

        if os.path.isfile(self.cache_filename):
            cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
            (self.year, self.month, self.day, self.hour, self.minute) = cache_file.readline().strip().split(' ', 4)
            self.date_minutes = self._calculate_minutes(int(self.year), int(self.month), int(self.day), int(self.hour), int(self.minute))
            cache_file.close()

    # End of read_cache_feed() function


    def write_cache_feed(self):
        """
        Overwrites the cache file with values stored in this class
        """
        cache_file = open_and_truncate_file(self.cache_filename)

        cache_file.write('%s %s %s %s %s' % (self.year, self.month, self.day, self.hour, self.minute))

        cache_file.close()

    # End of write_cache_feed() function


    def update_cache_feed(self, date):
        """
        Updates the values stored in the class with the date which should
        be a time.struct_time
        """

        self.year = date.tm_year
        self.month = date.tm_mon
        self.day = date.tm_mday
        self.hour = date.tm_hour
        self.minute = date.tm_min
        self.date_minutes = self._calculate_minutes_from_date(date)

    # End of update_cache_feed() function


    def _calculate_minutes(self, year, mon, day, hour, mins):
        """
        Calculate a number of minutes with all parameters and returns
        this.
        >>> fc = FeedCache('localdir','filename')
        >>> fc._calculate_minutes(2016, 5, 1, 0, 0)
        1059827040
        """

        minutes = (year * 365 * 24 * 60) + \
                  (mon * 30 * 24 * 60) + \
                  (day * 24 * 60) + \
                  (hour * 60) + \
                  (mins)

        return minutes

    # End of _calculate_minutes() function


    def _calculate_minutes_from_date(self, date):
        """
        Transforms a date in a number of minutes to ease comparisons
        and returns this number of minutes
        """

        return self._calculate_minutes(date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min)

    # End of _calculate_minutes() function


    def is_newer(self, date):
        """
        Tells wether "date" is newer than the one in the cache (returns True
        or not (returns False)
        """

        minutes = self._calculate_minutes_from_date(date)

        if minutes > self.date_minutes:
            return True

        else:
            return False

    # End of is_newer() function
# End of FeedCache class


######## Below are some utility functions used by classes above ########


def make_directories(path):
    """
    Makes all directories in path if possible. It is not an error if
    path already exists.
    """

    try:
        os.makedirs(path)

    except OSError as exc:

        if exc.errno != errno.EEXIST or os.path.isdir(path) is not True:
            raise

# End of make_directories() function


def open_and_truncate_file(filename):
    """
    Opens filename for writing truncating it to a zero length file
    and returns a python file object.
    """

    cache_file = codecs.open(filename, 'w', encoding='utf-8')
    cache_file.truncate(0)
    cache_file.flush()

    return cache_file

# End of open_and_truncate_file() function
####################### End of utility functions #######################


def get_latest_release_by_title(program, debug, feed_url):
    """
    Gets the latest release of a program on github. program must be a
    string of the form user/repository.
    """

    version = ''
    url = feed_url.format(program)
    feed = feedparser.parse(url)

    if len(feed.entries) > 0:
        version = feed.entries[0].title

    print_debug(debug, u'\tProject {}: {}'.format(program, version))

    return version

# End of get_latest_github_release() function


def check_versions_feeds_by_projects(project_list, local_dir, debug, feed_url, cache_filename):
    """
    Checks project's versions on feed_url if any are defined in the yaml
    file under the specified tag that got the project_list passed as an argument.
    """

    site_cache = FileCache(local_dir, cache_filename)

    for project in project_list:
        version = get_latest_release_by_title(project, debug, feed_url)
        if version != '':
            site_cache.update_cache_dict(project, version, debug)

    site_cache.write_cache_file()

# End of check_versions_for_github_projects() function


def make_list_of_newer_feeds(feed, feed_info, debug):
    """
    Compares feed entries and keep those that are newer than the latest
    check we've done and inserting the newer ones in reverse order in
    a list to be returned
    """

    feed_list = []

    # inserting into a list in reverse order to keep the most recent
    # version in case of multiple release of the same project in the
    # feeds
    for a_feed in feed.entries:
        (project, version) = a_feed.title.strip().split(' ', 1)
        print_debug(debug, u'\tFeed entry ({0}): project: {1:16} version: {2}'.format(time.strftime('%x %X', a_feed.published_parsed), project, version))
        if feed_info.is_newer(a_feed.published_parsed):
            feed_list.insert(0, a_feed)

    return feed_list

# End of make_list_of_newer_feeds() function


def lower_list_of_strings(project_list):
    """
    Lowers every string in the list to ease sorting and comparisons
    """

    project_list_low = [project.lower() for project in project_list]

    return project_list_low

# End of lower_list_of_strings() function


def check_and_update_feed(feed_list, project_list, cache, debug):
    """
    Checks every feed entry in the list against project list cache and
    then updates the dictionnary then writes the cache file to the disk.
     - feed_list    is a list of feed (from feedparser module)
     - project_list is the list of project as read from the yaml
                    configuration file
     - cache is an initialized instance of FileCache
    """

    # Lowers the list before searching in it
    project_list_low = lower_list_of_strings(project_list)

    # Checking every feed entry that are newer than the last check
    # and updates the dictionnary accordingly
    for entry in feed_list:
        (project, version) = entry.title.strip().split(' ', 1)
        print_debug(debug, u'\tChecking {0:16}: {1}'.format(project, version))

        if project.lower() in project_list_low:
            cache.update_cache_dict(project, version, debug)

    cache.write_cache_file()

# End of check_and_update_feed()


def check_versions_for_list_sites(freshcode_project_list, url, cache_filename, feed_filename, local_dir, debug):
    """
    Checks projects of list type sites such as freshcode's web site's RSS
    """

    freshcode_cache = FileCache(local_dir, cache_filename)

    feed = feedparser.parse(url)

    feed_info = FeedCache(local_dir, feed_filename)
    feed_info.read_cache_feed()

    length = len(feed.entries)

    if length > 0:
        print_debug(debug, u'\tFound {} entries'.format(length))

        feed_list = make_list_of_newer_feeds(feed, feed_info, debug)
        print_debug(debug, u'\tFound {} new entries (relative to {})'.format(len(feed_list), feed_info.date_minutes))

        check_and_update_feed(feed_list, freshcode_project_list, freshcode_cache, debug)

        # Updating feed_info with the latest parsed feed entry date
        feed_info.update_cache_feed(feed.entries[0].published_parsed)

    else:
        print_debug(debug, u'No entries found in feed')

    feed_info.write_cache_feed()

# End of check_versions_for_list_sites() function


def print_versions_from_cache(local_dir, cache_filename_list, debug):
    """
    Prints all projects and their associated data from the cache
    """
    for cache_filename in cache_filename_list:
        site_cache = FileCache(local_dir, cache_filename)
        site_cache.print_cache_dict(cache_filename)

# End of print_versions_from_cache()


def get_infos_for_site(versions_conf, site_name):
    """
    Returns informations about a site as a tuple
    (list of projects, url to check, filename of the cache)
    """

    project_list = versions_conf.extract_project_list_from_site_def(site_name)
    project_url = versions_conf.extract_project_url(site_name)
    cache_filename = u'{}.cache'.format(site_name)

    return (project_list, project_url, cache_filename)

# End of get_infos_for_site() function


def check_versions(versions_conf, debug):
    """
    Checks versions by parsing online feeds
    """

    # Checks projects from by project sites such as github and sourceforge
    byproject_site_list = versions_conf.extract_site_list('byproject')

    for site_name in byproject_site_list:

        print_debug(debug, u'Checking {} projects'.format(site_name))
        (project_list, project_url, cache_filename) = get_infos_for_site(versions_conf, site_name)
        check_versions_feeds_by_projects(project_list, versions_conf.local_dir, debug, project_url, cache_filename)

    # Checks projects from 'list' tupe sites such as freshcode.club
    list_site_list = versions_conf.extract_site_list('list')
    for site_name in list_site_list:
        print_debug(debug, u'Checking {} updates'.format(site_name))
        (project_list, project_url, cache_filename) = get_infos_for_site(versions_conf, site_name)
        feed_filename = u'{}.feed'.format(site_name)
        check_versions_for_list_sites(project_list, project_url, cache_filename, feed_filename, versions_conf.local_dir, debug)

# End of check_versions() function


def print_cache_or_check_versions(versions_conf):
    """
    Decide to pretty print projects and their associated version that
    are already in the cache or to check versions of that projects upon
    selections made at the command line
    """

    debug = versions_conf.options.debug
    print_debug(debug, u'Loading yaml config file')
    versions_conf.load_yaml_from_config_file(versions_conf.config_filename)

    if versions_conf.options.list_cache is True:
        # Pretty prints all caches.
        cache_list = versions_conf.get_site_cache_liste_name()
        print_versions_from_cache(versions_conf.local_dir, cache_list, debug)

    else:
        # Checks version from online feeds
        check_versions(versions_conf, debug)

# End of print_list_or_check_versions() function.


def main():
    """
    This is the where the program begins
    """

    versions_conf = Conf()  # Configuration options

    if versions_conf.options.debug:
        doctest.testmod(verbose=True)

    if os.path.isfile(versions_conf.config_filename):
        print_cache_or_check_versions(versions_conf)

    else:
        print('Error: file %s does not exist' % versions_conf.config_filename)

# End of main() function


def print_debug(debug, message):
    """
    Prints 'message' if debug mode is True
    """

    if debug:
        print(message)

# End of print_debug() function


if __name__ == "__main__":
    main()


1		#!/usr/bin/env python
2		# -- coding: utf8 --
3		#
4		# versions.py : checks releases and versions of programs through RSS
5		# or Atom feeds and tells you
6		#
7		# (C) Copyright 2016 Olivier Delhomme
8		# e-mail : [email protected]
9		#
10		# This program is free software; you can redistribute it and/or modify
11		# it under the terms of the GNU General Public License as published by
12		# the Free Software Foundation; either version 2, or (at your option)
13		# any later version.
14		#
15		# This program is distributed in the hope that it will be useful,
16		# but WITHOUT ANY WARRANTY; without even the implied warranty of
17		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18		# GNU General Public License for more details.
19		#
20		# You should have received a copy of the GNU General Public License
21		# along with this program; if not, write to the Free Software Foundation,
22		# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23		#
24	1	import codecs
25	1	import feedparser
26	1	import yaml
27	1	import argparse
28	1	import os
29	1	import errno
30	1	import time
31	1	import doctest
32
33
34	1	__author__ = "Olivier Delhomme <[email protected]>"
35	1	__date__ = "20.10.2017"
36	1	__version__ = "1.0.0"
37
38		"""
39		This program checks projects versions throught RSS and Atom feeds and
40		should only print those with new release version.
41
42		It implements checking for projects in github.com and freshcode.club.
43		Projects must be added to a YAML file (named by default
44		~/.config/versions/versions.yaml). One can use --file=FILENAME option
45		to specify an alternative YAML file.
46		github projects must be listed under a "github.com:" section and
47		freshcode ones must be listed under a "freshcode.club:" section.
48
49		Versions uses and produces text files. Those files are cache files
50		written into ~/.local/versions directory. "*.cache" are cache files
51		containing the project list and their associated version (the latest).
52		"*.feed" are information feed cache files containing on only one line
53		the latest parsed post of the feed.
54		"""
55
56
57	1	class Conf:
58		"""
59		Class to store configuration of the program
60		"""
61
62	1	config_dir = ''
63	1	local_dir = ''
64	1	config_filename = ''
65	1	description = {}
66	1	options = None
67
68	1	def __init__(self):
69		"""
70		Inits the class
71		"""
72	1	self.config_dir = os.path.expanduser("~/.config/versions")
73	1	self.local_dir = os.path.expanduser("~/.local/versions")
74	1	self.config_filename = '' # At this stage we do not know if a filename has been set on the command line
75	1	self.description = {}
76	1	self.options = None
77
78		# Make sure that the directories exists
79	1	make_directories(self.config_dir)
80	1	make_directories(self.local_dir)
81
82	1	self._get_command_line_arguments()
83
84		# End of init() function
85
86
87	1	def load_yaml_from_config_file(self, filename):
88		"""
89		Loads definitions from the YAML config file filename
90		"""
91
92	1	config_file = codecs.open(filename, 'r', encoding='utf-8')
93
94	1	self.description = yaml.safe_load(config_file)
95
96	1	config_file.close()
97
98		# End of load_yaml_from_config_file() function
99
100
101	1	def _get_command_line_arguments(self):
102		"""
103		Defines and gets all the arguments for the command line using
104		argparse module. This function is called in the __init__ function
105		of this class.
106		"""
107	1	str_version = 'versions.py - %s' % __version__
108
109	1	parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds', version=str_version)
110
111	1	parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='')
112	1	parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False)
113	1	parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False)
114
115	1	self.options = parser.parse_args()
116
117	1	if self.options.filename != '':
118	1	self.config_filename = self.options.filename
119		else:
120		self.config_filename = os.path.join(self.config_dir, 'versions.yaml')
121
122		# End of get_command_line_arguments() function
123
124	1	def extract_site_definition(self, site_name):
125		"""
126		extracts whole site definition
127		"""
128
129	1	if site_name in self.description:
130	1	return self.description[site_name]
131		else:
132		return dict()
133
134		# End of extract_site_definition()
135
136
137	1	def extract_project_list_from_site_def(self, site_name):
138		"""
139		Extracts a project list from a site by project definition
140		"""
141
142	1	site_definition = self.extract_site_definition(site_name)
143
144	1	if 'projects' in site_definition:
145	1	project_list = site_definition['projects']
146		else:
147	1	project_list = []
148
149	1	return project_list
150
151		# End of extract_project_list_from_site_def() function
152
153
154	1	def extract_project_url(self, site_name):
155		"""
156		Extracts the url definition where to check project version.
157		"""
158
159	1	site_definition = self.extract_site_definition(site_name)
160
161	1	if 'url' in site_definition:
162	1	project_url = site_definition['url']
163		else:
164		project_url = ''
165
166	1	return project_url
167
168		# End of extract_project_url() function
169
170
171	1	def is_site_of_type(self, site_name, type):
172		"""
173		Returns True if site_name is of type 'type'
174		"""
175
176	1	site_definition = self.extract_site_definition(site_name)
177	1	if 'type' in site_definition:
178	1	return (site_definition['type'] == type)
179		else:
180		return False
181
182		# End of is_site_of_type() function
183
184
185	1	def extract_site_list(self, type):
186		"""
187		Extracts all sites from a specific type (byproject or list)
188		"""
189
190	1	all_site_list = list(self.description.keys())
191	1	site_list = []
192	1	for site_name in all_site_list:
193	1	if self.is_site_of_type(site_name, type):
194	1	site_list.insert(0, site_name)
195
196	1	return site_list
197
198		# End of extract_site_list() function
199
200
201	1	def get_site_cache_liste_name(self):
202		"""
203		Formats list of cache filenames for all sites
204		"""
205
206		all_site_list = list(self.description.keys())
207		cache_list = []
208		for site_name in all_site_list:
209		site_cache = u'{}.cache'.format(site_name)
210		cache_list.insert(0, site_cache)
211
212		return cache_list
213
214		# End of get_site_cache_liste_name() function
215
216		# End of Conf class
217
218
219	1	class FileCache:
220		"""
221		This class should help in managing cache files
222		"""
223
224	1	cache_filename = ''
225	1	cache_dict = {} # Dictionnary of projects and their associated version
226
227	1	def __init__(self, local_dir, filename):
228		"""
229		Inits the class. 'local_dir' must be a directory where we want to
230		store the cache file named 'filename'
231		"""
232
233	1	self.cache_filename = os.path.join(local_dir, filename)
234	1	self.cache_dict = {}
235	1	self._read_cache_file()
236
237		# End of __init__() function
238
239
240	1	def _return_project_and_version_from_line(self, line):
241		"""
242		Splits the line into a project and a version if possible (the line
243		must contain a whitespace.
244		"""
245
246		line = line.strip()
247
248		if line.count(' ') > 0:
249		(project, version) = line.split(' ', 1)
250
251		elif line != '':
252		project = line
253		version = ''
254
255		return (project, version)
256
257		# End of _return_project_and_version_from_line() function
258
259
260	1	def _read_cache_file(self):
261		"""
262		Reads the cache file and puts it into a dictionnary of project with
263		their associated version
264		"""
265
266	1	if os.path.isfile(self.cache_filename):
267		cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
268
269		for line in cache_file:
270		(project, version) = self._return_project_and_version_from_line(line)
271		self.cache_dict[project] = version
272
273		cache_file.close()
274
275		# End of _read_cache_file() function
276
277
278	1	def write_cache_file(self):
279		"""
280		Owerwrites dictionnary cache to the cache file
281		"""
282
283	1	cache_file = open_and_truncate_file(self.cache_filename)
284
285	1	for (project, version) in self.cache_dict.iteritems():
286	1	cache_file.write('%s %s\n' % (project, version))
287
288	1	cache_file.close()
289
290		# End of write_cache_file() function
291
292
293	1	def update_cache_dict(self, project, version, debug):
294		"""
295		Updates cache dictionnary if needed
296		"""
297
298	1	try:
299	1	version_cache = self.cache_dict[project]
300		print_debug(debug, u'\t\tIn cache: {}'.format(version_cache))
301
302		if version != version_cache:
303		print('%s %s' % (project, version))
304		self.cache_dict[project] = version
305
306	1	except KeyError:
307	1	print('%s %s' % (project, version))
308	1	self.cache_dict[project] = version
309
310		# End of update_cache_dict() function
311
312
313	1	def print_cache_dict(self, sitename):
314		"""
315		Pretty prints the cache dictionary as it is recorded in the files.
316		"""
317
318		print('%s:' % sitename)
319
320		# Gets project and version tuple sorted by project lowered while sorting
321		for project, version in sorted(self.cache_dict.iteritems(), key=lambda proj: proj[0].lower()):
322		print('\t%s %s' % (project, version))
323
324		print('')
325
326		# End of print_cache_dict() function
327		# End of FileCache class
328
329
330	1	class FeedCache:
331
332	1	cache_filename = ''
333	1	year = 2016
334	1	month = 05
335	1	day = 1
336	1	hour = 0
337	1	minute = 0
338	1	date_minutes = 0
339
340
341	1	def __init__(self, local_dir, filename):
342		"""
343		Inits the class. 'local_dir' must be a directory where we want to
344		store the cache file named 'filename'
345		"""
346
347	1	self.cache_filename = os.path.join(local_dir, filename)
348	1	self.read_cache_feed()
349
350		# End of __init__() function
351
352
353	1	def read_cache_feed(self):
354		"""
355		Reads the cache file which should only contain one date on the
356		first line
357		"""
358
359	1	if os.path.isfile(self.cache_filename):
360		cache_file = codecs.open(self.cache_filename, 'r', encoding='utf-8')
361		(self.year, self.month, self.day, self.hour, self.minute) = cache_file.readline().strip().split(' ', 4)
362		self.date_minutes = self._calculate_minutes(int(self.year), int(self.month), int(self.day), int(self.hour), int(self.minute))
363		cache_file.close()
364
365		# End of read_cache_feed() function
366
367
368	1	def write_cache_feed(self):
369		"""
370		Overwrites the cache file with values stored in this class
371		"""
372	1	cache_file = open_and_truncate_file(self.cache_filename)
373
374	1	cache_file.write('%s %s %s %s %s' % (self.year, self.month, self.day, self.hour, self.minute))
375
376	1	cache_file.close()
377
378		# End of write_cache_feed() function
379
380
381	1	def update_cache_feed(self, date):
382		"""
383		Updates the values stored in the class with the date which should
384		be a time.struct_time
385		"""
386
387	1	self.year = date.tm_year
388	1	self.month = date.tm_mon
389	1	self.day = date.tm_mday
390	1	self.hour = date.tm_hour
391	1	self.minute = date.tm_min
392	1	self.date_minutes = self._calculate_minutes_from_date(date)
393
394		# End of update_cache_feed() function
395
396
397	1	def _calculate_minutes(self, year, mon, day, hour, mins):
398		"""
399		Calculate a number of minutes with all parameters and returns
400		this.
401		>>> fc = FeedCache('localdir','filename')
402		>>> fc._calculate_minutes(2016, 5, 1, 0, 0)
403		1059827040
404		"""
405
406	1	minutes = (year * 365 * 24 * 60) + \
407		(mon * 30 * 24 * 60) + \
408		(day * 24 * 60) + \
409		(hour * 60) + \
410		(mins)
411
412	1	return minutes
413
414		# End of _calculate_minutes() function
415
416
417	1	def _calculate_minutes_from_date(self, date):
418		"""
419		Transforms a date in a number of minutes to ease comparisons
420		and returns this number of minutes
421		"""
422
423	1	return self._calculate_minutes(date.tm_year, date.tm_mon, date.tm_mday, date.tm_hour, date.tm_min)
424
425		# End of _calculate_minutes() function
426
427
428	1	def is_newer(self, date):
429		"""
430		Tells wether "date" is newer than the one in the cache (returns True
431		or not (returns False)
432		"""
433
434	1	minutes = self._calculate_minutes_from_date(date)
435
436	1	if minutes > self.date_minutes:
437	1	return True
438
439		else:
440		return False
441
442		# End of is_newer() function
443		# End of FeedCache class
444
445
446		######## Below are some utility functions used by classes above ########
447
448
449	1	def make_directories(path):
450		"""
451		Makes all directories in path if possible. It is not an error if
452		path already exists.
453		"""
454
455	1	try:
456	1	os.makedirs(path)
457
458		except OSError as exc:
459
460		if exc.errno != errno.EEXIST or os.path.isdir(path) is not True:
461		raise
462
463		# End of make_directories() function
464
465
466	1	def open_and_truncate_file(filename):
467		"""
468		Opens filename for writing truncating it to a zero length file
469		and returns a python file object.
470		"""
471
472	1	cache_file = codecs.open(filename, 'w', encoding='utf-8')
473	1	cache_file.truncate(0)
474	1	cache_file.flush()
475
476	1	return cache_file
477
478		# End of open_and_truncate_file() function
479		####################### End of utility functions #######################
480
481
482	1	def get_latest_release_by_title(program, debug, feed_url):
483		"""
484		Gets the latest release of a program on github. program must be a
485		string of the form user/repository.
486		"""
487
488	1	version = ''
489	1	url = feed_url.format(program)
490	1	feed = feedparser.parse(url)
491
492	1	if len(feed.entries) > 0:
493	1	version = feed.entries[0].title
494
495	1	print_debug(debug, u'\tProject {}: {}'.format(program, version))
496
497	1	return version
498
499		# End of get_latest_github_release() function
500
501
502	1	def check_versions_feeds_by_projects(project_list, local_dir, debug, feed_url, cache_filename):
503		"""
504		Checks project's versions on feed_url if any are defined in the yaml
505		file under the specified tag that got the project_list passed as an argument.
506		"""
507
508	1	site_cache = FileCache(local_dir, cache_filename)
509
510	1	for project in project_list:
511	1	version = get_latest_release_by_title(project, debug, feed_url)
512	1	if version != '':
513	1	site_cache.update_cache_dict(project, version, debug)
514
515	1	site_cache.write_cache_file()
516
517		# End of check_versions_for_github_projects() function
518
519
520	1	def make_list_of_newer_feeds(feed, feed_info, debug):
521		"""
522		Compares feed entries and keep those that are newer than the latest
523		check we've done and inserting the newer ones in reverse order in
524		a list to be returned
525		"""
526
527	1	feed_list = []
528
529		# inserting into a list in reverse order to keep the most recent
530		# version in case of multiple release of the same project in the
531		# feeds
532	1	for a_feed in feed.entries:
533	1	(project, version) = a_feed.title.strip().split(' ', 1)
534	1	print_debug(debug, u'\tFeed entry ({0}): project: {1:16} version: {2}'.format(time.strftime('%x %X', a_feed.published_parsed), project, version))
535	1	if feed_info.is_newer(a_feed.published_parsed):
536	1	feed_list.insert(0, a_feed)
537
538	1	return feed_list
539
540		# End of make_list_of_newer_feeds() function
541
542
543	1	def lower_list_of_strings(project_list):
544		"""
545		Lowers every string in the list to ease sorting and comparisons
546		"""
547
548	1	project_list_low = [project.lower() for project in project_list]
549
550	1	return project_list_low
551
552		# End of lower_list_of_strings() function
553
554
555	1	def check_and_update_feed(feed_list, project_list, cache, debug):
556		"""
557		Checks every feed entry in the list against project list cache and
558		then updates the dictionnary then writes the cache file to the disk.
559		- feed_list is a list of feed (from feedparser module)
560		- project_list is the list of project as read from the yaml
561		configuration file
562		- cache is an initialized instance of FileCache
563		"""
564
565		# Lowers the list before searching in it
566	1	project_list_low = lower_list_of_strings(project_list)
567
568		# Checking every feed entry that are newer than the last check
569		# and updates the dictionnary accordingly
570	1	for entry in feed_list:
571	1	(project, version) = entry.title.strip().split(' ', 1)
572	1	print_debug(debug, u'\tChecking {0:16}: {1}'.format(project, version))
573
574	1	if project.lower() in project_list_low:
575		cache.update_cache_dict(project, version, debug)
576
577	1	cache.write_cache_file()
578
579		# End of check_and_update_feed()
580
581
582	1	def check_versions_for_list_sites(freshcode_project_list, url, cache_filename, feed_filename, local_dir, debug):
583		"""
584		Checks projects of list type sites such as freshcode's web site's RSS
585		"""
586
587	1	freshcode_cache = FileCache(local_dir, cache_filename)
588
589	1	feed = feedparser.parse(url)
590
591	1	feed_info = FeedCache(local_dir, feed_filename)
592	1	feed_info.read_cache_feed()
593
594	1	length = len(feed.entries)
595
596	1	if length > 0:
597	1	print_debug(debug, u'\tFound {} entries'.format(length))
598
599	1	feed_list = make_list_of_newer_feeds(feed, feed_info, debug)
600	1	print_debug(debug, u'\tFound {} new entries (relative to {})'.format(len(feed_list), feed_info.date_minutes))
601
602	1	check_and_update_feed(feed_list, freshcode_project_list, freshcode_cache, debug)
603
604		# Updating feed_info with the latest parsed feed entry date
605	1	feed_info.update_cache_feed(feed.entries[0].published_parsed)
606
607		else:
608		print_debug(debug, u'No entries found in feed')
609
610	1	feed_info.write_cache_feed()
611
612		# End of check_versions_for_list_sites() function
613
614
615	1	def print_versions_from_cache(local_dir, cache_filename_list, debug):
616		"""
617		Prints all projects and their associated data from the cache
618		"""
619		for cache_filename in cache_filename_list:
620		site_cache = FileCache(local_dir, cache_filename)
621		site_cache.print_cache_dict(cache_filename)
622
623		# End of print_versions_from_cache()
624
625
626	1	def get_infos_for_site(versions_conf, site_name):
627		"""
628		Returns informations about a site as a tuple
629		(list of projects, url to check, filename of the cache)
630		"""
631
632	1	project_list = versions_conf.extract_project_list_from_site_def(site_name)
633	1	project_url = versions_conf.extract_project_url(site_name)
634	1	cache_filename = u'{}.cache'.format(site_name)
635
636	1	return (project_list, project_url, cache_filename)
637
638		# End of get_infos_for_site() function
639
640
641	1	def check_versions(versions_conf, debug):
642		"""
643		Checks versions by parsing online feeds
644		"""
645
646		# Checks projects from by project sites such as github and sourceforge
647	1	byproject_site_list = versions_conf.extract_site_list('byproject')
648
649	1	for site_name in byproject_site_list:
650
651	1	print_debug(debug, u'Checking {} projects'.format(site_name))
652	1	(project_list, project_url, cache_filename) = get_infos_for_site(versions_conf, site_name)
653	1	check_versions_feeds_by_projects(project_list, versions_conf.local_dir, debug, project_url, cache_filename)
654
655		# Checks projects from 'list' tupe sites such as freshcode.club
656	1	list_site_list = versions_conf.extract_site_list('list')
657	1	for site_name in list_site_list:
658	1	print_debug(debug, u'Checking {} updates'.format(site_name))
659	1	(project_list, project_url, cache_filename) = get_infos_for_site(versions_conf, site_name)
660	1	feed_filename = u'{}.feed'.format(site_name)
661	1	check_versions_for_list_sites(project_list, project_url, cache_filename, feed_filename, versions_conf.local_dir, debug)
662
663		# End of check_versions() function
664
665
666	1	def print_cache_or_check_versions(versions_conf):
667		"""
668		Decide to pretty print projects and their associated version that
669		are already in the cache or to check versions of that projects upon
670		selections made at the command line
671		"""
672
673	1	debug = versions_conf.options.debug
674	1	print_debug(debug, u'Loading yaml config file')
675	1	versions_conf.load_yaml_from_config_file(versions_conf.config_filename)
676
677	1	if versions_conf.options.list_cache is True:
678		# Pretty prints all caches.
679		cache_list = versions_conf.get_site_cache_liste_name()
680		print_versions_from_cache(versions_conf.local_dir, cache_list, debug)
681
682		else:
683		# Checks version from online feeds
684	1	check_versions(versions_conf, debug)
685
686		# End of print_list_or_check_versions() function.
687
688
689	1	def main():
690		"""
691		This is the where the program begins
692		"""
693
694	1	versions_conf = Conf() # Configuration options
695
696	1	if versions_conf.options.debug:
697	1	doctest.testmod(verbose=True)
698
699	1	if os.path.isfile(versions_conf.config_filename):
700	1	print_cache_or_check_versions(versions_conf)
701
702		else:
703		print('Error: file %s does not exist' % versions_conf.config_filename)
704
705		# End of main() function
706
707
708	1	def print_debug(debug, message):
709		"""
710		Prints 'message' if debug mode is True
711		"""
712
713	1	if debug:
714	1	print(message)
715
716		# End of print_debug() function
717
718
719	1	if __name__ == "__main__":
720		main()
721

dupgit / versions

GitHub Access Token became invalid

Push — master ( 5562bc...7165b1 )

check_versions_feeds_by_projects() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like