|
1
|
|
|
#!/usr/bin/env python |
|
2
|
|
|
# -*- coding: utf8 -*- |
|
3
|
|
|
# |
|
4
|
|
|
# versions.py : checks releases and versions of programs through RSS |
|
5
|
|
|
# or Atom feeds and tells you |
|
6
|
|
|
# |
|
7
|
|
|
# (C) Copyright 2016 - 2018 Olivier Delhomme |
|
8
|
|
|
# e-mail : [email protected] |
|
9
|
|
|
# |
|
10
|
|
|
# This program is free software; you can redistribute it and/or modify |
|
11
|
|
|
# it under the terms of the GNU General Public License as published by |
|
12
|
|
|
# the Free Software Foundation; either version 3, or (at your option) |
|
13
|
|
|
# any later version. |
|
14
|
|
|
# |
|
15
|
|
|
# This program is distributed in the hope that it will be useful, |
|
16
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
17
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
18
|
|
|
# GNU General Public License for more details. |
|
19
|
|
|
# |
|
20
|
|
|
# You should have received a copy of the GNU General Public License |
|
21
|
|
|
# along with this program; if not, write to the Free Software Foundation, |
|
22
|
|
|
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
|
23
|
|
|
# |
|
24
|
|
|
|
|
25
|
1 |
|
import codecs |
|
26
|
1 |
|
import sys |
|
27
|
1 |
|
import locale |
|
28
|
1 |
|
import argparse |
|
29
|
1 |
|
import os |
|
30
|
1 |
|
import re |
|
31
|
1 |
|
import errno |
|
32
|
1 |
|
import time |
|
33
|
1 |
|
import doctest |
|
34
|
1 |
|
import feedparser |
|
35
|
1 |
|
import yaml |
|
36
|
1 |
|
import operator |
|
37
|
1 |
|
import common |
|
38
|
1 |
|
import caches |
|
39
|
|
|
|
|
40
|
1 |
|
__author__ = "Olivier Delhomme <[email protected]>" |
|
41
|
1 |
|
__date__ = "06.11.2018" |
|
42
|
1 |
|
__version__ = "1.5.2" |
|
43
|
|
|
|
|
44
|
|
|
|
|
45
|
1 |
|
def make_directories(path): |
|
46
|
|
|
""" |
|
47
|
|
|
Makes all directories in path if possible. It is not an error if |
|
48
|
|
|
path already exists. |
|
49
|
|
|
""" |
|
50
|
|
|
|
|
51
|
1 |
|
try: |
|
52
|
1 |
|
os.makedirs(path) |
|
53
|
|
|
|
|
54
|
1 |
|
except OSError as exc: |
|
55
|
|
|
|
|
56
|
1 |
|
if exc.errno != errno.EEXIST or os.path.isdir(path) is not True: |
|
57
|
|
|
raise |
|
58
|
|
|
|
|
59
|
|
|
# End of make_directories() function |
|
60
|
|
|
|
|
61
|
1 |
|
class Conf: |
|
62
|
|
|
""" |
|
63
|
|
|
Class to store configuration of the program and check version. |
|
64
|
|
|
""" |
|
65
|
|
|
|
|
66
|
1 |
|
config_dir = '' |
|
67
|
1 |
|
local_dir = '' |
|
68
|
1 |
|
config_filename = '' |
|
69
|
1 |
|
description = {} |
|
70
|
1 |
|
options = None |
|
71
|
|
|
|
|
72
|
1 |
|
def __init__(self): |
|
73
|
|
|
""" |
|
74
|
|
|
Inits the class |
|
75
|
|
|
""" |
|
76
|
1 |
|
self.config_dir = os.path.expanduser("~/.config/versions") |
|
77
|
1 |
|
self.local_dir = os.path.expanduser("~/.local/versions") |
|
78
|
1 |
|
self.config_filename = '' # At this stage we do not know if a filename has been set on the command line |
|
79
|
1 |
|
self.description = {} |
|
80
|
1 |
|
self.options = None |
|
81
|
|
|
|
|
82
|
|
|
# Make sure that the directories exists |
|
83
|
1 |
|
make_directories(self.config_dir) |
|
84
|
1 |
|
make_directories(self.local_dir) |
|
85
|
|
|
|
|
86
|
1 |
|
self._get_command_line_arguments() |
|
87
|
|
|
|
|
88
|
|
|
# End of init() function |
|
89
|
|
|
|
|
90
|
|
|
|
|
91
|
1 |
|
def load_yaml_from_config_file(self, filename): |
|
92
|
|
|
""" |
|
93
|
|
|
Loads definitions from the YAML config file filename |
|
94
|
|
|
>>> conf = Conf() |
|
95
|
|
|
>>> conf.load_yaml_from_config_file('./bad_formatted.yaml') |
|
96
|
|
|
Error in configuration file ./bad_formatted.yaml at position: 9:1 |
|
97
|
|
|
""" |
|
98
|
|
|
|
|
99
|
1 |
|
config_file = codecs.open(filename, 'r', encoding='utf-8') |
|
100
|
|
|
|
|
101
|
1 |
|
try: |
|
102
|
1 |
|
self.description = yaml.safe_load(config_file) |
|
103
|
1 |
|
except yaml.YAMLError as err: |
|
104
|
1 |
|
if hasattr(err, 'problem_mark'): |
|
105
|
1 |
|
mark = err.problem_mark |
|
106
|
1 |
|
print(u'Error in configuration file {} at position: {}:{}'.format(filename, mark.line+1, mark.column+1)) |
|
107
|
|
|
else: |
|
108
|
|
|
print(u'Error in configuration file {}'.format(filename)) |
|
109
|
|
|
|
|
110
|
1 |
|
config_file.close() |
|
111
|
|
|
|
|
112
|
|
|
# End of load_yaml_from_config_file() function |
|
113
|
|
|
|
|
114
|
|
|
|
|
115
|
1 |
|
def _get_command_line_arguments(self): |
|
116
|
|
|
""" |
|
117
|
|
|
Defines and gets all the arguments for the command line using |
|
118
|
|
|
argparse module. This function is called in the __init__ function |
|
119
|
|
|
of this class. |
|
120
|
|
|
""" |
|
121
|
1 |
|
str_version = 'versions.py - %s' % __version__ |
|
122
|
|
|
|
|
123
|
1 |
|
parser = argparse.ArgumentParser(description='This program checks releases and versions of programs through RSS or Atom feeds') |
|
124
|
|
|
|
|
125
|
1 |
|
parser.add_argument('-v', '--version', action='version', version=str_version) |
|
126
|
1 |
|
parser.add_argument('-f', '--file', action='store', dest='filename', help='Configuration file with projects to check', default='') |
|
127
|
1 |
|
parser.add_argument('-l', '--list-cache', action='store_true', dest='list_cache', help='Lists all projects and their version in cache', default=False) |
|
128
|
1 |
|
parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Starts in debug mode and prints things that may help', default=False) |
|
129
|
|
|
|
|
130
|
1 |
|
self.options = parser.parse_args() |
|
131
|
|
|
|
|
132
|
1 |
|
if self.options.filename != '': |
|
133
|
1 |
|
self.config_filename = self.options.filename |
|
134
|
|
|
else: |
|
135
|
1 |
|
self.config_filename = os.path.join(self.config_dir, 'versions.yaml') |
|
136
|
|
|
|
|
137
|
|
|
# End of get_command_line_arguments() function |
|
138
|
|
|
|
|
139
|
|
|
|
|
140
|
1 |
|
def extract_site_definition(self, site_name): |
|
141
|
|
|
""" |
|
142
|
|
|
extracts whole site definition |
|
143
|
|
|
""" |
|
144
|
|
|
|
|
145
|
1 |
|
if site_name in self.description: |
|
146
|
1 |
|
return self.description[site_name] |
|
147
|
|
|
else: |
|
148
|
|
|
return dict() |
|
149
|
|
|
|
|
150
|
|
|
# End of extract_site_definition() |
|
151
|
|
|
|
|
152
|
|
|
|
|
153
|
1 |
|
def extract_regex_from_site(self, site_name): |
|
154
|
|
|
""" |
|
155
|
|
|
Extracts a regex from a site as defined in the YAML file. |
|
156
|
|
|
Returns the regex if it exists or None otherwise. |
|
157
|
|
|
""" |
|
158
|
|
|
|
|
159
|
1 |
|
return self.extract_variable_from_site(site_name, 'regex', None) |
|
160
|
|
|
|
|
161
|
|
|
# End of extract_regex_from_site() function |
|
162
|
|
|
|
|
163
|
|
|
|
|
164
|
1 |
|
def extract_multiproject_from_site(self, site_name): |
|
165
|
|
|
""" |
|
166
|
|
|
Extracts from a site its separator list for its multiple |
|
167
|
|
|
projects in one title. It returns None if multiproject |
|
168
|
|
|
is not defined and the list of separators instead |
|
169
|
|
|
""" |
|
170
|
|
|
|
|
171
|
1 |
|
return self.extract_variable_from_site(site_name, 'multiproject', None) |
|
172
|
|
|
|
|
173
|
|
|
# End of extract…multiproject_from_site() function |
|
174
|
|
|
|
|
175
|
|
|
|
|
176
|
1 |
|
def extract_variable_from_site(self, site_name, variable, default_return): |
|
177
|
|
|
""" |
|
178
|
|
|
Extracts variable from site site_name if it exists and return |
|
179
|
|
|
default_return otherwise |
|
180
|
|
|
""" |
|
181
|
|
|
|
|
182
|
1 |
|
site_definition = self.extract_site_definition(site_name) |
|
183
|
|
|
|
|
184
|
1 |
|
if variable in site_definition: |
|
185
|
1 |
|
value = site_definition[variable] |
|
186
|
1 |
|
if value is None: |
|
187
|
1 |
|
print(u'Warning: no variable "{}" for site "{}".'.format(variable, site_name)) |
|
188
|
1 |
|
value = default_return |
|
189
|
|
|
else: |
|
190
|
1 |
|
value = default_return |
|
191
|
|
|
|
|
192
|
1 |
|
return value |
|
193
|
|
|
|
|
194
|
|
|
# End of extract_variable_from_site() function |
|
195
|
|
|
|
|
196
|
|
|
|
|
197
|
1 |
|
def extract_project_list_from_site(self, site_name): |
|
198
|
|
|
""" |
|
199
|
|
|
Extracts a project list from a site as defined in the YAML file. |
|
200
|
|
|
""" |
|
201
|
|
|
|
|
202
|
1 |
|
return self.extract_variable_from_site(site_name, 'projects', []) |
|
203
|
|
|
|
|
204
|
|
|
# End of extract_project_list_from_site() function |
|
205
|
|
|
|
|
206
|
|
|
|
|
207
|
1 |
|
def extract_project_url(self, site_name): |
|
208
|
|
|
""" |
|
209
|
|
|
Extracts the url definition where to check project version. |
|
210
|
|
|
""" |
|
211
|
|
|
|
|
212
|
1 |
|
return self.extract_variable_from_site(site_name, 'url', '') |
|
213
|
|
|
|
|
214
|
|
|
# End of extract_project_url() function |
|
215
|
|
|
|
|
216
|
|
|
|
|
217
|
1 |
|
def extract_project_entry(self, site_name): |
|
218
|
|
|
""" |
|
219
|
|
|
Extracts the entry definition (if any) of a site. |
|
220
|
|
|
""" |
|
221
|
|
|
|
|
222
|
1 |
|
return self.extract_variable_from_site(site_name, 'entry', '') |
|
223
|
|
|
|
|
224
|
|
|
# End of extract_project_entry() function. |
|
225
|
|
|
|
|
226
|
|
|
|
|
227
|
1 |
|
def is_site_of_type(self, site_name, site_type): |
|
228
|
|
|
""" |
|
229
|
|
|
Returns True if site_name is of type 'site_type' |
|
230
|
|
|
""" |
|
231
|
|
|
|
|
232
|
1 |
|
site_definition = self.extract_site_definition(site_name) |
|
233
|
1 |
|
if 'type' in site_definition: |
|
234
|
1 |
|
return (site_definition['type'] == site_type) |
|
235
|
|
|
else: |
|
236
|
|
|
return False |
|
237
|
|
|
|
|
238
|
|
|
# End of is_site_of_type() function |
|
239
|
|
|
|
|
240
|
|
|
|
|
241
|
1 |
|
def extract_site_list(self, site_type): |
|
242
|
|
|
""" |
|
243
|
|
|
Extracts all sites from a specific type (byproject or list) |
|
244
|
|
|
""" |
|
245
|
|
|
|
|
246
|
1 |
|
all_site_list = list(self.description.keys()) |
|
247
|
1 |
|
site_list = [] |
|
248
|
1 |
|
for site_name in all_site_list: |
|
249
|
1 |
|
if self.is_site_of_type(site_name, site_type): |
|
250
|
1 |
|
site_list.insert(0, site_name) |
|
251
|
|
|
|
|
252
|
1 |
|
return site_list |
|
253
|
|
|
|
|
254
|
|
|
# End of extract_site_list() function |
|
255
|
|
|
|
|
256
|
|
|
|
|
257
|
1 |
|
def make_site_cache_list_name(self): |
|
258
|
|
|
""" |
|
259
|
|
|
Formats list of cache filenames for all sites. |
|
260
|
|
|
""" |
|
261
|
|
|
|
|
262
|
1 |
|
all_site_list = list(self.description.keys()) |
|
263
|
1 |
|
cache_list = [] |
|
264
|
1 |
|
for site_name in all_site_list: |
|
265
|
1 |
|
site_cache = u'{}.cache'.format(site_name) |
|
266
|
1 |
|
cache_list.insert(0, site_cache) |
|
267
|
|
|
|
|
268
|
1 |
|
return cache_list |
|
269
|
|
|
|
|
270
|
|
|
# End of make_site_cache_list_name() function |
|
271
|
|
|
|
|
272
|
|
|
|
|
273
|
1 |
|
def print_cache_or_check_versions(self): |
|
274
|
|
|
""" |
|
275
|
|
|
Decide to pretty print projects and their associated version that |
|
276
|
|
|
are already in the cache or to check versions of that projects upon |
|
277
|
|
|
selections made at the command line |
|
278
|
|
|
""" |
|
279
|
|
|
|
|
280
|
1 |
|
common.print_debug(self.options.debug, u'Loading yaml config file') |
|
281
|
1 |
|
self.load_yaml_from_config_file(self.config_filename) |
|
282
|
|
|
|
|
283
|
1 |
|
if self.options.list_cache is True: |
|
284
|
|
|
# Pretty prints all caches. |
|
285
|
1 |
|
cache_list = self.make_site_cache_list_name() |
|
286
|
1 |
|
caches.print_versions_from_cache(self.local_dir, cache_list) |
|
287
|
|
|
|
|
288
|
|
|
else: |
|
289
|
|
|
# Checks version from online feeds |
|
290
|
1 |
|
self.check_versions() |
|
291
|
|
|
|
|
292
|
|
|
# End of print_list_or_check_versions() function. |
|
293
|
|
|
|
|
294
|
|
|
|
|
295
|
1 |
|
def check_versions(self): |
|
296
|
|
|
""" |
|
297
|
|
|
Checks versions by parsing online feeds. |
|
298
|
|
|
""" |
|
299
|
|
|
|
|
300
|
|
|
# Checks projects from by project sites such as github and sourceforge |
|
301
|
1 |
|
byproject_site_list = self.extract_site_list('byproject') |
|
302
|
|
|
|
|
303
|
1 |
|
for site_name in byproject_site_list: |
|
304
|
1 |
|
common.print_debug(self.options.debug, u'Checking {} projects'.format(site_name)) |
|
305
|
1 |
|
(project_list, project_url, cache_filename, project_entry) = self.get_infos_for_site(site_name) |
|
306
|
1 |
|
feed_filename = u'{}.feed'.format(site_name) |
|
307
|
1 |
|
check_versions_feeds_by_projects(project_list, self.local_dir, self.options.debug, project_url, cache_filename, feed_filename, project_entry) |
|
308
|
|
|
|
|
309
|
|
|
# Checks projects from 'list' tupe sites such as freshcode.club |
|
310
|
1 |
|
list_site_list = self.extract_site_list('list') |
|
311
|
1 |
|
for site_name in list_site_list: |
|
312
|
1 |
|
common.print_debug(self.options.debug, u'Checking {} updates'.format(site_name)) |
|
313
|
1 |
|
(project_list, project_url, cache_filename, project_entry) = self.get_infos_for_site(site_name) |
|
314
|
1 |
|
regex = self.extract_regex_from_site(site_name) |
|
315
|
1 |
|
multiproject = self.extract_multiproject_from_site(site_name) |
|
316
|
1 |
|
feed_filename = u'{}.feed'.format(site_name) |
|
317
|
1 |
|
check_versions_for_list_sites(project_list, project_url, cache_filename, feed_filename, self.local_dir, self.options.debug, regex, multiproject) |
|
318
|
|
|
|
|
319
|
|
|
# End of check_versions() function |
|
320
|
|
|
|
|
321
|
|
|
|
|
322
|
1 |
|
def get_infos_for_site(self, site_name): |
|
323
|
|
|
""" |
|
324
|
|
|
Returns informations about a site as a tuple |
|
325
|
|
|
(list of projects, url to check, filename of the cache) |
|
326
|
|
|
""" |
|
327
|
|
|
|
|
328
|
1 |
|
project_list = self.extract_project_list_from_site(site_name) |
|
329
|
1 |
|
project_url = self.extract_project_url(site_name) |
|
330
|
1 |
|
project_entry = self.extract_project_entry(site_name) |
|
331
|
1 |
|
cache_filename = u'{}.cache'.format(site_name) |
|
332
|
|
|
|
|
333
|
1 |
|
return (project_list, project_url, cache_filename, project_entry) |
|
334
|
|
|
|
|
335
|
|
|
# End of get_infos_for_site() function |
|
336
|
|
|
# End of Conf class |
|
337
|
|
|
|
|
338
|
|
|
|
|
339
|
|
|
|
|
340
|
|
|
|
|
341
|
|
|
|
|
342
|
|
|
|
|
343
|
|
|
|
|
344
|
1 |
|
def manage_http_status(feed, url): |
|
345
|
|
|
""" |
|
346
|
|
|
Manages http status code present in feed and prints |
|
347
|
|
|
an error in case of a 3xx, 4xx or 5xx and stops |
|
348
|
|
|
doing anything for the feed by returning None. |
|
349
|
|
|
""" |
|
350
|
|
|
|
|
351
|
1 |
|
err = feed.status / 100 |
|
352
|
|
|
|
|
353
|
1 |
|
if err > 2: |
|
354
|
1 |
|
print(u'Error {} while fetching "{}".'.format(feed.status, url)) |
|
355
|
1 |
|
feed = None |
|
356
|
|
|
|
|
357
|
1 |
|
return feed |
|
358
|
|
|
|
|
359
|
|
|
# End of manage_http_status() function |
|
360
|
|
|
|
|
361
|
|
|
|
|
362
|
1 |
|
def manage_non_http_errors(feed, url): |
|
363
|
|
|
""" |
|
364
|
|
|
Tries to manage non http errors and gives |
|
365
|
|
|
a message to the user. |
|
366
|
|
|
""" |
|
367
|
|
|
|
|
368
|
1 |
|
if feed.bozo: |
|
369
|
1 |
|
if feed.bozo_exception: |
|
370
|
1 |
|
exc = feed.bozo_exception |
|
371
|
1 |
|
if hasattr(exc, 'reason'): |
|
372
|
1 |
|
message = exc.reason |
|
373
|
|
|
else: |
|
374
|
|
|
message = 'unaddressed' |
|
375
|
|
|
|
|
376
|
1 |
|
print(u'Error {} while fetching "{}".'.format(message, url)) |
|
377
|
|
|
|
|
378
|
|
|
else: |
|
379
|
|
|
print(u'Error while fetching url "{}".'.format(url)) |
|
380
|
|
|
|
|
381
|
|
|
# End of manage_non_http_errors() function |
|
382
|
|
|
|
|
383
|
|
|
|
|
384
|
1 |
|
def get_feed_entries_from_url(url): |
|
385
|
|
|
""" |
|
386
|
|
|
Gets feed entries from an url that should be an |
|
387
|
|
|
RSS or Atom feed. |
|
388
|
|
|
>>> get_feed_entries_from_url("http://delhomme.org/notfound.html") |
|
389
|
|
|
Error 404 while fetching "http://delhomme.org/notfound.html". |
|
390
|
|
|
>>> feed = get_feed_entries_from_url("http://blog.delhomme.org/index.php?feed/atom") |
|
391
|
|
|
>>> feed.status |
|
392
|
|
|
200 |
|
393
|
|
|
""" |
|
394
|
|
|
|
|
395
|
1 |
|
feed = feedparser.parse(url) |
|
396
|
|
|
|
|
397
|
1 |
|
if 'status' in feed: |
|
398
|
1 |
|
feed = manage_http_status(feed, url) |
|
399
|
|
|
else: |
|
400
|
|
|
# An error happened such that the feed does not contain an HTTP response |
|
401
|
1 |
|
manage_non_http_errors(feed, url) |
|
402
|
1 |
|
feed = None |
|
403
|
|
|
|
|
404
|
1 |
|
return feed |
|
405
|
|
|
|
|
406
|
|
|
# End of get_feed_entries_from_url() function |
|
407
|
|
|
|
|
408
|
|
|
|
|
409
|
1 |
|
def format_project_feed_filename(feed_filename, name): |
|
410
|
|
|
""" |
|
411
|
|
|
Returns a valid filename formatted based on feed_filename (the site name) |
|
412
|
|
|
and name the name of the project |
|
413
|
|
|
""" |
|
414
|
|
|
|
|
415
|
1 |
|
(root, ext) = os.path.splitext(feed_filename) |
|
416
|
1 |
|
norm_name = name.replace('/', '_') |
|
417
|
|
|
|
|
418
|
1 |
|
filename = "{}_{}{}".format(root, norm_name, ext) |
|
419
|
|
|
|
|
420
|
1 |
|
return filename |
|
421
|
|
|
|
|
422
|
|
|
# End of format_project_feed_filename() function |
|
423
|
|
|
|
|
424
|
|
|
|
|
425
|
1 |
|
def is_entry_last_checked(entry): |
|
426
|
|
|
""" |
|
427
|
|
|
Returns true if entry is equal to last checked and |
|
428
|
|
|
false otherwise. |
|
429
|
|
|
>>> is_entry_last_checked('last checked') |
|
430
|
|
|
True |
|
431
|
|
|
>>> is_entry_last_checked('') |
|
432
|
|
|
False |
|
433
|
|
|
>>> is_entry_last_checked('latest') |
|
434
|
|
|
False |
|
435
|
|
|
""" |
|
436
|
|
|
|
|
437
|
1 |
|
return entry == 'last checked' |
|
438
|
|
|
|
|
439
|
|
|
# End of is_entry_last_checked() function |
|
440
|
|
|
|
|
441
|
|
|
|
|
442
|
1 |
|
def get_values_from_project(project): |
|
443
|
|
|
""" |
|
444
|
|
|
Gets the values of 'regex' and 'name' keys if found and |
|
445
|
|
|
returns a tuple (valued, name, regex, entry) |
|
446
|
|
|
""" |
|
447
|
|
|
|
|
448
|
1 |
|
regex = '' |
|
449
|
1 |
|
entry = '' |
|
450
|
1 |
|
name = project |
|
451
|
1 |
|
valued = False |
|
452
|
|
|
|
|
453
|
1 |
|
if type(project) is dict: |
|
454
|
1 |
|
if 'name' in project: |
|
455
|
1 |
|
name = project['name'] |
|
456
|
|
|
|
|
457
|
1 |
|
if 'regex' in project: |
|
458
|
1 |
|
regex = project['regex'] |
|
459
|
1 |
|
valued = True |
|
460
|
|
|
|
|
461
|
1 |
|
if 'entry' in project: |
|
462
|
1 |
|
entry = project['entry'] |
|
463
|
1 |
|
valued = True |
|
464
|
|
|
|
|
465
|
1 |
|
return (valued, name, regex, entry) |
|
466
|
|
|
|
|
467
|
|
|
# End of get_values_from_project() function |
|
468
|
|
|
|
|
469
|
|
|
|
|
470
|
|
|
|
|
471
|
|
|
|
|
472
|
|
|
|
|
473
|
|
|
|
|
474
|
|
|
|
|
475
|
|
|
|
|
476
|
|
|
|
|
477
|
1 |
|
def sort_feed_list(feed_list, feed): |
|
478
|
|
|
""" |
|
479
|
|
|
Sorts the feed list with the right attribute which depends on the feed. |
|
480
|
|
|
sort is reversed because feed_list is build by inserting ahead when |
|
481
|
|
|
parsing the feed from the most recent to the oldest entry. |
|
482
|
|
|
Returns a sorted list (by date) the first entry is the newest one. |
|
483
|
|
|
""" |
|
484
|
|
|
|
|
485
|
1 |
|
if feed.entries[0]: |
|
486
|
1 |
|
if 'published_parsed' in feed.entries[0]: |
|
487
|
|
|
feed_list = sorted(feed_list, key=operator.attrgetter('published_parsed'), reverse=True) |
|
488
|
1 |
|
elif 'updated_parsed' in feed.entries[0]: |
|
489
|
1 |
|
feed_list = sorted(feed_list, key=operator.attrgetter('updated_parsed'), reverse=True) |
|
490
|
|
|
|
|
491
|
1 |
|
return feed_list |
|
492
|
|
|
|
|
493
|
|
|
# End of sort_feed_list() function |
|
494
|
|
|
|
|
495
|
|
|
|
|
496
|
1 |
|
def get_releases_filtering_feed(debug, local_dir, filename, feed, entry): |
|
497
|
|
|
""" |
|
498
|
|
|
Filters the feed and returns a list of releases with one |
|
499
|
|
|
or more elements |
|
500
|
|
|
""" |
|
501
|
|
|
|
|
502
|
1 |
|
feed_list = [] |
|
503
|
|
|
|
|
504
|
1 |
|
if is_entry_last_checked(entry): |
|
505
|
1 |
|
feed_info = caches.FeedCache(local_dir, filename) |
|
506
|
1 |
|
feed_info.read_cache_feed() |
|
507
|
1 |
|
feed_list = make_list_of_newer_feeds(feed, feed_info, debug) |
|
508
|
1 |
|
feed_list = sort_feed_list(feed_list, feed) |
|
509
|
|
|
|
|
510
|
|
|
# Updating feed_info with the latest parsed feed entry date |
|
511
|
1 |
|
if len(feed_list) >= 1: |
|
512
|
1 |
|
published_date = get_entry_published_date(feed_list[0]) |
|
513
|
1 |
|
feed_info.update_cache_feed(published_date) |
|
514
|
|
|
|
|
515
|
1 |
|
feed_info.write_cache_feed() |
|
516
|
|
|
|
|
517
|
|
|
else: |
|
518
|
1 |
|
feed_list.insert(0, feed.entries[0]) |
|
519
|
|
|
|
|
520
|
1 |
|
return feed_list |
|
521
|
|
|
|
|
522
|
|
|
|
|
523
|
1 |
|
def get_latest_release_by_title(project, debug, feed_url, local_dir, feed_filename, project_entry): |
|
524
|
|
|
""" |
|
525
|
|
|
Gets the latest release or the releases between the last checked time of |
|
526
|
|
|
a program on a site of type 'byproject'. |
|
527
|
|
|
project must be a string that represents the project (user/repository in |
|
528
|
|
|
github for instance). |
|
529
|
|
|
Returns a tuple which contains the name of the project, a list of versions |
|
530
|
|
|
and a boolean that indicates if we checked by last checked time (True) or |
|
531
|
|
|
by release (False). |
|
532
|
|
|
""" |
|
533
|
|
|
|
|
534
|
1 |
|
feed_list = [] |
|
535
|
|
|
|
|
536
|
1 |
|
(valued, name, regex, entry) = get_values_from_project(project) |
|
537
|
|
|
|
|
538
|
1 |
|
if is_entry_last_checked(project_entry): |
|
539
|
1 |
|
last_checked = True |
|
540
|
1 |
|
entry = project_entry |
|
541
|
|
|
else: |
|
542
|
1 |
|
last_checked = is_entry_last_checked(entry) |
|
543
|
1 |
|
filename = format_project_feed_filename(feed_filename, name) |
|
544
|
|
|
|
|
545
|
1 |
|
url = feed_url.format(name) |
|
546
|
1 |
|
feed = get_feed_entries_from_url(url) |
|
547
|
|
|
|
|
548
|
1 |
|
if feed is not None and len(feed.entries) > 0: |
|
549
|
1 |
|
feed_list = get_releases_filtering_feed(debug, local_dir, filename, feed, entry) |
|
550
|
|
|
|
|
551
|
1 |
|
if valued and regex != '': |
|
552
|
|
|
# Here we match the whole list against the regex and replace the |
|
553
|
|
|
# title's entry of the result of that match upon success. |
|
554
|
1 |
|
for entry in feed_list: |
|
555
|
1 |
|
res = re.match(regex, entry.title) |
|
556
|
|
|
# Here we should make a new list with the matched entries and leave tho other ones |
|
557
|
1 |
|
if res: |
|
558
|
1 |
|
entry.title = res.group(1) |
|
559
|
1 |
|
common.print_debug(debug, u'\tname: {}\n\tversion: {}\n\tregex: {} : {}'.format(name, entry.title, regex, res)) |
|
560
|
|
|
|
|
561
|
1 |
|
common.print_debug(debug, u'\tProject {}: {}'.format(name, entry.title)) |
|
562
|
|
|
|
|
563
|
1 |
|
return (name, feed_list, last_checked) |
|
564
|
|
|
|
|
565
|
|
|
# End of get_latest_release_by_title() function |
|
566
|
|
|
|
|
567
|
|
|
|
|
568
|
1 |
|
def check_versions_feeds_by_projects(project_list, local_dir, debug, feed_url, cache_filename, feed_filename, project_entry): |
|
569
|
|
|
""" |
|
570
|
|
|
Checks project's versions on feed_url if any are defined in the yaml |
|
571
|
|
|
file under the specified tag that got the project_list passed as an argument. |
|
572
|
|
|
""" |
|
573
|
|
|
|
|
574
|
1 |
|
site_cache = caches.FileCache(local_dir, cache_filename) |
|
575
|
|
|
|
|
576
|
1 |
|
for project in project_list: |
|
577
|
1 |
|
(name, feed_list, last_checked) = get_latest_release_by_title(project, debug, feed_url, local_dir, feed_filename, project_entry) |
|
578
|
|
|
|
|
579
|
|
|
|
|
580
|
1 |
|
if len(feed_list) >= 1: |
|
581
|
|
|
# Updating the cache with the latest version (the first entry) |
|
582
|
1 |
|
version = feed_list[0].title |
|
583
|
|
|
|
|
584
|
1 |
|
if not last_checked: |
|
585
|
|
|
# printing only for latest release as last checked is |
|
586
|
|
|
# already filtered and to be printed entirely |
|
587
|
1 |
|
site_cache.print_if_newest_version(name, version, debug) |
|
588
|
|
|
|
|
589
|
1 |
|
site_cache.update_cache_dict(name, version, debug) |
|
590
|
|
|
|
|
591
|
1 |
|
if not last_checked: |
|
592
|
|
|
# we already printed this. |
|
593
|
1 |
|
del feed_list[0] |
|
594
|
|
|
|
|
595
|
1 |
|
for entry in feed_list: |
|
596
|
1 |
|
common.print_project_version(name, entry.title) |
|
597
|
|
|
|
|
598
|
1 |
|
site_cache.write_cache_file() |
|
599
|
|
|
|
|
600
|
|
|
# End of check_versions_feeds_by_projects() function |
|
601
|
|
|
|
|
602
|
|
|
#####################################################################$$ |
|
603
|
|
|
|
|
604
|
1 |
|
def cut_title_with_default_method(title): |
|
605
|
|
|
""" |
|
606
|
|
|
Cuts title with a default method and a fallback |
|
607
|
|
|
>>> cut_title_with_default_method('versions 1.3.2') |
|
608
|
|
|
('versions', '1.3.2') |
|
609
|
|
|
>>> cut_title_with_default_method('no_version_project') |
|
610
|
|
|
('no_version_project', '') |
|
611
|
|
|
""" |
|
612
|
|
|
|
|
613
|
1 |
|
try: |
|
614
|
1 |
|
(project, version) = title.strip().split(' ', 1) |
|
615
|
|
|
|
|
616
|
1 |
|
except ValueError: |
|
617
|
1 |
|
project = title.strip() |
|
618
|
1 |
|
version = '' |
|
619
|
|
|
|
|
620
|
1 |
|
return (project, version) |
|
621
|
|
|
|
|
622
|
|
|
# End of cut_title_with_default_method() function |
|
623
|
|
|
|
|
624
|
|
|
|
|
625
|
1 |
|
def cut_title_with_regex_method(title, regex): |
|
626
|
|
|
""" |
|
627
|
|
|
Cuts title using a regex. If it does not success |
|
628
|
|
|
fallback to default. |
|
629
|
|
|
>>> cut_title_with_regex_method('versions 1.3.2', '([\w]+)\s([\d\.]+)') |
|
630
|
|
|
('versions', '1.3.2', False) |
|
631
|
|
|
>>> cut_title_with_regex_method('versions 1.3.2', '([\w]+)notgood\s([\d\.]+)') |
|
632
|
|
|
('', '', True) |
|
633
|
|
|
""" |
|
634
|
|
|
|
|
635
|
1 |
|
default = False |
|
636
|
1 |
|
project = '' |
|
637
|
1 |
|
version = '' |
|
638
|
|
|
|
|
639
|
1 |
|
res = re.match(regex, title) |
|
640
|
1 |
|
if res: |
|
641
|
1 |
|
project = res.group(1) |
|
642
|
1 |
|
version = res.group(2) |
|
643
|
|
|
else: |
|
644
|
1 |
|
default = True |
|
645
|
|
|
|
|
646
|
1 |
|
return (project, version, default) |
|
647
|
|
|
|
|
648
|
|
|
# End of cut_title_with_regex_method() function |
|
649
|
|
|
|
|
650
|
|
|
|
|
651
|
1 |
|
def cut_title_in_project_version(title, regex): |
|
652
|
|
|
""" |
|
653
|
|
|
Cuts the title into a tuple (project, version) where possible with a regex |
|
654
|
|
|
or if there is no regex or the regex did not match cuts the title with a |
|
655
|
|
|
default method |
|
656
|
|
|
""" |
|
657
|
1 |
|
default = False |
|
658
|
|
|
|
|
659
|
1 |
|
if regex is not None: |
|
660
|
1 |
|
(project, version, default) = cut_title_with_regex_method(title, regex) |
|
661
|
|
|
else: |
|
662
|
1 |
|
default = True |
|
663
|
|
|
|
|
664
|
1 |
|
if default: |
|
665
|
1 |
|
(project, version) = cut_title_with_default_method(title) |
|
666
|
|
|
|
|
667
|
1 |
|
return (project, version) |
|
668
|
|
|
|
|
669
|
|
|
# End of cut_title_in_project_version() function |
|
670
|
|
|
|
|
671
|
|
|
|
|
672
|
1 |
|
def get_entry_published_date(entry): |
|
673
|
|
|
""" |
|
674
|
|
|
Returns the published date of an entry. |
|
675
|
|
|
Selects the right field to do so |
|
676
|
|
|
""" |
|
677
|
|
|
|
|
678
|
1 |
|
if 'published_parsed' in entry: |
|
679
|
1 |
|
published_date = entry.published_parsed |
|
680
|
1 |
|
elif 'updated_parsed' in entry: |
|
681
|
1 |
|
published_date = entry.updated_parsed |
|
682
|
|
|
elif 'pubDate' in entry: # rss-0.91.dtd (netscape) |
|
683
|
|
|
published_date = entry.pubDate |
|
684
|
|
|
|
|
685
|
1 |
|
return published_date |
|
686
|
|
|
|
|
687
|
|
|
# End of get_entry_published_date() function |
|
688
|
|
|
|
|
689
|
|
|
|
|
690
|
1 |
|
def make_list_of_newer_feeds(feed, feed_info, debug): |
|
691
|
|
|
""" |
|
692
|
|
|
Compares feed entries and keep those that are newer than the latest |
|
693
|
|
|
check we've done and inserting the newer ones in reverse order in |
|
694
|
|
|
a list to be returned |
|
695
|
|
|
""" |
|
696
|
|
|
|
|
697
|
1 |
|
feed_list = [] |
|
698
|
|
|
|
|
699
|
|
|
# inserting into a list in reverse order to keep the most recent |
|
700
|
|
|
# version in case of multiple release of the same project in the |
|
701
|
|
|
# feeds |
|
702
|
1 |
|
for a_feed in feed.entries: |
|
703
|
|
|
|
|
704
|
1 |
|
if a_feed: |
|
705
|
1 |
|
published_date = get_entry_published_date(a_feed) |
|
706
|
|
|
|
|
707
|
1 |
|
common.print_debug(debug, u'\tFeed entry ({0}): Feed title: "{1:16}"'.format(time.strftime('%x %X', published_date), a_feed.title)) |
|
708
|
|
|
|
|
709
|
1 |
|
if feed_info.is_newer(published_date): |
|
710
|
1 |
|
feed_list.insert(0, a_feed) |
|
711
|
|
|
else: |
|
712
|
1 |
|
print(u'Warning: empty feed in {}'.format(feed)) |
|
713
|
|
|
|
|
714
|
1 |
|
return feed_list |
|
715
|
|
|
|
|
716
|
|
|
# End of make_list_of_newer_feeds() function |
|
717
|
|
|
|
|
718
|
|
|
|
|
719
|
1 |
|
def lower_list_of_strings(project_list): |
|
720
|
|
|
""" |
|
721
|
|
|
Lowers every string in the list to ease sorting and comparisons |
|
722
|
|
|
""" |
|
723
|
|
|
|
|
724
|
1 |
|
project_list_low = [project.lower() for project in project_list] |
|
725
|
|
|
|
|
726
|
1 |
|
return project_list_low |
|
727
|
|
|
|
|
728
|
|
|
# End of lower_list_of_strings() function |
|
729
|
|
|
|
|
730
|
|
|
|
|
731
|
1 |
|
def split_multiproject_title_into_list(title, multiproject): |
|
732
|
|
|
""" |
|
733
|
|
|
Splits title into a list of projects according to multiproject being |
|
734
|
|
|
a list of separators |
|
735
|
|
|
""" |
|
736
|
|
|
|
|
737
|
1 |
|
if multiproject is not None: |
|
738
|
1 |
|
titles = re.split(multiproject, title) |
|
739
|
|
|
else: |
|
740
|
1 |
|
titles = [title] |
|
741
|
|
|
|
|
742
|
1 |
|
return titles |
|
743
|
|
|
|
|
744
|
|
|
# End of split_multiproject_title_into_list() function |
|
745
|
|
|
|
|
746
|
|
|
|
|
747
|
|
|
|
|
748
|
|
|
|
|
749
|
1 |
|
def check_and_update_feed(feed_list, project_list, cache, debug, regex, multiproject): |
|
750
|
|
|
""" |
|
751
|
|
|
Checks every feed entry in the list against project list cache and |
|
752
|
|
|
then updates the dictionnary then writes the cache file to the disk. |
|
753
|
|
|
- feed_list is a list of feed (from feedparser module) |
|
754
|
|
|
- project_list is the list of project as read from the yaml |
|
755
|
|
|
configuration file |
|
756
|
|
|
- cache is an initialized instance of FileCache |
|
757
|
|
|
""" |
|
758
|
|
|
|
|
759
|
|
|
# Lowers the list before searching in it |
|
760
|
1 |
|
project_list_low = lower_list_of_strings(project_list) |
|
761
|
|
|
|
|
762
|
|
|
# Checking every feed entry that are newer than the last check |
|
763
|
|
|
# and updates the dictionnary accordingly |
|
764
|
1 |
|
for entry in feed_list: |
|
765
|
|
|
|
|
766
|
1 |
|
titles = split_multiproject_title_into_list(entry.title, multiproject) |
|
767
|
|
|
|
|
768
|
1 |
|
for title in titles: |
|
769
|
1 |
|
(project, version) = cut_title_in_project_version(title, regex) |
|
770
|
1 |
|
common.print_debug(debug, u'\tChecking {0:16}: {1}'.format(project, version)) |
|
771
|
1 |
|
if project.lower() in project_list_low: |
|
772
|
1 |
|
cache.print_if_newest_version(project, version, debug) |
|
773
|
1 |
|
cache.update_cache_dict(project, version, debug) |
|
774
|
|
|
|
|
775
|
1 |
|
cache.write_cache_file() |
|
776
|
|
|
|
|
777
|
|
|
# End of check_and_update_feed() function |
|
778
|
|
|
|
|
779
|
|
|
|
|
780
|
|
|
|
|
781
|
|
|
|
|
782
|
1 |
|
def check_versions_for_list_sites(feed_project_list, url, cache_filename, feed_filename, local_dir, debug, regex, multiproject): |
|
783
|
|
|
""" |
|
784
|
|
|
Checks projects of 'list' type sites such as freshcode's web site's RSS |
|
785
|
|
|
""" |
|
786
|
|
|
|
|
787
|
1 |
|
freshcode_cache = caches.FileCache(local_dir, cache_filename) |
|
788
|
|
|
|
|
789
|
1 |
|
feed_info = caches.FeedCache(local_dir, feed_filename) |
|
790
|
1 |
|
feed_info.read_cache_feed() |
|
791
|
|
|
|
|
792
|
1 |
|
feed = get_feed_entries_from_url(url) |
|
793
|
|
|
|
|
794
|
1 |
|
if feed is not None: |
|
795
|
1 |
|
common.print_debug(debug, u'\tFound {} entries'.format(len(feed.entries))) |
|
796
|
1 |
|
feed_list = make_list_of_newer_feeds(feed, feed_info, debug) |
|
797
|
1 |
|
common.print_debug(debug, u'\tFound {} new entries (relative to {})'.format(len(feed_list), feed_info.date_minutes)) |
|
798
|
|
|
|
|
799
|
1 |
|
check_and_update_feed(feed_list, feed_project_list, freshcode_cache, debug, regex, multiproject) |
|
800
|
|
|
|
|
801
|
|
|
# Updating feed_info with the latest parsed feed entry date |
|
802
|
1 |
|
feed_info.update_cache_feed(feed.entries[0].published_parsed) |
|
803
|
|
|
|
|
804
|
1 |
|
feed_info.write_cache_feed() |
|
805
|
|
|
|
|
806
|
|
|
# End of check_versions_for_list_sites() function |
|
807
|
|
|
|
|
808
|
|
|
|
|
809
|
|
|
|
dupgit /
versions
