1
|
|
|
#!/usr/bin/env python |
2
|
|
|
# -*- coding: utf8 -*- |
3
|
|
|
# |
4
|
|
|
# byproject.py : related to sites that gives an RSS/Atom feed for |
5
|
|
|
# each project (such as github) |
6
|
|
|
# |
7
|
|
|
# (C) Copyright 2016 - 2018 Olivier Delhomme |
8
|
|
|
# e-mail : [email protected] |
9
|
|
|
# |
10
|
|
|
# This program is free software; you can redistribute it and/or modify |
11
|
|
|
# it under the terms of the GNU General Public License as published by |
12
|
|
|
# the Free Software Foundation; either version 3, or (at your option) |
13
|
|
|
# any later version. |
14
|
|
|
# |
15
|
|
|
# This program is distributed in the hope that it will be useful, |
16
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
17
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
18
|
|
|
# GNU General Public License for more details. |
19
|
|
|
# |
20
|
|
|
# You should have received a copy of the GNU General Public License |
21
|
|
|
# along with this program; if not, write to the Free Software Foundation, |
22
|
|
|
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
23
|
|
|
# |
24
|
|
|
import os |
25
|
|
|
import operator |
26
|
|
|
import re |
27
|
|
|
import caches |
28
|
|
|
import common |
29
|
|
|
|
30
|
|
|
|
31
|
|
|
def format_project_feed_filename(feed_filename, name): |
32
|
|
|
""" |
33
|
|
|
Returns a valid filename formatted based on feed_filename (the site name) |
34
|
|
|
and name (the name of the project). |
35
|
|
|
""" |
36
|
|
|
|
37
|
|
|
(root, ext) = os.path.splitext(feed_filename) |
38
|
|
|
norm_name = name.replace('/', '_') |
39
|
|
|
|
40
|
|
|
filename = "{}_{}{}".format(root, norm_name, ext) |
41
|
|
|
|
42
|
|
|
return filename |
43
|
|
|
|
44
|
|
|
# End of format_project_feed_filename() function |
45
|
|
|
|
46
|
|
|
|
47
|
|
|
def is_entry_last_checked(entry): |
48
|
|
|
""" |
49
|
|
|
Returns true if entry is equal to last checked and |
50
|
|
|
false otherwise. |
51
|
|
|
>>> is_entry_last_checked('last checked') |
52
|
|
|
True |
53
|
|
|
>>> is_entry_last_checked('') |
54
|
|
|
False |
55
|
|
|
>>> is_entry_last_checked('latest') |
56
|
|
|
False |
57
|
|
|
""" |
58
|
|
|
|
59
|
|
|
return entry == 'last checked' |
60
|
|
|
|
61
|
|
|
# End of is_entry_last_checked() function |
62
|
|
|
|
63
|
|
|
|
64
|
|
|
def get_values_from_project(project): |
65
|
|
|
""" |
66
|
|
|
Gets the values of 'regex' and 'name' keys if found and |
67
|
|
|
returns a tuple (valued, name, regex, entry) |
68
|
|
|
>>> project = {'name': 'version', 'regex': 'v([\d\.]+)\s*:.*', 'entry': 'entry'} |
69
|
|
|
>>> get_values_from_project(project) |
70
|
|
|
(True, 'version', 'v([\\\\d\\\\.]+)\\\\s*:.*', 'entry') |
71
|
|
|
>>> project = {'name': 'version'} |
72
|
|
|
>>> get_values_from_project(project) |
73
|
|
|
(False, 'version', '', '') |
74
|
|
|
""" |
75
|
|
|
|
76
|
|
|
regex = '' |
77
|
|
|
entry = '' |
78
|
|
|
name = project |
79
|
|
|
valued = False |
80
|
|
|
|
81
|
|
|
if type(project) is dict: |
82
|
|
|
if 'name' in project: |
83
|
|
|
name = project['name'] |
84
|
|
|
|
85
|
|
|
if 'regex' in project: |
86
|
|
|
regex = project['regex'] |
87
|
|
|
valued = True |
88
|
|
|
|
89
|
|
|
if 'entry' in project: |
90
|
|
|
entry = project['entry'] |
91
|
|
|
valued = True |
92
|
|
|
|
93
|
|
|
return (valued, name, regex, entry) |
94
|
|
|
|
95
|
|
|
# End of get_values_from_project() function |
96
|
|
|
|
97
|
|
|
|
98
|
|
|
def sort_feed_list(feed_list, feed): |
99
|
|
|
""" |
100
|
|
|
Sorts the feed list with the right attribute which depends on the feed. |
101
|
|
|
sort is reversed because feed_list is build by inserting ahead when |
102
|
|
|
parsing the feed from the most recent to the oldest entry. |
103
|
|
|
Returns a sorted list (by date) the first entry is the newest one. |
104
|
|
|
""" |
105
|
|
|
|
106
|
|
|
if feed.entries[0]: |
107
|
|
|
(published_date, field_name) = common.get_entry_published_date(feed.entries[0]) |
108
|
|
|
if field_name != '': |
109
|
|
|
feed_list = sorted(feed_list, key=operator.attrgetter(field_name), reverse=True) |
110
|
|
|
|
111
|
|
|
return feed_list |
112
|
|
|
|
113
|
|
|
# End of sort_feed_list() function |
114
|
|
|
|
115
|
|
|
|
116
|
|
|
def get_releases_filtering_feed(debug, local_dir, filename, feed, last_checked): |
117
|
|
|
""" |
118
|
|
|
Filters the feed and returns a list of releases with one |
119
|
|
|
or more elements |
120
|
|
|
""" |
121
|
|
|
|
122
|
|
|
feed_list = [] |
123
|
|
|
|
124
|
|
|
if last_checked: |
125
|
|
|
feed_info = caches.FeedCache(local_dir, filename) |
126
|
|
|
feed_info.read_cache_feed() |
127
|
|
|
feed_list = common.make_list_of_newer_feeds(feed, feed_info, debug) |
128
|
|
|
feed_list = sort_feed_list(feed_list, feed) |
129
|
|
|
|
130
|
|
|
# Updating feed_info with the latest parsed feed entry date |
131
|
|
|
if len(feed_list) >= 1: |
132
|
|
|
(published_date, field_name) = common.get_entry_published_date(feed_list[0]) |
133
|
|
|
feed_info.update_cache_feed(published_date) |
134
|
|
|
|
135
|
|
|
feed_info.write_cache_feed() |
136
|
|
|
|
137
|
|
|
else: |
138
|
|
|
feed_list.insert(0, feed.entries[0]) |
139
|
|
|
|
140
|
|
|
return feed_list |
141
|
|
|
|
142
|
|
|
# End of get_releases_filtering_feed() function |
143
|
|
|
|
144
|
|
|
|
145
|
|
|
def is_one_entry_field_value_egal_to_last_check(site_entry, entry): |
146
|
|
|
""" |
147
|
|
|
Returns True if the value of 'entry' field in the yaml file |
148
|
|
|
is last_checked and False otherwise. |
149
|
|
|
It checks firstly the 'entry' field for the whole site and if not |
150
|
|
|
found it then checks it for the project itself. |
151
|
|
|
""" |
152
|
|
|
|
153
|
|
|
if is_entry_last_checked(site_entry): |
154
|
|
|
last_checked = True |
155
|
|
|
else: |
156
|
|
|
last_checked = is_entry_last_checked(entry) |
157
|
|
|
|
158
|
|
|
return last_checked |
159
|
|
|
|
160
|
|
|
# End of get_relevant_entry_field_value() function |
161
|
|
|
|
162
|
|
|
|
163
|
|
|
def get_latest_release_by_title(project, debug, feed_url, local_dir, feed_filename, site_entry): |
164
|
|
|
""" |
165
|
|
|
Gets the latest release or the releases between the last checked time of |
166
|
|
|
a program on a site of type 'byproject'. |
167
|
|
|
project must be a string that represents the project (user/repository in |
168
|
|
|
github for instance). |
169
|
|
|
Returns a tuple which contains the name of the project, a list of versions |
170
|
|
|
and a boolean that indicates if we checked by last checked time (True) or |
171
|
|
|
by release (False). |
172
|
|
|
""" |
173
|
|
|
|
174
|
|
|
feed_list = [] |
175
|
|
|
|
176
|
|
|
(valued, name, regex, entry) = get_values_from_project(project) |
177
|
|
|
|
178
|
|
|
last_checked = is_one_entry_field_value_egal_to_last_check(site_entry, entry) |
179
|
|
|
filename = format_project_feed_filename(feed_filename, name) |
180
|
|
|
url = feed_url.format(name) |
181
|
|
|
feed = common.get_feed_entries_from_url(url) |
182
|
|
|
|
183
|
|
|
if feed is not None and len(feed.entries) > 0: |
184
|
|
|
feed_list = get_releases_filtering_feed(debug, local_dir, filename, feed, last_checked) |
185
|
|
|
|
186
|
|
|
if valued and regex != '': |
187
|
|
|
# Here we match the whole list against the regex and replace the |
188
|
|
|
# title's entry of the result of that match upon success. |
189
|
|
|
for feed_entry in feed_list: |
190
|
|
|
res = re.match(regex, feed_entry.title) |
191
|
|
|
# Here we should make a new list with the matched entries and leave the other ones |
192
|
|
|
if res: |
193
|
|
|
feed_entry.title = res.group(1) |
194
|
|
|
common.print_debug(debug, u'\tname: {}\n\tversion: {}\n\tregex: {} : {}'.format(name, feed_entry.title, regex, res)) |
195
|
|
|
|
196
|
|
|
common.print_debug(debug, u'\tProject {}: {}'.format(name, feed.entries[0].title)) |
197
|
|
|
|
198
|
|
|
return (name, feed_list, last_checked) |
199
|
|
|
|
200
|
|
|
# End of get_latest_release_by_title() function |
201
|
|
|
|
202
|
|
|
|
203
|
|
|
def check_versions_feeds_by_projects(project_list, local_dir, debug, feed_url, cache_filename, feed_filename, site_entry): |
204
|
|
|
""" |
205
|
|
|
Checks project's versions on feed_url if any are defined in the yaml |
206
|
|
|
file under the specified tag that got the project_list passed as an argument. |
207
|
|
|
""" |
208
|
|
|
|
209
|
|
|
site_cache = caches.FileCache(local_dir, cache_filename) |
210
|
|
|
|
211
|
|
|
for project in project_list: |
212
|
|
|
(name, feed_list, last_checked) = get_latest_release_by_title(project, debug, feed_url, local_dir, feed_filename, site_entry) |
213
|
|
|
|
214
|
|
|
if len(feed_list) >= 1: |
215
|
|
|
# Updating the cache with the latest version (the first feed entry) |
216
|
|
|
version = feed_list[0].title |
217
|
|
|
|
218
|
|
|
if not last_checked: |
219
|
|
|
# printing only for latest release as last checked is |
220
|
|
|
# already filtered and to be printed entirely |
221
|
|
|
site_cache.print_if_newest_version(name, version, debug) |
222
|
|
|
# we already printed this. |
223
|
|
|
del feed_list[0] |
224
|
|
|
|
225
|
|
|
site_cache.update_cache_dict(name, version, debug) |
226
|
|
|
|
227
|
|
|
# Printing all entries in the list. |
228
|
|
|
for feed_entry in feed_list: |
229
|
|
|
common.print_project_version(name, feed_entry.title) |
230
|
|
|
|
231
|
|
|
site_cache.write_cache_file() |
232
|
|
|
|
233
|
|
|
# End of check_versions_feeds_by_projects() function |
234
|
|
|
|
235
|
|
|
|
236
|
|
|
def check_versions(versions_conf, byproject_site_list): |
237
|
|
|
""" |
238
|
|
|
Checks version by checking each project's feed. |
239
|
|
|
""" |
240
|
|
|
|
241
|
|
|
for site_name in byproject_site_list: |
242
|
|
|
common.print_debug(versions_conf.options.debug, u'Checking {} projects'.format(site_name)) |
243
|
|
|
(project_list, project_url, cache_filename, site_entry) = versions_conf.get_infos_for_site(site_name) |
244
|
|
|
feed_filename = u'{}.feed'.format(site_name) |
245
|
|
|
check_versions_feeds_by_projects(project_list, versions_conf.local_dir, versions_conf.options.debug, project_url, cache_filename, feed_filename, site_entry) |
246
|
|
|
|
247
|
|
|
# End of check_versions() function. |
248
|
|
|
|