|
1
|
|
|
#!/usr/bin/env python |
|
2
|
|
|
# -*- coding: utf8 -*- |
|
3
|
|
|
# |
|
4
|
|
|
# byproject.py : related to sites that gives an RSS/Atom feed for |
|
5
|
|
|
# each project (such as github) |
|
6
|
|
|
# |
|
7
|
|
|
# (C) Copyright 2016 - 2018 Olivier Delhomme |
|
8
|
|
|
# e-mail : [email protected] |
|
9
|
|
|
# |
|
10
|
|
|
# This program is free software; you can redistribute it and/or modify |
|
11
|
|
|
# it under the terms of the GNU General Public License as published by |
|
12
|
|
|
# the Free Software Foundation; either version 3, or (at your option) |
|
13
|
|
|
# any later version. |
|
14
|
|
|
# |
|
15
|
|
|
# This program is distributed in the hope that it will be useful, |
|
16
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
17
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
18
|
|
|
# GNU General Public License for more details. |
|
19
|
|
|
# |
|
20
|
|
|
# You should have received a copy of the GNU General Public License |
|
21
|
|
|
# along with this program; if not, write to the Free Software Foundation, |
|
22
|
|
|
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
|
23
|
|
|
# |
|
24
|
1 |
|
import os |
|
25
|
1 |
|
import operator |
|
26
|
1 |
|
import re |
|
27
|
1 |
|
import caches |
|
28
|
1 |
|
import common |
|
29
|
|
|
|
|
30
|
|
|
|
|
31
|
1 |
|
def format_project_feed_filename(feed_filename, name): |
|
32
|
|
|
""" |
|
33
|
|
|
Returns a valid filename formatted based on feed_filename (the site name) |
|
34
|
|
|
and name (the name of the project). |
|
35
|
|
|
""" |
|
36
|
|
|
|
|
37
|
1 |
|
(root, ext) = os.path.splitext(feed_filename) |
|
38
|
1 |
|
norm_name = name.replace('/', '_') |
|
39
|
|
|
|
|
40
|
1 |
|
filename = "{}_{}{}".format(root, norm_name, ext) |
|
41
|
|
|
|
|
42
|
1 |
|
return filename |
|
43
|
|
|
|
|
44
|
|
|
# End of format_project_feed_filename() function |
|
45
|
|
|
|
|
46
|
|
|
|
|
47
|
1 |
|
def is_entry_last_checked(entry): |
|
48
|
|
|
""" |
|
49
|
|
|
Returns true if entry is equal to last checked and |
|
50
|
|
|
false otherwise. |
|
51
|
|
|
>>> is_entry_last_checked('last checked') |
|
52
|
|
|
True |
|
53
|
|
|
>>> is_entry_last_checked('') |
|
54
|
|
|
False |
|
55
|
|
|
>>> is_entry_last_checked('latest') |
|
56
|
|
|
False |
|
57
|
|
|
""" |
|
58
|
|
|
|
|
59
|
1 |
|
return entry == 'last checked' |
|
60
|
|
|
|
|
61
|
|
|
# End of is_entry_last_checked() function |
|
62
|
|
|
|
|
63
|
|
|
|
|
64
|
1 |
|
def get_values_from_project(project): |
|
65
|
|
|
""" |
|
66
|
|
|
Gets the values of 'regex' and 'name' keys if found and |
|
67
|
|
|
returns a tuple (valued, name, regex, entry) |
|
68
|
|
|
>>> project = {'name': 'version', 'regex': 'v([\d\.]+)\s*:.*', 'entry': 'entry'} |
|
69
|
|
|
>>> get_values_from_project(project) |
|
70
|
|
|
(True, 'version', 'v([\\\\d\\\\.]+)\\\\s*:.*', 'entry') |
|
71
|
|
|
>>> project = {'name': 'version'} |
|
72
|
|
|
>>> get_values_from_project(project) |
|
73
|
|
|
(False, 'version', '', '') |
|
74
|
|
|
""" |
|
75
|
|
|
|
|
76
|
1 |
|
regex = '' |
|
77
|
1 |
|
entry = '' |
|
78
|
1 |
|
name = project |
|
79
|
1 |
|
valued = False |
|
80
|
|
|
|
|
81
|
1 |
|
if isinstance(project, dict): |
|
82
|
1 |
|
if 'name' in project: |
|
83
|
1 |
|
name = project['name'] |
|
84
|
|
|
|
|
85
|
1 |
|
if 'regex' in project: |
|
86
|
1 |
|
regex = project['regex'] |
|
87
|
1 |
|
valued = True |
|
88
|
|
|
|
|
89
|
1 |
|
if 'entry' in project: |
|
90
|
1 |
|
entry = project['entry'] |
|
91
|
1 |
|
valued = True |
|
92
|
|
|
|
|
93
|
1 |
|
return (valued, name, regex, entry) |
|
94
|
|
|
|
|
95
|
|
|
# End of get_values_from_project() function |
|
96
|
|
|
|
|
97
|
|
|
|
|
98
|
1 |
|
def sort_feed_list(feed_list, feed): |
|
99
|
|
|
""" |
|
100
|
|
|
Sorts the feed list with the right attribute which depends on the feed. |
|
101
|
|
|
sort is reversed because feed_list is build by inserting ahead when |
|
102
|
|
|
parsing the feed from the most recent to the oldest entry. |
|
103
|
|
|
Returns a sorted list (by date) the first entry is the newest one. |
|
104
|
|
|
""" |
|
105
|
|
|
|
|
106
|
|
|
if feed.entries[0]: |
|
107
|
|
|
(published_date, field_name) = common.get_entry_published_date(feed.entries[0]) |
|
108
|
|
|
if field_name != '': |
|
109
|
|
|
feed_list = sorted(feed_list, key=operator.attrgetter(field_name), reverse=True) |
|
110
|
|
|
|
|
111
|
|
|
return feed_list |
|
112
|
|
|
|
|
113
|
|
|
# End of sort_feed_list() function |
|
114
|
|
|
|
|
115
|
|
|
|
|
116
|
1 |
|
def get_releases_filtering_feed(debug, local_dir, filename, feed, last_checked): |
|
117
|
|
|
""" |
|
118
|
|
|
Filters the feed and returns a list of releases with one |
|
119
|
|
|
or more elements |
|
120
|
|
|
""" |
|
121
|
|
|
|
|
122
|
1 |
|
feed_list = [] |
|
123
|
|
|
|
|
124
|
1 |
|
if last_checked: |
|
125
|
|
|
feed_info = caches.FeedCache(local_dir, filename) |
|
126
|
|
|
feed_info.read_cache_feed() |
|
127
|
|
|
feed_list = common.make_list_of_newer_feeds(feed, feed_info, debug) |
|
128
|
|
|
feed_list = sort_feed_list(feed_list, feed) |
|
129
|
|
|
|
|
130
|
|
|
# Updating feed_info with the latest parsed feed entry date |
|
131
|
|
|
if len(feed_list) >= 1: |
|
132
|
|
|
(published_date, field_name) = common.get_entry_published_date(feed_list[0]) |
|
133
|
|
|
feed_info.update_cache_feed(published_date) |
|
134
|
|
|
|
|
135
|
|
|
feed_info.write_cache_feed() |
|
136
|
|
|
|
|
137
|
|
|
else: |
|
138
|
1 |
|
feed_list.insert(0, feed.entries[0]) |
|
139
|
|
|
|
|
140
|
1 |
|
return feed_list |
|
141
|
|
|
|
|
142
|
|
|
# End of get_releases_filtering_feed() function |
|
143
|
|
|
|
|
144
|
|
|
|
|
145
|
1 |
|
def is_one_entry_field_value_equal_to_last_check(site_entry, entry): |
|
146
|
|
|
""" |
|
147
|
|
|
Returns True if the value of 'entry' field in the yaml file |
|
148
|
|
|
is last_checked and False otherwise. |
|
149
|
|
|
It checks firstly the 'entry' field for the whole site and if not |
|
150
|
|
|
found it then checks it for the project itself. |
|
151
|
|
|
""" |
|
152
|
|
|
|
|
153
|
1 |
|
if is_entry_last_checked(site_entry): |
|
154
|
1 |
|
last_checked = True |
|
155
|
|
|
else: |
|
156
|
1 |
|
last_checked = is_entry_last_checked(entry) |
|
157
|
|
|
|
|
158
|
1 |
|
return last_checked |
|
159
|
|
|
|
|
160
|
|
|
# End of get_relevant_entry_field_value() function |
|
161
|
|
|
|
|
162
|
|
|
|
|
163
|
1 |
|
def get_latest_release_by_title(project, debug, feed_url, local_dir, feed_filename, site_entry): |
|
164
|
|
|
""" |
|
165
|
|
|
Gets the latest release or the releases between the last checked time of |
|
166
|
|
|
a program on a site of type 'byproject'. |
|
167
|
|
|
project must be a string that represents the project (user/repository in |
|
168
|
|
|
github for instance). |
|
169
|
|
|
Returns a tuple which contains the name of the project, a list of versions |
|
170
|
|
|
and a boolean that indicates if we checked by last checked time (True) or |
|
171
|
|
|
by release (False). |
|
172
|
|
|
""" |
|
173
|
|
|
|
|
174
|
1 |
|
feed_list = [] |
|
175
|
|
|
|
|
176
|
1 |
|
(valued, name, regex, entry) = get_values_from_project(project) |
|
177
|
|
|
|
|
178
|
1 |
|
last_checked = is_one_entry_field_value_equal_to_last_check(site_entry, entry) |
|
179
|
1 |
|
filename = format_project_feed_filename(feed_filename, name) |
|
180
|
1 |
|
url = feed_url.format(name) |
|
181
|
1 |
|
feed = common.get_feed_entries_from_url(url) |
|
182
|
|
|
|
|
183
|
1 |
|
if feed is not None and len(feed.entries) > 0: |
|
184
|
1 |
|
feed_list = get_releases_filtering_feed(debug, local_dir, filename, feed, last_checked) |
|
185
|
|
|
|
|
186
|
1 |
|
if valued and regex != '': |
|
187
|
|
|
# Here we match the whole list against the regex and replace the |
|
188
|
|
|
# title's entry of the result of that match upon success. |
|
189
|
|
|
for feed_entry in feed_list: |
|
190
|
|
|
res = re.match(regex, feed_entry.title) |
|
191
|
|
|
# Here we should make a new list with the matched entries and leave the other ones |
|
192
|
|
|
if res: |
|
193
|
|
|
feed_entry.title = res.group(1) |
|
194
|
|
|
common.print_debug(debug, u'\tname: {}\n\tversion: {}\n\tregex: {} : {}'.format(name, feed_entry.title, regex, res)) |
|
195
|
|
|
|
|
196
|
|
|
common.print_debug(debug, u'\tProject {}: {}'.format(name, feed.entries[0].title)) |
|
197
|
|
|
|
|
198
|
1 |
|
return (name, feed_list, last_checked) |
|
199
|
|
|
|
|
200
|
|
|
# End of get_latest_release_by_title() function |
|
201
|
|
|
|
|
202
|
|
|
|
|
203
|
1 |
|
def check_versions_feeds_by_projects(project_list, local_dir, debug, feed_url, cache_filename, feed_filename, site_entry): |
|
204
|
|
|
""" |
|
205
|
|
|
Checks project's versions on feed_url if any are defined in the yaml |
|
206
|
|
|
file under the specified tag that got the project_list passed as an argument. |
|
207
|
|
|
""" |
|
208
|
|
|
|
|
209
|
1 |
|
site_cache = caches.FileCache(local_dir, cache_filename) |
|
210
|
|
|
|
|
211
|
1 |
|
for project in project_list: |
|
212
|
1 |
|
(name, feed_list, last_checked) = get_latest_release_by_title(project, debug, feed_url, local_dir, feed_filename, site_entry) |
|
213
|
|
|
|
|
214
|
1 |
|
if len(feed_list) >= 1: |
|
215
|
|
|
# Updating the cache with the latest version (the first feed entry) |
|
216
|
1 |
|
version = feed_list[0].title |
|
217
|
|
|
|
|
218
|
1 |
|
if not last_checked: |
|
219
|
|
|
# printing only for latest release as last checked is |
|
220
|
|
|
# already filtered and to be printed entirely |
|
221
|
1 |
|
site_cache.print_if_newest_version(name, version, debug) |
|
222
|
|
|
# we already printed this. |
|
223
|
1 |
|
del feed_list[0] |
|
224
|
|
|
|
|
225
|
1 |
|
site_cache.update_cache_dict(name, version, debug) |
|
226
|
|
|
|
|
227
|
|
|
# Printing all entries in the list. |
|
228
|
1 |
|
for feed_entry in feed_list: |
|
229
|
|
|
common.print_project_version(name, feed_entry.title) |
|
230
|
|
|
|
|
231
|
1 |
|
site_cache.write_cache_file() |
|
232
|
|
|
|
|
233
|
|
|
# End of check_versions_feeds_by_projects() function |
|
234
|
|
|
|
|
235
|
|
|
|
|
236
|
1 |
|
def check_versions(versions_conf, byproject_site_list): |
|
237
|
|
|
""" |
|
238
|
|
|
Checks version by checking each project's feed. |
|
239
|
|
|
""" |
|
240
|
|
|
|
|
241
|
1 |
|
for site_name in byproject_site_list: |
|
242
|
1 |
|
common.print_debug(versions_conf.options.debug, u'Checking {} projects'.format(site_name)) |
|
243
|
1 |
|
(project_list, project_url, cache_filename, site_entry) = versions_conf.get_infos_for_site(site_name) |
|
244
|
1 |
|
feed_filename = u'{}.feed'.format(site_name) |
|
245
|
1 |
|
check_versions_feeds_by_projects(project_list, versions_conf.local_dir, versions_conf.options.debug, project_url, cache_filename, feed_filename, site_entry) |
|
246
|
|
|
|
|
247
|
|
|
# End of check_versions() function. |
|
248
|
|
|
|
dupgit /
versions
