|
1
|
|
|
#!/usr/bin/env python |
|
2
|
|
|
# -*- coding: utf8 -*- |
|
3
|
|
|
# |
|
4
|
|
|
# bylist.py : related to sites that gives one RSS/Atom feed for |
|
5
|
|
|
# all the projects (such as freshcode.club) |
|
6
|
|
|
# |
|
7
|
|
|
# (C) Copyright 2016 - 2018 Olivier Delhomme |
|
8
|
|
|
# e-mail : [email protected] |
|
9
|
|
|
# |
|
10
|
|
|
# This program is free software; you can redistribute it and/or modify |
|
11
|
|
|
# it under the terms of the GNU General Public License as published by |
|
12
|
|
|
# the Free Software Foundation; either version 3, or (at your option) |
|
13
|
|
|
# any later version. |
|
14
|
|
|
# |
|
15
|
|
|
# This program is distributed in the hope that it will be useful, |
|
16
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
17
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
18
|
|
|
# GNU General Public License for more details. |
|
19
|
|
|
# |
|
20
|
|
|
# You should have received a copy of the GNU General Public License |
|
21
|
|
|
# along with this program; if not, write to the Free Software Foundation, |
|
22
|
|
|
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
|
23
|
|
|
# |
|
24
|
|
|
import re |
|
25
|
|
|
import caches |
|
26
|
|
|
import common |
|
27
|
|
|
|
|
28
|
|
|
|
|
29
|
|
|
def cut_title_with_default_method(title): |
|
30
|
|
|
""" |
|
31
|
|
|
Cuts title with a default method and a fallback |
|
32
|
|
|
>>> cut_title_with_default_method('versions 1.3.2') |
|
33
|
|
|
('versions', '1.3.2') |
|
34
|
|
|
>>> cut_title_with_default_method('no_version_project') |
|
35
|
|
|
('no_version_project', '') |
|
36
|
|
|
""" |
|
37
|
|
|
|
|
38
|
|
|
try: |
|
39
|
|
|
(project, version) = title.strip().split(' ', 1) |
|
40
|
|
|
|
|
41
|
|
|
except ValueError: |
|
42
|
|
|
project = title.strip() |
|
43
|
|
|
version = '' |
|
44
|
|
|
|
|
45
|
|
|
return (project, version) |
|
46
|
|
|
|
|
47
|
|
|
# End of cut_title_with_default_method() function |
|
48
|
|
|
|
|
49
|
|
|
|
|
50
|
|
|
def cut_title_with_regex_method(title, regex): |
|
51
|
|
|
""" |
|
52
|
|
|
Cuts title using a regex. If it does not success |
|
53
|
|
|
fallback to default. |
|
54
|
|
|
>>> cut_title_with_regex_method('versions 1.3.2', '([\w]+)\s([\d\.]+)') |
|
55
|
|
|
('versions', '1.3.2', False) |
|
56
|
|
|
>>> cut_title_with_regex_method('versions 1.3.2', '([\w]+)notgood\s([\d\.]+)') |
|
57
|
|
|
('', '', True) |
|
58
|
|
|
""" |
|
59
|
|
|
|
|
60
|
|
|
default = False |
|
61
|
|
|
project = '' |
|
62
|
|
|
version = '' |
|
63
|
|
|
|
|
64
|
|
|
res = re.match(regex, title) |
|
65
|
|
|
if res: |
|
66
|
|
|
project = res.group(1) |
|
67
|
|
|
version = res.group(2) |
|
68
|
|
|
else: |
|
69
|
|
|
default = True |
|
70
|
|
|
|
|
71
|
|
|
return (project, version, default) |
|
72
|
|
|
|
|
73
|
|
|
# End of cut_title_with_regex_method() function |
|
74
|
|
|
|
|
75
|
|
|
|
|
76
|
|
|
def cut_title_in_project_version(title, regex): |
|
77
|
|
|
""" |
|
78
|
|
|
Cuts the title into a tuple (project, version) where possible with a regex |
|
79
|
|
|
or if there is no regex or the regex did not match cuts the title with a |
|
80
|
|
|
default method |
|
81
|
|
|
>>> cut_title_in_project_version('versions 1.3.2', None) |
|
82
|
|
|
('versions', '1.3.2') |
|
83
|
|
|
>>> cut_title_in_project_version('no_version_project', None) |
|
84
|
|
|
('no_version_project', '') |
|
85
|
|
|
>>> cut_title_in_project_version('versions 1.3.2', '([\w]+)badregex\s([\d\.]+)') |
|
86
|
|
|
('versions', '1.3.2') |
|
87
|
|
|
>>> cut_title_in_project_version('versions 1.3.2', '([\w]+)\s([\d\.i\-rcbetaRCBETA]+)') |
|
88
|
|
|
('versions', '1.3.2') |
|
89
|
|
|
""" |
|
90
|
|
|
default = False |
|
91
|
|
|
project = '' |
|
92
|
|
|
version = '' |
|
93
|
|
|
|
|
94
|
|
|
if regex is not None: |
|
95
|
|
|
(project, version, default) = cut_title_with_regex_method(title, regex) |
|
96
|
|
|
else: |
|
97
|
|
|
default = True |
|
98
|
|
|
|
|
99
|
|
|
if default: |
|
100
|
|
|
(project, version) = cut_title_with_default_method(title) |
|
101
|
|
|
|
|
102
|
|
|
return (project, version) |
|
103
|
|
|
|
|
104
|
|
|
# End of cut_title_in_project_version() function |
|
105
|
|
|
|
|
106
|
|
|
|
|
107
|
|
|
def lower_list_of_strings(project_list): |
|
108
|
|
|
""" |
|
109
|
|
|
Lowers every string in the list to ease sorting and comparisons |
|
110
|
|
|
>>> lower_list_of_strings(['TEST', 'LoweRed']) |
|
111
|
|
|
['test', 'lowered'] |
|
112
|
|
|
""" |
|
113
|
|
|
|
|
114
|
|
|
project_list_low = [project.lower() for project in project_list] |
|
115
|
|
|
|
|
116
|
|
|
return project_list_low |
|
117
|
|
|
|
|
118
|
|
|
# End of lower_list_of_strings() function |
|
119
|
|
|
|
|
120
|
|
|
|
|
121
|
|
|
def split_multiproject_title_into_list(title, multiproject): |
|
122
|
|
|
""" |
|
123
|
|
|
Splits title into a list of projects according to multiproject being |
|
124
|
|
|
a list of separators |
|
125
|
|
|
""" |
|
126
|
|
|
|
|
127
|
|
|
if multiproject is not None: |
|
128
|
|
|
titles = re.split(multiproject, title) |
|
129
|
|
|
else: |
|
130
|
|
|
titles = [title] |
|
131
|
|
|
|
|
132
|
|
|
return titles |
|
133
|
|
|
|
|
134
|
|
|
# End of split_multiproject_title_into_list() function |
|
135
|
|
|
|
|
136
|
|
|
|
|
137
|
|
|
def check_and_update_feed(feed_list, project_list, cache, debug, regex, multiproject): |
|
138
|
|
|
""" |
|
139
|
|
|
Checks every feed entry in the list against project list cache and |
|
140
|
|
|
then updates the dictionary then writes the cache file to the disk. |
|
141
|
|
|
- feed_list is a list of feed (from feedparser module) |
|
142
|
|
|
- project_list is the list of project as read from the yaml |
|
143
|
|
|
configuration file |
|
144
|
|
|
- cache is an initialized instance of FileCache |
|
145
|
|
|
""" |
|
146
|
|
|
|
|
147
|
|
|
# Lowers the list before searching in it |
|
148
|
|
|
project_list_low = lower_list_of_strings(project_list) |
|
149
|
|
|
|
|
150
|
|
|
# Checking every feed entry that are newer than the last check |
|
151
|
|
|
# and updates the dictionary accordingly |
|
152
|
|
|
for entry in feed_list: |
|
153
|
|
|
|
|
154
|
|
|
titles = split_multiproject_title_into_list(entry.title, multiproject) |
|
155
|
|
|
|
|
156
|
|
|
for title in titles: |
|
157
|
|
|
(project, version) = cut_title_in_project_version(title, regex) |
|
158
|
|
|
common.print_debug(debug, u'\tChecking {0:16}: {1}'.format(project, version)) |
|
159
|
|
|
if project.lower() in project_list_low: |
|
160
|
|
|
cache.print_if_newest_version(project, version, debug) |
|
161
|
|
|
cache.update_cache_dict(project, version, debug) |
|
162
|
|
|
|
|
163
|
|
|
cache.write_cache_file() |
|
164
|
|
|
|
|
165
|
|
|
# End of check_and_update_feed() function |
|
166
|
|
|
|
|
167
|
|
|
|
|
168
|
|
|
def check_versions_for_list_sites(feed_project_list, url, cache_filename, feed_filename, local_dir, debug, regex, multiproject): |
|
169
|
|
|
""" |
|
170
|
|
|
Checks projects of 'list' type sites such as freshcode's web site's RSS |
|
171
|
|
|
""" |
|
172
|
|
|
|
|
173
|
|
|
freshcode_cache = caches.FileCache(local_dir, cache_filename) |
|
174
|
|
|
|
|
175
|
|
|
feed_info = caches.FeedCache(local_dir, feed_filename) |
|
176
|
|
|
feed_info.read_cache_feed() |
|
177
|
|
|
|
|
178
|
|
|
feed = common.get_feed_entries_from_url(url) |
|
179
|
|
|
|
|
180
|
|
|
if feed is not None: |
|
181
|
|
|
common.print_debug(debug, u'\tFound {} entries'.format(len(feed.entries))) |
|
182
|
|
|
feed_list = common.make_list_of_newer_feeds(feed, feed_info, debug) |
|
183
|
|
|
common.print_debug(debug, u'\tFound {} new entries (relative to {})'.format(len(feed_list), feed_info.date_minutes)) |
|
184
|
|
|
|
|
185
|
|
|
check_and_update_feed(feed_list, feed_project_list, freshcode_cache, debug, regex, multiproject) |
|
186
|
|
|
|
|
187
|
|
|
# Updating feed_info with the latest parsed feed entry date |
|
188
|
|
|
feed_info.update_cache_feed(feed.entries[0].published_parsed) |
|
189
|
|
|
|
|
190
|
|
|
feed_info.write_cache_feed() |
|
191
|
|
|
|
|
192
|
|
|
# End of check_versions_for_list_sites() function |
|
193
|
|
|
|
|
194
|
|
|
|
|
195
|
|
|
def check_versions(versions_conf, list_site_list): |
|
196
|
|
|
""" |
|
197
|
|
|
Checks version by checking each project's feed. |
|
198
|
|
|
""" |
|
199
|
|
|
|
|
200
|
|
|
for site_name in list_site_list: |
|
201
|
|
|
common.print_debug(versions_conf.options.debug, u'Checking {} updates'.format(site_name)) |
|
202
|
|
|
(project_list, project_url, cache_filename, project_entry) = versions_conf.get_infos_for_site(site_name) |
|
203
|
|
|
regex = versions_conf.extract_regex_from_site(site_name) |
|
204
|
|
|
multiproject = versions_conf.extract_multiproject_from_site(site_name) |
|
205
|
|
|
feed_filename = u'{}.feed'.format(site_name) |
|
206
|
|
|
check_versions_for_list_sites(project_list, project_url, cache_filename, feed_filename, versions_conf.local_dir, versions_conf.options.debug, regex, multiproject) |
|
207
|
|
|
|
|
208
|
|
|
# End of check_versions() function |
|
209
|
|
|
|
dupgit /
versions
