1 | <?php |
||||||
2 | class RSSUtils { |
||||||
3 | public static function calculate_article_hash($article, $pluginhost) { |
||||||
4 | $tmp = ""; |
||||||
5 | |||||||
6 | foreach ($article as $k => $v) { |
||||||
7 | if ($k != "feed" && isset($v)) { |
||||||
8 | $x = strip_tags(is_array($v) ? implode(",", $v) : $v); |
||||||
9 | |||||||
10 | $tmp .= sha1("$k:".sha1($x)); |
||||||
11 | } |
||||||
12 | } |
||||||
13 | |||||||
14 | return sha1(implode(",", $pluginhost->get_plugin_names()).$tmp); |
||||||
15 | } |
||||||
16 | |||||||
17 | // Strips utf8mb4 characters (i.e. emoji) for mysql |
||||||
18 | public static function strip_utf8mb4($str) { |
||||||
19 | return preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $str); |
||||||
20 | } |
||||||
21 | |||||||
22 | public static function cleanup_feed_browser() { |
||||||
23 | $pdo = Db::pdo(); |
||||||
24 | $pdo->query("DELETE FROM ttrss_feedbrowser_cache"); |
||||||
25 | } |
||||||
26 | |||||||
27 | public static function update_daemon_common($limit = DAEMON_FEED_LIMIT) { |
||||||
0 ignored issues
–
show
Bug
introduced
by
![]() |
|||||||
28 | $schema_version = get_schema_version(); |
||||||
29 | |||||||
30 | if ($schema_version != SCHEMA_VERSION) { |
||||||
31 | die("Schema version is wrong, please upgrade the database.\n"); |
||||||
0 ignored issues
–
show
|
|||||||
32 | } |
||||||
33 | |||||||
34 | $pdo = Db::pdo(); |
||||||
35 | |||||||
36 | if (!SINGLE_USER_MODE && DAEMON_UPDATE_LOGIN_LIMIT > 0) { |
||||||
0 ignored issues
–
show
|
|||||||
37 | if (DB_TYPE == "pgsql") { |
||||||
0 ignored issues
–
show
|
|||||||
38 | $login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".DAEMON_UPDATE_LOGIN_LIMIT." days'"; |
||||||
39 | } else { |
||||||
40 | $login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".DAEMON_UPDATE_LOGIN_LIMIT." DAY)"; |
||||||
41 | } |
||||||
42 | } else { |
||||||
43 | $login_thresh_qpart = ""; |
||||||
44 | } |
||||||
45 | |||||||
46 | if (DB_TYPE == "pgsql") { |
||||||
47 | $update_limit_qpart = "AND (( |
||||||
48 | ttrss_feeds.update_interval = 0 |
||||||
49 | AND ttrss_user_prefs.value != '-1' |
||||||
50 | AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_user_prefs.value || ' minutes') AS INTERVAL) |
||||||
51 | ) OR ( |
||||||
52 | ttrss_feeds.update_interval > 0 |
||||||
53 | AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_feeds.update_interval || ' minutes') AS INTERVAL) |
||||||
54 | ) OR (ttrss_feeds.last_updated IS NULL |
||||||
55 | AND ttrss_user_prefs.value != '-1') |
||||||
56 | OR (last_updated = '1970-01-01 00:00:00' |
||||||
57 | AND ttrss_user_prefs.value != '-1'))"; |
||||||
58 | } else { |
||||||
59 | $update_limit_qpart = "AND (( |
||||||
60 | ttrss_feeds.update_interval = 0 |
||||||
61 | AND ttrss_user_prefs.value != '-1' |
||||||
62 | AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL CONVERT(ttrss_user_prefs.value, SIGNED INTEGER) MINUTE) |
||||||
63 | ) OR ( |
||||||
64 | ttrss_feeds.update_interval > 0 |
||||||
65 | AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ttrss_feeds.update_interval MINUTE) |
||||||
66 | ) OR (ttrss_feeds.last_updated IS NULL |
||||||
67 | AND ttrss_user_prefs.value != '-1') |
||||||
68 | OR (last_updated = '1970-01-01 00:00:00' |
||||||
69 | AND ttrss_user_prefs.value != '-1'))"; |
||||||
70 | } |
||||||
71 | |||||||
72 | // Test if feed is currently being updated by another process. |
||||||
73 | if (DB_TYPE == "pgsql") { |
||||||
74 | $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '10 minutes')"; |
||||||
75 | } else { |
||||||
76 | $updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 10 MINUTE))"; |
||||||
77 | } |
||||||
78 | |||||||
79 | $query_limit = $limit ? sprintf("LIMIT %d", $limit) : ""; |
||||||
80 | |||||||
81 | // Update the least recently updated feeds first |
||||||
82 | $query_order = "ORDER BY last_updated"; |
||||||
83 | if (DB_TYPE == "pgsql") { |
||||||
84 | $query_order .= " NULLS FIRST"; |
||||||
85 | } |
||||||
86 | |||||||
87 | $query = "SELECT DISTINCT ttrss_feeds.feed_url, ttrss_feeds.last_updated |
||||||
88 | FROM |
||||||
89 | ttrss_feeds, ttrss_users, ttrss_user_prefs |
||||||
90 | WHERE |
||||||
91 | ttrss_feeds.owner_uid = ttrss_users.id |
||||||
92 | AND ttrss_user_prefs.profile IS NULL |
||||||
93 | AND ttrss_users.id = ttrss_user_prefs.owner_uid |
||||||
94 | AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' |
||||||
95 | $login_thresh_qpart $update_limit_qpart |
||||||
96 | $updstart_thresh_qpart |
||||||
97 | $query_order $query_limit"; |
||||||
98 | |||||||
99 | $res = $pdo->query($query); |
||||||
100 | |||||||
101 | $feeds_to_update = array(); |
||||||
102 | while ($line = $res->fetch()) { |
||||||
103 | array_push($feeds_to_update, $line['feed_url']); |
||||||
104 | } |
||||||
105 | |||||||
106 | Debug::log(sprintf("Scheduled %d feeds to update...", count($feeds_to_update))); |
||||||
107 | |||||||
108 | // Update last_update_started before actually starting the batch |
||||||
109 | // in order to minimize collision risk for parallel daemon tasks |
||||||
110 | if (count($feeds_to_update) > 0) { |
||||||
111 | $feeds_qmarks = arr_qmarks($feeds_to_update); |
||||||
112 | |||||||
113 | $tmph = $pdo->prepare("UPDATE ttrss_feeds SET last_update_started = NOW() |
||||||
114 | WHERE feed_url IN ($feeds_qmarks)"); |
||||||
115 | $tmph->execute($feeds_to_update); |
||||||
116 | } |
||||||
117 | |||||||
118 | $nf = 0; |
||||||
119 | $bstarted = microtime(true); |
||||||
120 | |||||||
121 | $batch_owners = array(); |
||||||
122 | |||||||
123 | // since we have the data cached, we can deal with other feeds with the same url |
||||||
124 | $usth = $pdo->prepare("SELECT DISTINCT ttrss_feeds.id,last_updated,ttrss_feeds.owner_uid |
||||||
125 | FROM ttrss_feeds, ttrss_users, ttrss_user_prefs WHERE |
||||||
126 | ttrss_user_prefs.owner_uid = ttrss_feeds.owner_uid AND |
||||||
127 | ttrss_users.id = ttrss_user_prefs.owner_uid AND |
||||||
128 | ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' AND |
||||||
129 | ttrss_user_prefs.profile IS NULL AND |
||||||
130 | feed_url = ? |
||||||
131 | $update_limit_qpart |
||||||
132 | $login_thresh_qpart |
||||||
133 | ORDER BY ttrss_feeds.id $query_limit"); |
||||||
134 | |||||||
135 | foreach ($feeds_to_update as $feed) { |
||||||
136 | Debug::log("Base feed: $feed"); |
||||||
137 | |||||||
138 | $usth->execute([$feed]); |
||||||
139 | //update_rss_feed($line["id"], true); |
||||||
140 | |||||||
141 | if ($tline = $usth->fetch()) { |
||||||
142 | Debug::log(" => ".$tline["last_updated"].", ".$tline["id"]." ".$tline["owner_uid"]); |
||||||
143 | |||||||
144 | if (array_search($tline["owner_uid"], $batch_owners) === false) { |
||||||
145 | array_push($batch_owners, $tline["owner_uid"]); |
||||||
146 | } |
||||||
147 | |||||||
148 | $fstarted = microtime(true); |
||||||
149 | |||||||
150 | try { |
||||||
151 | RSSUtils::update_rss_feed($tline["id"], true, false); |
||||||
0 ignored issues
–
show
The call to
RSSUtils::update_rss_feed() has too many arguments starting with false .
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
This check compares calls to functions or methods with their respective definitions. If the call has more arguments than are defined, it raises an issue. If a function is defined several times with a different number of parameters, the check may pick up the wrong definition and report false positives. One codebase where this has been known to happen is Wordpress. Please note the @ignore annotation hint above. ![]() |
|||||||
152 | } catch (PDOException $e) { |
||||||
153 | Logger::get()->log_error(E_USER_NOTICE, $e->getMessage(), $e->getFile(), $e->getLine(), $e->getTraceAsString()); |
||||||
154 | |||||||
155 | try { |
||||||
156 | $pdo->rollback(); |
||||||
157 | } catch (PDOException $e) { |
||||||
158 | // it doesn't matter if there wasn't actually anything to rollback, PDO Exception can be |
||||||
159 | // thrown outside of an active transaction during feed update |
||||||
160 | } |
||||||
161 | } |
||||||
162 | |||||||
163 | Debug::log(sprintf(" %.4f (sec)", microtime(true) - $fstarted)); |
||||||
164 | |||||||
165 | ++$nf; |
||||||
166 | } |
||||||
167 | } |
||||||
168 | |||||||
169 | if ($nf > 0) { |
||||||
170 | Debug::log(sprintf("Processed %d feeds in %.4f (sec), %.4f (sec/feed avg)", $nf, |
||||||
171 | microtime(true) - $bstarted, (microtime(true) - $bstarted) / $nf)); |
||||||
172 | } |
||||||
173 | |||||||
174 | foreach ($batch_owners as $owner_uid) { |
||||||
175 | Debug::log("Running housekeeping tasks for user $owner_uid..."); |
||||||
176 | |||||||
177 | RSSUtils::housekeeping_user($owner_uid); |
||||||
178 | } |
||||||
179 | |||||||
180 | // Send feed digests by email if needed. |
||||||
181 | Digest::send_headlines_digests(); |
||||||
182 | |||||||
183 | return $nf; |
||||||
184 | } |
||||||
185 | |||||||
186 | // this is used when subscribing |
||||||
187 | public static function set_basic_feed_info($feed) { |
||||||
188 | |||||||
189 | $pdo = Db::pdo(); |
||||||
190 | |||||||
191 | $sth = $pdo->prepare("SELECT owner_uid,feed_url,auth_pass,auth_login |
||||||
192 | FROM ttrss_feeds WHERE id = ?"); |
||||||
193 | $sth->execute([$feed]); |
||||||
194 | |||||||
195 | if ($row = $sth->fetch()) { |
||||||
196 | |||||||
197 | $owner_uid = $row["owner_uid"]; |
||||||
198 | $auth_login = $row["auth_login"]; |
||||||
199 | $auth_pass = $row["auth_pass"]; |
||||||
200 | $fetch_url = $row["feed_url"]; |
||||||
201 | |||||||
202 | $pluginhost = new PluginHost(); |
||||||
203 | $user_plugins = get_pref("_ENABLED_PLUGINS", $owner_uid); |
||||||
204 | |||||||
205 | $pluginhost->load(PLUGINS, PluginHost::KIND_ALL); |
||||||
0 ignored issues
–
show
|
|||||||
206 | $pluginhost->load($user_plugins, PluginHost::KIND_USER, $owner_uid); |
||||||
207 | $pluginhost->load_data(); |
||||||
208 | |||||||
209 | $basic_info = array(); |
||||||
210 | foreach ($pluginhost->get_hooks(PluginHost::HOOK_FEED_BASIC_INFO) as $plugin) { |
||||||
211 | $basic_info = $plugin->hook_feed_basic_info($basic_info, $fetch_url, $owner_uid, $feed, $auth_login, $auth_pass); |
||||||
212 | } |
||||||
213 | |||||||
214 | if (!$basic_info) { |
||||||
215 | $feed_data = fetch_file_contents($fetch_url, false, |
||||||
216 | $auth_login, $auth_pass, false, |
||||||
217 | FEED_FETCH_TIMEOUT, |
||||||
0 ignored issues
–
show
|
|||||||
218 | 0); |
||||||
219 | |||||||
220 | $feed_data = trim($feed_data); |
||||||
221 | |||||||
222 | $rss = new FeedParser($feed_data); |
||||||
223 | $rss->init(); |
||||||
224 | |||||||
225 | if (!$rss->error()) { |
||||||
226 | $basic_info = array( |
||||||
227 | 'title' => mb_substr(clean($rss->get_title()), 0, 199), |
||||||
0 ignored issues
–
show
It seems like
clean($rss->get_title()) can also be of type array ; however, parameter $str of mb_substr() does only seem to accept string , maybe add an additional type check?
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
![]() |
|||||||
228 | 'site_url' => mb_substr(rewrite_relative_url($fetch_url, clean($rss->get_link())), 0, 245) |
||||||
0 ignored issues
–
show
It seems like
clean($rss->get_link()) can also be of type array ; however, parameter $rel_url of rewrite_relative_url() does only seem to accept string , maybe add an additional type check?
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
![]() |
|||||||
229 | ); |
||||||
230 | } |
||||||
231 | } |
||||||
232 | |||||||
233 | if ($basic_info && is_array($basic_info)) { |
||||||
234 | $sth = $pdo->prepare("SELECT title, site_url FROM ttrss_feeds WHERE id = ?"); |
||||||
235 | $sth->execute([$feed]); |
||||||
236 | |||||||
237 | if ($row = $sth->fetch()) { |
||||||
238 | |||||||
239 | $registered_title = $row["title"]; |
||||||
240 | $orig_site_url = $row["site_url"]; |
||||||
241 | |||||||
242 | if ($basic_info['title'] && (!$registered_title || $registered_title == "[Unknown]")) { |
||||||
243 | |||||||
244 | $sth = $pdo->prepare("UPDATE ttrss_feeds SET |
||||||
245 | title = ? WHERE id = ?"); |
||||||
246 | $sth->execute([$basic_info['title'], $feed]); |
||||||
247 | } |
||||||
248 | |||||||
249 | if ($basic_info['site_url'] && $orig_site_url != $basic_info['site_url']) { |
||||||
250 | $sth = $pdo->prepare("UPDATE ttrss_feeds SET |
||||||
251 | site_url = ? WHERE id = ?"); |
||||||
252 | $sth->execute([$basic_info['site_url'], $feed]); |
||||||
253 | } |
||||||
254 | |||||||
255 | } |
||||||
256 | } |
||||||
257 | } |
||||||
258 | } |
||||||
259 | |||||||
260 | /** |
||||||
261 | * @SuppressWarnings(PHPMD.UnusedFormalParameter) |
||||||
262 | */ |
||||||
263 | public static function update_rss_feed($feed, $no_cache = false) { |
||||||
264 | |||||||
265 | reset_fetch_domain_quota(); |
||||||
266 | |||||||
267 | Debug::log("start", Debug::$LOG_VERBOSE); |
||||||
268 | |||||||
269 | $pdo = Db::pdo(); |
||||||
270 | |||||||
271 | $sth = $pdo->prepare("SELECT title, site_url FROM ttrss_feeds WHERE id = ?"); |
||||||
272 | $sth->execute([$feed]); |
||||||
273 | |||||||
274 | if (!$row = $sth->fetch()) { |
||||||
275 | Debug::log("feed $feed not found, skipping."); |
||||||
276 | user_error("Attempt to update unknown/invalid feed $feed", E_USER_WARNING); |
||||||
277 | return false; |
||||||
278 | } |
||||||
279 | |||||||
280 | $title = $row["title"]; |
||||||
281 | $site_url = $row["site_url"]; |
||||||
282 | |||||||
283 | // feed was batch-subscribed or something, we need to get basic info |
||||||
284 | // this is not optimal currently as it fetches stuff separately TODO: optimize |
||||||
285 | if ($title == "[Unknown]" || !$title || !$site_url) { |
||||||
286 | Debug::log("setting basic feed info for $feed [$title, $site_url]..."); |
||||||
287 | RSSUtils::set_basic_feed_info($feed); |
||||||
288 | } |
||||||
289 | |||||||
290 | $sth = $pdo->prepare("SELECT id,update_interval,auth_login, |
||||||
291 | feed_url,auth_pass,cache_images, |
||||||
292 | mark_unread_on_update, owner_uid, |
||||||
293 | auth_pass_encrypted, feed_language, |
||||||
294 | last_modified, |
||||||
295 | ".SUBSTRING_FOR_DATE."(last_unconditional, 1, 19) AS last_unconditional |
||||||
296 | FROM ttrss_feeds WHERE id = ?"); |
||||||
297 | $sth->execute([$feed]); |
||||||
298 | |||||||
299 | if ($row = $sth->fetch()) { |
||||||
300 | |||||||
301 | $owner_uid = $row["owner_uid"]; |
||||||
302 | $mark_unread_on_update = $row["mark_unread_on_update"]; |
||||||
303 | |||||||
304 | $sth = $pdo->prepare("UPDATE ttrss_feeds SET last_update_started = NOW() |
||||||
305 | WHERE id = ?"); |
||||||
306 | $sth->execute([$feed]); |
||||||
307 | |||||||
308 | $auth_login = $row["auth_login"]; |
||||||
309 | $auth_pass = $row["auth_pass"]; |
||||||
310 | $stored_last_modified = $row["last_modified"]; |
||||||
311 | $last_unconditional = $row["last_unconditional"]; |
||||||
312 | $cache_images = $row["cache_images"]; |
||||||
313 | $fetch_url = $row["feed_url"]; |
||||||
314 | |||||||
315 | $feed_language = mb_strtolower($row["feed_language"]); |
||||||
316 | |||||||
317 | if (!$feed_language) { |
||||||
318 | $feed_language = mb_strtolower(get_pref('DEFAULT_SEARCH_LANGUAGE', $owner_uid)); |
||||||
319 | } |
||||||
320 | |||||||
321 | if (!$feed_language) { |
||||||
322 | $feed_language = 'simple'; |
||||||
323 | } |
||||||
324 | |||||||
325 | } else { |
||||||
326 | return false; |
||||||
327 | } |
||||||
328 | |||||||
329 | $date_feed_processed = date('Y-m-d H:i'); |
||||||
330 | |||||||
331 | $cache_filename = CACHE_DIR."/feeds/".sha1($fetch_url).".xml"; |
||||||
0 ignored issues
–
show
|
|||||||
332 | |||||||
333 | $pluginhost = new PluginHost(); |
||||||
334 | $user_plugins = get_pref("_ENABLED_PLUGINS", $owner_uid); |
||||||
335 | |||||||
336 | $pluginhost->load(PLUGINS, PluginHost::KIND_ALL); |
||||||
0 ignored issues
–
show
|
|||||||
337 | $pluginhost->load($user_plugins, PluginHost::KIND_USER, $owner_uid); |
||||||
338 | $pluginhost->load_data(); |
||||||
339 | |||||||
340 | $rss_hash = false; |
||||||
341 | |||||||
342 | $force_refetch = isset($_REQUEST["force_refetch"]); |
||||||
343 | $feed_data = ""; |
||||||
344 | |||||||
345 | Debug::log("running HOOK_FETCH_FEED handlers...", Debug::$LOG_VERBOSE); |
||||||
346 | |||||||
347 | foreach ($pluginhost->get_hooks(PluginHost::HOOK_FETCH_FEED) as $plugin) { |
||||||
348 | Debug::log("... ".get_class($plugin), Debug::$LOG_VERBOSE); |
||||||
349 | $start = microtime(true); |
||||||
350 | $feed_data = $plugin->hook_fetch_feed($feed_data, $fetch_url, $owner_uid, $feed, 0, $auth_login, $auth_pass); |
||||||
351 | Debug::log(sprintf("=== %.4f (sec)", microtime(true) - $start), Debug::$LOG_VERBOSE); |
||||||
352 | } |
||||||
353 | |||||||
354 | if ($feed_data) { |
||||||
355 | Debug::log("feed data has been modified by a plugin.", Debug::$LOG_VERBOSE); |
||||||
356 | } else { |
||||||
357 | Debug::log("feed data has not been modified by a plugin.", Debug::$LOG_VERBOSE); |
||||||
358 | } |
||||||
359 | |||||||
360 | // try cache |
||||||
361 | if (!$feed_data && |
||||||
362 | file_exists($cache_filename) && |
||||||
363 | is_readable($cache_filename) && |
||||||
364 | !$auth_login && !$auth_pass && |
||||||
365 | filemtime($cache_filename) > time() - 30) { |
||||||
366 | |||||||
367 | Debug::log("using local cache [$cache_filename].", Debug::$LOG_VERBOSE); |
||||||
368 | |||||||
369 | @$feed_data = file_get_contents($cache_filename); |
||||||
370 | |||||||
371 | if ($feed_data) { |
||||||
372 | $rss_hash = sha1($feed_data); |
||||||
373 | } |
||||||
374 | |||||||
375 | } else { |
||||||
376 | Debug::log("local cache will not be used for this feed", Debug::$LOG_VERBOSE); |
||||||
377 | } |
||||||
378 | |||||||
379 | global $fetch_last_modified; |
||||||
380 | |||||||
381 | // fetch feed from source |
||||||
382 | if (!$feed_data) { |
||||||
383 | Debug::log("last unconditional update request: $last_unconditional", Debug::$LOG_VERBOSE); |
||||||
384 | |||||||
385 | if (ini_get("open_basedir") && function_exists("curl_init")) { |
||||||
386 | Debug::log("not using CURL due to open_basedir restrictions", Debug::$LOG_VERBOSE); |
||||||
387 | } |
||||||
388 | |||||||
389 | if (time() - strtotime($last_unconditional) > MAX_CONDITIONAL_INTERVAL) { |
||||||
0 ignored issues
–
show
|
|||||||
390 | Debug::log("maximum allowed interval for conditional requests exceeded, forcing refetch", Debug::$LOG_VERBOSE); |
||||||
391 | |||||||
392 | $force_refetch = true; |
||||||
393 | } else { |
||||||
394 | Debug::log("stored last modified for conditional request: $stored_last_modified", Debug::$LOG_VERBOSE); |
||||||
395 | } |
||||||
396 | |||||||
397 | Debug::log("fetching [$fetch_url] (force_refetch: $force_refetch)...", Debug::$LOG_VERBOSE); |
||||||
398 | |||||||
399 | $feed_data = fetch_file_contents([ |
||||||
400 | "url" => $fetch_url, |
||||||
401 | "login" => $auth_login, |
||||||
402 | "pass" => $auth_pass, |
||||||
403 | "timeout" => $no_cache ? FEED_FETCH_NO_CACHE_TIMEOUT : FEED_FETCH_TIMEOUT, |
||||||
0 ignored issues
–
show
|
|||||||
404 | "last_modified" => $force_refetch ? "" : $stored_last_modified |
||||||
405 | ]); |
||||||
406 | |||||||
407 | $feed_data = trim($feed_data); |
||||||
408 | |||||||
409 | Debug::log("fetch done.", Debug::$LOG_VERBOSE); |
||||||
410 | Debug::log("source last modified: ".$fetch_last_modified, Debug::$LOG_VERBOSE); |
||||||
411 | |||||||
412 | if ($feed_data && $fetch_last_modified != $stored_last_modified) { |
||||||
413 | $sth = $pdo->prepare("UPDATE ttrss_feeds SET last_modified = ? WHERE id = ?"); |
||||||
414 | $sth->execute([substr($fetch_last_modified, 0, 245), $feed]); |
||||||
415 | } |
||||||
416 | |||||||
417 | // cache vanilla feed data for re-use |
||||||
418 | if ($feed_data && !$auth_pass && !$auth_login && is_writable(CACHE_DIR."/feeds")) { |
||||||
419 | $new_rss_hash = sha1($feed_data); |
||||||
420 | |||||||
421 | if ($new_rss_hash != $rss_hash) { |
||||||
422 | Debug::log("saving $cache_filename", Debug::$LOG_VERBOSE); |
||||||
423 | @file_put_contents($cache_filename, $feed_data); |
||||||
0 ignored issues
–
show
It seems like you do not handle an error condition for
file_put_contents() . This can introduce security issues, and is generally not recommended.
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
If you suppress an error, we recommend checking for the error condition explicitly: // For example instead of
@mkdir($dir);
// Better use
if (@mkdir($dir) === false) {
throw new \RuntimeException('The directory '.$dir.' could not be created.');
}
![]() |
|||||||
424 | } |
||||||
425 | } |
||||||
426 | } |
||||||
427 | |||||||
428 | if (!$feed_data) { |
||||||
429 | global $fetch_last_error; |
||||||
430 | global $fetch_last_error_code; |
||||||
431 | |||||||
432 | Debug::log("unable to fetch: $fetch_last_error [$fetch_last_error_code]", Debug::$LOG_VERBOSE); |
||||||
433 | |||||||
434 | // If-Modified-Since |
||||||
435 | if ($fetch_last_error_code != 304) { |
||||||
436 | $error_message = $fetch_last_error; |
||||||
437 | } else { |
||||||
438 | Debug::log("source claims data not modified, nothing to do.", Debug::$LOG_VERBOSE); |
||||||
439 | $error_message = ""; |
||||||
440 | } |
||||||
441 | |||||||
442 | $sth = $pdo->prepare("UPDATE ttrss_feeds SET last_error = ?, |
||||||
443 | last_updated = NOW() WHERE id = ?"); |
||||||
444 | $sth->execute([$error_message, $feed]); |
||||||
445 | |||||||
446 | return; |
||||||
447 | } |
||||||
448 | |||||||
449 | Debug::log("running HOOK_FEED_FETCHED handlers...", Debug::$LOG_VERBOSE); |
||||||
450 | $feed_data_checksum = md5($feed_data); |
||||||
451 | |||||||
452 | foreach ($pluginhost->get_hooks(PluginHost::HOOK_FEED_FETCHED) as $plugin) { |
||||||
453 | Debug::log("... ".get_class($plugin), Debug::$LOG_VERBOSE); |
||||||
454 | $start = microtime(true); |
||||||
455 | $feed_data = $plugin->hook_feed_fetched($feed_data, $fetch_url, $owner_uid, $feed); |
||||||
456 | Debug::log(sprintf("=== %.4f (sec)", microtime(true) - $start), Debug::$LOG_VERBOSE); |
||||||
457 | } |
||||||
458 | |||||||
459 | if (md5($feed_data) != $feed_data_checksum) { |
||||||
460 | Debug::log("feed data has been modified by a plugin.", Debug::$LOG_VERBOSE); |
||||||
461 | } else { |
||||||
462 | Debug::log("feed data has not been modified by a plugin.", Debug::$LOG_VERBOSE); |
||||||
463 | } |
||||||
464 | |||||||
465 | $rss = new FeedParser($feed_data); |
||||||
466 | $rss->init(); |
||||||
467 | |||||||
468 | if (!$rss->error()) { |
||||||
469 | |||||||
470 | Debug::log("running HOOK_FEED_PARSED handlers...", Debug::$LOG_VERBOSE); |
||||||
471 | |||||||
472 | // We use local pluginhost here because we need to load different per-user feed plugins |
||||||
473 | |||||||
474 | foreach ($pluginhost->get_hooks(PluginHost::HOOK_FEED_PARSED) as $plugin) { |
||||||
475 | Debug::log("... ".get_class($plugin), Debug::$LOG_VERBOSE); |
||||||
476 | $start = microtime(true); |
||||||
477 | $plugin->hook_feed_parsed($rss); |
||||||
478 | Debug::log(sprintf("=== %.4f (sec)", microtime(true) - $start), Debug::$LOG_VERBOSE); |
||||||
479 | } |
||||||
480 | |||||||
481 | Debug::log("language: $feed_language", Debug::$LOG_VERBOSE); |
||||||
482 | Debug::log("processing feed data...", Debug::$LOG_VERBOSE); |
||||||
483 | |||||||
484 | if (DB_TYPE == "pgsql") { |
||||||
0 ignored issues
–
show
|
|||||||
485 | $favicon_interval_qpart = "favicon_last_checked < NOW() - INTERVAL '12 hour'"; |
||||||
486 | } else { |
||||||
487 | $favicon_interval_qpart = "favicon_last_checked < DATE_SUB(NOW(), INTERVAL 12 HOUR)"; |
||||||
488 | } |
||||||
489 | |||||||
490 | $sth = $pdo->prepare("SELECT owner_uid,favicon_avg_color, |
||||||
491 | (favicon_last_checked IS NULL OR $favicon_interval_qpart) AS |
||||||
492 | favicon_needs_check |
||||||
493 | FROM ttrss_feeds WHERE id = ?"); |
||||||
494 | $sth->execute([$feed]); |
||||||
495 | |||||||
496 | if ($row = $sth->fetch()) { |
||||||
497 | $favicon_needs_check = $row["favicon_needs_check"]; |
||||||
498 | $favicon_avg_color = $row["favicon_avg_color"]; |
||||||
499 | $owner_uid = $row["owner_uid"]; |
||||||
500 | } else { |
||||||
501 | return false; |
||||||
502 | } |
||||||
503 | |||||||
504 | $site_url = mb_substr(rewrite_relative_url($fetch_url, clean($rss->get_link())), 0, 245); |
||||||
0 ignored issues
–
show
It seems like
clean($rss->get_link()) can also be of type array ; however, parameter $rel_url of rewrite_relative_url() does only seem to accept string , maybe add an additional type check?
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
![]() |
|||||||
505 | |||||||
506 | Debug::log("site_url: $site_url", Debug::$LOG_VERBOSE); |
||||||
507 | Debug::log("feed_title: ".clean($rss->get_title()), Debug::$LOG_VERBOSE); |
||||||
0 ignored issues
–
show
Are you sure
clean($rss->get_title()) of type array|mixed|string can be used in concatenation ?
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
![]() |
|||||||
508 | |||||||
509 | if ($favicon_needs_check || $force_refetch) { |
||||||
510 | |||||||
511 | /* terrible hack: if we crash on floicon shit here, we won't check |
||||||
512 | * the icon avgcolor again (unless the icon got updated) */ |
||||||
513 | |||||||
514 | $favicon_file = ICONS_DIR."/$feed.ico"; |
||||||
0 ignored issues
–
show
|
|||||||
515 | $favicon_modified = @filemtime($favicon_file); |
||||||
516 | |||||||
517 | Debug::log("checking favicon...", Debug::$LOG_VERBOSE); |
||||||
518 | |||||||
519 | RSSUtils::check_feed_favicon($site_url, $feed); |
||||||
520 | $favicon_modified_new = @filemtime($favicon_file); |
||||||
521 | |||||||
522 | if ($favicon_modified_new > $favicon_modified) { |
||||||
523 | $favicon_avg_color = ''; |
||||||
524 | } |
||||||
525 | |||||||
526 | $favicon_colorstring = ""; |
||||||
527 | if (file_exists($favicon_file) && function_exists("imagecreatefromstring") && $favicon_avg_color == '') { |
||||||
528 | require_once "colors.php"; |
||||||
529 | |||||||
530 | $sth = $pdo->prepare("UPDATE ttrss_feeds SET favicon_avg_color = 'fail' WHERE |
||||||
531 | id = ?"); |
||||||
532 | $sth->execute([$feed]); |
||||||
533 | |||||||
534 | $favicon_color = calculate_avg_color($favicon_file); |
||||||
535 | |||||||
536 | $favicon_colorstring = ",favicon_avg_color = ".$pdo->quote($favicon_color); |
||||||
537 | |||||||
538 | } else if ($favicon_avg_color == 'fail') { |
||||||
539 | Debug::log("floicon failed on this file, not trying to recalculate avg color", Debug::$LOG_VERBOSE); |
||||||
540 | } |
||||||
541 | |||||||
542 | $sth = $pdo->prepare("UPDATE ttrss_feeds SET favicon_last_checked = NOW() |
||||||
543 | $favicon_colorstring WHERE id = ?"); |
||||||
544 | $sth->execute([$feed]); |
||||||
545 | } |
||||||
546 | |||||||
547 | Debug::log("loading filters & labels...", Debug::$LOG_VERBOSE); |
||||||
548 | |||||||
549 | $filters = RSSUtils::load_filters($feed, $owner_uid); |
||||||
550 | |||||||
551 | if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { |
||||||
552 | print_r($filters); |
||||||
553 | } |
||||||
554 | |||||||
555 | Debug::log("".count($filters)." filters loaded.", Debug::$LOG_VERBOSE); |
||||||
556 | |||||||
557 | $items = $rss->get_items(); |
||||||
558 | |||||||
559 | if (!is_array($items)) { |
||||||
0 ignored issues
–
show
|
|||||||
560 | Debug::log("no articles found.", Debug::$LOG_VERBOSE); |
||||||
561 | |||||||
562 | $sth = $pdo->prepare("UPDATE ttrss_feeds |
||||||
563 | SET last_updated = NOW(), last_unconditional = NOW(), last_error = '' WHERE id = ?"); |
||||||
564 | $sth->execute([$feed]); |
||||||
565 | |||||||
566 | return true; // no articles |
||||||
567 | } |
||||||
568 | |||||||
569 | Debug::log("processing articles...", Debug::$LOG_VERBOSE); |
||||||
570 | |||||||
571 | $tstart = time(); |
||||||
572 | |||||||
573 | foreach ($items as $item) { |
||||||
574 | $pdo->beginTransaction(); |
||||||
575 | |||||||
576 | if (Debug::get_loglevel() >= 3) { |
||||||
577 | print_r($item); |
||||||
578 | } |
||||||
579 | |||||||
580 | if (ini_get("max_execution_time") > 0 && time() - $tstart >= ini_get("max_execution_time") * 0.7) { |
||||||
581 | Debug::log("looks like there's too many articles to process at once, breaking out", Debug::$LOG_VERBOSE); |
||||||
582 | $pdo->commit(); |
||||||
583 | break; |
||||||
584 | } |
||||||
585 | |||||||
586 | $entry_guid = strip_tags($item->get_id()); |
||||||
587 | if (!$entry_guid) { |
||||||
588 | $entry_guid = strip_tags($item->get_link()); |
||||||
589 | } |
||||||
590 | if (!$entry_guid) { |
||||||
591 | $entry_guid = RSSUtils::make_guid_from_title($item->get_title()); |
||||||
592 | } |
||||||
593 | |||||||
594 | if (!$entry_guid) { |
||||||
595 | $pdo->commit(); |
||||||
596 | continue; |
||||||
597 | } |
||||||
598 | |||||||
599 | $entry_guid = "$owner_uid,$entry_guid"; |
||||||
600 | |||||||
601 | $entry_guid_hashed = 'SHA1:'.sha1($entry_guid); |
||||||
602 | |||||||
603 | Debug::log("guid $entry_guid / $entry_guid_hashed", Debug::$LOG_VERBOSE); |
||||||
604 | |||||||
605 | $entry_timestamp = (int) $item->get_date(); |
||||||
606 | |||||||
607 | Debug::log("orig date: ".$item->get_date(), Debug::$LOG_VERBOSE); |
||||||
608 | |||||||
609 | $entry_title = strip_tags($item->get_title()); |
||||||
610 | |||||||
611 | $entry_link = rewrite_relative_url($site_url, clean($item->get_link())); |
||||||
612 | |||||||
613 | $entry_language = mb_substr(trim($item->get_language()), 0, 2); |
||||||
614 | |||||||
615 | Debug::log("title $entry_title", Debug::$LOG_VERBOSE); |
||||||
616 | Debug::log("link $entry_link", Debug::$LOG_VERBOSE); |
||||||
617 | Debug::log("language $entry_language", Debug::$LOG_VERBOSE); |
||||||
618 | |||||||
619 | if (!$entry_title) { |
||||||
620 | $entry_title = date("Y-m-d H:i:s", $entry_timestamp); |
||||||
621 | } |
||||||
622 | ; |
||||||
623 | |||||||
624 | $entry_content = $item->get_content(); |
||||||
625 | if (!$entry_content) { |
||||||
626 | $entry_content = $item->get_description(); |
||||||
627 | } |
||||||
628 | |||||||
629 | if (Debug::get_loglevel() >= 3) { |
||||||
630 | print "content: "; |
||||||
631 | print htmlspecialchars($entry_content); |
||||||
632 | print "\n"; |
||||||
633 | } |
||||||
634 | |||||||
635 | $entry_comments = mb_substr(strip_tags($item->get_comments_url()), 0, 245); |
||||||
636 | $num_comments = (int) $item->get_comments_count(); |
||||||
637 | |||||||
638 | $entry_author = strip_tags($item->get_author()); |
||||||
639 | $entry_guid = mb_substr($entry_guid, 0, 245); |
||||||
640 | |||||||
641 | Debug::log("author $entry_author", Debug::$LOG_VERBOSE); |
||||||
642 | Debug::log("looking for tags...", Debug::$LOG_VERBOSE); |
||||||
643 | |||||||
644 | $entry_tags = $item->get_categories(); |
||||||
645 | Debug::log("tags found: ".join(", ", $entry_tags), Debug::$LOG_VERBOSE); |
||||||
646 | |||||||
647 | Debug::log("done collecting data.", Debug::$LOG_VERBOSE); |
||||||
648 | |||||||
649 | $sth = $pdo->prepare("SELECT id, content_hash, lang FROM ttrss_entries |
||||||
650 | WHERE guid = ? OR guid = ?"); |
||||||
651 | $sth->execute([$entry_guid, $entry_guid_hashed]); |
||||||
652 | |||||||
653 | if ($row = $sth->fetch()) { |
||||||
654 | $base_entry_id = $row["id"]; |
||||||
655 | $entry_stored_hash = $row["content_hash"]; |
||||||
656 | $article_labels = Article::get_article_labels($base_entry_id, $owner_uid); |
||||||
657 | |||||||
658 | $existing_tags = Article::get_article_tags($base_entry_id, $owner_uid); |
||||||
659 | $entry_tags = array_unique(array_merge($entry_tags, $existing_tags)); |
||||||
660 | } else { |
||||||
661 | $base_entry_id = false; |
||||||
662 | $entry_stored_hash = ""; |
||||||
663 | $article_labels = array(); |
||||||
664 | } |
||||||
665 | |||||||
666 | $article = array("owner_uid" => $owner_uid, // read only |
||||||
667 | "guid" => $entry_guid, // read only |
||||||
668 | "guid_hashed" => $entry_guid_hashed, // read only |
||||||
669 | "title" => $entry_title, |
||||||
670 | "content" => $entry_content, |
||||||
671 | "link" => $entry_link, |
||||||
672 | "labels" => $article_labels, // current limitation: can add labels to article, can't remove them |
||||||
673 | "tags" => $entry_tags, |
||||||
674 | "author" => $entry_author, |
||||||
675 | "force_catchup" => false, // ugly hack for the time being |
||||||
676 | "score_modifier" => 0, // no previous value, plugin should recalculate score modifier based on content if needed |
||||||
677 | "language" => $entry_language, |
||||||
678 | "timestamp" => $entry_timestamp, |
||||||
679 | "num_comments" => $num_comments, |
||||||
680 | "feed" => array("id" => $feed, |
||||||
681 | "fetch_url" => $fetch_url, |
||||||
682 | "site_url" => $site_url, |
||||||
683 | "cache_images" => $cache_images) |
||||||
684 | ); |
||||||
685 | |||||||
686 | $entry_plugin_data = ""; |
||||||
687 | $entry_current_hash = RSSUtils::calculate_article_hash($article, $pluginhost); |
||||||
688 | |||||||
689 | Debug::log("article hash: $entry_current_hash [stored=$entry_stored_hash]", Debug::$LOG_VERBOSE); |
||||||
690 | |||||||
691 | if ($entry_current_hash == $entry_stored_hash && !isset($_REQUEST["force_rehash"])) { |
||||||
692 | Debug::log("stored article seems up to date [IID: $base_entry_id], updating timestamp only", Debug::$LOG_VERBOSE); |
||||||
693 | |||||||
694 | // we keep encountering the entry in feeds, so we need to |
||||||
695 | // update date_updated column so that we don't get horrible |
||||||
696 | // dupes when the entry gets purged and reinserted again e.g. |
||||||
697 | // in the case of SLOW SLOW OMG SLOW updating feeds |
||||||
698 | |||||||
699 | $sth = $pdo->prepare("UPDATE ttrss_entries SET date_updated = NOW() |
||||||
700 | WHERE id = ?"); |
||||||
701 | $sth->execute([$base_entry_id]); |
||||||
702 | |||||||
703 | $pdo->commit(); |
||||||
704 | continue; |
||||||
705 | } |
||||||
706 | |||||||
707 | Debug::log("hash differs, applying plugin filters:", Debug::$LOG_VERBOSE); |
||||||
708 | |||||||
709 | foreach ($pluginhost->get_hooks(PluginHost::HOOK_ARTICLE_FILTER) as $plugin) { |
||||||
710 | Debug::log("... ".get_class($plugin), Debug::$LOG_VERBOSE); |
||||||
711 | |||||||
712 | $start = microtime(true); |
||||||
713 | $article = $plugin->hook_article_filter($article); |
||||||
714 | |||||||
715 | Debug::log(sprintf("=== %.4f (sec)", microtime(true) - $start), Debug::$LOG_VERBOSE); |
||||||
716 | |||||||
717 | $entry_plugin_data .= mb_strtolower(get_class($plugin)).","; |
||||||
718 | } |
||||||
719 | |||||||
720 | if (Debug::get_loglevel() >= 3) { |
||||||
721 | print "processed content: "; |
||||||
722 | print htmlspecialchars($article["content"]); |
||||||
723 | print "\n"; |
||||||
724 | } |
||||||
725 | |||||||
726 | Debug::log("plugin data: $entry_plugin_data", Debug::$LOG_VERBOSE); |
||||||
727 | |||||||
728 | // Workaround: 4-byte unicode requires utf8mb4 in MySQL. See https://tt-rss.org/forum/viewtopic.php?f=1&t=3377&p=20077#p20077 |
||||||
729 | if (DB_TYPE == "mysql" && MYSQL_CHARSET != "UTF8MB4") { |
||||||
0 ignored issues
–
show
|
|||||||
730 | foreach ($article as $k => $v) { |
||||||
731 | // i guess we'll have to take the risk of 4byte unicode labels & tags here |
||||||
732 | if (is_string($article[$k])) { |
||||||
733 | $article[$k] = RSSUtils::strip_utf8mb4($v); |
||||||
734 | } |
||||||
735 | } |
||||||
736 | } |
||||||
737 | |||||||
738 | /* Collect article tags here so we could filter by them: */ |
||||||
739 | |||||||
740 | $matched_rules = []; |
||||||
741 | $matched_filters = []; |
||||||
742 | |||||||
743 | $article_filters = RSSUtils::get_article_filters($filters, $article["title"], |
||||||
744 | $article["content"], $article["link"], $article["author"], |
||||||
745 | $article["tags"], $matched_rules, $matched_filters); |
||||||
746 | |||||||
747 | // $article_filters should be renamed to something like $filter_actions; actual filter objects are in $matched_filters |
||||||
748 | foreach ($pluginhost->get_hooks(PluginHost::HOOK_FILTER_TRIGGERED) as $plugin) { |
||||||
749 | $plugin->hook_filter_triggered($feed, $owner_uid, $article, $matched_filters, $matched_rules, $article_filters); |
||||||
750 | } |
||||||
751 | |||||||
752 | $matched_filter_ids = array_map(function($f) { return $f['id']; }, $matched_filters); |
||||||
753 | |||||||
754 | if (count($matched_filter_ids) > 0) { |
||||||
755 | $filter_ids_qmarks = arr_qmarks($matched_filter_ids); |
||||||
756 | |||||||
757 | $fsth = $pdo->prepare("UPDATE ttrss_filters2 SET last_triggered = NOW() WHERE |
||||||
758 | id IN ($filter_ids_qmarks) AND owner_uid = ?"); |
||||||
759 | |||||||
760 | $fsth->execute(array_merge($matched_filter_ids, [$owner_uid])); |
||||||
761 | } |
||||||
762 | |||||||
763 | if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { |
||||||
764 | Debug::log("matched filters: ", Debug::$LOG_VERBOSE); |
||||||
765 | |||||||
766 | if (count($matched_filters != 0)) { |
||||||
0 ignored issues
–
show
$matched_filters != 0 of type boolean is incompatible with the type Countable|array expected by parameter $var of count() .
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
![]() |
|||||||
767 | print_r($matched_filters); |
||||||
768 | } |
||||||
769 | |||||||
770 | Debug::log("matched filter rules: ", Debug::$LOG_VERBOSE); |
||||||
771 | |||||||
772 | if (count($matched_rules) != 0) { |
||||||
773 | print_r($matched_rules); |
||||||
774 | } |
||||||
775 | |||||||
776 | Debug::log("filter actions: ", Debug::$LOG_VERBOSE); |
||||||
777 | |||||||
778 | if (count($article_filters) != 0) { |
||||||
779 | print_r($article_filters); |
||||||
780 | } |
||||||
781 | } |
||||||
782 | |||||||
783 | $plugin_filter_names = RSSUtils::find_article_filters($article_filters, "plugin"); |
||||||
784 | $plugin_filter_actions = $pluginhost->get_filter_actions(); |
||||||
785 | |||||||
786 | if (count($plugin_filter_names) > 0) { |
||||||
787 | Debug::log("applying plugin filter actions...", Debug::$LOG_VERBOSE); |
||||||
788 | |||||||
789 | foreach ($plugin_filter_names as $pfn) { |
||||||
790 | list($pfclass, $pfaction) = explode(":", $pfn["param"]); |
||||||
791 | |||||||
792 | if (isset($plugin_filter_actions[$pfclass])) { |
||||||
793 | $plugin = $pluginhost->get_plugin($pfclass); |
||||||
794 | |||||||
795 | Debug::log("... $pfclass: $pfaction", Debug::$LOG_VERBOSE); |
||||||
796 | |||||||
797 | if ($plugin) { |
||||||
798 | $start = microtime(true); |
||||||
799 | $article = $plugin->hook_article_filter_action($article, $pfaction); |
||||||
800 | |||||||
801 | Debug::log(sprintf("=== %.4f (sec)", microtime(true) - $start), Debug::$LOG_VERBOSE); |
||||||
802 | } else { |
||||||
803 | Debug::log("??? $pfclass: plugin object not found.", Debug::$LOG_VERBOSE); |
||||||
804 | } |
||||||
805 | } else { |
||||||
806 | Debug::log("??? $pfclass: filter plugin not registered.", Debug::$LOG_VERBOSE); |
||||||
807 | } |
||||||
808 | } |
||||||
809 | } |
||||||
810 | |||||||
811 | $entry_tags = $article["tags"]; |
||||||
812 | $entry_title = strip_tags($article["title"]); |
||||||
813 | $entry_author = mb_substr(strip_tags($article["author"]), 0, 245); |
||||||
814 | $entry_link = strip_tags($article["link"]); |
||||||
815 | $entry_content = $article["content"]; // escaped below |
||||||
816 | $entry_force_catchup = $article["force_catchup"]; |
||||||
817 | $article_labels = $article["labels"]; |
||||||
818 | $entry_score_modifier = (int) $article["score_modifier"]; |
||||||
819 | $entry_language = $article["language"]; |
||||||
820 | $entry_timestamp = $article["timestamp"]; |
||||||
821 | $num_comments = $article["num_comments"]; |
||||||
822 | |||||||
823 | if ($entry_timestamp == -1 || !$entry_timestamp || $entry_timestamp > time()) { |
||||||
824 | $entry_timestamp = time(); |
||||||
825 | } |
||||||
826 | |||||||
827 | $entry_timestamp_fmt = strftime("%Y/%m/%d %H:%M:%S", $entry_timestamp); |
||||||
828 | |||||||
829 | Debug::log("date $entry_timestamp [$entry_timestamp_fmt]", Debug::$LOG_VERBOSE); |
||||||
830 | Debug::log("num_comments: $num_comments", Debug::$LOG_VERBOSE); |
||||||
831 | |||||||
832 | if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { |
||||||
833 | Debug::log("article labels:", Debug::$LOG_VERBOSE); |
||||||
834 | |||||||
835 | if (count($article_labels) != 0) { |
||||||
836 | print_r($article_labels); |
||||||
837 | } |
||||||
838 | } |
||||||
839 | |||||||
840 | Debug::log("force catchup: $entry_force_catchup", Debug::$LOG_VERBOSE); |
||||||
841 | |||||||
842 | if ($cache_images) { |
||||||
843 | RSSUtils::cache_media($entry_content, $site_url); |
||||||
844 | } |
||||||
845 | |||||||
846 | $csth = $pdo->prepare("SELECT id FROM ttrss_entries |
||||||
847 | WHERE guid = ? OR guid = ?"); |
||||||
848 | $csth->execute([$entry_guid, $entry_guid_hashed]); |
||||||
849 | |||||||
850 | if (!$row = $csth->fetch()) { |
||||||
0 ignored issues
–
show
|
|||||||
851 | |||||||
852 | Debug::log("base guid [$entry_guid or $entry_guid_hashed] not found, creating...", Debug::$LOG_VERBOSE); |
||||||
853 | |||||||
854 | // base post entry does not exist, create it |
||||||
855 | |||||||
856 | $usth = $pdo->prepare( |
||||||
857 | "INSERT INTO ttrss_entries |
||||||
858 | (title, |
||||||
859 | guid, |
||||||
860 | link, |
||||||
861 | updated, |
||||||
862 | content, |
||||||
863 | content_hash, |
||||||
864 | no_orig_date, |
||||||
865 | date_updated, |
||||||
866 | date_entered, |
||||||
867 | comments, |
||||||
868 | num_comments, |
||||||
869 | plugin_data, |
||||||
870 | lang, |
||||||
871 | author) |
||||||
872 | VALUES |
||||||
873 | (?, ?, ?, ?, ?, ?, |
||||||
874 | false, |
||||||
875 | NOW(), |
||||||
876 | ?, ?, ?, ?, ?, ?)"); |
||||||
877 | |||||||
878 | $usth->execute([$entry_title, |
||||||
879 | $entry_guid_hashed, |
||||||
880 | $entry_link, |
||||||
881 | $entry_timestamp_fmt, |
||||||
882 | "$entry_content", |
||||||
883 | $entry_current_hash, |
||||||
884 | $date_feed_processed, |
||||||
885 | $entry_comments, |
||||||
886 | (int) $num_comments, |
||||||
887 | $entry_plugin_data, |
||||||
888 | "$entry_language", |
||||||
889 | "$entry_author"]); |
||||||
890 | |||||||
891 | } |
||||||
892 | |||||||
893 | $csth->execute([$entry_guid, $entry_guid_hashed]); |
||||||
894 | |||||||
895 | $entry_ref_id = 0; |
||||||
896 | $entry_int_id = 0; |
||||||
897 | |||||||
898 | if ($row = $csth->fetch()) { |
||||||
899 | |||||||
900 | Debug::log("base guid found, checking for user record", Debug::$LOG_VERBOSE); |
||||||
901 | |||||||
902 | $ref_id = $row['id']; |
||||||
903 | $entry_ref_id = $ref_id; |
||||||
904 | |||||||
905 | if (RSSUtils::find_article_filter($article_filters, "filter")) { |
||||||
906 | Debug::log("article is filtered out, nothing to do.", Debug::$LOG_VERBOSE); |
||||||
907 | $pdo->commit(); |
||||||
908 | continue; |
||||||
909 | } |
||||||
910 | |||||||
911 | $score = RSSUtils::calculate_article_score($article_filters) + $entry_score_modifier; |
||||||
912 | |||||||
913 | Debug::log("initial score: $score [including plugin modifier: $entry_score_modifier]", Debug::$LOG_VERBOSE); |
||||||
914 | |||||||
915 | // check for user post link to main table |
||||||
916 | |||||||
917 | $sth = $pdo->prepare("SELECT ref_id, int_id FROM ttrss_user_entries WHERE |
||||||
918 | ref_id = ? AND owner_uid = ?"); |
||||||
919 | $sth->execute([$ref_id, $owner_uid]); |
||||||
920 | |||||||
921 | // okay it doesn't exist - create user entry |
||||||
922 | if ($row = $sth->fetch()) { |
||||||
923 | $entry_ref_id = $row["ref_id"]; |
||||||
924 | $entry_int_id = $row["int_id"]; |
||||||
925 | |||||||
926 | Debug::log("user record FOUND: RID: $entry_ref_id, IID: $entry_int_id", Debug::$LOG_VERBOSE); |
||||||
927 | } else { |
||||||
928 | |||||||
929 | Debug::log("user record not found, creating...", Debug::$LOG_VERBOSE); |
||||||
930 | |||||||
931 | if ($score >= -500 && !RSSUtils::find_article_filter($article_filters, 'catchup') && !$entry_force_catchup) { |
||||||
932 | $unread = 1; |
||||||
933 | $last_read_qpart = null; |
||||||
934 | } else { |
||||||
935 | $unread = 0; |
||||||
936 | $last_read_qpart = date("Y-m-d H:i"); // we can't use NOW() here because it gets quoted |
||||||
937 | } |
||||||
938 | |||||||
939 | if (RSSUtils::find_article_filter($article_filters, 'mark') || $score > 1000) { |
||||||
940 | $marked = 1; |
||||||
941 | } else { |
||||||
942 | $marked = 0; |
||||||
943 | } |
||||||
944 | |||||||
945 | if (RSSUtils::find_article_filter($article_filters, 'publish')) { |
||||||
946 | $published = 1; |
||||||
947 | } else { |
||||||
948 | $published = 0; |
||||||
949 | } |
||||||
950 | |||||||
951 | $last_marked = ($marked == 1) ? 'NOW()' : 'NULL'; |
||||||
952 | $last_published = ($published == 1) ? 'NOW()' : 'NULL'; |
||||||
953 | |||||||
954 | $sth = $pdo->prepare( |
||||||
955 | "INSERT INTO ttrss_user_entries |
||||||
956 | (ref_id, owner_uid, feed_id, unread, last_read, marked, |
||||||
957 | published, score, tag_cache, label_cache, uuid, |
||||||
958 | last_marked, last_published) |
||||||
959 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, '', '', '', ".$last_marked.", ".$last_published.")"); |
||||||
960 | |||||||
961 | $sth->execute([$ref_id, $owner_uid, $feed, $unread, $last_read_qpart, $marked, |
||||||
962 | $published, $score]); |
||||||
963 | |||||||
964 | $sth = $pdo->prepare("SELECT int_id FROM ttrss_user_entries WHERE |
||||||
965 | ref_id = ? AND owner_uid = ? AND |
||||||
966 | feed_id = ? LIMIT 1"); |
||||||
967 | |||||||
968 | $sth->execute([$ref_id, $owner_uid, $feed]); |
||||||
969 | |||||||
970 | if ($row = $sth->fetch()) { |
||||||
971 | $entry_int_id = $row['int_id']; |
||||||
972 | } |
||||||
973 | } |
||||||
974 | |||||||
975 | Debug::log("resulting RID: $entry_ref_id, IID: $entry_int_id", Debug::$LOG_VERBOSE); |
||||||
976 | |||||||
977 | if (DB_TYPE == "pgsql") { |
||||||
978 | $tsvector_qpart = "tsvector_combined = to_tsvector(:ts_lang, :ts_content),"; |
||||||
979 | } else { |
||||||
980 | $tsvector_qpart = ""; |
||||||
981 | } |
||||||
982 | |||||||
983 | $sth = $pdo->prepare("UPDATE ttrss_entries |
||||||
984 | SET title = :title, |
||||||
985 | $tsvector_qpart |
||||||
986 | content = :content, |
||||||
987 | content_hash = :content_hash, |
||||||
988 | updated = :updated, |
||||||
989 | date_updated = NOW(), |
||||||
990 | num_comments = :num_comments, |
||||||
991 | plugin_data = :plugin_data, |
||||||
992 | author = :author, |
||||||
993 | lang = :lang |
||||||
994 | WHERE id = :id"); |
||||||
995 | |||||||
996 | $params = [":title" => $entry_title, |
||||||
997 | ":content" => "$entry_content", |
||||||
998 | ":content_hash" => $entry_current_hash, |
||||||
999 | ":updated" => $entry_timestamp_fmt, |
||||||
1000 | ":num_comments" => (int) $num_comments, |
||||||
1001 | ":plugin_data" => $entry_plugin_data, |
||||||
1002 | ":author" => "$entry_author", |
||||||
1003 | ":lang" => $entry_language, |
||||||
1004 | ":id" => $ref_id]; |
||||||
1005 | |||||||
1006 | if (DB_TYPE == "pgsql") { |
||||||
1007 | $params[":ts_lang"] = $feed_language; |
||||||
1008 | $params[":ts_content"] = mb_substr(strip_tags($entry_title." ".$entry_content), 0, 900000); |
||||||
1009 | } |
||||||
1010 | |||||||
1011 | $sth->execute($params); |
||||||
1012 | |||||||
1013 | // update aux data |
||||||
1014 | $sth = $pdo->prepare("UPDATE ttrss_user_entries |
||||||
1015 | SET score = ? WHERE ref_id = ?"); |
||||||
1016 | $sth->execute([$score, $ref_id]); |
||||||
1017 | |||||||
1018 | if ($mark_unread_on_update && |
||||||
1019 | !$entry_force_catchup && |
||||||
1020 | !RSSUtils::find_article_filter($article_filters, 'catchup')) { |
||||||
1021 | |||||||
1022 | Debug::log("article updated, marking unread as requested.", Debug::$LOG_VERBOSE); |
||||||
1023 | |||||||
1024 | $sth = $pdo->prepare("UPDATE ttrss_user_entries |
||||||
1025 | SET last_read = null, unread = true WHERE ref_id = ?"); |
||||||
1026 | $sth->execute([$ref_id]); |
||||||
1027 | } else { |
||||||
1028 | Debug::log("article updated, but we're forbidden to mark it unread.", Debug::$LOG_VERBOSE); |
||||||
1029 | } |
||||||
1030 | } |
||||||
1031 | |||||||
1032 | Debug::log("assigning labels [other]...", Debug::$LOG_VERBOSE); |
||||||
1033 | |||||||
1034 | foreach ($article_labels as $label) { |
||||||
1035 | Labels::add_article($entry_ref_id, $label[1], $owner_uid); |
||||||
1036 | } |
||||||
1037 | |||||||
1038 | Debug::log("assigning labels [filters]...", Debug::$LOG_VERBOSE); |
||||||
1039 | |||||||
1040 | RSSUtils::assign_article_to_label_filters($entry_ref_id, $article_filters, |
||||||
1041 | $owner_uid, $article_labels); |
||||||
1042 | |||||||
1043 | Debug::log("looking for enclosures...", Debug::$LOG_VERBOSE); |
||||||
1044 | |||||||
1045 | // enclosures |
||||||
1046 | |||||||
1047 | $enclosures = array(); |
||||||
1048 | |||||||
1049 | $encs = $item->get_enclosures(); |
||||||
1050 | |||||||
1051 | if (is_array($encs)) { |
||||||
1052 | foreach ($encs as $e) { |
||||||
1053 | $e_item = array( |
||||||
1054 | rewrite_relative_url($site_url, $e->link), |
||||||
1055 | $e->type, $e->length, $e->title, $e->width, $e->height); |
||||||
1056 | |||||||
1057 | // Yet another episode of "mysql utf8_general_ci is gimped" |
||||||
1058 | if (DB_TYPE == "mysql" && MYSQL_CHARSET != "UTF8MB4") { |
||||||
1059 | for ($i = 0; $i < count($e_item); $i++) { |
||||||
0 ignored issues
–
show
It seems like you are calling the size function
count() as part of the test condition. You might want to compute the size beforehand, and not on each iteration.
If the size of the collection does not change during the iteration, it is generally a good practice to compute it beforehand, and not on each iteration: for ($i=0; $i<count($array); $i++) { // calls count() on each iteration
}
// Better
for ($i=0, $c=count($array); $i<$c; $i++) { // calls count() just once
}
![]() |
|||||||
1060 | if (is_string($e_item[$i])) { |
||||||
1061 | $e_item[$i] = RSSUtils::strip_utf8mb4($e_item[$i]); |
||||||
1062 | } |
||||||
1063 | } |
||||||
1064 | } |
||||||
1065 | |||||||
1066 | array_push($enclosures, $e_item); |
||||||
1067 | } |
||||||
1068 | } |
||||||
1069 | |||||||
1070 | if ($cache_images) { |
||||||
1071 | RSSUtils::cache_enclosures($enclosures, $site_url); |
||||||
1072 | } |
||||||
1073 | |||||||
1074 | if (Debug::get_loglevel() >= Debug::$LOG_EXTENDED) { |
||||||
1075 | Debug::log("article enclosures:", Debug::$LOG_VERBOSE); |
||||||
1076 | print_r($enclosures); |
||||||
1077 | } |
||||||
1078 | |||||||
1079 | $esth = $pdo->prepare("SELECT id FROM ttrss_enclosures |
||||||
1080 | WHERE content_url = ? AND content_type = ? AND post_id = ?"); |
||||||
1081 | |||||||
1082 | $usth = $pdo->prepare("INSERT INTO ttrss_enclosures |
||||||
1083 | (content_url, content_type, title, duration, post_id, width, height) VALUES |
||||||
1084 | (?, ?, ?, ?, ?, ?, ?)"); |
||||||
1085 | |||||||
1086 | foreach ($enclosures as $enc) { |
||||||
1087 | $enc_url = $enc[0]; |
||||||
1088 | $enc_type = $enc[1]; |
||||||
1089 | $enc_dur = (int) $enc[2]; |
||||||
1090 | $enc_title = $enc[3]; |
||||||
1091 | $enc_width = intval($enc[4]); |
||||||
1092 | $enc_height = intval($enc[5]); |
||||||
1093 | |||||||
1094 | $esth->execute([$enc_url, $enc_type, $entry_ref_id]); |
||||||
1095 | |||||||
1096 | if (!$esth->fetch()) { |
||||||
1097 | $usth->execute([$enc_url, $enc_type, (string) $enc_title, $enc_dur, $entry_ref_id, $enc_width, $enc_height]); |
||||||
1098 | } |
||||||
1099 | } |
||||||
1100 | |||||||
1101 | // check for manual tags (we have to do it here since they're loaded from filters) |
||||||
1102 | |||||||
1103 | foreach ($article_filters as $f) { |
||||||
1104 | if ($f["type"] == "tag") { |
||||||
1105 | |||||||
1106 | $manual_tags = trim_array(explode(",", $f["param"])); |
||||||
1107 | |||||||
1108 | foreach ($manual_tags as $tag) { |
||||||
1109 | array_push($entry_tags, $tag); |
||||||
1110 | } |
||||||
1111 | } |
||||||
1112 | } |
||||||
1113 | |||||||
1114 | // Skip boring tags |
||||||
1115 | |||||||
1116 | $boring_tags = trim_array(explode(",", mb_strtolower(get_pref( |
||||||
1117 | 'BLACKLISTED_TAGS', $owner_uid, ''), 'utf-8'))); |
||||||
1118 | |||||||
1119 | $filtered_tags = array(); |
||||||
1120 | $tags_to_cache = array(); |
||||||
1121 | |||||||
1122 | foreach ($entry_tags as $tag) { |
||||||
1123 | if (array_search($tag, $boring_tags) === false) { |
||||||
1124 | array_push($filtered_tags, $tag); |
||||||
1125 | } |
||||||
1126 | } |
||||||
1127 | |||||||
1128 | $filtered_tags = array_unique($filtered_tags); |
||||||
1129 | |||||||
1130 | if (Debug::get_loglevel() >= Debug::$LOG_VERBOSE) { |
||||||
1131 | Debug::log("filtered tags: ".implode(", ", $filtered_tags), Debug::$LOG_VERBOSE); |
||||||
1132 | |||||||
1133 | } |
||||||
1134 | |||||||
1135 | // Save article tags in the database |
||||||
1136 | |||||||
1137 | if (count($filtered_tags) > 0) { |
||||||
1138 | |||||||
1139 | $tsth = $pdo->prepare("SELECT id FROM ttrss_tags |
||||||
1140 | WHERE tag_name = ? AND post_int_id = ? AND |
||||||
1141 | owner_uid = ? LIMIT 1"); |
||||||
1142 | |||||||
1143 | $usth = $pdo->prepare("INSERT INTO ttrss_tags |
||||||
1144 | (owner_uid,tag_name,post_int_id) |
||||||
1145 | VALUES (?, ?, ?)"); |
||||||
1146 | |||||||
1147 | $filtered_tags = FeedItem_Common::normalize_categories($filtered_tags); |
||||||
1148 | |||||||
1149 | foreach ($filtered_tags as $tag) { |
||||||
1150 | $tsth->execute([$tag, $entry_int_id, $owner_uid]); |
||||||
1151 | |||||||
1152 | if (!$tsth->fetch()) { |
||||||
1153 | $usth->execute([$owner_uid, $tag, $entry_int_id]); |
||||||
1154 | } |
||||||
1155 | |||||||
1156 | array_push($tags_to_cache, $tag); |
||||||
1157 | } |
||||||
1158 | |||||||
1159 | /* update the cache */ |
||||||
1160 | $tags_str = join(",", $tags_to_cache); |
||||||
1161 | |||||||
1162 | $tsth = $pdo->prepare("UPDATE ttrss_user_entries |
||||||
1163 | SET tag_cache = ? WHERE ref_id = ? |
||||||
1164 | AND owner_uid = ?"); |
||||||
1165 | $tsth->execute([$tags_str, $entry_ref_id, $owner_uid]); |
||||||
1166 | } |
||||||
1167 | |||||||
1168 | Debug::log("article processed", Debug::$LOG_VERBOSE); |
||||||
1169 | |||||||
1170 | $pdo->commit(); |
||||||
1171 | } |
||||||
1172 | |||||||
1173 | Debug::log("purging feed...", Debug::$LOG_VERBOSE); |
||||||
1174 | |||||||
1175 | Feeds::purge_feed($feed, 0); |
||||||
1176 | |||||||
1177 | $sth = $pdo->prepare("UPDATE ttrss_feeds |
||||||
1178 | SET last_updated = NOW(), last_unconditional = NOW(), last_error = '' WHERE id = ?"); |
||||||
1179 | $sth->execute([$feed]); |
||||||
1180 | |||||||
1181 | } else { |
||||||
1182 | |||||||
1183 | $error_msg = mb_substr($rss->error(), 0, 245); |
||||||
1184 | |||||||
1185 | Debug::log("fetch error: $error_msg", Debug::$LOG_VERBOSE); |
||||||
1186 | |||||||
1187 | if (count($rss->errors()) > 1) { |
||||||
1188 | foreach ($rss->errors() as $error) { |
||||||
1189 | Debug::log("+ $error", Debug::$LOG_VERBOSE); |
||||||
1190 | } |
||||||
1191 | } |
||||||
1192 | |||||||
1193 | $sth = $pdo->prepare("UPDATE ttrss_feeds SET last_error = ?, |
||||||
1194 | last_updated = NOW(), last_unconditional = NOW() WHERE id = ?"); |
||||||
1195 | $sth->execute([$error_msg, $feed]); |
||||||
1196 | |||||||
1197 | unset($rss); |
||||||
1198 | |||||||
1199 | Debug::log("update failed.", Debug::$LOG_VERBOSE); |
||||||
1200 | return false; |
||||||
1201 | } |
||||||
1202 | |||||||
1203 | Debug::log("update done.", Debug::$LOG_VERBOSE); |
||||||
1204 | |||||||
1205 | return true; |
||||||
1206 | } |
||||||
1207 | |||||||
1208 | public static function cache_enclosures($enclosures, $site_url) { |
||||||
1209 | $cache = new DiskCache("images"); |
||||||
1210 | |||||||
1211 | if ($cache->isWritable()) { |
||||||
1212 | foreach ($enclosures as $enc) { |
||||||
1213 | |||||||
1214 | if (preg_match("/(image|audio|video)/", $enc[1])) { |
||||||
1215 | $src = rewrite_relative_url($site_url, $enc[0]); |
||||||
1216 | |||||||
1217 | $local_filename = sha1($src); |
||||||
1218 | |||||||
1219 | Debug::log("cache_enclosures: downloading: $src to $local_filename", Debug::$LOG_VERBOSE); |
||||||
1220 | |||||||
1221 | if (!$cache->exists($local_filename)) { |
||||||
1222 | |||||||
1223 | global $fetch_last_error_code; |
||||||
1224 | global $fetch_last_error; |
||||||
1225 | |||||||
1226 | $file_content = fetch_file_contents(array("url" => $src, |
||||||
1227 | "http_referrer" => $src, |
||||||
1228 | "max_size" => MAX_CACHE_FILE_SIZE)); |
||||||
0 ignored issues
–
show
|
|||||||
1229 | |||||||
1230 | if ($file_content) { |
||||||
1231 | $cache->put($local_filename, $file_content); |
||||||
1232 | } else { |
||||||
1233 | Debug::log("cache_enclosures: failed with $fetch_last_error_code: $fetch_last_error"); |
||||||
1234 | } |
||||||
1235 | } else if (is_writable($local_filename)) { |
||||||
1236 | $cache->touch($local_filename); |
||||||
1237 | } |
||||||
1238 | } |
||||||
1239 | } |
||||||
1240 | } |
||||||
1241 | } |
||||||
1242 | |||||||
1243 | public static function cache_media($html, $site_url) { |
||||||
1244 | $cache = new DiskCache("images"); |
||||||
1245 | |||||||
1246 | if ($cache->isWritable()) { |
||||||
1247 | $doc = new DOMDocument(); |
||||||
1248 | if ($doc->loadHTML($html)) { |
||||||
1249 | $xpath = new DOMXPath($doc); |
||||||
1250 | |||||||
1251 | $entries = $xpath->query('(//img[@src])|(//video/source[@src])|(//audio/source[@src])'); |
||||||
1252 | |||||||
1253 | foreach ($entries as $entry) { |
||||||
1254 | if ($entry->hasAttribute('src') && strpos($entry->getAttribute('src'), "data:") !== 0) { |
||||||
1255 | $src = rewrite_relative_url($site_url, $entry->getAttribute('src')); |
||||||
1256 | |||||||
1257 | $local_filename = sha1($src); |
||||||
1258 | |||||||
1259 | Debug::log("cache_media: checking $src", Debug::$LOG_VERBOSE); |
||||||
1260 | |||||||
1261 | if (!$cache->exists($local_filename)) { |
||||||
1262 | Debug::log("cache_media: downloading: $src to $local_filename", Debug::$LOG_VERBOSE); |
||||||
1263 | |||||||
1264 | global $fetch_last_error_code; |
||||||
1265 | global $fetch_last_error; |
||||||
1266 | |||||||
1267 | $file_content = fetch_file_contents(array("url" => $src, |
||||||
1268 | "http_referrer" => $src, |
||||||
1269 | "max_size" => MAX_CACHE_FILE_SIZE)); |
||||||
0 ignored issues
–
show
|
|||||||
1270 | |||||||
1271 | if ($file_content) { |
||||||
1272 | $cache->put($local_filename, $file_content); |
||||||
1273 | } else { |
||||||
1274 | Debug::log("cache_media: failed with $fetch_last_error_code: $fetch_last_error"); |
||||||
1275 | } |
||||||
1276 | } else if ($cache->isWritable($local_filename)) { |
||||||
1277 | $cache->touch($local_filename); |
||||||
1278 | } |
||||||
1279 | } |
||||||
1280 | } |
||||||
1281 | } |
||||||
1282 | } |
||||||
1283 | } |
||||||
1284 | |||||||
1285 | public static function expire_error_log() { |
||||||
1286 | Debug::log("Removing old error log entries..."); |
||||||
1287 | |||||||
1288 | $pdo = Db::pdo(); |
||||||
1289 | |||||||
1290 | if (DB_TYPE == "pgsql") { |
||||||
0 ignored issues
–
show
|
|||||||
1291 | $pdo->query("DELETE FROM ttrss_error_log |
||||||
1292 | WHERE created_at < NOW() - INTERVAL '7 days'"); |
||||||
1293 | } else { |
||||||
1294 | $pdo->query("DELETE FROM ttrss_error_log |
||||||
1295 | WHERE created_at < DATE_SUB(NOW(), INTERVAL 7 DAY)"); |
||||||
1296 | } |
||||||
1297 | } |
||||||
1298 | |||||||
1299 | public static function expire_feed_archive() { |
||||||
1300 | Debug::log("Removing old archived feeds..."); |
||||||
1301 | |||||||
1302 | $pdo = Db::pdo(); |
||||||
1303 | |||||||
1304 | if (DB_TYPE == "pgsql") { |
||||||
0 ignored issues
–
show
|
|||||||
1305 | $pdo->query("DELETE FROM ttrss_archived_feeds |
||||||
1306 | WHERE created < NOW() - INTERVAL '1 month'"); |
||||||
1307 | } else { |
||||||
1308 | $pdo->query("DELETE FROM ttrss_archived_feeds |
||||||
1309 | WHERE created < DATE_SUB(NOW(), INTERVAL 1 MONTH)"); |
||||||
1310 | } |
||||||
1311 | } |
||||||
1312 | |||||||
1313 | public static function expire_lock_files() { |
||||||
1314 | Debug::log("Removing old lock files...", Debug::$LOG_VERBOSE); |
||||||
1315 | |||||||
1316 | $num_deleted = 0; |
||||||
1317 | |||||||
1318 | if (is_writable(LOCK_DIRECTORY)) { |
||||||
0 ignored issues
–
show
|
|||||||
1319 | $files = glob(LOCK_DIRECTORY."/*.lock"); |
||||||
1320 | |||||||
1321 | if ($files) { |
||||||
1322 | foreach ($files as $file) { |
||||||
1323 | if (!file_is_locked(basename($file)) && time() - filemtime($file) > 86400 * 2) { |
||||||
1324 | unlink($file); |
||||||
1325 | ++$num_deleted; |
||||||
1326 | } |
||||||
1327 | } |
||||||
1328 | } |
||||||
1329 | } |
||||||
1330 | |||||||
1331 | Debug::log("removed $num_deleted old lock files."); |
||||||
1332 | } |
||||||
1333 | |||||||
1334 | /** |
||||||
1335 | * Source: http://www.php.net/manual/en/function.parse-url.php#104527 |
||||||
1336 | * Returns the url query as associative array |
||||||
1337 | * |
||||||
1338 | * @param string query |
||||||
1339 | * @return array params |
||||||
1340 | */ |
||||||
1341 | public static function convertUrlQuery($query) { |
||||||
1342 | $queryParts = explode('&', $query); |
||||||
1343 | |||||||
1344 | $params = array(); |
||||||
1345 | |||||||
1346 | foreach ($queryParts as $param) { |
||||||
1347 | $item = explode('=', $param); |
||||||
1348 | $params[$item[0]] = $item[1]; |
||||||
1349 | } |
||||||
1350 | |||||||
1351 | return $params; |
||||||
1352 | } |
||||||
1353 | |||||||
1354 | public static function get_article_filters($filters, $title, $content, $link, $author, $tags, &$matched_rules = false, &$matched_filters = false) { |
||||||
1355 | $matches = array(); |
||||||
1356 | |||||||
1357 | foreach ($filters as $filter) { |
||||||
1358 | $match_any_rule = $filter["match_any_rule"]; |
||||||
1359 | $inverse = $filter["inverse"]; |
||||||
1360 | $filter_match = false; |
||||||
1361 | |||||||
1362 | foreach ($filter["rules"] as $rule) { |
||||||
1363 | $match = false; |
||||||
1364 | $reg_exp = str_replace('/', '\/', $rule["reg_exp"]); |
||||||
1365 | $rule_inverse = $rule["inverse"]; |
||||||
1366 | |||||||
1367 | if (!$reg_exp) { |
||||||
1368 | continue; |
||||||
1369 | } |
||||||
1370 | |||||||
1371 | switch ($rule["type"]) { |
||||||
1372 | case "title": |
||||||
1373 | $match = @preg_match("/$reg_exp/iu", $title); |
||||||
1374 | break; |
||||||
1375 | case "content": |
||||||
1376 | // we don't need to deal with multiline regexps |
||||||
1377 | $content = preg_replace("/[\r\n\t]/", "", $content); |
||||||
1378 | |||||||
1379 | $match = @preg_match("/$reg_exp/iu", $content); |
||||||
1380 | break; |
||||||
1381 | case "both": |
||||||
1382 | // we don't need to deal with multiline regexps |
||||||
1383 | $content = preg_replace("/[\r\n\t]/", "", $content); |
||||||
1384 | |||||||
1385 | $match = (@preg_match("/$reg_exp/iu", $title) || @preg_match("/$reg_exp/iu", $content)); |
||||||
1386 | break; |
||||||
1387 | case "link": |
||||||
1388 | $match = @preg_match("/$reg_exp/iu", $link); |
||||||
1389 | break; |
||||||
1390 | case "author": |
||||||
1391 | $match = @preg_match("/$reg_exp/iu", $author); |
||||||
1392 | break; |
||||||
1393 | case "tag": |
||||||
1394 | foreach ($tags as $tag) { |
||||||
1395 | if (@preg_match("/$reg_exp/iu", $tag)) { |
||||||
1396 | $match = true; |
||||||
1397 | break; |
||||||
1398 | } |
||||||
1399 | } |
||||||
1400 | break; |
||||||
1401 | } |
||||||
1402 | |||||||
1403 | if ($rule_inverse) { |
||||||
1404 | $match = !$match; |
||||||
1405 | } |
||||||
1406 | |||||||
1407 | if ($match_any_rule) { |
||||||
1408 | if ($match) { |
||||||
1409 | $filter_match = true; |
||||||
1410 | break; |
||||||
1411 | } |
||||||
1412 | } else { |
||||||
1413 | $filter_match = $match; |
||||||
1414 | if (!$match) { |
||||||
1415 | break; |
||||||
1416 | } |
||||||
1417 | } |
||||||
1418 | } |
||||||
1419 | |||||||
1420 | if ($inverse) { |
||||||
1421 | $filter_match = !$filter_match; |
||||||
1422 | } |
||||||
1423 | |||||||
1424 | if ($filter_match) { |
||||||
1425 | if (is_array($matched_rules)) { |
||||||
1426 | array_push($matched_rules, $rule); |
||||||
0 ignored issues
–
show
Comprehensibility
Best Practice
introduced
by
|
|||||||
1427 | } |
||||||
1428 | if (is_array($matched_filters)) { |
||||||
1429 | array_push($matched_filters, $filter); |
||||||
1430 | } |
||||||
1431 | |||||||
1432 | foreach ($filter["actions"] as $action) { |
||||||
1433 | array_push($matches, $action); |
||||||
1434 | |||||||
1435 | // if Stop action encountered, perform no further processing |
||||||
1436 | if (isset($action["type"]) && $action["type"] == "stop") { |
||||||
1437 | return $matches; |
||||||
1438 | } |
||||||
1439 | } |
||||||
1440 | } |
||||||
1441 | } |
||||||
1442 | |||||||
1443 | return $matches; |
||||||
1444 | } |
||||||
1445 | |||||||
1446 | public static function find_article_filter($filters, $filter_name) { |
||||||
1447 | foreach ($filters as $f) { |
||||||
1448 | if ($f["type"] == $filter_name) { |
||||||
1449 | return $f; |
||||||
1450 | }; |
||||||
1451 | } |
||||||
1452 | return false; |
||||||
1453 | } |
||||||
1454 | |||||||
1455 | public static function find_article_filters($filters, $filter_name) { |
||||||
1456 | $results = array(); |
||||||
1457 | |||||||
1458 | foreach ($filters as $f) { |
||||||
1459 | if ($f["type"] == $filter_name) { |
||||||
1460 | array_push($results, $f); |
||||||
1461 | }; |
||||||
1462 | } |
||||||
1463 | return $results; |
||||||
1464 | } |
||||||
1465 | |||||||
1466 | public static function calculate_article_score($filters) { |
||||||
1467 | $score = 0; |
||||||
1468 | |||||||
1469 | foreach ($filters as $f) { |
||||||
1470 | if ($f["type"] == "score") { |
||||||
1471 | $score += $f["param"]; |
||||||
1472 | }; |
||||||
1473 | } |
||||||
1474 | return $score; |
||||||
1475 | } |
||||||
1476 | |||||||
1477 | public static function labels_contains_caption($labels, $caption) { |
||||||
1478 | foreach ($labels as $label) { |
||||||
1479 | if ($label[1] == $caption) { |
||||||
1480 | return true; |
||||||
1481 | } |
||||||
1482 | } |
||||||
1483 | |||||||
1484 | return false; |
||||||
1485 | } |
||||||
1486 | |||||||
1487 | public static function assign_article_to_label_filters($id, $filters, $owner_uid, $article_labels) { |
||||||
1488 | foreach ($filters as $f) { |
||||||
1489 | if ($f["type"] == "label") { |
||||||
1490 | if (!RSSUtils::labels_contains_caption($article_labels, $f["param"])) { |
||||||
1491 | Labels::add_article($id, $f["param"], $owner_uid); |
||||||
1492 | } |
||||||
1493 | } |
||||||
1494 | } |
||||||
1495 | } |
||||||
1496 | |||||||
1497 | public static function make_guid_from_title($title) { |
||||||
1498 | return preg_replace("/[ \"\',.:;]/", "-", |
||||||
1499 | mb_strtolower(strip_tags($title), 'utf-8')); |
||||||
1500 | } |
||||||
1501 | |||||||
1502 | public static function cleanup_counters_cache() { |
||||||
1503 | $pdo = Db::pdo(); |
||||||
1504 | |||||||
1505 | $res = $pdo->query("DELETE FROM ttrss_counters_cache |
||||||
1506 | WHERE feed_id > 0 AND |
||||||
1507 | (SELECT COUNT(id) FROM ttrss_feeds WHERE |
||||||
1508 | id = feed_id AND |
||||||
1509 | ttrss_counters_cache.owner_uid = ttrss_feeds.owner_uid) = 0"); |
||||||
1510 | |||||||
1511 | $frows = $res->rowCount(); |
||||||
1512 | |||||||
1513 | $res = $pdo->query("DELETE FROM ttrss_cat_counters_cache |
||||||
1514 | WHERE feed_id > 0 AND |
||||||
1515 | (SELECT COUNT(id) FROM ttrss_feed_categories WHERE |
||||||
1516 | id = feed_id AND |
||||||
1517 | ttrss_cat_counters_cache.owner_uid = ttrss_feed_categories.owner_uid) = 0"); |
||||||
1518 | |||||||
1519 | $crows = $res->rowCount(); |
||||||
1520 | |||||||
1521 | Debug::log("removed $frows (feeds) $crows (cats) orphaned counter cache entries."); |
||||||
1522 | } |
||||||
1523 | |||||||
1524 | public static function housekeeping_user($owner_uid) { |
||||||
1525 | $tmph = new PluginHost(); |
||||||
1526 | |||||||
1527 | load_user_plugins($owner_uid, $tmph); |
||||||
1528 | |||||||
1529 | $tmph->run_hooks(PluginHost::HOOK_HOUSE_KEEPING, "hook_house_keeping", ""); |
||||||
1530 | } |
||||||
1531 | |||||||
1532 | public static function housekeeping_common() { |
||||||
1533 | DiskCache::expire(); |
||||||
1534 | |||||||
1535 | RSSUtils::expire_lock_files(); |
||||||
1536 | RSSUtils::expire_error_log(); |
||||||
1537 | RSSUtils::expire_feed_archive(); |
||||||
1538 | RSSUtils::cleanup_feed_browser(); |
||||||
1539 | |||||||
1540 | Article::purge_orphans(); |
||||||
1541 | RSSUtils::cleanup_counters_cache(); |
||||||
1542 | |||||||
1543 | PluginHost::getInstance()->run_hooks(PluginHost::HOOK_HOUSE_KEEPING, "hook_house_keeping", ""); |
||||||
1544 | } |
||||||
1545 | |||||||
1546 | public static function check_feed_favicon($site_url, $feed) { |
||||||
1547 | # print "FAVICON [$site_url]: $favicon_url\n"; |
||||||
1548 | |||||||
1549 | $icon_file = ICONS_DIR."/$feed.ico"; |
||||||
0 ignored issues
–
show
|
|||||||
1550 | |||||||
1551 | if (!file_exists($icon_file)) { |
||||||
1552 | $favicon_url = RSSUtils::get_favicon_url($site_url); |
||||||
1553 | |||||||
1554 | if ($favicon_url) { |
||||||
1555 | // Limiting to "image" type misses those served with text/plain |
||||||
1556 | $contents = fetch_file_contents($favicon_url); // , "image"); |
||||||
1557 | |||||||
1558 | if ($contents) { |
||||||
1559 | // Crude image type matching. |
||||||
1560 | // Patterns gleaned from the file(1) source code. |
||||||
1561 | if (preg_match('/^\x00\x00\x01\x00/', $contents)) { |
||||||
0 ignored issues
–
show
It seems like
$contents can also be of type true ; however, parameter $subject of preg_match() does only seem to accept string , maybe add an additional type check?
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
![]() |
|||||||
1562 | // 0 string \000\000\001\000 MS Windows icon resource |
||||||
1563 | //error_log("check_feed_favicon: favicon_url=$favicon_url isa MS Windows icon resource"); |
||||||
1564 | } elseif (preg_match('/^GIF8/', $contents)) { |
||||||
1565 | // 0 string GIF8 GIF image data |
||||||
1566 | //error_log("check_feed_favicon: favicon_url=$favicon_url isa GIF image"); |
||||||
1567 | } elseif (preg_match('/^\x89PNG\x0d\x0a\x1a\x0a/', $contents)) { |
||||||
1568 | // 0 string \x89PNG\x0d\x0a\x1a\x0a PNG image data |
||||||
1569 | //error_log("check_feed_favicon: favicon_url=$favicon_url isa PNG image"); |
||||||
1570 | } elseif (preg_match('/^\xff\xd8/', $contents)) { |
||||||
1571 | // 0 beshort 0xffd8 JPEG image data |
||||||
1572 | //error_log("check_feed_favicon: favicon_url=$favicon_url isa JPG image"); |
||||||
1573 | } elseif (preg_match('/^BM/', $contents)) { |
||||||
1574 | // 0 string BM PC bitmap (OS2, Windows BMP files) |
||||||
1575 | //error_log("check_feed_favicon, favicon_url=$favicon_url isa BMP image"); |
||||||
1576 | } else { |
||||||
1577 | //error_log("check_feed_favicon: favicon_url=$favicon_url isa UNKNOWN type"); |
||||||
1578 | $contents = ""; |
||||||
1579 | } |
||||||
1580 | } |
||||||
1581 | |||||||
1582 | if ($contents) { |
||||||
1583 | $fp = @fopen($icon_file, "w"); |
||||||
1584 | |||||||
1585 | if ($fp) { |
||||||
0 ignored issues
–
show
|
|||||||
1586 | fwrite($fp, $contents); |
||||||
1587 | fclose($fp); |
||||||
1588 | chmod($icon_file, 0644); |
||||||
1589 | } |
||||||
1590 | } |
||||||
1591 | } |
||||||
1592 | return $icon_file; |
||||||
1593 | } |
||||||
1594 | } |
||||||
1595 | |||||||
1596 | public static function is_gzipped($feed_data) { |
||||||
1597 | return strpos(substr($feed_data, 0, 3), |
||||||
1598 | "\x1f"."\x8b"."\x08", 0) === 0; |
||||||
1599 | } |
||||||
1600 | |||||||
1601 | public static function load_filters($feed_id, $owner_uid) { |
||||||
1602 | $filters = array(); |
||||||
1603 | |||||||
1604 | $feed_id = (int) $feed_id; |
||||||
1605 | $cat_id = (int) Feeds::getFeedCategory($feed_id); |
||||||
1606 | |||||||
1607 | if ($cat_id == 0) { |
||||||
1608 | $null_cat_qpart = "cat_id IS NULL OR"; |
||||||
1609 | } else { |
||||||
1610 | $null_cat_qpart = ""; |
||||||
1611 | } |
||||||
1612 | |||||||
1613 | $pdo = Db::pdo(); |
||||||
1614 | |||||||
1615 | $sth = $pdo->prepare("SELECT * FROM ttrss_filters2 WHERE |
||||||
1616 | owner_uid = ? AND enabled = true ORDER BY order_id, title"); |
||||||
1617 | $sth->execute([$owner_uid]); |
||||||
1618 | |||||||
1619 | $check_cats = array_merge( |
||||||
1620 | Feeds::getParentCategories($cat_id, $owner_uid), |
||||||
1621 | [$cat_id]); |
||||||
1622 | |||||||
1623 | $check_cats_str = join(",", $check_cats); |
||||||
1624 | $check_cats_fullids = array_map(function($a) { return "CAT:$a"; }, $check_cats); |
||||||
1625 | |||||||
1626 | while ($line = $sth->fetch()) { |
||||||
1627 | $filter_id = $line["id"]; |
||||||
1628 | |||||||
1629 | $match_any_rule = sql_bool_to_bool($line["match_any_rule"]); |
||||||
1630 | |||||||
1631 | $sth2 = $pdo->prepare("SELECT |
||||||
1632 | r.reg_exp, r.inverse, r.feed_id, r.cat_id, r.cat_filter, r.match_on, t.name AS type_name |
||||||
1633 | FROM ttrss_filters2_rules AS r, |
||||||
1634 | ttrss_filter_types AS t |
||||||
1635 | WHERE |
||||||
1636 | (match_on IS NOT NULL OR |
||||||
1637 | (($null_cat_qpart (cat_id IS NULL AND cat_filter = false) OR cat_id IN ($check_cats_str)) AND |
||||||
1638 | (feed_id IS NULL OR feed_id = ?))) AND |
||||||
1639 | filter_type = t.id AND filter_id = ?"); |
||||||
1640 | $sth2->execute([$feed_id, $filter_id]); |
||||||
1641 | |||||||
1642 | $rules = array(); |
||||||
1643 | $actions = array(); |
||||||
1644 | |||||||
1645 | while ($rule_line = $sth2->fetch()) { |
||||||
1646 | # print_r($rule_line); |
||||||
1647 | |||||||
1648 | if ($rule_line["match_on"]) { |
||||||
1649 | $match_on = json_decode($rule_line["match_on"], true); |
||||||
1650 | |||||||
1651 | if (in_array("0", $match_on) || in_array($feed_id, $match_on) || count(array_intersect($check_cats_fullids, $match_on)) > 0) { |
||||||
1652 | |||||||
1653 | $rule = array(); |
||||||
1654 | $rule["reg_exp"] = $rule_line["reg_exp"]; |
||||||
1655 | $rule["type"] = $rule_line["type_name"]; |
||||||
1656 | $rule["inverse"] = sql_bool_to_bool($rule_line["inverse"]); |
||||||
1657 | |||||||
1658 | array_push($rules, $rule); |
||||||
1659 | } else if (!$match_any_rule) { |
||||||
1660 | // this filter contains a rule that doesn't match to this feed/category combination |
||||||
1661 | // thus filter has to be rejected |
||||||
1662 | |||||||
1663 | $rules = []; |
||||||
1664 | break; |
||||||
1665 | } |
||||||
1666 | |||||||
1667 | } else { |
||||||
1668 | |||||||
1669 | $rule = array(); |
||||||
1670 | $rule["reg_exp"] = $rule_line["reg_exp"]; |
||||||
1671 | $rule["type"] = $rule_line["type_name"]; |
||||||
1672 | $rule["inverse"] = sql_bool_to_bool($rule_line["inverse"]); |
||||||
1673 | |||||||
1674 | array_push($rules, $rule); |
||||||
1675 | } |
||||||
1676 | } |
||||||
1677 | |||||||
1678 | if (count($rules) > 0) { |
||||||
1679 | $sth2 = $pdo->prepare("SELECT a.action_param,t.name AS type_name |
||||||
1680 | FROM ttrss_filters2_actions AS a, |
||||||
1681 | ttrss_filter_actions AS t |
||||||
1682 | WHERE |
||||||
1683 | action_id = t.id AND filter_id = ?"); |
||||||
1684 | $sth2->execute([$filter_id]); |
||||||
1685 | |||||||
1686 | while ($action_line = $sth2->fetch()) { |
||||||
1687 | # print_r($action_line); |
||||||
1688 | |||||||
1689 | $action = array(); |
||||||
1690 | $action["type"] = $action_line["type_name"]; |
||||||
1691 | $action["param"] = $action_line["action_param"]; |
||||||
1692 | |||||||
1693 | array_push($actions, $action); |
||||||
1694 | } |
||||||
1695 | } |
||||||
1696 | |||||||
1697 | $filter = []; |
||||||
1698 | $filter["id"] = $filter_id; |
||||||
1699 | $filter["match_any_rule"] = sql_bool_to_bool($line["match_any_rule"]); |
||||||
1700 | $filter["inverse"] = sql_bool_to_bool($line["inverse"]); |
||||||
1701 | $filter["rules"] = $rules; |
||||||
1702 | $filter["actions"] = $actions; |
||||||
1703 | |||||||
1704 | if (count($rules) > 0 && count($actions) > 0) { |
||||||
1705 | array_push($filters, $filter); |
||||||
1706 | } |
||||||
1707 | } |
||||||
1708 | |||||||
1709 | return $filters; |
||||||
1710 | } |
||||||
1711 | |||||||
1712 | /** |
||||||
1713 | * Try to determine the favicon URL for a feed. |
||||||
1714 | * adapted from wordpress favicon plugin by Jeff Minard (http://thecodepro.com/) |
||||||
1715 | * http://dev.wp-plugins.org/file/favatars/trunk/favatars.php |
||||||
1716 | * |
||||||
1717 | * @param string $url A feed or page URL |
||||||
1718 | * @access public |
||||||
1719 | * @return mixed The favicon URL, or false if none was found. |
||||||
1720 | */ |
||||||
1721 | public static function get_favicon_url($url) { |
||||||
1722 | |||||||
1723 | $favicon_url = false; |
||||||
1724 | |||||||
1725 | if ($html = @fetch_file_contents($url)) { |
||||||
1726 | |||||||
1727 | $doc = new DOMDocument(); |
||||||
1728 | if ($doc->loadHTML($html)) { |
||||||
0 ignored issues
–
show
It seems like
$html can also be of type true ; however, parameter $source of DOMDocument::loadHTML() does only seem to accept string , maybe add an additional type check?
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
![]() |
|||||||
1729 | $xpath = new DOMXPath($doc); |
||||||
1730 | |||||||
1731 | $base = $xpath->query('/html/head/base[@href]'); |
||||||
1732 | foreach ($base as $b) { |
||||||
1733 | $url = rewrite_relative_url($url, $b->getAttribute("href")); |
||||||
1734 | break; |
||||||
1735 | } |
||||||
1736 | |||||||
1737 | $entries = $xpath->query('/html/head/link[@rel="shortcut icon" or @rel="icon"]'); |
||||||
1738 | if (count($entries) > 0) { |
||||||
0 ignored issues
–
show
It seems like
$entries can also be of type false ; however, parameter $var of count() does only seem to accept Countable|array , maybe add an additional type check?
(
Ignorable by Annotation
)
If this is a false-positive, you can also ignore this issue in your code via the
![]() |
|||||||
1739 | foreach ($entries as $entry) { |
||||||
1740 | $favicon_url = rewrite_relative_url($url, $entry->getAttribute("href")); |
||||||
1741 | break; |
||||||
1742 | } |
||||||
1743 | } |
||||||
1744 | } |
||||||
1745 | } |
||||||
1746 | |||||||
1747 | if (!$favicon_url) { |
||||||
1748 | $favicon_url = rewrite_relative_url($url, "/favicon.ico"); |
||||||
1749 | } |
||||||
1750 | |||||||
1751 | return $favicon_url; |
||||||
1752 | } |
||||||
1753 | |||||||
1754 | } |
||||||
1755 |