Issues (1270)

update.php (11 issues)

1
#!/usr/bin/env php
2
<?php
3
    set_include_path(dirname(__FILE__)."/include".PATH_SEPARATOR.
4
        get_include_path());
5
6
    define('DISABLE_SESSIONS', true);
7
8
    chdir(dirname(__FILE__));
9
10
    require_once "autoload.php";
11
    require_once "functions.php";
12
    require_once "config.php";
13
    require_once "sanity_check.php";
14
    require_once "db.php";
15
    require_once "db-prefs.php";
16
17
    function cleanup_tags($days = 14, $limit = 1000) {
18
19
        $days = (int) $days;
20
21
        if (DB_TYPE == "pgsql") {
0 ignored issues
show
The constant DB_TYPE was not found. Maybe you did not declare it correctly or list all dependencies?
Loading history...
22
            $interval_query = "date_updated < NOW() - INTERVAL '$days days'";
23
        } else if (DB_TYPE == "mysql") {
24
            $interval_query = "date_updated < DATE_SUB(NOW(), INTERVAL $days DAY)";
25
        }
26
27
        $tags_deleted = 0;
28
29
        $pdo = Db::pdo();
30
31
        while ($limit > 0) {
32
            $limit_part = 500;
33
34
            $sth = $pdo->prepare("SELECT ttrss_tags.id AS id
35
						FROM ttrss_tags, ttrss_user_entries, ttrss_entries
36
						WHERE post_int_id = int_id AND $interval_query AND
0 ignored issues
show
Comprehensibility Best Practice introduced by
The variable $interval_query does not seem to be defined for all execution paths leading up to this point.
Loading history...
37
						ref_id = ttrss_entries.id AND tag_cache != '' LIMIT ?");
38
            $sth->bindValue(1, $limit_part, PDO::PARAM_INT);
39
            $sth->execute();
40
41
            $ids = array();
42
43
            while ($line = $sth->fetch()) {
44
                array_push($ids, $line['id']);
45
            }
46
47
            if (count($ids) > 0) {
48
                $ids = join(",", $ids);
49
50
                $usth = $pdo->query("DELETE FROM ttrss_tags WHERE id IN ($ids)");
51
                $tags_deleted = $usth->rowCount();
52
            } else {
53
                break;
54
            }
55
56
            $limit -= $limit_part;
57
        }
58
59
        return $tags_deleted;
60
    }
61
62
    if (!defined('PHP_EXECUTABLE')) {
63
            define('PHP_EXECUTABLE', '/usr/bin/php');
64
    }
65
66
    $pdo = Db::pdo();
67
68
    init_plugins();
69
70
    $longopts = array("feeds",
71
            "daemon",
72
            "daemon-loop",
73
            "send-digests",
74
            "task:",
75
            "cleanup-tags",
76
            "quiet",
77
            "log:",
78
            "log-level:",
79
            "indexes",
80
            "pidlock:",
81
            "update-schema",
82
            "convert-filters",
83
            "force-update",
84
            "gen-search-idx",
85
            "list-plugins",
86
            "debug-feed:",
87
            "force-refetch",
88
            "force-rehash",
89
            "help");
90
91
    foreach (PluginHost::getInstance()->get_commands() as $command => $data) {
92
        array_push($longopts, $command.$data["suffix"]);
93
    }
94
95
    $options = getopt("", $longopts);
96
97
    if (!is_array($options)) {
0 ignored issues
show
The condition is_array($options) is always true.
Loading history...
98
        die("error: getopt() failed. ".
99
            "Most probably you are using PHP CGI to run this script ".
100
            "instead of required PHP CLI. Check tt-rss wiki page on updating feeds for ".
101
            "additional information.\n");
102
    }
103
104
    if (count($options) == 0 && !defined('STDIN')) {
105
        ?>
106
		<!DOCTYPE html>
107
		<html>
108
		<head>
109
		<title>Tiny Tiny RSS data update script.</title>
110
		<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
111
		</head>
112
113
		<body>
114
		<h1><?php echo __("Tiny Tiny RSS data update script.") ?></h1>
115
116
		<?php print_error("Please run this script from the command line. Use option \"--help\" to display command help if this error is displayed erroneously."); ?>
0 ignored issues
show
The call to print_error() has too many arguments starting with 'Please run this script ...displayed erroneously.'. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-call  annotation

116
		<?php /** @scrutinizer ignore-call */ print_error("Please run this script from the command line. Use option \"--help\" to display command help if this error is displayed erroneously."); ?>

This check compares calls to functions or methods with their respective definitions. If the call has more arguments than are defined, it raises an issue.

If a function is defined several times with a different number of parameters, the check may pick up the wrong definition and report false positives. One codebase where this has been known to happen is Wordpress. Please note the @ignore annotation hint above.

Loading history...
Deprecated Code introduced by
The function print_error() has been deprecated: Use twig function errorMessage ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

116
		<?php /** @scrutinizer ignore-deprecated */ print_error("Please run this script from the command line. Use option \"--help\" to display command help if this error is displayed erroneously."); ?>

This function has been deprecated. The supplier of the function has supplied an explanatory message.

The explanatory message should give you some clue as to whether and when the function will be removed and what other function to use instead.

Loading history...
117
118
		</body></html>
119
	<?php
120
        exit;
121
    }
122
123
    if (count($options) == 0 || isset($options["help"])) {
124
        print "Tiny Tiny RSS data update script.\n\n";
125
        print "Options:\n";
126
        print "  --feeds              - update feeds\n";
127
        print "  --daemon             - start single-process update daemon\n";
128
        print "  --task N             - create lockfile using this task id\n";
129
        print "  --cleanup-tags       - perform tags table maintenance\n";
130
        print "  --quiet              - don't output messages to stdout\n";
131
        print "  --log FILE           - log messages to FILE\n";
132
        print "  --log-level N        - log verbosity level\n";
133
        print "  --indexes            - recreate missing schema indexes\n";
134
        print "  --update-schema      - update database schema\n";
135
        print "  --gen-search-idx     - generate basic PostgreSQL fulltext search index\n";
136
        print "  --convert-filters    - convert type1 filters to type2\n";
137
        print "  --send-digests       - send pending email digests\n";
138
        print "  --force-update       - force update of all feeds\n";
139
        print "  --list-plugins       - list all available plugins\n";
140
        print "  --debug-feed N       - perform debug update of feed N\n";
141
        print "  --force-refetch      - debug update: force refetch feed data\n";
142
        print "  --force-rehash       - debug update: force rehash articles\n";
143
        print "  --help               - show this help\n";
144
        print "Plugin options:\n";
145
146
        foreach (PluginHost::getInstance()->get_commands() as $command => $data) {
147
            $args = $data['arghelp'];
148
            printf(" --%-19s - %s\n", "$command $args", $data["description"]);
149
        }
150
151
        return;
152
    }
153
154
    if (!isset($options['daemon'])) {
155
        require_once "errorhandler.php";
156
    }
157
158
    if (!isset($options['update-schema'])) {
159
        $schema_version = get_schema_version();
160
161
        if ($schema_version != SCHEMA_VERSION) {
162
            die("Schema version is wrong, please upgrade the database (--update-schema).\n");
163
        }
164
    }
165
166
    Debug::set_enabled(true);
167
168
    if (isset($options["log-level"])) {
169
        Debug::set_loglevel((int) $options["log-level"]);
170
    }
171
172
    if (isset($options["log"])) {
173
        Debug::set_quiet(isset($options['quiet']));
174
        Debug::set_logfile($options["log"]);
175
        Debug::log("Logging to ".$options["log"]);
176
    } else {
177
        if (isset($options['quiet'])) {
178
            Debug::set_loglevel(Debug::$LOG_DISABLED);
179
        }
180
    }
181
182
    if (!isset($options["daemon"])) {
183
        $lock_filename = "update.lock";
184
    } else {
185
        $lock_filename = "update_daemon.lock";
186
    }
187
188
    if (isset($options["task"])) {
189
        Debug::log("Using task id ".$options["task"]);
190
        $lock_filename = $lock_filename."-task_".$options["task"];
191
    }
192
193
    if (isset($options["pidlock"])) {
194
        $my_pid = $options["pidlock"];
195
        $lock_filename = "update_daemon-$my_pid.lock";
196
197
    }
198
199
    Debug::log("Lock: $lock_filename");
200
201
    $lock_handle = make_lockfile($lock_filename);
202
    $must_exit = false;
203
204
    if (isset($options["task"]) && isset($options["pidlock"])) {
205
        $waits = $options["task"] * 5;
206
        Debug::log("Waiting before update ($waits)");
207
        sleep($waits);
208
    }
209
210
    // Try to lock a file in order to avoid concurrent update.
211
    if (!$lock_handle) {
0 ignored issues
show
The condition $lock_handle is always false.
Loading history...
212
        die("error: Can't create lockfile ($lock_filename). ".
213
            "Maybe another update process is already running.\n");
214
    }
215
216
    if (isset($options["force-update"])) {
217
        Debug::log("marking all feeds as needing update...");
218
219
        $pdo->query("UPDATE ttrss_feeds SET
220
          last_update_started = '1970-01-01', last_updated = '1970-01-01'");
221
    }
222
223
    if (isset($options["feeds"])) {
224
        RSSUtils::update_daemon_common();
225
        RSSUtils::housekeeping_common(true);
226
227
        PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK, "hook_update_task", $op);
228
    }
229
230
    if (isset($options["daemon"])) {
231
        while (true) {
232
            $quiet = (isset($options["quiet"])) ? "--quiet" : "";
233
            $log = isset($options['log']) ? '--log '.$options['log'] : '';
234
            $log_level = isset($options['log-level']) ? '--log-level '.$options['log-level'] : '';
235
236
            passthru(PHP_EXECUTABLE." ".$argv[0]." --daemon-loop $quiet $log $log_level");
237
238
            // let's enforce a minimum spawn interval as to not forkbomb the host
239
            $spawn_interval = max(60, DAEMON_SLEEP_INTERVAL);
0 ignored issues
show
The constant DAEMON_SLEEP_INTERVAL was not found. Maybe you did not declare it correctly or list all dependencies?
Loading history...
240
241
            Debug::log("Sleeping for $spawn_interval seconds...");
242
            sleep($spawn_interval);
243
        }
244
    }
245
246
    if (isset($options["daemon-loop"])) {
247
        if (!make_stampfile('update_daemon.stamp')) {
248
            Debug::log("warning: unable to create stampfile\n");
249
        }
250
251
        RSSUtils::update_daemon_common(isset($options["pidlock"]) ? 50 : DAEMON_FEED_LIMIT);
0 ignored issues
show
The constant DAEMON_FEED_LIMIT was not found. Maybe you did not declare it correctly or list all dependencies?
Loading history...
252
253
        if (!isset($options["pidlock"]) || $options["task"] == 0) {
254
                    RSSUtils::housekeeping_common(true);
255
        }
256
257
        PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK, "hook_update_task", $op);
258
    }
259
260
    if (isset($options["cleanup-tags"])) {
261
        $rc = cleanup_tags(14, 50000);
262
        Debug::log("$rc tags deleted.\n");
263
    }
264
265
    if (isset($options["indexes"])) {
266
        Debug::log("PLEASE BACKUP YOUR DATABASE BEFORE PROCEEDING!");
267
        Debug::log("Type 'yes' to continue.");
268
269
        if (read_stdin() != 'yes') {
270
                    exit;
271
        }
272
273
        Debug::log("clearing existing indexes...");
274
275
        if (DB_TYPE == "pgsql") {
0 ignored issues
show
The constant DB_TYPE was not found. Maybe you did not declare it correctly or list all dependencies?
Loading history...
276
            $sth = $pdo->query("SELECT relname FROM
277
				pg_catalog.pg_class WHERE relname LIKE 'ttrss_%'
278
					AND relname NOT LIKE '%_pkey'
279
				AND relkind = 'i'");
280
        } else {
281
            $sth = $pdo->query("SELECT index_name,table_name FROM
282
				information_schema.statistics WHERE index_name LIKE 'ttrss_%'");
283
        }
284
285
        while ($line = $sth->fetch()) {
286
            if (DB_TYPE == "pgsql") {
287
                $statement = "DROP INDEX ".$line["relname"];
288
                Debug::log($statement);
289
            } else {
290
                $statement = "ALTER TABLE ".
291
                    $line['table_name']." DROP INDEX ".$line['index_name'];
292
                Debug::log($statement);
293
            }
294
            $pdo->query($statement);
295
        }
296
297
        Debug::log("reading indexes from schema for: ".DB_TYPE);
298
299
        $fp = fopen("schema/ttrss_schema_".DB_TYPE.".sql", "r");
300
        if ($fp) {
301
            while ($line = fgets($fp)) {
302
                $matches = array();
303
304
                if (preg_match("/^create index ([^ ]+) on ([^ ]+)$/i", $line, $matches)) {
305
                    $index = $matches[1];
306
                    $table = $matches[2];
307
308
                    $statement = "CREATE INDEX $index ON $table";
309
310
                    Debug::log($statement);
311
                    $pdo->query($statement);
312
                }
313
            }
314
            fclose($fp);
315
        } else {
316
            Debug::log("unable to open schema file.");
317
        }
318
        Debug::log("all done.");
319
    }
320
321
    if (isset($options["convert-filters"])) {
322
        Debug::log("WARNING: this will remove all existing type2 filters.");
323
        Debug::log("Type 'yes' to continue.");
324
325
        if (read_stdin() != 'yes') {
326
                    exit;
327
        }
328
329
        Debug::log("converting filters...");
330
331
        $pdo->query("DELETE FROM ttrss_filters2");
332
333
        $res = $pdo->query("SELECT * FROM ttrss_filters ORDER BY id");
334
335
        while ($line = $res->fetch()) {
336
            $owner_uid = $line["owner_uid"];
337
338
            // date filters are removed
339
            if ($line["filter_type"] != 5) {
340
                $filter = array();
341
342
                if (sql_bool_to_bool($line["cat_filter"])) {
343
                    $feed_id = "CAT:".(int) $line["cat_id"];
344
                } else {
345
                    $feed_id = (int) $line["feed_id"];
346
                }
347
348
                $filter["enabled"] = $line["enabled"] ? "on" : "off";
349
                $filter["rule"] = array(
350
                    json_encode(array(
351
                        "reg_exp" => $line["reg_exp"],
352
                        "feed_id" => $feed_id,
353
                        "filter_type" => $line["filter_type"])));
354
355
                $filter["action"] = array(
356
                    json_encode(array(
357
                        "action_id" => $line["action_id"],
358
                        "action_param_label" => $line["action_param"],
359
                        "action_param" => $line["action_param"])));
360
361
                // Oh god it's full of hacks
362
363
                $_REQUEST = $filter;
364
                $_SESSION["uid"] = $owner_uid;
365
366
                $filters = new Pref_Filters($_REQUEST);
367
                $filters->add();
368
            }
369
        }
370
371
    }
372
373
    if (isset($options["update-schema"])) {
374
        Debug::log("Checking for updates (".DB_TYPE.")...");
375
376
        $updater = new DbUpdater(DB_TYPE, SCHEMA_VERSION);
377
378
        if ($updater->isUpdateRequired()) {
379
            Debug::log("Schema update required, version ".$updater->getSchemaVersion()." to ".SCHEMA_VERSION);
380
381
            if (DB_TYPE == "mysql") {
382
                            Debug::Log("READ THIS: Due to MySQL limitations, your database is not completely protected while updating.\n".
383
                    "Errors may put it in an inconsistent state requiring manual rollback.\nBACKUP YOUR DATABASE BEFORE CONTINUING.");
384
            } else {
385
                            Debug::log("WARNING: please backup your database before continuing.");
386
            }
387
388
            Debug::log("Type 'yes' to continue.");
389
390
            if (read_stdin() != 'yes') {
391
                            exit;
392
            }
393
394
            Debug::log("Performing updates to version ".SCHEMA_VERSION);
395
396
            for ($i = $updater->getSchemaVersion() + 1; $i <= SCHEMA_VERSION; $i++) {
397
                Debug::log("* Updating to version $i...");
398
399
                $result = $updater->performUpdateTo($i, false);
400
401
                if ($result) {
402
                    Debug::log("* Completed.");
403
                } else {
404
                    Debug::log("One of the updates failed. Either retry the process or perform updates manually.");
405
                    return;
406
                }
407
408
            }
409
        } else {
410
            Debug::log("Update not required.");
411
        }
412
413
    }
414
415
    if (isset($options["gen-search-idx"])) {
416
        echo "Generating search index (stemming set to English)...\n";
417
418
        $res = $pdo->query("SELECT COUNT(id) AS count FROM ttrss_entries WHERE tsvector_combined IS NULL");
419
        $row = $res->fetch();
420
        $count = $row['count'];
421
422
        print "Articles to process: $count.\n";
423
424
        $limit = 500;
425
        $processed = 0;
426
427
        $sth = $pdo->prepare("SELECT id, title, content FROM ttrss_entries WHERE
428
          tsvector_combined IS NULL ORDER BY id LIMIT ?");
429
        $sth->execute([$limit]);
430
431
        $usth = $pdo->prepare("UPDATE ttrss_entries
432
          SET tsvector_combined = to_tsvector('english', ?) WHERE id = ?");
433
434
        while (true) {
435
436
            while ($line = $sth->fetch()) {
437
                $tsvector_combined = mb_substr(strip_tags($line["title"]." ".$line["content"]), 0, 1000000);
438
439
                $usth->execute([$tsvector_combined, $line['id']]);
440
441
                $processed++;
442
            }
443
444
            print "processed $processed articles...\n";
445
446
            if ($processed < $limit) {
447
                echo "All done.\n";
448
                break;
449
            }
450
        }
451
    }
452
453
    if (isset($options["list-plugins"])) {
454
        $tmppluginhost = new PluginHost();
455
        $tmppluginhost->load_all($tmppluginhost::KIND_ALL, false);
456
        $enabled = array_map("trim", explode(",", PLUGINS));
0 ignored issues
show
The constant PLUGINS was not found. Maybe you did not declare it correctly or list all dependencies?
Loading history...
457
458
        echo "List of all available plugins:\n";
459
460
        foreach ($tmppluginhost->get_plugins() as $name => $plugin) {
461
            $about = $plugin->about();
462
463
            $status = $about[3] ? "system" : "user";
464
465
            if (in_array($name, $enabled)) {
466
                $name .= "*";
467
            }
468
469
            printf("%-50s %-10s v%.2f (by %s)\n%s\n\n",
470
                $name, $status, $about[0], $about[2], $about[1]);
471
        }
472
473
        echo "Plugins marked by * are currently enabled for all users.\n";
474
475
    }
476
477
    if (isset($options["debug-feed"])) {
478
        $feed = $options["debug-feed"];
479
480
        if (isset($options["force-refetch"])) {
481
            $_REQUEST["force_refetch"] = true;
482
        }
483
        if (isset($options["force-rehash"])) {
484
            $_REQUEST["force_rehash"] = true;
485
        }
486
487
        Debug::set_loglevel(Debug::$LOG_EXTENDED);
488
489
        $rc = RSSUtils::update_rss_feed($feed) != false ? 0 : 1;
490
491
        exit($rc);
492
    }
493
494
    if (isset($options["send-digests"])) {
495
        Digest::send_headlines_digests();
496
    }
497
498
    PluginHost::getInstance()->run_commands($options);
499
500
    if (file_exists(LOCK_DIRECTORY."/$lock_filename")) {
0 ignored issues
show
The constant LOCK_DIRECTORY was not found. Maybe you did not declare it correctly or list all dependencies?
Loading history...
501
            if (strtoupper(substr(PHP_OS, 0, 3)) == 'WIN') {
502
                        fclose($lock_handle);
503
            }
504
    }
505
        unlink(LOCK_DIRECTORY."/$lock_filename");
506