Passed
Push — issue/620 ( 5cb93c...533fdf )
by Tomas Norre
05:20
created

CrawlerController::expandParameters()   F

Complexity

Conditions 25
Paths 831

Size

Total Lines 129
Code Lines 74

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 58
CRAP Score 28.1471

Importance

Changes 1
Bugs 0 Features 0
Metric Value
eloc 74
nc 831
nop 2
dl 0
loc 129
c 1
b 0
f 0
cc 25
ccs 58
cts 70
cp 0.8286
crap 28.1471
rs 0.2347

How to fix   Long Method    Complexity   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

1
<?php
2
3
declare(strict_types=1);
4
5
namespace AOE\Crawler\Controller;
6
7
/***************************************************************
8
 *  Copyright notice
9
 *
10
 *  (c) 2020 AOE GmbH <[email protected]>
11
 *
12
 *  All rights reserved
13
 *
14
 *  This script is part of the TYPO3 project. The TYPO3 project is
15
 *  free software; you can redistribute it and/or modify
16
 *  it under the terms of the GNU General Public License as published by
17
 *  the Free Software Foundation; either version 3 of the License, or
18
 *  (at your option) any later version.
19
 *
20
 *  The GNU General Public License can be found at
21
 *  http://www.gnu.org/copyleft/gpl.html.
22
 *
23
 *  This script is distributed in the hope that it will be useful,
24
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
25
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26
 *  GNU General Public License for more details.
27
 *
28
 *  This copyright notice MUST APPEAR in all copies of the script!
29
 ***************************************************************/
30
31
use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
32
use AOE\Crawler\Converter\JsonCompatibilityConverter;
33
use AOE\Crawler\Crawler;
34
use AOE\Crawler\CrawlStrategy\CrawlStrategyFactory;
35
use AOE\Crawler\Domain\Model\Process;
36
use AOE\Crawler\Domain\Repository\ConfigurationRepository;
37
use AOE\Crawler\Domain\Repository\ProcessRepository;
38
use AOE\Crawler\Domain\Repository\QueueRepository;
39
use AOE\Crawler\QueueExecutor;
40
use AOE\Crawler\Service\UrlService;
41
use AOE\Crawler\Utility\SignalSlotUtility;
42
use AOE\Crawler\Value\QueueFilter;
43
use Psr\Http\Message\UriInterface;
44
use Psr\Log\LoggerAwareInterface;
45
use Psr\Log\LoggerAwareTrait;
46
use TYPO3\CMS\Backend\Tree\View\PageTreeView;
47
use TYPO3\CMS\Backend\Utility\BackendUtility;
48
use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
49
use TYPO3\CMS\Core\Compatibility\PublicMethodDeprecationTrait;
50
use TYPO3\CMS\Core\Compatibility\PublicPropertyDeprecationTrait;
51
use TYPO3\CMS\Core\Core\Bootstrap;
52
use TYPO3\CMS\Core\Core\Environment;
53
use TYPO3\CMS\Core\Database\Connection;
54
use TYPO3\CMS\Core\Database\ConnectionPool;
55
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
56
use TYPO3\CMS\Core\Imaging\Icon;
57
use TYPO3\CMS\Core\Imaging\IconFactory;
58
use TYPO3\CMS\Core\Site\Entity\Site;
59
use TYPO3\CMS\Core\Type\Bitmask\Permission;
60
use TYPO3\CMS\Core\TypoScript\Parser\TypoScriptParser;
61
use TYPO3\CMS\Core\Utility\DebugUtility;
62
use TYPO3\CMS\Core\Utility\GeneralUtility;
63
use TYPO3\CMS\Core\Utility\MathUtility;
64
use TYPO3\CMS\Extbase\Object\ObjectManager;
65
use TYPO3\CMS\Frontend\Page\PageRepository;
66
67
/**
68
 * Class CrawlerController
69
 *
70
 * @package AOE\Crawler\Controller
71
 */
72
class CrawlerController implements LoggerAwareInterface
73
{
74
    use LoggerAwareTrait;
75
    use PublicMethodDeprecationTrait;
76
    use PublicPropertyDeprecationTrait;
77
78
    public const CLI_STATUS_NOTHING_PROCCESSED = 0;
79
80
    //queue not empty
81
    public const CLI_STATUS_REMAIN = 1;
82
83
    //(some) queue items where processed
84
    public const CLI_STATUS_PROCESSED = 2;
85
86
    //instance didn't finish
87
    public const CLI_STATUS_ABORTED = 4;
88
89
    public const CLI_STATUS_POLLABLE_PROCESSED = 8;
90
91
    /**
92
     * @var integer
93
     */
94
    public $setID = 0;
95
96
    /**
97
     * @var string
98
     */
99
    public $processID = '';
100
101
    /**
102
     * @var array
103
     */
104
    public $duplicateTrack = [];
105
106
    /**
107
     * @var array
108
     */
109
    public $downloadUrls = [];
110
111
    /**
112
     * @var array
113
     */
114
    public $incomingProcInstructions = [];
115
116
    /**
117
     * @var array
118
     */
119
    public $incomingConfigurationSelection = [];
120
121
    /**
122
     * @var bool
123
     */
124
    public $registerQueueEntriesInternallyOnly = false;
125
126
    /**
127
     * @var array
128
     */
129
    public $queueEntries = [];
130
131
    /**
132
     * @var array
133
     */
134
    public $urlList = [];
135
136
    /**
137
     * @var array
138
     */
139
    public $extensionSettings = [];
140
141
    /**
142
     * Mount Point
143
     *
144
     * @var bool
145
     * Todo: Check what this is used for and adjust the type hint or code, as bool doesn't match the current code.
146
     */
147
    public $MP = false;
148
149
    /**
150
     * @var string
151
     * @deprecated
152
     */
153
    protected $processFilename;
154
155
    /**
156
     * Holds the internal access mode can be 'gui','cli' or 'cli_im'
157
     *
158
     * @var string
159
     * @deprecated
160
     */
161
    protected $accessMode;
162
163
    /**
164
     * @var QueueRepository
165
     */
166
    protected $queueRepository;
167
168
    /**
169
     * @var ProcessRepository
170
     */
171
    protected $processRepository;
172
173
    /**
174
     * @var ConfigurationRepository
175
     */
176
    protected $configurationRepository;
177
178
    /**
179
     * @var string
180
     */
181
    protected $tableName = 'tx_crawler_queue';
182
183
    /**
184
     * @var QueueExecutor
185
     */
186
    protected $queueExecutor;
187
188
    /**
189
     * @var int
190
     */
191
    protected $maximumUrlsToCompile = 10000;
192
193
    /**
194
     * @var IconFactory
195
     */
196
    protected $iconFactory;
197
198
    /**
199
     * @var string[]
200
     */
201
    private $deprecatedPublicMethods = [
0 ignored issues
show
introduced by
The private property $deprecatedPublicMethods is not used, and could be removed.
Loading history...
202
        'cleanUpOldQueueEntries' => 'Using CrawlerController::cleanUpOldQueueEntries() is deprecated since 9.0.1 and will be removed in v11.x, please use QueueRepository->cleanUpOldQueueEntries() instead.',
203
        'CLI_debug' => 'Using CrawlerController->CLI_debug() is deprecated since 9.1.3 and will be removed in v11.x',
204
        'getAccessMode' => 'Using CrawlerController->getAccessMode() is deprecated since 9.1.3 and will be removed in v11.x',
205
        'getLogEntriesForSetId' => 'Using crawlerController::getLogEntriesForSetId() is deprecated since 9.0.1 and will be removed in v11.x',
206
        'flushQueue' => 'Using CrawlerController::flushQueue() is deprecated since 9.0.1 and will be removed in v11.x, please use QueueRepository->flushQueue() instead.',
207
        'setAccessMode' => 'Using CrawlerController->setAccessMode() is deprecated since 9.1.3 and will be removed in v11.x',
208
        'getDisabled' => 'Using CrawlerController->getDisabled() is deprecated since 9.1.3 and will be removed in v11.x, please use Crawler->isDisabled() instead',
209
        'setDisabled' => 'Using CrawlerController->setDisabled() is deprecated since 9.1.3 and will be removed in v11.x, please use Crawler->setDisabled() instead',
210
        'getProcessFilename' => 'Using CrawlerController->getProcessFilename() is deprecated since 9.1.3 and will be removed in v11.x',
211
        'setProcessFilename' => 'Using CrawlerController->setProcessFilename() is deprecated since 9.1.3 and will be removed in v11.x',
212
        'getDuplicateRowsIfExist' => 'Using CrawlerController->getDuplicateRowsIfExist() is deprecated since 9.1.4 and will be remove in v11.x, please use QueueRepository->getDuplicateQueueItemsIfExists() instead',
213
214
    ];
215
216
    /**
217
     * @var string[]
218
     */
219
    private $deprecatedPublicProperties = [
1 ignored issue
show
introduced by
The private property $deprecatedPublicProperties is not used, and could be removed.
Loading history...
220
        'accessMode' => 'Using CrawlerController->accessMode is deprecated since 9.1.3 and will be removed in v11.x',
221
        'processFilename' => 'Using CrawlerController->accessMode is deprecated since 9.1.3 and will be removed in v11.x',
222
    ];
223
224
    /**
225
     * @var BackendUserAuthentication|null
226
     */
227
    private $backendUser;
228
229
    /**
230
     * @var integer
231
     */
232
    private $scheduledTime = 0;
233
234
    /**
235
     * @var integer
236
     */
237
    private $reqMinute = 0;
238
239
    /**
240
     * @var bool
241
     */
242
    private $submitCrawlUrls = false;
243
244
    /**
245
     * @var bool
246
     */
247
    private $downloadCrawlUrls = false;
248
249
    /**
250
     * @var PageRepository
251
     */
252
    private $pageRepository;
253
254
    /**
255
     * @var Crawler
256
     */
257
    private $crawler;
258
259
    /************************************
260
     *
261
     * Getting URLs based on Page TSconfig
262
     *
263
     ************************************/
264
265 36
    public function __construct()
266
    {
267 36
        $objectManager = GeneralUtility::makeInstance(ObjectManager::class);
268 36
        $crawlStrategyFactory = GeneralUtility::makeInstance(CrawlStrategyFactory::class);
269 36
        $this->queueRepository = $objectManager->get(QueueRepository::class);
270 36
        $this->processRepository = $objectManager->get(ProcessRepository::class);
271 36
        $this->configurationRepository = $objectManager->get(ConfigurationRepository::class);
272 36
        $this->pageRepository = $objectManager->get(PageRepository::class);
273 36
        $this->queueExecutor = GeneralUtility::makeInstance(QueueExecutor::class, $crawlStrategyFactory);
274 36
        $this->iconFactory = GeneralUtility::makeInstance(IconFactory::class);
275 36
        $this->crawler = GeneralUtility::makeInstance(Crawler::class);
276
277 36
        $this->processFilename = Environment::getVarPath() . '/lock/tx_crawler.proc';
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...oller::$processFilename has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

277
        /** @scrutinizer ignore-deprecated */ $this->processFilename = Environment::getVarPath() . '/lock/tx_crawler.proc';
Loading history...
278
279
        /** @var ExtensionConfigurationProvider $configurationProvider */
280 36
        $configurationProvider = GeneralUtility::makeInstance(ExtensionConfigurationProvider::class);
281 36
        $settings = $configurationProvider->getExtensionConfiguration();
282 36
        $this->extensionSettings = is_array($settings) ? $settings : [];
0 ignored issues
show
introduced by
The condition is_array($settings) is always true.
Loading history...
283
284
        // set defaults:
285 36
        if (MathUtility::convertToPositiveInteger($this->extensionSettings['countInARun']) === 0) {
286
            $this->extensionSettings['countInARun'] = 100;
287
        }
288
289 36
        $this->extensionSettings['processLimit'] = MathUtility::forceIntegerInRange($this->extensionSettings['processLimit'], 1, 99, 1);
290 36
        $this->maximumUrlsToCompile = MathUtility::forceIntegerInRange($this->extensionSettings['maxCompileUrls'], 1, 1000000000, 10000);
291 36
    }
292
293
    /**
294
     * Method to set the accessMode can be gui, cli or cli_im
295
     *
296
     * @return string
297
     * @deprecated
298
     */
299 1
    public function getAccessMode()
300
    {
301 1
        return $this->accessMode;
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...Controller::$accessMode has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

301
        return /** @scrutinizer ignore-deprecated */ $this->accessMode;
Loading history...
302
    }
303
304
    /**
305
     * @param string $accessMode
306
     * @deprecated
307
     */
308 1
    public function setAccessMode($accessMode): void
309
    {
310 1
        $this->accessMode = $accessMode;
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...Controller::$accessMode has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

310
        /** @scrutinizer ignore-deprecated */ $this->accessMode = $accessMode;
Loading history...
311 1
    }
312
313
    /**
314
     * Set disabled status to prevent processes from being processed
315
     *
316
     * @param bool $disabled (optional, defaults to true)
317
     * @deprecated
318
     */
319 2
    public function setDisabled($disabled = true): void
320
    {
321 2
        if ($disabled) {
322 1
            GeneralUtility::writeFile($this->processFilename, '');
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...oller::$processFilename has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

322
            GeneralUtility::writeFile(/** @scrutinizer ignore-deprecated */ $this->processFilename, '');
Loading history...
323
        } else {
324 1
            if (is_file($this->processFilename)) {
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...oller::$processFilename has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

324
            if (is_file(/** @scrutinizer ignore-deprecated */ $this->processFilename)) {
Loading history...
325 1
                unlink($this->processFilename);
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...oller::$processFilename has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

325
                unlink(/** @scrutinizer ignore-deprecated */ $this->processFilename);
Loading history...
326
            }
327
        }
328 2
    }
329
330
    /**
331
     * Get disable status
332
     *
333
     * @return bool true if disabled
334
     * @deprecated
335
     */
336 2
    public function getDisabled()
337
    {
338 2
        return is_file($this->processFilename);
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...oller::$processFilename has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

338
        return is_file(/** @scrutinizer ignore-deprecated */ $this->processFilename);
Loading history...
339
    }
340
341
    /**
342
     * @param string $filenameWithPath
343
     * @deprecated
344
     */
345 3
    public function setProcessFilename($filenameWithPath): void
346
    {
347 3
        $this->processFilename = $filenameWithPath;
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...oller::$processFilename has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

347
        /** @scrutinizer ignore-deprecated */ $this->processFilename = $filenameWithPath;
Loading history...
348 3
    }
349
350
    /**
351
     * @return string
352
     * @deprecated
353
     */
354 1
    public function getProcessFilename()
355
    {
356 1
        return $this->processFilename;
0 ignored issues
show
Deprecated Code introduced by
The property AOE\Crawler\Controller\C...oller::$processFilename has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

356
        return /** @scrutinizer ignore-deprecated */ $this->processFilename;
Loading history...
357
    }
358
359
    /**
360
     * Sets the extensions settings (unserialized pendant of $TYPO3_CONF_VARS['EXT']['extConf']['crawler']).
361
     */
362 14
    public function setExtensionSettings(array $extensionSettings): void
363
    {
364 14
        $this->extensionSettings = $extensionSettings;
365 14
    }
366
367
    /**
368
     * Check if the given page should be crawled
369
     *
370
     * @return false|string false if the page should be crawled (not excluded), true / skipMessage if it should be skipped
371
     */
372 12
    public function checkIfPageShouldBeSkipped(array $pageRow)
373
    {
374 12
        $skipPage = false;
375
        // message will be overwritten later
376 12
        $skipMessage = 'Skipped';
377
378
        // if page is hidden
379 12
        if (! $this->extensionSettings['crawlHiddenPages']) {
380 12
            if ($pageRow['hidden']) {
381 1
                $skipPage = true;
382 1
                $skipMessage = 'Because page is hidden';
383
            }
384
        }
385
386 12
        if (! $skipPage) {
387 11
            if (GeneralUtility::inList('3,4,199,254,255', $pageRow['doktype'])) {
388 3
                $skipPage = true;
389 3
                $skipMessage = 'Because doktype is not allowed';
390
            }
391
        }
392
393 12
        if (! $skipPage) {
394 8
            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['excludeDoktype'] ?? [] as $key => $doktypeList) {
395 1
                if (GeneralUtility::inList($doktypeList, $pageRow['doktype'])) {
396 1
                    $skipPage = true;
397 1
                    $skipMessage = 'Doktype was excluded by "' . $key . '"';
398 1
                    break;
399
                }
400
            }
401
        }
402
403 12
        if (! $skipPage) {
404
            // veto hook
405 7
            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['pageVeto'] ?? [] as $key => $func) {
406
                $params = [
407 2
                    'pageRow' => $pageRow,
408
                ];
409
                // expects "false" if page is ok and "true" or a skipMessage if this page should _not_ be crawled
410 2
                $veto = GeneralUtility::callUserFunction($func, $params, $this);
411 2
                if ($veto !== false) {
412 2
                    $skipPage = true;
413 2
                    if (is_string($veto)) {
414 1
                        $skipMessage = $veto;
415
                    } else {
416 1
                        $skipMessage = 'Veto from hook "' . htmlspecialchars($key) . '"';
417
                    }
418
                    // no need to execute other hooks if a previous one return a veto
419 2
                    break;
420
                }
421
            }
422
        }
423
424 12
        return $skipPage ? $skipMessage : false;
425
    }
426
427
    /**
428
     * Wrapper method for getUrlsForPageId()
429
     * It returns an array of configurations and no urls!
430
     *
431
     * @param array $pageRow Page record with at least dok-type and uid columns.
432
     * @param string $skipMessage
433
     * @return array
434
     * @see getUrlsForPageId()
435
     */
436 6
    public function getUrlsForPageRow(array $pageRow, &$skipMessage = '')
437
    {
438 6
        if (! is_int($pageRow['uid'])) {
439
            $skipMessage = 'PageUid ' . $pageRow['uid'] . ' was not an integer';
440
            return [];
441
        }
442
443 6
        $message = $this->checkIfPageShouldBeSkipped($pageRow);
444 6
        if ($message === false) {
445 5
            $res = $this->getUrlsForPageId($pageRow['uid']);
446 5
            $skipMessage = '';
447
        } else {
448 1
            $skipMessage = $message;
449 1
            $res = [];
450
        }
451
452 6
        return $res;
453
    }
454
455
    /**
456
     * Creates a list of URLs from input array (and submits them to queue if asked for)
457
     * See Web > Info module script + "indexed_search"'s crawler hook-client using this!
458
     *
459
     * @param array $vv Information about URLs from pageRow to crawl.
460
     * @param array $pageRow Page row
461
     * @param int $scheduledTime Unix time to schedule indexing to, typically time()
462
     * @param int $reqMinute Number of requests per minute (creates the interleave between requests)
463
     * @param bool $submitCrawlUrls If set, submits the URLs to queue
464
     * @param bool $downloadCrawlUrls If set (and submitcrawlUrls is false) will fill $downloadUrls with entries)
465
     * @param array $duplicateTrack Array which is passed by reference and contains the an id per url to secure we will not crawl duplicates
466
     * @param array $downloadUrls Array which will be filled with URLS for download if flag is set.
467
     * @param array $incomingProcInstructions Array of processing instructions
468
     * @return string List of URLs (meant for display in backend module)
469
     */
470 4
    public function urlListFromUrlArray(
471
        array $vv,
472
        array $pageRow,
473
        $scheduledTime,
474
        $reqMinute,
475
        $submitCrawlUrls,
476
        $downloadCrawlUrls,
477
        array &$duplicateTrack,
478
        array &$downloadUrls,
479
        array $incomingProcInstructions
480
    ) {
481 4
        if (! is_array($vv['URLs'])) {
482
            return 'ERROR - no URL generated';
483
        }
484 4
        $urlLog = [];
485 4
        $pageId = (int) $pageRow['uid'];
486 4
        $configurationHash = $this->getConfigurationHash($vv);
487 4
        $skipInnerCheck = $this->queueRepository->noUnprocessedQueueEntriesForPageWithConfigurationHashExist($pageId, $configurationHash);
488
489 4
        $urlService = new UrlService();
490
491 4
        foreach ($vv['URLs'] as $urlQuery) {
492 4
            if (! $this->drawURLs_PIfilter($vv['subCfg']['procInstrFilter'], $incomingProcInstructions)) {
493
                continue;
494
            }
495 4
            $url = (string) $urlService->getUrlFromPageAndQueryParameters(
496 4
                $pageId,
497 4
                $urlQuery,
498 4
                $vv['subCfg']['baseUrl'] ?? null,
499 4
                $vv['subCfg']['force_ssl'] ?? 0
500
            );
501
502
            // Create key by which to determine unique-ness:
503 4
            $uKey = $url . '|' . $vv['subCfg']['userGroups'] . '|' . $vv['subCfg']['procInstrFilter'];
504
505 4
            if (isset($duplicateTrack[$uKey])) {
506
                //if the url key is registered just display it and do not resubmit is
507
                $urlLog[] = '<em><span class="text-muted">' . htmlspecialchars($url) . '</span></em>';
508
            } else {
509
                // Scheduled time:
510 4
                $schTime = $scheduledTime + round(count($duplicateTrack) * (60 / $reqMinute));
511 4
                $schTime = intval($schTime / 60) * 60;
512 4
                $formattedDate = BackendUtility::datetime($schTime);
513 4
                $this->urlList[] = '[' . $formattedDate . '] ' . $url;
514 4
                $urlList = '[' . $formattedDate . '] ' . htmlspecialchars($url);
515
516
                // Submit for crawling!
517 4
                if ($submitCrawlUrls) {
518 4
                    $added = $this->addUrl(
519 4
                        $pageId,
520 4
                        $url,
521 4
                        $vv['subCfg'],
522 4
                        $scheduledTime,
523 4
                        $configurationHash,
524 4
                        $skipInnerCheck
525
                    );
526 4
                    if ($added === false) {
527 4
                        $urlList .= ' (URL already existed)';
528
                    }
529
                } elseif ($downloadCrawlUrls) {
530
                    $downloadUrls[$url] = $url;
531
                }
532 4
                $urlLog[] = $urlList;
533
            }
534 4
            $duplicateTrack[$uKey] = true;
535
        }
536
537 4
        return implode('<br>', $urlLog);
538
    }
539
540
    /**
541
     * Returns true if input processing instruction is among registered ones.
542
     *
543
     * @param string $piString PI to test
544
     * @param array $incomingProcInstructions Processing instructions
545
     * @return boolean
546
     */
547 5
    public function drawURLs_PIfilter($piString, array $incomingProcInstructions)
548
    {
549 5
        if (empty($incomingProcInstructions)) {
550 1
            return true;
551
        }
552
553 4
        foreach ($incomingProcInstructions as $pi) {
554 4
            if (GeneralUtility::inList($piString, $pi)) {
555 2
                return true;
556
            }
557
        }
558 2
        return false;
559
    }
560
561 5
    public function getPageTSconfigForId($id): array
562
    {
563 5
        if (! $this->MP) {
564 5
            $pageTSconfig = BackendUtility::getPagesTSconfig($id);
0 ignored issues
show
Deprecated Code introduced by
The function TYPO3\CMS\Backend\Utilit...ity::getPagesTSconfig() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

564
            $pageTSconfig = /** @scrutinizer ignore-deprecated */ BackendUtility::getPagesTSconfig($id);
Loading history...
565
        } else {
566
            // TODO: Please check, this makes no sense to split a boolean value.
567
            [, $mountPointId] = explode('-', $this->MP);
0 ignored issues
show
Bug introduced by
$this->MP of type true is incompatible with the type string expected by parameter $string of explode(). ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

567
            [, $mountPointId] = explode('-', /** @scrutinizer ignore-type */ $this->MP);
Loading history...
568
            $pageTSconfig = BackendUtility::getPagesTSconfig($mountPointId);
0 ignored issues
show
Deprecated Code introduced by
The function TYPO3\CMS\Backend\Utilit...ity::getPagesTSconfig() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

568
            $pageTSconfig = /** @scrutinizer ignore-deprecated */ BackendUtility::getPagesTSconfig($mountPointId);
Loading history...
Bug introduced by
$mountPointId of type string is incompatible with the type integer expected by parameter $id of TYPO3\CMS\Backend\Utilit...ity::getPagesTSconfig(). ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

568
            $pageTSconfig = BackendUtility::getPagesTSconfig(/** @scrutinizer ignore-type */ $mountPointId);
Loading history...
569
        }
570
571
        // Call a hook to alter configuration
572 5
        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['getPageTSconfigForId'])) {
573
            $params = [
574
                'pageId' => $id,
575
                'pageTSConfig' => &$pageTSconfig,
576
            ];
577
            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['getPageTSconfigForId'] as $userFunc) {
578
                GeneralUtility::callUserFunction($userFunc, $params, $this);
579
            }
580
        }
581 5
        return $pageTSconfig;
582
    }
583
584
    /**
585
     * This methods returns an array of configurations.
586
     * Adds no urls!
587
     */
588 4
    public function getUrlsForPageId(int $pageId): array
589
    {
590
        // Get page TSconfig for page ID
591 4
        $pageTSconfig = $this->getPageTSconfigForId($pageId);
592
593 4
        $res = [];
594
595
        // Fetch Crawler Configuration from pageTSconfig
596 4
        $crawlerCfg = $pageTSconfig['tx_crawler.']['crawlerCfg.']['paramSets.'] ?? [];
597 4
        foreach ($crawlerCfg as $key => $values) {
598 3
            if (! is_array($values)) {
599 3
                continue;
600
            }
601 3
            $key = str_replace('.', '', $key);
602
            // Sub configuration for a single configuration string:
603 3
            $subCfg = (array) $crawlerCfg[$key . '.'];
604 3
            $subCfg['key'] = $key;
605
606 3
            if (strcmp($subCfg['procInstrFilter'] ?? '', '')) {
607 3
                $subCfg['procInstrFilter'] = implode(',', GeneralUtility::trimExplode(',', $subCfg['procInstrFilter']));
608
            }
609 3
            $pidOnlyList = implode(',', GeneralUtility::trimExplode(',', $subCfg['pidsOnly'], true));
610
611
            // process configuration if it is not page-specific or if the specific page is the current page:
612
            // TODO: Check if $pidOnlyList can be kept as Array instead of imploded
613 3
            if (! strcmp((string) $subCfg['pidsOnly'], '') || GeneralUtility::inList($pidOnlyList, strval($pageId))) {
614
615
                // Explode, process etc.:
616 3
                $res[$key] = [];
617 3
                $res[$key]['subCfg'] = $subCfg;
618 3
                $res[$key]['paramParsed'] = GeneralUtility::explodeUrl2Array($crawlerCfg[$key]);
619 3
                $res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $pageId);
620 3
                $res[$key]['origin'] = 'pagets';
621
622
                // recognize MP value
623 3
                if (! $this->MP) {
624 3
                    $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id=' . $pageId]);
625
                } else {
626
                    $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id=' . $pageId . '&MP=' . $this->MP]);
0 ignored issues
show
Bug introduced by
Are you sure $this->MP of type true can be used in concatenation? ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

626
                    $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id=' . $pageId . '&MP=' . /** @scrutinizer ignore-type */ $this->MP]);
Loading history...
627
                }
628
            }
629
        }
630
631
        // Get configuration from tx_crawler_configuration records up the rootline
632 4
        $crawlerConfigurations = $this->configurationRepository->getCrawlerConfigurationRecordsFromRootLine($pageId);
633 4
        foreach ($crawlerConfigurations as $configurationRecord) {
634
635
            // check access to the configuration record
636 1
            if (empty($configurationRecord['begroups']) || $this->getBackendUser()->isAdmin() || $this->hasGroupAccess($this->getBackendUser()->user['usergroup_cached_list'], $configurationRecord['begroups'])) {
637 1
                $pidOnlyList = implode(',', GeneralUtility::trimExplode(',', $configurationRecord['pidsonly'], true));
638
639
                // process configuration if it is not page-specific or if the specific page is the current page:
640
                // TODO: Check if $pidOnlyList can be kept as Array instead of imploded
641 1
                if (! strcmp($configurationRecord['pidsonly'], '') || GeneralUtility::inList($pidOnlyList, strval($pageId))) {
642 1
                    $key = $configurationRecord['name'];
643
644
                    // don't overwrite previously defined paramSets
645 1
                    if (! isset($res[$key])) {
646
647
                        /* @var $TSparserObject \TYPO3\CMS\Core\TypoScript\Parser\TypoScriptParser */
648 1
                        $TSparserObject = GeneralUtility::makeInstance(TypoScriptParser::class);
649 1
                        $TSparserObject->parse($configurationRecord['processing_instruction_parameters_ts']);
650
651
                        $subCfg = [
652 1
                            'procInstrFilter' => $configurationRecord['processing_instruction_filter'],
653 1
                            'procInstrParams.' => $TSparserObject->setup,
654 1
                            'baseUrl' => $configurationRecord['base_url'],
655 1
                            'force_ssl' => (int) $configurationRecord['force_ssl'],
656 1
                            'userGroups' => $configurationRecord['fegroups'],
657 1
                            'exclude' => $configurationRecord['exclude'],
658 1
                            'key' => $key,
659
                        ];
660
661 1
                        if (! in_array($pageId, $this->expandExcludeString($subCfg['exclude']), true)) {
662 1
                            $res[$key] = [];
663 1
                            $res[$key]['subCfg'] = $subCfg;
664 1
                            $res[$key]['paramParsed'] = GeneralUtility::explodeUrl2Array($configurationRecord['configuration']);
665 1
                            $res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $pageId);
666 1
                            $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id=' . $pageId]);
667 1
                            $res[$key]['origin'] = 'tx_crawler_configuration_' . $configurationRecord['uid'];
668
                        }
669
                    }
670
                }
671
            }
672
        }
673
674 4
        foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['processUrls'] ?? [] as $func) {
675
            $params = [
676
                'res' => &$res,
677
            ];
678
            GeneralUtility::callUserFunction($func, $params, $this);
679
        }
680 4
        return $res;
681
    }
682
683
    /**
684
     * Find all configurations of subpages of a page
685
     * TODO: Write Functional Tests
686
     */
687 1
    public function getConfigurationsForBranch(int $rootid, int $depth): array
688
    {
689 1
        $configurationsForBranch = [];
690 1
        $pageTSconfig = $this->getPageTSconfigForId($rootid);
691 1
        $sets = $pageTSconfig['tx_crawler.']['crawlerCfg.']['paramSets.'] ?? [];
692 1
        foreach ($sets as $key => $value) {
693
            if (! is_array($value)) {
694
                continue;
695
            }
696
            $configurationsForBranch[] = substr($key, -1) === '.' ? substr($key, 0, -1) : $key;
697
        }
698 1
        $pids = [];
699 1
        $rootLine = BackendUtility::BEgetRootLine($rootid);
700 1
        foreach ($rootLine as $node) {
701 1
            $pids[] = $node['uid'];
702
        }
703
        /* @var PageTreeView $tree */
704 1
        $tree = GeneralUtility::makeInstance(PageTreeView::class);
705 1
        $perms_clause = $this->getBackendUser()->getPagePermsClause(Permission::PAGE_SHOW);
706 1
        $tree->init(empty($perms_clause) ? '' : ('AND ' . $perms_clause));
707 1
        $tree->getTree($rootid, $depth, '');
708 1
        foreach ($tree->tree as $node) {
709
            $pids[] = $node['row']['uid'];
710
        }
711
712 1
        $queryBuilder = $this->getQueryBuilder('tx_crawler_configuration');
713
        $statement = $queryBuilder
714 1
            ->select('name')
715 1
            ->from('tx_crawler_configuration')
716 1
            ->where(
717 1
                $queryBuilder->expr()->in('pid', $queryBuilder->createNamedParameter($pids, Connection::PARAM_INT_ARRAY))
718
            )
719 1
            ->execute();
720
721 1
        while ($row = $statement->fetch()) {
722 1
            $configurationsForBranch[] = $row['name'];
723
        }
724 1
        return $configurationsForBranch;
725
    }
726
727
    /**
728
     * Check if a user has access to an item
729
     * (e.g. get the group list of the current logged in user from $GLOBALS['TSFE']->gr_list)
730
     *
731
     * @param string $groupList Comma-separated list of (fe_)group UIDs from a user
732
     * @param string $accessList Comma-separated list of (fe_)group UIDs of the item to access
733
     * @return bool TRUE if at least one of the users group UIDs is in the access list or the access list is empty
734
     * @see \TYPO3\CMS\Frontend\Page\PageRepository::getMultipleGroupsWhereClause()
735
     */
736 3
    public function hasGroupAccess($groupList, $accessList)
737
    {
738 3
        if (empty($accessList)) {
739 1
            return true;
740
        }
741 2
        foreach (GeneralUtility::intExplode(',', $groupList) as $groupUid) {
742 2
            if (GeneralUtility::inList($accessList, $groupUid)) {
743 1
                return true;
744
            }
745
        }
746 1
        return false;
747
    }
748
749
    /**
750
     * Will expand the parameters configuration to individual values. This follows a certain syntax of the value of each parameter.
751
     * Syntax of values:
752
     * - Basically: If the value is wrapped in [...] it will be expanded according to the following syntax, otherwise the value is taken literally
753
     * - Configuration is splitted by "|" and the parts are processed individually and finally added together
754
     * - For each configuration part:
755
     *         - "[int]-[int]" = Integer range, will be expanded to all values in between, values included, starting from low to high (max. 1000). Example "1-34" or "-40--30"
756
     *         - "_TABLE:[TCA table name];[_PID:[optional page id, default is current page]];[_ENABLELANG:1]" = Look up of table records from PID, filtering out deleted records. Example "_TABLE:tt_content; _PID:123"
757
     *        _ENABLELANG:1 picks only original records without their language overlays
758
     *         - Default: Literal value
759
     *
760
     * @param array $paramArray Array with key (GET var name) and values (value of GET var which is configuration for expansion)
761
     * @param integer $pid Current page ID
762
     * @return array
763
     *
764
     * TODO: Write Functional Tests
765
     */
766 11
    public function expandParameters($paramArray, $pid)
767
    {
768
        // Traverse parameter names:
769 11
        foreach ($paramArray as $p => $v) {
770 11
            $v = trim($v);
771
772
            // If value is encapsulated in square brackets it means there are some ranges of values to find, otherwise the value is literal
773 11
            if (strpos($v, '[') === 0 && substr($v, -1) === ']') {
774
                // So, find the value inside brackets and reset the paramArray value as an array.
775 11
                $v = substr($v, 1, -1);
776 11
                $paramArray[$p] = [];
777
778
                // Explode parts and traverse them:
779 11
                $parts = explode('|', $v);
780 11
                foreach ($parts as $pV) {
781
782
                    // Look for integer range: (fx. 1-34 or -40--30 // reads minus 40 to minus 30)
783 11
                    if (preg_match('/^(-?[0-9]+)\s*-\s*(-?[0-9]+)$/', trim($pV), $reg)) {
784 1
                        $reg = $this->swapIfFirstIsLargerThanSecond($reg);
785
786
                        // Traverse range, add values:
787
                        // Limit to size of range!
788 1
                        $runAwayBrake = 1000;
789 1
                        for ($a = $reg[1]; $a <= $reg[2]; $a++) {
790 1
                            $paramArray[$p][] = $a;
791 1
                            $runAwayBrake--;
792 1
                            if ($runAwayBrake <= 0) {
793
                                break;
794
                            }
795
                        }
796 10
                    } elseif (strpos(trim($pV), '_TABLE:') === 0) {
797
798
                        // Parse parameters:
799 6
                        $subparts = GeneralUtility::trimExplode(';', $pV);
800 6
                        $subpartParams = [];
801 6
                        foreach ($subparts as $spV) {
802 6
                            [$pKey, $pVal] = GeneralUtility::trimExplode(':', $spV);
803 6
                            $subpartParams[$pKey] = $pVal;
804
                        }
805
806
                        // Table exists:
807 6
                        if (isset($GLOBALS['TCA'][$subpartParams['_TABLE']])) {
808 6
                            $lookUpPid = isset($subpartParams['_PID']) ? intval($subpartParams['_PID']) : intval($pid);
809 6
                            $recursiveDepth = isset($subpartParams['_RECURSIVE']) ? intval($subpartParams['_RECURSIVE']) : 0;
810 6
                            $pidField = isset($subpartParams['_PIDFIELD']) ? trim($subpartParams['_PIDFIELD']) : 'pid';
811 6
                            $where = $subpartParams['_WHERE'] ?? '';
812 6
                            $addTable = $subpartParams['_ADDTABLE'] ?? '';
813
814 6
                            $fieldName = $subpartParams['_FIELD'] ? $subpartParams['_FIELD'] : 'uid';
815 6
                            if ($fieldName === 'uid' || $GLOBALS['TCA'][$subpartParams['_TABLE']]['columns'][$fieldName]) {
816 6
                                $queryBuilder = $this->getQueryBuilder($subpartParams['_TABLE']);
817
818 6
                                if ($recursiveDepth > 0) {
819
                                    /** @var \TYPO3\CMS\Core\Database\QueryGenerator $queryGenerator */
820 2
                                    $queryGenerator = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Database\QueryGenerator::class);
821 2
                                    $pidList = $queryGenerator->getTreeList($lookUpPid, $recursiveDepth, 0, 1);
822 2
                                    $pidArray = GeneralUtility::intExplode(',', $pidList);
823
                                } else {
824 4
                                    $pidArray = [(string) $lookUpPid];
825
                                }
826
827 6
                                $queryBuilder->getRestrictions()
828 6
                                    ->removeAll()
829 6
                                    ->add(GeneralUtility::makeInstance(DeletedRestriction::class));
830
831
                                $queryBuilder
832 6
                                    ->select($fieldName)
833 6
                                    ->from($subpartParams['_TABLE'])
834 6
                                    ->where(
835 6
                                        $queryBuilder->expr()->in($pidField, $queryBuilder->createNamedParameter($pidArray, Connection::PARAM_INT_ARRAY)),
836 6
                                        $where
837
                                    );
838
839 6
                                if (! empty($addTable)) {
840
                                    // TODO: Check if this works as intended!
841
                                    $queryBuilder->add('from', $addTable);
842
                                }
843 6
                                $transOrigPointerField = $GLOBALS['TCA'][$subpartParams['_TABLE']]['ctrl']['transOrigPointerField'];
844
845 6
                                if ($subpartParams['_ENABLELANG'] && $transOrigPointerField) {
846
                                    $queryBuilder->andWhere(
847
                                        $queryBuilder->expr()->lte(
848
                                            $transOrigPointerField,
849
                                            0
850
                                        )
851
                                    );
852
                                }
853
854 6
                                $statement = $queryBuilder->execute();
855
856 6
                                $rows = [];
857 6
                                while ($row = $statement->fetch()) {
858 6
                                    $rows[$row[$fieldName]] = $row;
859
                                }
860
861 6
                                if (is_array($rows)) {
862 6
                                    $paramArray[$p] = array_merge($paramArray[$p], array_keys($rows));
863
                                }
864
                            }
865
                        }
866
                    } else {
867
                        // Just add value:
868 4
                        $paramArray[$p][] = $pV;
869
                    }
870
                    // Hook for processing own expandParameters place holder
871 11
                    if (is_array($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['crawler/class.tx_crawler_lib.php']['expandParameters'])) {
872
                        $_params = [
873
                            'pObj' => &$this,
874
                            'paramArray' => &$paramArray,
875
                            'currentKey' => $p,
876
                            'currentValue' => $pV,
877
                            'pid' => $pid,
878
                        ];
879
                        foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['crawler/class.tx_crawler_lib.php']['expandParameters'] as $_funcRef) {
880
                            GeneralUtility::callUserFunction($_funcRef, $_params, $this);
881
                        }
882
                    }
883
                }
884
885
                // Make unique set of values and sort array by key:
886 11
                $paramArray[$p] = array_unique($paramArray[$p]);
887 11
                ksort($paramArray);
888
            } else {
889
                // Set the literal value as only value in array:
890 4
                $paramArray[$p] = [$v];
891
            }
892
        }
893
894 11
        return $paramArray;
895
    }
896
897
    /**
898
     * Compiling URLs from parameter array (output of expandParameters())
899
     * The number of URLs will be the multiplication of the number of parameter values for each key
900
     *
901
     * @param array $paramArray Output of expandParameters(): Array with keys (GET var names) and for each an array of values
902
     * @param array $urls URLs accumulated in this array (for recursion)
903
     * @return array
904
     */
905 7
    public function compileUrls($paramArray, array $urls)
906
    {
907 7
        if (empty($paramArray)) {
908 7
            return $urls;
909
        }
910
        // shift first off stack:
911 6
        reset($paramArray);
912 6
        $varName = key($paramArray);
913 6
        $valueSet = array_shift($paramArray);
914
915
        // Traverse value set:
916 6
        $newUrls = [];
917 6
        foreach ($urls as $url) {
918 5
            foreach ($valueSet as $val) {
919 5
                $newUrls[] = $url . (strcmp((string) $val, '') ? '&' . rawurlencode($varName) . '=' . rawurlencode((string) $val) : '');
920
921 5
                if (count($newUrls) > $this->maximumUrlsToCompile) {
922
                    break;
923
                }
924
            }
925
        }
926 6
        return $this->compileUrls($paramArray, $newUrls);
927
    }
928
929
    /************************************
930
     *
931
     * Crawler log
932
     *
933
     ************************************/
934
935
    /**
936
     * Return array of records from crawler queue for input page ID
937
     *
938
     * @param integer $id Page ID for which to look up log entries.
939
     * @param boolean $doFlush If TRUE, then entries selected at DELETED(!) instead of selected!
940
     * @param boolean $doFullFlush
941
     * @param integer $itemsPerPage Limit the amount of entries per page default is 10
942
     * @return array
943
     */
944 4
    public function getLogEntriesForPageId($id, QueueFilter $queueFilter, $doFlush = false, $doFullFlush = false, $itemsPerPage = 10)
945
    {
946 4
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
947
        $queryBuilder
948 4
            ->select('*')
949 4
            ->from($this->tableName)
950 4
            ->where(
951 4
                $queryBuilder->expr()->eq('page_id', $queryBuilder->createNamedParameter($id, \PDO::PARAM_INT))
952
            )
953 4
            ->orderBy('scheduled', 'DESC');
954
955 4
        $expressionBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
956 4
            ->getConnectionForTable($this->tableName)
957 4
            ->getExpressionBuilder();
958 4
        $query = $expressionBuilder->andX();
0 ignored issues
show
Unused Code introduced by
The assignment to $query is dead and can be removed.
Loading history...
959
        // PHPStorm adds the highlight that the $addWhere is immediately overwritten,
960
        // but the $query = $expressionBuilder->andX() ensures that the $addWhere is written correctly with AND
961
        // between the statements, it's not a mistake in the code.
962 4
        switch ($queueFilter) {
963 4
            case 'pending':
964
                $queryBuilder->andWhere($queryBuilder->expr()->eq('exec_time', 0));
965
                break;
966 4
            case 'finished':
967
                $queryBuilder->andWhere($queryBuilder->expr()->gt('exec_time', 0));
968
                break;
969
        }
970
971 4
        if ($doFlush) {
972 2
            if ($doFullFlush) {
973 1
                $this->queueRepository->flushQueue($queueFilter);
974
            } else {
975 1
                $this->queueRepository->flushQueue($queueFilter);
976
            }
977
        }
978 4
        if ($itemsPerPage > 0) {
979
            $queryBuilder
980 4
                ->setMaxResults((int) $itemsPerPage);
981
        }
982
983 4
        return $queryBuilder->execute()->fetchAll();
984
    }
985
986
    /**
987
     * Return array of records from crawler queue for input set ID
988
     *
989
     * @param int $set_id Set ID for which to look up log entries.
990
     * @param string $filter Filter: "all" => all entries, "pending" => all that is not yet run, "finished" => all complete ones
991
     * @param bool $doFlush If TRUE, then entries selected at DELETED(!) instead of selected!
992
     * @param int $itemsPerPage Limit the amount of entries per page default is 10
993
     * @return array
994
     *
995
     * @deprecated
996
     */
997 6
    public function getLogEntriesForSetId(int $set_id, string $filter = '', bool $doFlush = false, bool $doFullFlush = false, int $itemsPerPage = 10)
998
    {
999 6
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
1000
        $queryBuilder
1001 6
            ->select('*')
1002 6
            ->from($this->tableName)
1003 6
            ->where(
1004 6
                $queryBuilder->expr()->eq('set_id', $queryBuilder->createNamedParameter($set_id, \PDO::PARAM_INT))
1005
            )
1006 6
            ->orderBy('scheduled', 'DESC');
1007
1008 6
        $expressionBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
1009 6
            ->getConnectionForTable($this->tableName)
1010 6
            ->getExpressionBuilder();
1011 6
        $query = $expressionBuilder->andX();
1012
        // PHPStorm adds the highlight that the $addWhere is immediately overwritten,
1013
        // but the $query = $expressionBuilder->andX() ensures that the $addWhere is written correctly with AND
1014
        // between the statements, it's not a mistake in the code.
1015 6
        $addWhere = '';
1016 6
        switch ($filter) {
1017 6
            case 'pending':
1018 1
                $queryBuilder->andWhere($queryBuilder->expr()->eq('exec_time', 0));
1019 1
                $addWhere = $query->add($expressionBuilder->eq('exec_time', 0));
0 ignored issues
show
Unused Code introduced by
The assignment to $addWhere is dead and can be removed.
Loading history...
1020 1
                break;
1021 5
            case 'finished':
1022 1
                $queryBuilder->andWhere($queryBuilder->expr()->gt('exec_time', 0));
1023 1
                $addWhere = $query->add($expressionBuilder->gt('exec_time', 0));
1024 1
                break;
1025
        }
1026 6
        if ($doFlush) {
1027 4
            $addWhere = $query->add($expressionBuilder->eq('set_id', (int) $set_id));
1028 4
            $this->flushQueue($doFullFlush ? '' : $addWhere);
0 ignored issues
show
Deprecated Code introduced by
The function AOE\Crawler\Controller\C...ontroller::flushQueue() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

1028
            /** @scrutinizer ignore-deprecated */ $this->flushQueue($doFullFlush ? '' : $addWhere);
Loading history...
1029 4
            return [];
1030
        }
1031 2
        if ($itemsPerPage > 0) {
1032
            $queryBuilder
1033 2
                ->setMaxResults((int) $itemsPerPage);
1034
        }
1035
1036 2
        return $queryBuilder->execute()->fetchAll();
1037
    }
1038
1039
    /**
1040
     * Adding call back entries to log (called from hooks typically, see indexed search class "class.crawler.php"
1041
     *
1042
     * @param integer $setId Set ID
1043
     * @param array $params Parameters to pass to call back function
1044
     * @param string $callBack Call back object reference, eg. 'EXT:indexed_search/class.crawler.php:&tx_indexedsearch_crawler'
1045
     * @param integer $page_id Page ID to attach it to
1046
     * @param integer $schedule Time at which to activate
1047
     */
1048
    public function addQueueEntry_callBack($setId, $params, $callBack, $page_id = 0, $schedule = 0): void
1049
    {
1050
        if (! is_array($params)) {
0 ignored issues
show
introduced by
The condition is_array($params) is always true.
Loading history...
1051
            $params = [];
1052
        }
1053
        $params['_CALLBACKOBJ'] = $callBack;
1054
1055
        GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_crawler_queue')
1056
            ->insert(
1057
                'tx_crawler_queue',
1058
                [
1059
                    'page_id' => (int) $page_id,
1060
                    'parameters' => json_encode($params),
1061
                    'scheduled' => (int) $schedule ?: $this->getCurrentTime(),
1062
                    'exec_time' => 0,
1063
                    'set_id' => (int) $setId,
1064
                    'result_data' => '',
1065
                ]
1066
            );
1067
    }
1068
1069
    /************************************
1070
     *
1071
     * URL setting
1072
     *
1073
     ************************************/
1074
1075
    /**
1076
     * Setting a URL for crawling:
1077
     *
1078
     * @param integer $id Page ID
1079
     * @param string $url Complete URL
1080
     * @param array $subCfg Sub configuration array (from TS config)
1081
     * @param integer $tstamp Scheduled-time
1082
     * @param string $configurationHash (optional) configuration hash
1083
     * @param bool $skipInnerDuplicationCheck (optional) skip inner duplication check
1084
     * @return bool
1085
     */
1086 8
    public function addUrl(
1087
        $id,
1088
        $url,
1089
        array $subCfg,
1090
        $tstamp,
1091
        $configurationHash = '',
1092
        $skipInnerDuplicationCheck = false
1093
    ) {
1094 8
        $urlAdded = false;
1095 8
        $rows = [];
1096
1097
        // Creating parameters:
1098
        $parameters = [
1099 8
            'url' => $url,
1100
        ];
1101
1102
        // fe user group simulation:
1103 8
        $uGs = implode(',', array_unique(GeneralUtility::intExplode(',', $subCfg['userGroups'], true)));
1104 8
        if ($uGs) {
1105 1
            $parameters['feUserGroupList'] = $uGs;
1106
        }
1107
1108
        // Setting processing instructions
1109 8
        $parameters['procInstructions'] = GeneralUtility::trimExplode(',', $subCfg['procInstrFilter']);
1110 8
        if (is_array($subCfg['procInstrParams.'])) {
1111 5
            $parameters['procInstrParams'] = $subCfg['procInstrParams.'];
1112
        }
1113
1114
        // Compile value array:
1115 8
        $parameters_serialized = json_encode($parameters);
1116
        $fieldArray = [
1117 8
            'page_id' => (int) $id,
1118 8
            'parameters' => $parameters_serialized,
1119 8
            'parameters_hash' => GeneralUtility::shortMD5($parameters_serialized),
1120 8
            'configuration_hash' => $configurationHash,
1121 8
            'scheduled' => $tstamp,
1122 8
            'exec_time' => 0,
1123 8
            'set_id' => (int) $this->setID,
1124 8
            'result_data' => '',
1125 8
            'configuration' => $subCfg['key'],
1126
        ];
1127
1128 8
        if ($this->registerQueueEntriesInternallyOnly) {
1129
            //the entries will only be registered and not stored to the database
1130 1
            $this->queueEntries[] = $fieldArray;
1131
        } else {
1132 7
            if (! $skipInnerDuplicationCheck) {
1133
                // check if there is already an equal entry
1134 6
                $rows = $this->queueRepository->getDuplicateQueueItemsIfExists(
1135 6
                    (bool) $this->extensionSettings['enableTimeslot'],
1136 6
                    $tstamp,
1137 6
                    $this->getCurrentTime(),
1138 6
                    $fieldArray['page_id'],
1139 6
                    $fieldArray['parameters_hash']
1140
                );
1141
            }
1142
1143 7
            if (empty($rows)) {
1144 6
                $connectionForCrawlerQueue = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_crawler_queue');
1145 6
                $connectionForCrawlerQueue->insert(
1146 6
                    'tx_crawler_queue',
1147 6
                    $fieldArray
1148
                );
1149 6
                $uid = $connectionForCrawlerQueue->lastInsertId('tx_crawler_queue', 'qid');
1150 6
                $rows[] = $uid;
1151 6
                $urlAdded = true;
1152
1153 6
                $signalPayload = ['uid' => $uid, 'fieldArray' => $fieldArray];
1154 6
                SignalSlotUtility::emitSignal(
1155 6
                    self::class,
1156 6
                    SignalSlotUtility::SIGNAL_URL_ADDED_TO_QUEUE,
1157 6
                    $signalPayload
1158
                );
1159
            } else {
1160 3
                $signalPayload = ['rows' => $rows, 'fieldArray' => $fieldArray];
1161 3
                SignalSlotUtility::emitSignal(
1162 3
                    self::class,
1163 3
                    SignalSlotUtility::SIGNAL_DUPLICATE_URL_IN_QUEUE,
1164 3
                    $signalPayload
1165
                );
1166
            }
1167
        }
1168
1169 8
        return $urlAdded;
1170
    }
1171
1172
    /**
1173
     * Returns the current system time
1174
     *
1175
     * @return int
1176
     */
1177 2
    public function getCurrentTime()
1178
    {
1179 2
        return time();
1180
    }
1181
1182
    /************************************
1183
     *
1184
     * URL reading
1185
     *
1186
     ************************************/
1187
1188
    /**
1189
     * Read URL for single queue entry
1190
     *
1191
     * @param integer $queueId
1192
     * @param boolean $force If set, will process even if exec_time has been set!
1193
     * @return integer
1194
     */
1195
    public function readUrl($queueId, $force = false)
1196
    {
1197
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
1198
        $ret = 0;
1199
        $this->logger->debug('crawler-readurl start ' . microtime(true));
1200
        // Get entry:
1201
        $queryBuilder
1202
            ->select('*')
1203
            ->from('tx_crawler_queue')
1204
            ->where(
1205
                $queryBuilder->expr()->eq('qid', $queryBuilder->createNamedParameter($queueId, \PDO::PARAM_INT))
1206
            );
1207
        if (! $force) {
1208
            $queryBuilder
1209
                ->andWhere('exec_time = 0')
1210
                ->andWhere('process_scheduled > 0');
1211
        }
1212
        $queueRec = $queryBuilder->execute()->fetch();
1213
1214
        if (! is_array($queueRec)) {
1215
            return;
1216
        }
1217
1218
        SignalSlotUtility::emitSignal(
1219
            self::class,
1220
            SignalSlotUtility::SIGNAL_QUEUEITEM_PREPROCESS,
1221
            [$queueId, &$queueRec]
1222
        );
1223
1224
        // Set exec_time to lock record:
1225
        $field_array = ['exec_time' => $this->getCurrentTime()];
1226
1227
        if (isset($this->processID)) {
1228
            //if mulitprocessing is used we need to store the id of the process which has handled this entry
1229
            $field_array['process_id_completed'] = $this->processID;
1230
        }
1231
1232
        GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_crawler_queue')
1233
            ->update(
1234
                'tx_crawler_queue',
1235
                $field_array,
1236
                ['qid' => (int) $queueId]
1237
            );
1238
1239
        $result = $this->queueExecutor->executeQueueItem($queueRec, $this);
1240
        if ($result['content'] === null) {
1241
            $resultData = 'An errors happened';
1242
        } else {
1243
            /** @var JsonCompatibilityConverter $jsonCompatibilityConverter */
1244
            $jsonCompatibilityConverter = GeneralUtility::makeInstance(JsonCompatibilityConverter::class);
1245
            $resultData = $jsonCompatibilityConverter->convert($result['content']);
1246
        }
1247
1248
        //atm there's no need to point to specific pollable extensions
1249
        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['pollSuccess'])) {
1250
            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['pollSuccess'] as $pollable) {
1251
                // only check the success value if the instruction is runnig
1252
                // it is important to name the pollSuccess key same as the procInstructions key
1253
                if (is_array($resultData['parameters']['procInstructions'])
1254
                    && in_array(
1255
                        $pollable,
1256
                        $resultData['parameters']['procInstructions'], true
1257
                    )
1258
                ) {
1259
                    if (! empty($resultData['success'][$pollable]) && $resultData['success'][$pollable]) {
1260
                        $ret |= self::CLI_STATUS_POLLABLE_PROCESSED;
1261
                    }
1262
                }
1263
            }
1264
        }
1265
1266
        // Set result in log which also denotes the end of the processing of this entry.
1267
        $field_array = ['result_data' => json_encode($result)];
1268
1269
        SignalSlotUtility::emitSignal(
1270
            self::class,
1271
            SignalSlotUtility::SIGNAL_QUEUEITEM_POSTPROCESS,
1272
            [$queueId, &$field_array]
1273
        );
1274
1275
        GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_crawler_queue')
1276
            ->update(
1277
                'tx_crawler_queue',
1278
                $field_array,
1279
                ['qid' => (int) $queueId]
1280
            );
1281
1282
        $this->logger->debug('crawler-readurl stop ' . microtime(true));
1283
        return $ret;
1284
    }
1285
1286
    /**
1287
     * Read URL for not-yet-inserted log-entry
1288
     *
1289
     * @param array $field_array Queue field array,
1290
     *
1291
     * @return array|bool|mixed|string
1292
     */
1293
    public function readUrlFromArray($field_array)
1294
    {
1295
        // Set exec_time to lock record:
1296
        $field_array['exec_time'] = $this->getCurrentTime();
1297
        $connectionForCrawlerQueue = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable($this->tableName);
1298
        $connectionForCrawlerQueue->insert(
1299
            $this->tableName,
1300
            $field_array
1301
        );
1302
        $queueId = $connectionForCrawlerQueue->lastInsertId($this->tableName, 'qid');
1303
        $result = $this->queueExecutor->executeQueueItem($field_array, $this);
1304
1305
        // Set result in log which also denotes the end of the processing of this entry.
1306
        $field_array = ['result_data' => json_encode($result)];
1307
1308
        SignalSlotUtility::emitSignal(
1309
            self::class,
1310
            SignalSlotUtility::SIGNAL_QUEUEITEM_POSTPROCESS,
1311
            [$queueId, &$field_array]
1312
        );
1313
1314
        $connectionForCrawlerQueue->update(
1315
            $this->tableName,
1316
            $field_array,
1317
            ['qid' => $queueId]
1318
        );
1319
1320
        return $result;
1321
    }
1322
1323
    /*****************************
1324
     *
1325
     * Compiling URLs to crawl - tools
1326
     *
1327
     *****************************/
1328
1329
    /**
1330
     * @param integer $id Root page id to start from.
1331
     * @param integer $depth Depth of tree, 0=only id-page, 1= on sublevel, 99 = infinite
1332
     * @param integer $scheduledTime Unix Time when the URL is timed to be visited when put in queue
1333
     * @param integer $reqMinute Number of requests per minute (creates the interleave between requests)
1334
     * @param boolean $submitCrawlUrls If set, submits the URLs to queue in database (real crawling)
1335
     * @param boolean $downloadCrawlUrls If set (and submitcrawlUrls is false) will fill $downloadUrls with entries)
1336
     * @param array $incomingProcInstructions Array of processing instructions
1337
     * @param array $configurationSelection Array of configuration keys
1338
     * @return string
1339
     */
1340
    public function getPageTreeAndUrls(
1341
        $id,
1342
        $depth,
1343
        $scheduledTime,
1344
        $reqMinute,
1345
        $submitCrawlUrls,
1346
        $downloadCrawlUrls,
1347
        array $incomingProcInstructions,
1348
        array $configurationSelection
1349
    ) {
1350
        $this->scheduledTime = $scheduledTime;
1351
        $this->reqMinute = $reqMinute;
1352
        $this->submitCrawlUrls = $submitCrawlUrls;
1353
        $this->downloadCrawlUrls = $downloadCrawlUrls;
1354
        $this->incomingProcInstructions = $incomingProcInstructions;
1355
        $this->incomingConfigurationSelection = $configurationSelection;
1356
1357
        $this->duplicateTrack = [];
1358
        $this->downloadUrls = [];
1359
1360
        // Drawing tree:
1361
        /* @var PageTreeView $tree */
1362
        $tree = GeneralUtility::makeInstance(PageTreeView::class);
1363
        $perms_clause = $this->getBackendUser()->getPagePermsClause(Permission::PAGE_SHOW);
1364
        $tree->init('AND ' . $perms_clause);
1365
1366
        $pageInfo = BackendUtility::readPageAccess($id, $perms_clause);
1367
        if (is_array($pageInfo)) {
0 ignored issues
show
introduced by
The condition is_array($pageInfo) is always true.
Loading history...
1368
            // Set root row:
1369
            $tree->tree[] = [
1370
                'row' => $pageInfo,
1371
                'HTML' => $this->iconFactory->getIconForRecord('pages', $pageInfo, Icon::SIZE_SMALL),
1372
            ];
1373
        }
1374
1375
        // Get branch beneath:
1376
        if ($depth) {
1377
            $tree->getTree($id, $depth, '');
1378
        }
1379
1380
        // Traverse page tree:
1381
        $code = '';
1382
1383
        foreach ($tree->tree as $data) {
1384
            $this->MP = false;
1385
1386
            // recognize mount points
1387
            if ($data['row']['doktype'] === PageRepository::DOKTYPE_MOUNTPOINT) {
1388
                $mountpage = $this->pageRepository->getPage($data['row']['uid']);
1389
1390
                // fetch mounted pages
1391
                $this->MP = $mountpage[0]['mount_pid'] . '-' . $data['row']['uid'];
0 ignored issues
show
Documentation Bug introduced by
The property $MP was declared of type boolean, but $mountpage[0]['mount_pid...' . $data['row']['uid'] is of type string. Maybe add a type cast?

This check looks for assignments to scalar types that may be of the wrong type.

To ensure the code behaves as expected, it may be a good idea to add an explicit type cast.

$answer = 42;

$correct = false;

$correct = (bool) $answer;
Loading history...
1392
1393
                $mountTree = GeneralUtility::makeInstance(PageTreeView::class);
1394
                $mountTree->init('AND ' . $perms_clause);
1395
                $mountTree->getTree($mountpage[0]['mount_pid'], $depth);
1396
1397
                foreach ($mountTree->tree as $mountData) {
1398
                    $code .= $this->drawURLs_addRowsForPage(
1399
                        $mountData['row'],
1400
                        $mountData['HTML'] . BackendUtility::getRecordTitle('pages', $mountData['row'], true)
1401
                    );
1402
                }
1403
1404
                // replace page when mount_pid_ol is enabled
1405
                if ($mountpage[0]['mount_pid_ol']) {
1406
                    $data['row']['uid'] = $mountpage[0]['mount_pid'];
1407
                } else {
1408
                    // if the mount_pid_ol is not set the MP must not be used for the mountpoint page
1409
                    $this->MP = false;
1410
                }
1411
            }
1412
1413
            $code .= $this->drawURLs_addRowsForPage(
1414
                $data['row'],
1415
                $data['HTML'] . BackendUtility::getRecordTitle('pages', $data['row'], true)
1416
            );
1417
        }
1418
1419
        return $code;
1420
    }
1421
1422
    /**
1423
     * Expands exclude string
1424
     *
1425
     * @param string $excludeString Exclude string
1426
     * @return array
1427
     */
1428 2
    public function expandExcludeString($excludeString)
1429
    {
1430
        // internal static caches;
1431 2
        static $expandedExcludeStringCache;
1432 2
        static $treeCache;
1433
1434 2
        if (empty($expandedExcludeStringCache[$excludeString])) {
1435 2
            $pidList = [];
1436
1437 2
            if (! empty($excludeString)) {
1438
                /** @var PageTreeView $tree */
1439 1
                $tree = GeneralUtility::makeInstance(PageTreeView::class);
1440 1
                $tree->init('AND ' . $this->getBackendUser()->getPagePermsClause(Permission::PAGE_SHOW));
1441
1442 1
                $excludeParts = GeneralUtility::trimExplode(',', $excludeString);
1443
1444 1
                foreach ($excludeParts as $excludePart) {
1445 1
                    [$pid, $depth] = GeneralUtility::trimExplode('+', $excludePart);
1446
1447
                    // default is "page only" = "depth=0"
1448 1
                    if (empty($depth)) {
1449 1
                        $depth = (stristr($excludePart, '+')) ? 99 : 0;
1450
                    }
1451
1452 1
                    $pidList[] = (int) $pid;
1453
1454 1
                    if ($depth > 0) {
1455
                        if (empty($treeCache[$pid][$depth])) {
1456
                            $tree->reset();
1457
                            $tree->getTree($pid, $depth);
0 ignored issues
show
Bug introduced by
$pid of type string is incompatible with the type integer expected by parameter $uid of TYPO3\CMS\Backend\Tree\V...ractTreeView::getTree(). ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

1457
                            $tree->getTree(/** @scrutinizer ignore-type */ $pid, $depth);
Loading history...
1458
                            $treeCache[$pid][$depth] = $tree->tree;
1459
                        }
1460
1461
                        foreach ($treeCache[$pid][$depth] as $data) {
1462
                            $pidList[] = (int) $data['row']['uid'];
1463
                        }
1464
                    }
1465
                }
1466
            }
1467
1468 2
            $expandedExcludeStringCache[$excludeString] = array_unique($pidList);
1469
        }
1470
1471 2
        return $expandedExcludeStringCache[$excludeString];
1472
    }
1473
1474
    /**
1475
     * Create the rows for display of the page tree
1476
     * For each page a number of rows are shown displaying GET variable configuration
1477
     */
1478
    public function drawURLs_addRowsForPage(array $pageRow, string $pageTitle): string
1479
    {
1480
        $skipMessage = '';
1481
1482
        // Get list of configurations
1483
        $configurations = $this->getUrlsForPageRow($pageRow, $skipMessage);
1484
1485
        if (! empty($this->incomingConfigurationSelection)) {
1486
            // remove configuration that does not match the current selection
1487
            foreach ($configurations as $confKey => $confArray) {
1488
                if (! in_array($confKey, $this->incomingConfigurationSelection, true)) {
1489
                    unset($configurations[$confKey]);
1490
                }
1491
            }
1492
        }
1493
1494
        // Traverse parameter combinations:
1495
        $c = 0;
1496
        $content = '';
1497
        if (! empty($configurations)) {
1498
            foreach ($configurations as $confKey => $confArray) {
1499
1500
                // Title column:
1501
                if (! $c) {
1502
                    $titleClm = '<td rowspan="' . count($configurations) . '">' . $pageTitle . '</td>';
1503
                } else {
1504
                    $titleClm = '';
1505
                }
1506
1507
                if (! in_array($pageRow['uid'], $this->expandExcludeString($confArray['subCfg']['exclude']), true)) {
1508
1509
                    // URL list:
1510
                    $urlList = $this->urlListFromUrlArray(
1511
                        $confArray,
1512
                        $pageRow,
1513
                        $this->scheduledTime,
1514
                        $this->reqMinute,
1515
                        $this->submitCrawlUrls,
1516
                        $this->downloadCrawlUrls,
1517
                        $this->duplicateTrack,
1518
                        $this->downloadUrls,
1519
                        // if empty the urls won't be filtered by processing instructions
1520
                        $this->incomingProcInstructions
1521
                    );
1522
1523
                    // Expanded parameters:
1524
                    $paramExpanded = '';
1525
                    $calcAccu = [];
1526
                    $calcRes = 1;
1527
                    foreach ($confArray['paramExpanded'] as $gVar => $gVal) {
1528
                        $paramExpanded .= '
1529
                            <tr>
1530
                                <td>' . htmlspecialchars('&' . $gVar . '=') . '<br/>' .
1531
                            '(' . count($gVal) . ')' .
1532
                            '</td>
1533
                                <td nowrap="nowrap">' . nl2br(htmlspecialchars(implode(chr(10), $gVal))) . '</td>
1534
                            </tr>
1535
                        ';
1536
                        $calcRes *= count($gVal);
1537
                        $calcAccu[] = count($gVal);
1538
                    }
1539
                    $paramExpanded = '<table>' . $paramExpanded . '</table>';
1540
                    $paramExpanded .= 'Comb: ' . implode('*', $calcAccu) . '=' . $calcRes;
1541
1542
                    // Options
1543
                    $optionValues = '';
1544
                    if ($confArray['subCfg']['userGroups']) {
1545
                        $optionValues .= 'User Groups: ' . $confArray['subCfg']['userGroups'] . '<br/>';
1546
                    }
1547
                    if ($confArray['subCfg']['procInstrFilter']) {
1548
                        $optionValues .= 'ProcInstr: ' . $confArray['subCfg']['procInstrFilter'] . '<br/>';
1549
                    }
1550
1551
                    // Compile row:
1552
                    $content .= '
1553
                        <tr>
1554
                            ' . $titleClm . '
1555
                            <td>' . htmlspecialchars($confKey) . '</td>
1556
                            <td>' . nl2br(htmlspecialchars(rawurldecode(trim(str_replace('&', chr(10) . '&', GeneralUtility::implodeArrayForUrl('', $confArray['paramParsed'])))))) . '</td>
1557
                            <td>' . $paramExpanded . '</td>
1558
                            <td nowrap="nowrap">' . $urlList . '</td>
1559
                            <td nowrap="nowrap">' . $optionValues . '</td>
1560
                            <td nowrap="nowrap">' . DebugUtility::viewArray($confArray['subCfg']['procInstrParams.']) . '</td>
1561
                        </tr>';
1562
                } else {
1563
                    $content .= '<tr>
1564
                            ' . $titleClm . '
1565
                            <td>' . htmlspecialchars($confKey) . '</td>
1566
                            <td colspan="5"><em>No entries</em> (Page is excluded in this configuration)</td>
1567
                        </tr>';
1568
                }
1569
1570
                $c++;
1571
            }
1572
        } else {
1573
            $message = ! empty($skipMessage) ? ' (' . $skipMessage . ')' : '';
1574
1575
            // Compile row:
1576
            $content .= '
1577
                <tr>
1578
                    <td>' . $pageTitle . '</td>
1579
                    <td colspan="6"><em>No entries</em>' . $message . '</td>
1580
                </tr>';
1581
        }
1582
1583
        return $content;
1584
    }
1585
1586
    /*****************************
1587
     *
1588
     * CLI functions
1589
     *
1590
     *****************************/
1591
1592
    /**
1593
     * Running the functionality of the CLI (crawling URLs from queue)
1594
     */
1595
    public function CLI_run(int $countInARun, int $sleepTime, int $sleepAfterFinish): int
1596
    {
1597
        $result = 0;
1598
        $counter = 0;
1599
1600
        // First, run hooks:
1601
        $this->CLI_runHooks();
1602
1603
        // Clean up the queue
1604
        $this->queueRepository->cleanupQueue();
1605
1606
        // Select entries:
1607
        $rows = $this->queueRepository->fetchRecordsToBeCrawled($countInARun);
1608
1609
        if (! empty($rows)) {
1610
            $quidList = [];
1611
1612
            foreach ($rows as $r) {
1613
                $quidList[] = $r['qid'];
1614
            }
1615
1616
            $processId = $this->CLI_buildProcessId();
1617
1618
            //save the number of assigned queue entries to determine how many have been processed later
1619
            $numberOfAffectedRows = $this->queueRepository->updateProcessIdAndSchedulerForQueueIds($quidList, $processId);
1620
            $this->processRepository->updateProcessAssignItemsCount($numberOfAffectedRows, $processId);
1621
1622
            if ($numberOfAffectedRows !== count($quidList)) {
1623
                $this->CLI_debug('Nothing processed due to multi-process collision (' . $this->CLI_buildProcessId() . ')');
0 ignored issues
show
Deprecated Code introduced by
The function AOE\Crawler\Controller\C...Controller::CLI_debug() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

1623
                /** @scrutinizer ignore-deprecated */ $this->CLI_debug('Nothing processed due to multi-process collision (' . $this->CLI_buildProcessId() . ')');
Loading history...
1624
                return ($result | self::CLI_STATUS_ABORTED);
1625
            }
1626
1627
            foreach ($rows as $r) {
1628
                $result |= $this->readUrl($r['qid']);
1629
1630
                $counter++;
1631
                // Just to relax the system
1632
                usleep((int) $sleepTime);
1633
1634
                // if during the start and the current read url the cli has been disable we need to return from the function
1635
                // mark the process NOT as ended.
1636
                if ($this->crawler->isDisabled()) {
1637
                    return ($result | self::CLI_STATUS_ABORTED);
1638
                }
1639
1640
                if (! $this->processRepository->isProcessActive($this->CLI_buildProcessId())) {
1641
                    $this->CLI_debug('conflict / timeout (' . $this->CLI_buildProcessId() . ')');
0 ignored issues
show
Deprecated Code introduced by
The function AOE\Crawler\Controller\C...Controller::CLI_debug() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

1641
                    /** @scrutinizer ignore-deprecated */ $this->CLI_debug('conflict / timeout (' . $this->CLI_buildProcessId() . ')');
Loading history...
1642
                    $result |= self::CLI_STATUS_ABORTED;
1643
                    //possible timeout
1644
                    break;
1645
                }
1646
            }
1647
1648
            sleep((int) $sleepAfterFinish);
1649
1650
            $msg = 'Rows: ' . $counter;
1651
            $this->CLI_debug($msg . ' (' . $this->CLI_buildProcessId() . ')');
0 ignored issues
show
Deprecated Code introduced by
The function AOE\Crawler\Controller\C...Controller::CLI_debug() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

1651
            /** @scrutinizer ignore-deprecated */ $this->CLI_debug($msg . ' (' . $this->CLI_buildProcessId() . ')');
Loading history...
1652
        } else {
1653
            $this->CLI_debug('Nothing within queue which needs to be processed (' . $this->CLI_buildProcessId() . ')');
0 ignored issues
show
Deprecated Code introduced by
The function AOE\Crawler\Controller\C...Controller::CLI_debug() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

1653
            /** @scrutinizer ignore-deprecated */ $this->CLI_debug('Nothing within queue which needs to be processed (' . $this->CLI_buildProcessId() . ')');
Loading history...
1654
        }
1655
1656
        if ($counter > 0) {
1657
            $result |= self::CLI_STATUS_PROCESSED;
1658
        }
1659
1660
        return $result;
1661
    }
1662
1663
    /**
1664
     * Activate hooks
1665
     */
1666
    public function CLI_runHooks(): void
1667
    {
1668
        foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['cli_hooks'] ?? [] as $objRef) {
1669
            $hookObj = GeneralUtility::makeInstance($objRef);
1670
            if (is_object($hookObj)) {
1671
                $hookObj->crawler_init($this);
1672
            }
1673
        }
1674
    }
1675
1676
    /**
1677
     * Try to acquire a new process with the given id
1678
     * also performs some auto-cleanup for orphan processes
1679
     * @param string $id identification string for the process
1680
     * @return boolean
1681
     * @todo preemption might not be the most elegant way to clean up
1682
     */
1683
    public function CLI_checkAndAcquireNewProcess($id)
1684
    {
1685
        $ret = true;
1686
1687
        $systemProcessId = getmypid();
1688
        if (! $systemProcessId) {
1689
            return false;
1690
        }
1691
1692
        $processCount = 0;
1693
        $orphanProcesses = [];
1694
1695
        $activeProcesses = $this->processRepository->findAllActive();
1696
        $currentTime = $this->getCurrentTime();
1697
1698
        /** @var Process $process */
1699
        foreach ($activeProcesses as $process) {
1700
            if ($process->getTtl() < $currentTime) {
1701
                $orphanProcesses[] = $process->getProcessId();
1702
            } else {
1703
                $processCount++;
1704
            }
1705
        }
1706
1707
        // if there are less than allowed active processes then add a new one
1708
        if ($processCount < (int) $this->extensionSettings['processLimit']) {
1709
            $this->CLI_debug('add process ' . $this->CLI_buildProcessId() . ' (' . ($processCount + 1) . '/' . (int) $this->extensionSettings['processLimit'] . ')');
0 ignored issues
show
Deprecated Code introduced by
The function AOE\Crawler\Controller\C...Controller::CLI_debug() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

1709
            /** @scrutinizer ignore-deprecated */ $this->CLI_debug('add process ' . $this->CLI_buildProcessId() . ' (' . ($processCount + 1) . '/' . (int) $this->extensionSettings['processLimit'] . ')');
Loading history...
1710
1711
            GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable('tx_crawler_process')->insert(
1712
                'tx_crawler_process',
1713
                [
1714
                    'process_id' => $id,
1715
                    'active' => 1,
1716
                    'ttl' => $currentTime + (int) $this->extensionSettings['processMaxRunTime'],
1717
                    'system_process_id' => $systemProcessId,
1718
                ]
1719
            );
1720
        } else {
1721
            $this->CLI_debug('Processlimit reached (' . ($processCount) . '/' . (int) $this->extensionSettings['processLimit'] . ')');
0 ignored issues
show
Deprecated Code introduced by
The function AOE\Crawler\Controller\C...Controller::CLI_debug() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

1721
            /** @scrutinizer ignore-deprecated */ $this->CLI_debug('Processlimit reached (' . ($processCount) . '/' . (int) $this->extensionSettings['processLimit'] . ')');
Loading history...
1722
            $ret = false;
1723
        }
1724
1725
        $this->processRepository->deleteProcessesMarkedAsDeleted();
1726
        $this->CLI_releaseProcesses($orphanProcesses);
1727
1728
        return $ret;
1729
    }
1730
1731
    /**
1732
     * Release a process and the required resources
1733
     *
1734
     * @param mixed $releaseIds string with a single process-id or array with multiple process-ids
1735
     * @return boolean
1736
     */
1737
    public function CLI_releaseProcesses($releaseIds)
1738
    {
1739
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
1740
1741
        if (! is_array($releaseIds)) {
1742
            $releaseIds = [$releaseIds];
1743
        }
1744
1745
        if (empty($releaseIds)) {
1746
            //nothing to release
1747
            return false;
1748
        }
1749
1750
        // some kind of 2nd chance algo - this way you need at least 2 processes to have a real cleanup
1751
        // this ensures that a single process can't mess up the entire process table
1752
1753
        // mark all processes as deleted which have no "waiting" queue-entires and which are not active
1754
1755
        $queryBuilder
1756
            ->update($this->tableName, 'q')
1757
            ->where(
1758
                'q.process_id IN(SELECT p.process_id FROM tx_crawler_process as p WHERE p.active = 0)'
1759
            )
1760
            ->set('q.process_scheduled', 0)
1761
            ->set('q.process_id', '')
1762
            ->execute();
1763
1764
        // FIXME: Not entirely sure that this is equivalent to the previous version
1765
        $queryBuilder->resetQueryPart('set');
1766
1767
        $queryBuilder
1768
            ->update('tx_crawler_process')
1769
            ->where(
1770
                $queryBuilder->expr()->eq('active', 0),
1771
                'process_id IN(SELECT q.process_id FROM tx_crawler_queue as q WHERE q.exec_time = 0)'
1772
            )
1773
            ->set('system_process_id', 0)
1774
            ->execute();
1775
1776
        $this->processRepository->markRequestedProcessesAsNotActive($releaseIds);
1777
        $this->queueRepository->unsetProcessScheduledAndProcessIdForQueueEntries($releaseIds);
1778
1779
        return true;
1780
    }
1781
1782
    /**
1783
     * Create a unique Id for the current process
1784
     *
1785
     * @return string the ID
1786
     */
1787 1
    public function CLI_buildProcessId()
1788
    {
1789 1
        if (! $this->processID) {
1790
            $this->processID = GeneralUtility::shortMD5(microtime(true));
1791
        }
1792 1
        return $this->processID;
1793
    }
1794
1795
    /**
1796
     * Prints a message to the stdout (only if debug-mode is enabled)
1797
     *
1798
     * @param string $msg the message
1799
     * @deprecated
1800
     * @codeCoverageIgnore
1801
     */
1802
    public function CLI_debug($msg): void
1803
    {
1804
        if ((int) $this->extensionSettings['processDebug']) {
1805
            echo $msg . "\n";
1806
            flush();
1807
        }
1808
    }
1809
1810
    /**
1811
     * Cleans up entries that stayed for too long in the queue. These are:
1812
     * - processed entries that are over 1.5 days in age
1813
     * - scheduled entries that are over 7 days old
1814
     *
1815
     * @deprecated
1816
     */
1817 1
    public function cleanUpOldQueueEntries(): void
1818
    {
1819
        // 24*60*60 Seconds in 24 hours
1820 1
        $processedAgeInSeconds = $this->extensionSettings['cleanUpProcessedAge'] * 86400;
1821 1
        $scheduledAgeInSeconds = $this->extensionSettings['cleanUpScheduledAge'] * 86400;
1822
1823 1
        $now = time();
1824 1
        $condition = '(exec_time<>0 AND exec_time<' . ($now - $processedAgeInSeconds) . ') OR scheduled<=' . ($now - $scheduledAgeInSeconds);
1825 1
        $this->flushQueue($condition);
0 ignored issues
show
Deprecated Code introduced by
The function AOE\Crawler\Controller\C...ontroller::flushQueue() has been deprecated. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-deprecated  annotation

1825
        /** @scrutinizer ignore-deprecated */ $this->flushQueue($condition);
Loading history...
1826 1
    }
1827
1828
    /**
1829
     * Removes queue entries
1830
     *
1831
     * @param string $where SQL related filter for the entries which should be removed
1832
     *
1833
     * @deprecated
1834
     */
1835 5
    protected function flushQueue($where = ''): void
1836
    {
1837 5
        $realWhere = strlen((string) $where) > 0 ? $where : '1=1';
1838
1839 5
        $queryBuilder = $this->getQueryBuilder($this->tableName);
1840
1841
        $groups = $queryBuilder
1842 5
            ->selectLiteral('DISTINCT set_id')
1843 5
            ->from($this->tableName)
1844 5
            ->where($realWhere)
1845 5
            ->execute()
1846 5
            ->fetchAll();
1847 5
        if (is_array($groups)) {
0 ignored issues
show
introduced by
The condition is_array($groups) is always true.
Loading history...
1848 5
            foreach ($groups as $group) {
1849
                $subSet = $queryBuilder
1850 4
                    ->select('qid', 'set_id')
1851 4
                    ->from($this->tableName)
1852 4
                    ->where(
1853 4
                        $realWhere,
1854 4
                        $queryBuilder->expr()->eq('set_id', $group['set_id'])
1855
                    )
1856 4
                    ->execute()
1857 4
                    ->fetchAll();
1858
1859 4
                $payLoad = ['subSet' => $subSet];
1860 4
                SignalSlotUtility::emitSignal(
1861 4
                    self::class,
1862 4
                    SignalSlotUtility::SIGNAL_QUEUE_ENTRY_FLUSH,
1863 4
                    $payLoad
1864
                );
1865
            }
1866
        }
1867
1868
        $queryBuilder
1869 5
            ->delete($this->tableName)
1870 5
            ->where($realWhere)
1871 5
            ->execute();
1872 5
    }
1873
1874
    /**
1875
     * This method determines duplicates for a queue entry with the same parameters and this timestamp.
1876
     * If the timestamp is in the past, it will check if there is any unprocessed queue entry in the past.
1877
     * If the timestamp is in the future it will check, if the queued entry has exactly the same timestamp
1878
     *
1879
     * @param int $tstamp
1880
     * @param array $fieldArray
1881
     *
1882
     * @return array
1883
     * @deprecated
1884
     */
1885 5
    protected function getDuplicateRowsIfExist($tstamp, $fieldArray)
1886
    {
1887 5
        $rows = [];
1888
1889 5
        $currentTime = $this->getCurrentTime();
1890
1891 5
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
1892
        $queryBuilder
1893 5
            ->select('qid')
1894 5
            ->from('tx_crawler_queue');
1895
        //if this entry is scheduled with "now"
1896 5
        if ($tstamp <= $currentTime) {
1897 2
            if ($this->extensionSettings['enableTimeslot']) {
1898 1
                $timeBegin = $currentTime - 100;
1899 1
                $timeEnd = $currentTime + 100;
1900
                $queryBuilder
1901 1
                    ->where(
1902 1
                        'scheduled BETWEEN ' . $timeBegin . ' AND ' . $timeEnd . ''
1903
                    )
1904 1
                    ->orWhere(
1905 1
                        $queryBuilder->expr()->lte('scheduled', $currentTime)
1906
                    );
1907
            } else {
1908
                $queryBuilder
1909 1
                    ->where(
1910 2
                        $queryBuilder->expr()->lte('scheduled', $currentTime)
1911
                    );
1912
            }
1913 3
        } elseif ($tstamp > $currentTime) {
1914
            //entry with a timestamp in the future need to have the same schedule time
1915
            $queryBuilder
1916 3
                ->where(
1917 3
                    $queryBuilder->expr()->eq('scheduled', $tstamp)
1918
                );
1919
        }
1920
1921
        $queryBuilder
1922 5
            ->andWhere('NOT exec_time')
1923 5
            ->andWhere('NOT process_id')
1924 5
            ->andWhere($queryBuilder->expr()->eq('page_id', $queryBuilder->createNamedParameter($fieldArray['page_id'], \PDO::PARAM_INT)))
1925 5
            ->andWhere($queryBuilder->expr()->eq('parameters_hash', $queryBuilder->createNamedParameter($fieldArray['parameters_hash'], \PDO::PARAM_STR)));
1926
1927 5
        $statement = $queryBuilder->execute();
1928
1929 5
        while ($row = $statement->fetch()) {
1930 5
            $rows[] = $row['qid'];
1931
        }
1932
1933 5
        return $rows;
1934
    }
1935
1936
    /**
1937
     * Returns a md5 hash generated from a serialized configuration array.
1938
     *
1939
     * @return string
1940
     */
1941 10
    protected function getConfigurationHash(array $configuration)
1942
    {
1943 10
        unset($configuration['paramExpanded']);
1944 10
        unset($configuration['URLs']);
1945 10
        return md5(serialize($configuration));
1946
    }
1947
1948
    /**
1949
     * Build a URL from a Page and the Query String. If the page has a Site configuration, it can be built by using
1950
     * the Site instance.
1951
     *
1952
     * @param int $httpsOrHttp see tx_crawler_configuration.force_ssl
1953
     * @throws \TYPO3\CMS\Core\Exception\SiteNotFoundException
1954
     * @throws \TYPO3\CMS\Core\Routing\InvalidRouteArgumentsException
1955
     *
1956
     * @deprecated Using CrawlerController::getUrlFromPageAndQueryParameters() is deprecated since 9.1.1 and will be removed in v11.x, please use UrlService->getUrlFromPageAndQueryParameters() instead.
1957
     * @codeCoverageIgnore
1958
     */
1959
    protected function getUrlFromPageAndQueryParameters(int $pageId, string $queryString, ?string $alternativeBaseUrl, int $httpsOrHttp): UriInterface
1960
    {
1961
        $urlService = new UrlService();
1962
        return $urlService->getUrlFromPageAndQueryParameters($pageId, $queryString, $alternativeBaseUrl, $httpsOrHttp);
1963
    }
1964
1965 1
    protected function swapIfFirstIsLargerThanSecond(array $reg): array
1966
    {
1967
        // Swap if first is larger than last:
1968 1
        if ($reg[1] > $reg[2]) {
1969
            $temp = $reg[2];
1970
            $reg[2] = $reg[1];
1971
            $reg[1] = $temp;
1972
        }
1973
1974 1
        return $reg;
1975
    }
1976
1977
    /**
1978
     * @return BackendUserAuthentication
1979
     */
1980 2
    private function getBackendUser()
1981
    {
1982
        // Make sure the _cli_ user is loaded
1983 2
        Bootstrap::initializeBackendAuthentication();
1984 2
        if ($this->backendUser === null) {
1985 2
            $this->backendUser = $GLOBALS['BE_USER'];
1986
        }
1987 2
        return $this->backendUser;
1988
    }
1989
1990
    /**
1991
     * Get querybuilder for given table
1992
     *
1993
     * @return \TYPO3\CMS\Core\Database\Query\QueryBuilder
1994
     */
1995 12
    private function getQueryBuilder(string $table)
1996
    {
1997 12
        return GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
1998
    }
1999
}
2000