@@ -87,7 +87,7 @@ discard block |
||
87 | 87 | public function multiProcess($timeout) |
88 | 88 | { |
89 | 89 | if ($this->processLimit <= 1) { |
90 | - throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.' . PHP_EOL); |
|
90 | + throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.'.PHP_EOL); |
|
91 | 91 | } |
92 | 92 | |
93 | 93 | $pendingItemsStart = $this->queueRepository->countAllPendingItems(); |
@@ -105,7 +105,7 @@ discard block |
||
105 | 105 | } |
106 | 106 | if ($currentPendingItems == 0) { |
107 | 107 | if ($this->verbose) { |
108 | - echo 'Finished...' . chr(10); |
|
108 | + echo 'Finished...'.chr(10); |
|
109 | 109 | } |
110 | 110 | break; |
111 | 111 | } |
@@ -117,16 +117,16 @@ discard block |
||
117 | 117 | } |
118 | 118 | sleep(1); |
119 | 119 | if ($nextTimeOut < time()) { |
120 | - $timedOutProcesses = $this->processRepository->findAll('', 'DESC', null, 0, 'ttl >' . $nextTimeOut); |
|
120 | + $timedOutProcesses = $this->processRepository->findAll('', 'DESC', null, 0, 'ttl >'.$nextTimeOut); |
|
121 | 121 | $nextTimeOut = time() + $this->timeToLive; |
122 | 122 | if ($this->verbose) { |
123 | - echo 'Cleanup' . implode(',', $timedOutProcesses->getProcessIds()) . chr(10); |
|
123 | + echo 'Cleanup'.implode(',', $timedOutProcesses->getProcessIds()).chr(10); |
|
124 | 124 | } |
125 | 125 | $this->crawlerObj->CLI_releaseProcesses($timedOutProcesses->getProcessIds(), true); |
126 | 126 | } |
127 | 127 | } |
128 | 128 | if ($currentPendingItems > 0 && $this->verbose) { |
129 | - echo 'Stop with timeout' . chr(10); |
|
129 | + echo 'Stop with timeout'.chr(10); |
|
130 | 130 | } |
131 | 131 | } |
132 | 132 | |
@@ -135,7 +135,7 @@ discard block |
||
135 | 135 | */ |
136 | 136 | protected function reportItemStatus() |
137 | 137 | { |
138 | - echo 'Pending:' . $this->queueRepository->countAllPendingItems() . ' / Assigned:' . $this->queueRepository->countAllAssignedPendingItems() . chr(10); |
|
138 | + echo 'Pending:'.$this->queueRepository->countAllPendingItems().' / Assigned:'.$this->queueRepository->countAllAssignedPendingItems().chr(10); |
|
139 | 139 | } |
140 | 140 | |
141 | 141 | /** |
@@ -149,14 +149,14 @@ discard block |
||
149 | 149 | $currentProcesses = $this->processRepository->countActive(); |
150 | 150 | $availableProcessesCount = $this->processLimit - $currentProcesses; |
151 | 151 | $requiredProcessesCount = ceil($this->queueRepository->countAllUnassignedPendingItems() / $this->countInARun); |
152 | - $startProcessCount = min([$availableProcessesCount,$requiredProcessesCount]); |
|
152 | + $startProcessCount = min([$availableProcessesCount, $requiredProcessesCount]); |
|
153 | 153 | if ($startProcessCount <= 0) { |
154 | 154 | return $ret; |
155 | 155 | } |
156 | 156 | if ($startProcessCount && $this->verbose) { |
157 | - echo 'Start ' . $startProcessCount . ' new processes (Running:' . $currentProcesses . ')'; |
|
157 | + echo 'Start '.$startProcessCount.' new processes (Running:'.$currentProcesses.')'; |
|
158 | 158 | } |
159 | - for ($i = 0;$i < $startProcessCount;$i++) { |
|
159 | + for ($i = 0; $i < $startProcessCount; $i++) { |
|
160 | 160 | usleep(100); |
161 | 161 | if ($this->startProcess()) { |
162 | 162 | if ($this->verbose) { |
@@ -182,16 +182,16 @@ discard block |
||
182 | 182 | |
183 | 183 | // Check whether OS is Windows |
184 | 184 | if (TYPO3_OS === 'WIN') { |
185 | - $completePath = escapeshellcmd('start ' . $this->getCrawlerCliPath()); |
|
185 | + $completePath = escapeshellcmd('start '.$this->getCrawlerCliPath()); |
|
186 | 186 | } else { |
187 | - $completePath = '(' . escapeshellcmd($this->getCrawlerCliPath()) . ' &) > /dev/null'; |
|
187 | + $completePath = '('.escapeshellcmd($this->getCrawlerCliPath()).' &) > /dev/null'; |
|
188 | 188 | } |
189 | 189 | |
190 | 190 | $fileHandler = system($completePath); |
191 | 191 | if ($fileHandler === false) { |
192 | 192 | throw new Exception('could not start process!'); |
193 | 193 | } else { |
194 | - for ($i = 0;$i < 10;$i++) { |
|
194 | + for ($i = 0; $i < 10; $i++) { |
|
195 | 195 | if ($this->processRepository->countNotTimeouted($ttl) > $current) { |
196 | 196 | return true; |
197 | 197 | } |
@@ -208,11 +208,11 @@ discard block |
||
208 | 208 | */ |
209 | 209 | public function getCrawlerCliPath() |
210 | 210 | { |
211 | - $phpPath = $this->crawlerObj->extensionSettings['phpPath'] . ' '; |
|
211 | + $phpPath = $this->crawlerObj->extensionSettings['phpPath'].' '; |
|
212 | 212 | $pathToTypo3 = rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_DOCUMENT_ROOT'), '/'); |
213 | 213 | $pathToTypo3 .= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_PATH'), '/'); |
214 | 214 | $cliPart = '/typo3/cli_dispatch.phpsh crawler'; |
215 | - $scriptPath = $phpPath . $pathToTypo3 . $cliPart; |
|
215 | + $scriptPath = $phpPath.$pathToTypo3.$cliPart; |
|
216 | 216 | |
217 | 217 | if (TYPO3_OS === 'WIN') { |
218 | 218 | $scriptPath = str_replace('/', '\\', $scriptPath); |
@@ -204,7 +204,7 @@ discard block |
||
204 | 204 | { |
205 | 205 | $this->db = $GLOBALS['TYPO3_DB']; |
206 | 206 | $this->backendUser = $GLOBALS['BE_USER']; |
207 | - $this->processFilename = PATH_site . 'typo3temp/tx_crawler.proc'; |
|
207 | + $this->processFilename = PATH_site.'typo3temp/tx_crawler.proc'; |
|
208 | 208 | |
209 | 209 | $settings = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['crawler']); |
210 | 210 | $settings = is_array($settings) ? $settings : []; |
@@ -262,7 +262,7 @@ discard block |
||
262 | 262 | foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['excludeDoktype'] as $key => $doktypeList) { |
263 | 263 | if (\TYPO3\CMS\Core\Utility\GeneralUtility::inList($doktypeList, $pageRow['doktype'])) { |
264 | 264 | $skipPage = true; |
265 | - $skipMessage = 'Doktype was excluded by "' . $key . '"'; |
|
265 | + $skipMessage = 'Doktype was excluded by "'.$key.'"'; |
|
266 | 266 | break; |
267 | 267 | } |
268 | 268 | } |
@@ -283,7 +283,7 @@ discard block |
||
283 | 283 | if (is_string($veto)) { |
284 | 284 | $skipMessage = $veto; |
285 | 285 | } else { |
286 | - $skipMessage = 'Veto from hook "' . htmlspecialchars($key) . '"'; |
|
286 | + $skipMessage = 'Veto from hook "'.htmlspecialchars($key).'"'; |
|
287 | 287 | } |
288 | 288 | // no need to execute other hooks if a previous one return a veto |
289 | 289 | break; |
@@ -332,7 +332,7 @@ discard block |
||
332 | 332 | protected function noUnprocessedQueueEntriesForPageWithConfigurationHashExist($uid, $configurationHash) |
333 | 333 | { |
334 | 334 | $configurationHash = $this->db->fullQuoteStr($configurationHash, 'tx_crawler_queue'); |
335 | - $res = $this->db->exec_SELECTquery('count(*) as anz', 'tx_crawler_queue', "page_id=" . intval($uid) . " AND configuration_hash=" . $configurationHash . " AND exec_time=0"); |
|
335 | + $res = $this->db->exec_SELECTquery('count(*) as anz', 'tx_crawler_queue', "page_id=".intval($uid)." AND configuration_hash=".$configurationHash." AND exec_time=0"); |
|
336 | 336 | $row = $this->db->sql_fetch_assoc($res); |
337 | 337 | |
338 | 338 | return ($row['anz'] == 0); |
@@ -408,14 +408,14 @@ discard block |
||
408 | 408 | if ($vv['subCfg']['cHash']) { |
409 | 409 | /* @var $cacheHash \TYPO3\CMS\Frontend\Page\CacheHashCalculator */ |
410 | 410 | $cacheHash = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Frontend\Page\CacheHashCalculator'); |
411 | - $urlQuery .= '&cHash=' . $cacheHash->generateForParameters($urlQuery); |
|
411 | + $urlQuery .= '&cHash='.$cacheHash->generateForParameters($urlQuery); |
|
412 | 412 | } |
413 | 413 | |
414 | 414 | // Create key by which to determine unique-ness: |
415 | - $uKey = $urlQuery . '|' . $vv['subCfg']['userGroups'] . '|' . $vv['subCfg']['baseUrl'] . '|' . $vv['subCfg']['procInstrFilter']; |
|
415 | + $uKey = $urlQuery.'|'.$vv['subCfg']['userGroups'].'|'.$vv['subCfg']['baseUrl'].'|'.$vv['subCfg']['procInstrFilter']; |
|
416 | 416 | |
417 | 417 | // realurl support (thanks to Ingo Renner) |
418 | - $urlQuery = 'index.php' . $urlQuery; |
|
418 | + $urlQuery = 'index.php'.$urlQuery; |
|
419 | 419 | if (\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded('realurl') && $vv['subCfg']['realurl']) { |
420 | 420 | $params = [ |
421 | 421 | 'LD' => [ |
@@ -434,12 +434,12 @@ discard block |
||
434 | 434 | if (isset($duplicateTrack[$uKey])) { |
435 | 435 | |
436 | 436 | //if the url key is registered just display it and do not resubmit is |
437 | - $urlList = '<em><span class="typo3-dimmed">' . htmlspecialchars($urlQuery) . '</span></em><br/>'; |
|
437 | + $urlList = '<em><span class="typo3-dimmed">'.htmlspecialchars($urlQuery).'</span></em><br/>'; |
|
438 | 438 | } else { |
439 | - $urlList = '[' . date('d.m.y H:i', $schTime) . '] ' . htmlspecialchars($urlQuery); |
|
440 | - $this->urlList[] = '[' . date('d.m.y H:i', $schTime) . '] ' . $urlQuery; |
|
439 | + $urlList = '['.date('d.m.y H:i', $schTime).'] '.htmlspecialchars($urlQuery); |
|
440 | + $this->urlList[] = '['.date('d.m.y H:i', $schTime).'] '.$urlQuery; |
|
441 | 441 | |
442 | - $theUrl = ($vv['subCfg']['baseUrl'] ? $vv['subCfg']['baseUrl'] : \TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_URL')) . $urlQuery; |
|
442 | + $theUrl = ($vv['subCfg']['baseUrl'] ? $vv['subCfg']['baseUrl'] : \TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_URL')).$urlQuery; |
|
443 | 443 | |
444 | 444 | // Submit for crawling! |
445 | 445 | if ($submitCrawlUrls) { |
@@ -541,7 +541,7 @@ discard block |
||
541 | 541 | if (!is_array($values)) { |
542 | 542 | |
543 | 543 | // Sub configuration for a single configuration string: |
544 | - $subCfg = (array)$crawlerCfg['paramSets.'][$key . '.']; |
|
544 | + $subCfg = (array) $crawlerCfg['paramSets.'][$key.'.']; |
|
545 | 545 | $subCfg['key'] = $key; |
546 | 546 | |
547 | 547 | if (strcmp($subCfg['procInstrFilter'], '')) { |
@@ -566,9 +566,9 @@ discard block |
||
566 | 566 | |
567 | 567 | // recognize MP value |
568 | 568 | if (!$this->MP) { |
569 | - $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id=' . $id]); |
|
569 | + $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id='.$id]); |
|
570 | 570 | } else { |
571 | - $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id=' . $id . '&MP=' . $this->MP]); |
|
571 | + $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id='.$id.'&MP='.$this->MP]); |
|
572 | 572 | } |
573 | 573 | } |
574 | 574 | } |
@@ -588,7 +588,7 @@ discard block |
||
588 | 588 | 'tx_crawler_configuration', |
589 | 589 | 'pid', |
590 | 590 | intval($page['uid']), |
591 | - \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('tx_crawler_configuration') . \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('tx_crawler_configuration') |
|
591 | + \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('tx_crawler_configuration').\TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('tx_crawler_configuration') |
|
592 | 592 | ); |
593 | 593 | |
594 | 594 | if (is_array($configurationRecordsForCurrentPage)) { |
@@ -634,8 +634,8 @@ discard block |
||
634 | 634 | $res[$key]['subCfg'] = $subCfg; |
635 | 635 | $res[$key]['paramParsed'] = $this->parseParams($configurationRecord['configuration']); |
636 | 636 | $res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $id); |
637 | - $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id=' . $id]); |
|
638 | - $res[$key]['origin'] = 'tx_crawler_configuration_' . $configurationRecord['uid']; |
|
637 | + $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], ['?id='.$id]); |
|
638 | + $res[$key]['origin'] = 'tx_crawler_configuration_'.$configurationRecord['uid']; |
|
639 | 639 | } |
640 | 640 | } |
641 | 641 | } |
@@ -673,13 +673,13 @@ discard block |
||
673 | 673 | $res = $this->db->exec_SELECTquery( |
674 | 674 | '*', |
675 | 675 | 'sys_domain', |
676 | - 'uid = ' . $sysDomainUid . |
|
677 | - \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_domain') . |
|
676 | + 'uid = '.$sysDomainUid. |
|
677 | + \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_domain'). |
|
678 | 678 | \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('sys_domain') |
679 | 679 | ); |
680 | 680 | $row = $this->db->sql_fetch_assoc($res); |
681 | 681 | if ($row['domainName'] != '') { |
682 | - return $urlScheme . '://' . $row['domainName']; |
|
682 | + return $urlScheme.'://'.$row['domainName']; |
|
683 | 683 | } |
684 | 684 | } |
685 | 685 | return $baseUrl; |
@@ -709,7 +709,7 @@ discard block |
||
709 | 709 | /* @var \TYPO3\CMS\Backend\Tree\View\PageTreeView */ |
710 | 710 | $tree = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Backend\Tree\View\PageTreeView'); |
711 | 711 | $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1); |
712 | - $tree->init('AND ' . $perms_clause); |
|
712 | + $tree->init('AND '.$perms_clause); |
|
713 | 713 | $tree->getTree($rootid, $depth, ''); |
714 | 714 | foreach ($tree->tree as $node) { |
715 | 715 | $pids[] = $node['row']['uid']; |
@@ -718,10 +718,10 @@ discard block |
||
718 | 718 | $res = $this->db->exec_SELECTquery( |
719 | 719 | '*', |
720 | 720 | 'tx_crawler_configuration', |
721 | - 'pid IN (' . implode(',', $pids) . ') ' . |
|
722 | - \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('tx_crawler_configuration') . |
|
723 | - \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('tx_crawler_configuration') . ' ' . |
|
724 | - \TYPO3\CMS\Backend\Utility\BackendUtility::versioningPlaceholderClause('tx_crawler_configuration') . ' ' |
|
721 | + 'pid IN ('.implode(',', $pids).') '. |
|
722 | + \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('tx_crawler_configuration'). |
|
723 | + \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('tx_crawler_configuration').' '. |
|
724 | + \TYPO3\CMS\Backend\Utility\BackendUtility::versioningPlaceholderClause('tx_crawler_configuration').' ' |
|
725 | 725 | ); |
726 | 726 | |
727 | 727 | while ($row = $this->db->sql_fetch_assoc($res)) { |
@@ -820,7 +820,7 @@ discard block |
||
820 | 820 | |
821 | 821 | // Traverse range, add values: |
822 | 822 | $runAwayBrake = 1000; // Limit to size of range! |
823 | - for ($a = $reg[1]; $a <= $reg[2];$a++) { |
|
823 | + for ($a = $reg[1]; $a <= $reg[2]; $a++) { |
|
824 | 824 | $paramArray[$p][] = $a; |
825 | 825 | $runAwayBrake--; |
826 | 826 | if ($runAwayBrake <= 0) { |
@@ -850,16 +850,16 @@ discard block |
||
850 | 850 | $transOrigPointerField = $TCA[$subpartParams['_TABLE']]['ctrl']['transOrigPointerField']; |
851 | 851 | |
852 | 852 | if ($subpartParams['_ENABLELANG'] && $transOrigPointerField) { |
853 | - $andWhereLanguage = ' AND ' . $this->db->quoteStr($transOrigPointerField, $subpartParams['_TABLE']) . ' <= 0 '; |
|
853 | + $andWhereLanguage = ' AND '.$this->db->quoteStr($transOrigPointerField, $subpartParams['_TABLE']).' <= 0 '; |
|
854 | 854 | } |
855 | 855 | |
856 | - $where = $this->db->quoteStr($pidField, $subpartParams['_TABLE']) . '=' . intval($lookUpPid) . ' ' . |
|
857 | - $andWhereLanguage . $where; |
|
856 | + $where = $this->db->quoteStr($pidField, $subpartParams['_TABLE']).'='.intval($lookUpPid).' '. |
|
857 | + $andWhereLanguage.$where; |
|
858 | 858 | |
859 | 859 | $rows = $this->db->exec_SELECTgetRows( |
860 | 860 | $fieldName, |
861 | - $subpartParams['_TABLE'] . $addTable, |
|
862 | - $where . \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause($subpartParams['_TABLE']), |
|
861 | + $subpartParams['_TABLE'].$addTable, |
|
862 | + $where.\TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause($subpartParams['_TABLE']), |
|
863 | 863 | '', |
864 | 864 | '', |
865 | 865 | '', |
@@ -921,7 +921,7 @@ discard block |
||
921 | 921 | $newUrls = []; |
922 | 922 | foreach ($urls as $url) { |
923 | 923 | foreach ($valueSet as $val) { |
924 | - $newUrls[] = $url . (strcmp($val, '') ? '&' . rawurlencode($varName) . '=' . rawurlencode($val) : ''); |
|
924 | + $newUrls[] = $url.(strcmp($val, '') ? '&'.rawurlencode($varName).'='.rawurlencode($val) : ''); |
|
925 | 925 | |
926 | 926 | if (count($newUrls) > \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($this->extensionSettings['maxCompileUrls'], 1, 1000000000, 10000)) { |
927 | 927 | break; |
@@ -968,13 +968,13 @@ discard block |
||
968 | 968 | |
969 | 969 | // FIXME: Write unit test that ensures that the right records are deleted. |
970 | 970 | if ($doFlush) { |
971 | - $this->flushQueue(($doFullFlush ? '1=1' : ('page_id=' . intval($id))) . $addWhere); |
|
971 | + $this->flushQueue(($doFullFlush ? '1=1' : ('page_id='.intval($id))).$addWhere); |
|
972 | 972 | return []; |
973 | 973 | } else { |
974 | 974 | return $this->db->exec_SELECTgetRows( |
975 | 975 | '*', |
976 | 976 | 'tx_crawler_queue', |
977 | - 'page_id=' . intval($id) . $addWhere, |
|
977 | + 'page_id='.intval($id).$addWhere, |
|
978 | 978 | '', |
979 | 979 | 'scheduled DESC', |
980 | 980 | (intval($itemsPerPage) > 0 ? intval($itemsPerPage) : '') |
@@ -1007,13 +1007,13 @@ discard block |
||
1007 | 1007 | } |
1008 | 1008 | // FIXME: Write unit test that ensures that the right records are deleted. |
1009 | 1009 | if ($doFlush) { |
1010 | - $this->flushQueue($doFullFlush ? '' : ('set_id=' . intval($set_id) . $addWhere)); |
|
1010 | + $this->flushQueue($doFullFlush ? '' : ('set_id='.intval($set_id).$addWhere)); |
|
1011 | 1011 | return []; |
1012 | 1012 | } else { |
1013 | 1013 | return $this->db->exec_SELECTgetRows( |
1014 | 1014 | '*', |
1015 | 1015 | 'tx_crawler_queue', |
1016 | - 'set_id=' . intval($set_id) . $addWhere, |
|
1016 | + 'set_id='.intval($set_id).$addWhere, |
|
1017 | 1017 | '', |
1018 | 1018 | 'scheduled DESC', |
1019 | 1019 | (intval($itemsPerPage) > 0 ? intval($itemsPerPage) : '') |
@@ -1034,7 +1034,7 @@ discard block |
||
1034 | 1034 | if (tx_crawler_domain_events_dispatcher::getInstance()->hasObserver('queueEntryFlush')) { |
1035 | 1035 | $groups = $this->db->exec_SELECTgetRows('DISTINCT set_id', 'tx_crawler_queue', $realWhere); |
1036 | 1036 | foreach ($groups as $group) { |
1037 | - tx_crawler_domain_events_dispatcher::getInstance()->post('queueEntryFlush', $group['set_id'], $this->db->exec_SELECTgetRows('uid, set_id', 'tx_crawler_queue', $realWhere . ' AND set_id="' . $group['set_id'] . '"')); |
|
1037 | + tx_crawler_domain_events_dispatcher::getInstance()->post('queueEntryFlush', $group['set_id'], $this->db->exec_SELECTgetRows('uid, set_id', 'tx_crawler_queue', $realWhere.' AND set_id="'.$group['set_id'].'"')); |
|
1038 | 1038 | } |
1039 | 1039 | } |
1040 | 1040 | |
@@ -1176,24 +1176,24 @@ discard block |
||
1176 | 1176 | if ($this->extensionSettings['enableTimeslot']) { |
1177 | 1177 | $timeBegin = $currentTime - 100; |
1178 | 1178 | $timeEnd = $currentTime + 100; |
1179 | - $where = ' ((scheduled BETWEEN ' . $timeBegin . ' AND ' . $timeEnd . ' ) OR scheduled <= ' . $currentTime . ') '; |
|
1179 | + $where = ' ((scheduled BETWEEN '.$timeBegin.' AND '.$timeEnd.' ) OR scheduled <= '.$currentTime.') '; |
|
1180 | 1180 | } else { |
1181 | - $where = 'scheduled <= ' . $currentTime; |
|
1181 | + $where = 'scheduled <= '.$currentTime; |
|
1182 | 1182 | } |
1183 | 1183 | } elseif ($tstamp > $currentTime) { |
1184 | 1184 | //entry with a timestamp in the future need to have the same schedule time |
1185 | - $where = 'scheduled = ' . $tstamp ; |
|
1185 | + $where = 'scheduled = '.$tstamp; |
|
1186 | 1186 | } |
1187 | 1187 | |
1188 | 1188 | if (!empty($where)) { |
1189 | 1189 | $result = $this->db->exec_SELECTgetRows( |
1190 | 1190 | 'qid', |
1191 | 1191 | 'tx_crawler_queue', |
1192 | - $where . |
|
1193 | - ' AND NOT exec_time' . |
|
1194 | - ' AND NOT process_id ' . |
|
1195 | - ' AND page_id=' . intval($fieldArray['page_id']) . |
|
1196 | - ' AND parameters_hash = ' . $this->db->fullQuoteStr($fieldArray['parameters_hash'], 'tx_crawler_queue') |
|
1192 | + $where. |
|
1193 | + ' AND NOT exec_time'. |
|
1194 | + ' AND NOT process_id '. |
|
1195 | + ' AND page_id='.intval($fieldArray['page_id']). |
|
1196 | + ' AND parameters_hash = '.$this->db->fullQuoteStr($fieldArray['parameters_hash'], 'tx_crawler_queue') |
|
1197 | 1197 | ); |
1198 | 1198 | |
1199 | 1199 | if (is_array($result)) { |
@@ -1233,13 +1233,13 @@ discard block |
||
1233 | 1233 | { |
1234 | 1234 | $ret = 0; |
1235 | 1235 | if ($this->debugMode) { |
1236 | - \TYPO3\CMS\Core\Utility\GeneralUtility::devlog('crawler-readurl start ' . microtime(true), __FUNCTION__); |
|
1236 | + \TYPO3\CMS\Core\Utility\GeneralUtility::devlog('crawler-readurl start '.microtime(true), __FUNCTION__); |
|
1237 | 1237 | } |
1238 | 1238 | // Get entry: |
1239 | 1239 | list($queueRec) = $this->db->exec_SELECTgetRows( |
1240 | 1240 | '*', |
1241 | 1241 | 'tx_crawler_queue', |
1242 | - 'qid=' . intval($queueId) . ($force ? '' : ' AND exec_time=0 AND process_scheduled > 0') |
|
1242 | + 'qid='.intval($queueId).($force ? '' : ' AND exec_time=0 AND process_scheduled > 0') |
|
1243 | 1243 | ); |
1244 | 1244 | |
1245 | 1245 | if (!is_array($queueRec)) { |
@@ -1248,10 +1248,10 @@ discard block |
||
1248 | 1248 | |
1249 | 1249 | $parameters = unserialize($queueRec['parameters']); |
1250 | 1250 | if ($parameters['rootTemplatePid']) { |
1251 | - $this->initTSFE((int)$parameters['rootTemplatePid']); |
|
1251 | + $this->initTSFE((int) $parameters['rootTemplatePid']); |
|
1252 | 1252 | } else { |
1253 | 1253 | \TYPO3\CMS\Core\Utility\GeneralUtility::sysLog( |
1254 | - 'Page with (' . $queueRec['page_id'] . ') could not be crawled, please check your crawler configuration. Perhaps no Root Template Pid is set', |
|
1254 | + 'Page with ('.$queueRec['page_id'].') could not be crawled, please check your crawler configuration. Perhaps no Root Template Pid is set', |
|
1255 | 1255 | 'crawler', |
1256 | 1256 | \TYPO3\CMS\Core\Utility\GeneralUtility::SYSLOG_SEVERITY_WARNING |
1257 | 1257 | ); |
@@ -1270,7 +1270,7 @@ discard block |
||
1270 | 1270 | //if mulitprocessing is used we need to store the id of the process which has handled this entry |
1271 | 1271 | $field_array['process_id_completed'] = $this->processID; |
1272 | 1272 | } |
1273 | - $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid=' . intval($queueId), $field_array); |
|
1273 | + $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid='.intval($queueId), $field_array); |
|
1274 | 1274 | |
1275 | 1275 | $result = $this->readUrl_exec($queueRec); |
1276 | 1276 | $resultData = unserialize($result['content']); |
@@ -1301,10 +1301,10 @@ discard block |
||
1301 | 1301 | [$queueId, &$field_array] |
1302 | 1302 | ); |
1303 | 1303 | |
1304 | - $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid=' . intval($queueId), $field_array); |
|
1304 | + $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid='.intval($queueId), $field_array); |
|
1305 | 1305 | |
1306 | 1306 | if ($this->debugMode) { |
1307 | - \TYPO3\CMS\Core\Utility\GeneralUtility::devlog('crawler-readurl stop ' . microtime(true), __FUNCTION__); |
|
1307 | + \TYPO3\CMS\Core\Utility\GeneralUtility::devlog('crawler-readurl stop '.microtime(true), __FUNCTION__); |
|
1308 | 1308 | } |
1309 | 1309 | |
1310 | 1310 | return $ret; |
@@ -1335,7 +1335,7 @@ discard block |
||
1335 | 1335 | [$queueId, &$field_array] |
1336 | 1336 | ); |
1337 | 1337 | |
1338 | - $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid=' . intval($queueId), $field_array); |
|
1338 | + $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid='.intval($queueId), $field_array); |
|
1339 | 1339 | |
1340 | 1340 | return $result; |
1341 | 1341 | } |
@@ -1359,12 +1359,12 @@ discard block |
||
1359 | 1359 | unset($parameters['_CALLBACKOBJ']); |
1360 | 1360 | $result = ['content' => serialize($callBackObj->crawler_execute($parameters, $this))]; |
1361 | 1361 | } else { |
1362 | - $result = ['content' => 'No object: ' . $objRef]; |
|
1362 | + $result = ['content' => 'No object: '.$objRef]; |
|
1363 | 1363 | } |
1364 | 1364 | } else { // Regular FE request: |
1365 | 1365 | |
1366 | 1366 | // Prepare: |
1367 | - $crawlerId = $queueRec['qid'] . ':' . md5($queueRec['qid'] . '|' . $queueRec['set_id'] . '|' . $GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey']); |
|
1367 | + $crawlerId = $queueRec['qid'].':'.md5($queueRec['qid'].'|'.$queueRec['set_id'].'|'.$GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey']); |
|
1368 | 1368 | |
1369 | 1369 | // Get result: |
1370 | 1370 | $result = $this->requestUrl($parameters['url'], $crawlerId); |
@@ -1401,7 +1401,7 @@ discard block |
||
1401 | 1401 | return false; |
1402 | 1402 | } |
1403 | 1403 | |
1404 | - if (!in_array($url['scheme'], ['','http','https'])) { |
|
1404 | + if (!in_array($url['scheme'], ['', 'http', 'https'])) { |
|
1405 | 1405 | if (TYPO3_DLOG) { |
1406 | 1406 | \TYPO3\CMS\Core\Utility\GeneralUtility::devLog(sprintf('Scheme does not match for url "%s"', $url), 'crawler', 4, ['crawlerId' => $crawlerId]); |
1407 | 1407 | } |
@@ -1421,14 +1421,14 @@ discard block |
||
1421 | 1421 | |
1422 | 1422 | if ($GLOBALS['TYPO3_CONF_VARS']['SYS']['curlUse'] && $GLOBALS['TYPO3_CONF_VARS']['SYS']['curlProxyServer']) { |
1423 | 1423 | $rurl = parse_url($GLOBALS['TYPO3_CONF_VARS']['SYS']['curlProxyServer']); |
1424 | - $url['path'] = $url['scheme'] . '://' . $url['host'] . ($url['port'] > 0 ? ':' . $url['port'] : '') . $url['path']; |
|
1424 | + $url['path'] = $url['scheme'].'://'.$url['host'].($url['port'] > 0 ? ':'.$url['port'] : '').$url['path']; |
|
1425 | 1425 | $reqHeaders = $this->buildRequestHeaderArray($url, $crawlerId); |
1426 | 1426 | } |
1427 | 1427 | |
1428 | 1428 | $host = $rurl['host']; |
1429 | 1429 | |
1430 | 1430 | if ($url['scheme'] == 'https') { |
1431 | - $host = 'ssl://' . $host; |
|
1431 | + $host = 'ssl://'.$host; |
|
1432 | 1432 | $port = ($rurl['port'] > 0) ? $rurl['port'] : 443; |
1433 | 1433 | } else { |
1434 | 1434 | $port = ($rurl['port'] > 0) ? $rurl['port'] : 80; |
@@ -1444,7 +1444,7 @@ discard block |
||
1444 | 1444 | return false; |
1445 | 1445 | } else { |
1446 | 1446 | // Request message: |
1447 | - $msg = implode("\r\n", $reqHeaders) . "\r\n\r\n"; |
|
1447 | + $msg = implode("\r\n", $reqHeaders)."\r\n\r\n"; |
|
1448 | 1448 | fputs($fp, $msg); |
1449 | 1449 | |
1450 | 1450 | // Read response: |
@@ -1452,13 +1452,13 @@ discard block |
||
1452 | 1452 | fclose($fp); |
1453 | 1453 | |
1454 | 1454 | $time = microtime(true) - $startTime; |
1455 | - $this->log($originalUrl . ' ' . $time); |
|
1455 | + $this->log($originalUrl.' '.$time); |
|
1456 | 1456 | |
1457 | 1457 | // Implode content and headers: |
1458 | 1458 | $result = [ |
1459 | 1459 | 'request' => $msg, |
1460 | 1460 | 'headers' => implode('', $d['headers']), |
1461 | - 'content' => implode('', (array)$d['content']) |
|
1461 | + 'content' => implode('', (array) $d['content']) |
|
1462 | 1462 | ]; |
1463 | 1463 | |
1464 | 1464 | if (($this->extensionSettings['follow30x']) && ($newUrl = $this->getRequestUrlFrom302Header($d['headers'], $url['user'], $url['pass']))) { |
@@ -1503,8 +1503,8 @@ discard block |
||
1503 | 1503 | |
1504 | 1504 | // Base path must be '/<pathSegements>/': |
1505 | 1505 | if ($frontendBasePath != '/') { |
1506 | - $frontendBasePath = '/' . ltrim($frontendBasePath, '/'); |
|
1507 | - $frontendBasePath = rtrim($frontendBasePath, '/') . '/'; |
|
1506 | + $frontendBasePath = '/'.ltrim($frontendBasePath, '/'); |
|
1507 | + $frontendBasePath = rtrim($frontendBasePath, '/').'/'; |
|
1508 | 1508 | } |
1509 | 1509 | |
1510 | 1510 | return $frontendBasePath; |
@@ -1560,7 +1560,7 @@ discard block |
||
1560 | 1560 | protected function log($message) |
1561 | 1561 | { |
1562 | 1562 | if (!empty($this->extensionSettings['logFileName'])) { |
1563 | - @file_put_contents($this->extensionSettings['logFileName'], date('Ymd His') . ' ' . $message . PHP_EOL, FILE_APPEND); |
|
1563 | + @file_put_contents($this->extensionSettings['logFileName'], date('Ymd His').' '.$message.PHP_EOL, FILE_APPEND); |
|
1564 | 1564 | } |
1565 | 1565 | } |
1566 | 1566 | |
@@ -1575,16 +1575,16 @@ discard block |
||
1575 | 1575 | protected function buildRequestHeaderArray(array $url, $crawlerId) |
1576 | 1576 | { |
1577 | 1577 | $reqHeaders = []; |
1578 | - $reqHeaders[] = 'GET ' . $url['path'] . ($url['query'] ? '?' . $url['query'] : '') . ' HTTP/1.0'; |
|
1579 | - $reqHeaders[] = 'Host: ' . $url['host']; |
|
1578 | + $reqHeaders[] = 'GET '.$url['path'].($url['query'] ? '?'.$url['query'] : '').' HTTP/1.0'; |
|
1579 | + $reqHeaders[] = 'Host: '.$url['host']; |
|
1580 | 1580 | if (stristr($url['query'], 'ADMCMD_previewWS')) { |
1581 | 1581 | $reqHeaders[] = 'Cookie: $Version="1"; be_typo_user="1"; $Path=/'; |
1582 | 1582 | } |
1583 | 1583 | $reqHeaders[] = 'Connection: close'; |
1584 | 1584 | if ($url['user'] != '') { |
1585 | - $reqHeaders[] = 'Authorization: Basic ' . base64_encode($url['user'] . ':' . $url['pass']); |
|
1585 | + $reqHeaders[] = 'Authorization: Basic '.base64_encode($url['user'].':'.$url['pass']); |
|
1586 | 1586 | } |
1587 | - $reqHeaders[] = 'X-T3crawler: ' . $crawlerId; |
|
1587 | + $reqHeaders[] = 'X-T3crawler: '.$crawlerId; |
|
1588 | 1588 | $reqHeaders[] = 'User-Agent: TYPO3 crawler'; |
1589 | 1589 | return $reqHeaders; |
1590 | 1590 | } |
@@ -1621,9 +1621,9 @@ discard block |
||
1621 | 1621 | if (!($tmp = parse_url($header['Location']))) { |
1622 | 1622 | return false; |
1623 | 1623 | } |
1624 | - $newUrl = $tmp['scheme'] . '://' . $user . ':' . $pass . '@' . $tmp['host'] . $tmp['path']; |
|
1624 | + $newUrl = $tmp['scheme'].'://'.$user.':'.$pass.'@'.$tmp['host'].$tmp['path']; |
|
1625 | 1625 | if ($tmp['query'] != '') { |
1626 | - $newUrl .= '?' . $tmp['query']; |
|
1626 | + $newUrl .= '?'.$tmp['query']; |
|
1627 | 1627 | } |
1628 | 1628 | } else { |
1629 | 1629 | $newUrl = $header['Location']; |
@@ -1651,10 +1651,10 @@ discard block |
||
1651 | 1651 | // Authenticate crawler request: |
1652 | 1652 | if (isset($_SERVER['HTTP_X_T3CRAWLER'])) { |
1653 | 1653 | list($queueId, $hash) = explode(':', $_SERVER['HTTP_X_T3CRAWLER']); |
1654 | - list($queueRec) = $this->db->exec_SELECTgetRows('*', 'tx_crawler_queue', 'qid=' . intval($queueId)); |
|
1654 | + list($queueRec) = $this->db->exec_SELECTgetRows('*', 'tx_crawler_queue', 'qid='.intval($queueId)); |
|
1655 | 1655 | |
1656 | 1656 | // If a crawler record was found and hash was matching, set it up: |
1657 | - if (is_array($queueRec) && $hash === md5($queueRec['qid'] . '|' . $queueRec['set_id'] . '|' . $GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey'])) { |
|
1657 | + if (is_array($queueRec) && $hash === md5($queueRec['qid'].'|'.$queueRec['set_id'].'|'.$GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey'])) { |
|
1658 | 1658 | $params['pObj']->applicationData['tx_crawler']['running'] = true; |
1659 | 1659 | $params['pObj']->applicationData['tx_crawler']['parameters'] = unserialize($queueRec['parameters']); |
1660 | 1660 | $params['pObj']->applicationData['tx_crawler']['log'] = []; |
@@ -1711,7 +1711,7 @@ discard block |
||
1711 | 1711 | /* @var $tree \TYPO3\CMS\Backend\Tree\View\PageTreeView */ |
1712 | 1712 | $tree = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Backend\Tree\View\PageTreeView'); |
1713 | 1713 | $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1); |
1714 | - $tree->init('AND ' . $perms_clause); |
|
1714 | + $tree->init('AND '.$perms_clause); |
|
1715 | 1715 | |
1716 | 1716 | $pageinfo = \TYPO3\CMS\Backend\Utility\BackendUtility::readPageAccess($id, $perms_clause); |
1717 | 1717 | |
@@ -1734,19 +1734,19 @@ discard block |
||
1734 | 1734 | |
1735 | 1735 | // recognize mount points |
1736 | 1736 | if ($data['row']['doktype'] == 7) { |
1737 | - $mountpage = $this->db->exec_SELECTgetRows('*', 'pages', 'uid = ' . $data['row']['uid']); |
|
1737 | + $mountpage = $this->db->exec_SELECTgetRows('*', 'pages', 'uid = '.$data['row']['uid']); |
|
1738 | 1738 | |
1739 | 1739 | // fetch mounted pages |
1740 | - $this->MP = $mountpage[0]['mount_pid'] . '-' . $data['row']['uid']; |
|
1740 | + $this->MP = $mountpage[0]['mount_pid'].'-'.$data['row']['uid']; |
|
1741 | 1741 | |
1742 | 1742 | $mountTree = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Backend\Tree\View\PageTreeView'); |
1743 | - $mountTree->init('AND ' . $perms_clause); |
|
1743 | + $mountTree->init('AND '.$perms_clause); |
|
1744 | 1744 | $mountTree->getTree($mountpage[0]['mount_pid'], $depth, ''); |
1745 | 1745 | |
1746 | 1746 | foreach ($mountTree->tree as $mountData) { |
1747 | 1747 | $code .= $this->drawURLs_addRowsForPage( |
1748 | 1748 | $mountData['row'], |
1749 | - $mountData['HTML'] . \TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages', $mountData['row'], true) |
|
1749 | + $mountData['HTML'].\TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages', $mountData['row'], true) |
|
1750 | 1750 | ); |
1751 | 1751 | } |
1752 | 1752 | |
@@ -1761,7 +1761,7 @@ discard block |
||
1761 | 1761 | |
1762 | 1762 | $code .= $this->drawURLs_addRowsForPage( |
1763 | 1763 | $data['row'], |
1764 | - $data['HTML'] . \TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages', $data['row'], true) |
|
1764 | + $data['HTML'].\TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages', $data['row'], true) |
|
1765 | 1765 | ); |
1766 | 1766 | } |
1767 | 1767 | |
@@ -1786,7 +1786,7 @@ discard block |
||
1786 | 1786 | if (!empty($excludeString)) { |
1787 | 1787 | /* @var $tree \TYPO3\CMS\Backend\Tree\View\PageTreeView */ |
1788 | 1788 | $tree = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Backend\Tree\View\PageTreeView'); |
1789 | - $tree->init('AND ' . $this->backendUser->getPagePermsClause(1)); |
|
1789 | + $tree->init('AND '.$this->backendUser->getPagePermsClause(1)); |
|
1790 | 1790 | |
1791 | 1791 | $excludeParts = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $excludeString); |
1792 | 1792 | |
@@ -1853,7 +1853,7 @@ discard block |
||
1853 | 1853 | |
1854 | 1854 | // Title column: |
1855 | 1855 | if (!$c) { |
1856 | - $titleClm = '<td rowspan="' . count($configurations) . '">' . $pageTitleAndIcon . '</td>'; |
|
1856 | + $titleClm = '<td rowspan="'.count($configurations).'">'.$pageTitleAndIcon.'</td>'; |
|
1857 | 1857 | } else { |
1858 | 1858 | $titleClm = ''; |
1859 | 1859 | } |
@@ -1880,45 +1880,45 @@ discard block |
||
1880 | 1880 | foreach ($confArray['paramExpanded'] as $gVar => $gVal) { |
1881 | 1881 | $paramExpanded .= ' |
1882 | 1882 | <tr> |
1883 | - <td class="bgColor4-20">' . htmlspecialchars('&' . $gVar . '=') . '<br/>' . |
|
1884 | - '(' . count($gVal) . ')' . |
|
1883 | + <td class="bgColor4-20">' . htmlspecialchars('&'.$gVar.'=').'<br/>'. |
|
1884 | + '('.count($gVal).')'. |
|
1885 | 1885 | '</td> |
1886 | - <td class="bgColor4" nowrap="nowrap">' . nl2br(htmlspecialchars(implode(chr(10), $gVal))) . '</td> |
|
1886 | + <td class="bgColor4" nowrap="nowrap">' . nl2br(htmlspecialchars(implode(chr(10), $gVal))).'</td> |
|
1887 | 1887 | </tr> |
1888 | 1888 | '; |
1889 | 1889 | $calcRes *= count($gVal); |
1890 | 1890 | $calcAccu[] = count($gVal); |
1891 | 1891 | } |
1892 | - $paramExpanded = '<table class="lrPadding c-list param-expanded">' . $paramExpanded . '</table>'; |
|
1893 | - $paramExpanded .= 'Comb: ' . implode('*', $calcAccu) . '=' . $calcRes; |
|
1892 | + $paramExpanded = '<table class="lrPadding c-list param-expanded">'.$paramExpanded.'</table>'; |
|
1893 | + $paramExpanded .= 'Comb: '.implode('*', $calcAccu).'='.$calcRes; |
|
1894 | 1894 | |
1895 | 1895 | // Options |
1896 | 1896 | $optionValues = ''; |
1897 | 1897 | if ($confArray['subCfg']['userGroups']) { |
1898 | - $optionValues .= 'User Groups: ' . $confArray['subCfg']['userGroups'] . '<br/>'; |
|
1898 | + $optionValues .= 'User Groups: '.$confArray['subCfg']['userGroups'].'<br/>'; |
|
1899 | 1899 | } |
1900 | 1900 | if ($confArray['subCfg']['baseUrl']) { |
1901 | - $optionValues .= 'Base Url: ' . $confArray['subCfg']['baseUrl'] . '<br/>'; |
|
1901 | + $optionValues .= 'Base Url: '.$confArray['subCfg']['baseUrl'].'<br/>'; |
|
1902 | 1902 | } |
1903 | 1903 | if ($confArray['subCfg']['procInstrFilter']) { |
1904 | - $optionValues .= 'ProcInstr: ' . $confArray['subCfg']['procInstrFilter'] . '<br/>'; |
|
1904 | + $optionValues .= 'ProcInstr: '.$confArray['subCfg']['procInstrFilter'].'<br/>'; |
|
1905 | 1905 | } |
1906 | 1906 | |
1907 | 1907 | // Compile row: |
1908 | 1908 | $content .= ' |
1909 | - <tr class="bgColor' . ($c % 2 ? '-20' : '-10') . '"> |
|
1910 | - ' . $titleClm . ' |
|
1911 | - <td>' . htmlspecialchars($confKey) . '</td> |
|
1912 | - <td>' . nl2br(htmlspecialchars(rawurldecode(trim(str_replace('&', chr(10) . '&', \TYPO3\CMS\Core\Utility\GeneralUtility::implodeArrayForUrl('', $confArray['paramParsed'])))))) . '</td> |
|
1913 | - <td>' . $paramExpanded . '</td> |
|
1914 | - <td nowrap="nowrap">' . $urlList . '</td> |
|
1915 | - <td nowrap="nowrap">' . $optionValues . '</td> |
|
1916 | - <td nowrap="nowrap">' . \TYPO3\CMS\Core\Utility\DebugUtility::viewArray($confArray['subCfg']['procInstrParams.']) . '</td> |
|
1909 | + <tr class="bgColor' . ($c % 2 ? '-20' : '-10').'"> |
|
1910 | + ' . $titleClm.' |
|
1911 | + <td>' . htmlspecialchars($confKey).'</td> |
|
1912 | + <td>' . nl2br(htmlspecialchars(rawurldecode(trim(str_replace('&', chr(10).'&', \TYPO3\CMS\Core\Utility\GeneralUtility::implodeArrayForUrl('', $confArray['paramParsed'])))))).'</td> |
|
1913 | + <td>' . $paramExpanded.'</td> |
|
1914 | + <td nowrap="nowrap">' . $urlList.'</td> |
|
1915 | + <td nowrap="nowrap">' . $optionValues.'</td> |
|
1916 | + <td nowrap="nowrap">' . \TYPO3\CMS\Core\Utility\DebugUtility::viewArray($confArray['subCfg']['procInstrParams.']).'</td> |
|
1917 | 1917 | </tr>'; |
1918 | 1918 | } else { |
1919 | - $content .= '<tr class="bgColor' . ($c % 2 ? '-20' : '-10') . '"> |
|
1920 | - ' . $titleClm . ' |
|
1921 | - <td>' . htmlspecialchars($confKey) . '</td> |
|
1919 | + $content .= '<tr class="bgColor'.($c % 2 ? '-20' : '-10').'"> |
|
1920 | + ' . $titleClm.' |
|
1921 | + <td>' . htmlspecialchars($confKey).'</td> |
|
1922 | 1922 | <td colspan="5"><em>No entries</em> (Page is excluded in this configuration)</td> |
1923 | 1923 | </tr>'; |
1924 | 1924 | } |
@@ -1926,13 +1926,13 @@ discard block |
||
1926 | 1926 | $c++; |
1927 | 1927 | } |
1928 | 1928 | } else { |
1929 | - $message = !empty($skipMessage) ? ' (' . $skipMessage . ')' : ''; |
|
1929 | + $message = !empty($skipMessage) ? ' ('.$skipMessage.')' : ''; |
|
1930 | 1930 | |
1931 | 1931 | // Compile row: |
1932 | 1932 | $content .= ' |
1933 | 1933 | <tr class="bgColor-20" style="border-bottom: 1px solid black;"> |
1934 | - <td>' . $pageTitleAndIcon . '</td> |
|
1935 | - <td colspan="6"><em>No entries</em>' . $message . '</td> |
|
1934 | + <td>' . $pageTitleAndIcon.'</td> |
|
1935 | + <td colspan="6"><em>No entries</em>' . $message.'</td> |
|
1936 | 1936 | </tr>'; |
1937 | 1937 | } |
1938 | 1938 | |
@@ -1947,7 +1947,7 @@ discard block |
||
1947 | 1947 | $res = $this->db->exec_SELECTquery( |
1948 | 1948 | 'count(*) as num', |
1949 | 1949 | 'tx_crawler_queue', |
1950 | - 'exec_time=0 AND process_scheduled=0 AND scheduled<=' . $this->getCurrentTime() |
|
1950 | + 'exec_time=0 AND process_scheduled=0 AND scheduled<='.$this->getCurrentTime() |
|
1951 | 1951 | ); |
1952 | 1952 | |
1953 | 1953 | $count = $this->db->sql_fetch_assoc($res); |
@@ -1989,7 +1989,7 @@ discard block |
||
1989 | 1989 | // Run process: |
1990 | 1990 | $result = $this->CLI_run($countInARun, $sleepTime, $sleepAfterFinish); |
1991 | 1991 | } catch (Exception $e) { |
1992 | - $this->CLI_debug(get_class($e) . ': ' . $e->getMessage()); |
|
1992 | + $this->CLI_debug(get_class($e).': '.$e->getMessage()); |
|
1993 | 1993 | $result = self::CLI_STATUS_ABORTED; |
1994 | 1994 | } |
1995 | 1995 | |
@@ -1999,7 +1999,7 @@ discard block |
||
1999 | 1999 | //TODO can't we do that in a clean way? |
2000 | 2000 | $releaseStatus = $this->CLI_releaseProcesses($this->CLI_buildProcessId()); |
2001 | 2001 | |
2002 | - $this->CLI_debug("Unprocessed Items remaining:" . $this->getUnprocessedItemsCount() . " (" . $this->CLI_buildProcessId() . ")"); |
|
2002 | + $this->CLI_debug("Unprocessed Items remaining:".$this->getUnprocessedItemsCount()." (".$this->CLI_buildProcessId().")"); |
|
2003 | 2003 | $result |= ($this->getUnprocessedItemsCount() > 0 ? self::CLI_STATUS_REMAIN : self::CLI_STATUS_NOTHING_PROCCESSED); |
2004 | 2004 | } else { |
2005 | 2005 | $result |= self::CLI_STATUS_ABORTED; |
@@ -2083,32 +2083,32 @@ discard block |
||
2083 | 2083 | ); |
2084 | 2084 | |
2085 | 2085 | if ($cliObj->cli_argValue('-o') === 'url') { |
2086 | - $cliObj->cli_echo(implode(chr(10), $this->downloadUrls) . chr(10), 1); |
|
2086 | + $cliObj->cli_echo(implode(chr(10), $this->downloadUrls).chr(10), 1); |
|
2087 | 2087 | } elseif ($cliObj->cli_argValue('-o') === 'exec') { |
2088 | - $cliObj->cli_echo("Executing " . count($this->urlList) . " requests right away:\n\n"); |
|
2089 | - $cliObj->cli_echo(implode(chr(10), $this->urlList) . chr(10)); |
|
2088 | + $cliObj->cli_echo("Executing ".count($this->urlList)." requests right away:\n\n"); |
|
2089 | + $cliObj->cli_echo(implode(chr(10), $this->urlList).chr(10)); |
|
2090 | 2090 | $cliObj->cli_echo("\nProcessing:\n"); |
2091 | 2091 | |
2092 | 2092 | foreach ($this->queueEntries as $queueRec) { |
2093 | 2093 | $p = unserialize($queueRec['parameters']); |
2094 | - $cliObj->cli_echo($p['url'] . ' (' . implode(',', $p['procInstructions']) . ') => '); |
|
2094 | + $cliObj->cli_echo($p['url'].' ('.implode(',', $p['procInstructions']).') => '); |
|
2095 | 2095 | |
2096 | 2096 | $result = $this->readUrlFromArray($queueRec); |
2097 | 2097 | |
2098 | 2098 | $requestResult = unserialize($result['content']); |
2099 | 2099 | if (is_array($requestResult)) { |
2100 | - $resLog = is_array($requestResult['log']) ? chr(10) . chr(9) . chr(9) . implode(chr(10) . chr(9) . chr(9), $requestResult['log']) : ''; |
|
2101 | - $cliObj->cli_echo('OK: ' . $resLog . chr(10)); |
|
2100 | + $resLog = is_array($requestResult['log']) ? chr(10).chr(9).chr(9).implode(chr(10).chr(9).chr(9), $requestResult['log']) : ''; |
|
2101 | + $cliObj->cli_echo('OK: '.$resLog.chr(10)); |
|
2102 | 2102 | } else { |
2103 | - $cliObj->cli_echo('Error checking Crawler Result: ' . substr(preg_replace('/\s+/', ' ', strip_tags($result['content'])), 0, 30000) . '...' . chr(10)); |
|
2103 | + $cliObj->cli_echo('Error checking Crawler Result: '.substr(preg_replace('/\s+/', ' ', strip_tags($result['content'])), 0, 30000).'...'.chr(10)); |
|
2104 | 2104 | } |
2105 | 2105 | } |
2106 | 2106 | } elseif ($cliObj->cli_argValue('-o') === 'queue') { |
2107 | - $cliObj->cli_echo("Putting " . count($this->urlList) . " entries in queue:\n\n"); |
|
2108 | - $cliObj->cli_echo(implode(chr(10), $this->urlList) . chr(10)); |
|
2107 | + $cliObj->cli_echo("Putting ".count($this->urlList)." entries in queue:\n\n"); |
|
2108 | + $cliObj->cli_echo(implode(chr(10), $this->urlList).chr(10)); |
|
2109 | 2109 | } else { |
2110 | - $cliObj->cli_echo(count($this->urlList) . " entries found for processing. (Use -o to decide action):\n\n", 1); |
|
2111 | - $cliObj->cli_echo(implode(chr(10), $this->urlList) . chr(10), 1); |
|
2110 | + $cliObj->cli_echo(count($this->urlList)." entries found for processing. (Use -o to decide action):\n\n", 1); |
|
2111 | + $cliObj->cli_echo(implode(chr(10), $this->urlList).chr(10), 1); |
|
2112 | 2112 | } |
2113 | 2113 | } |
2114 | 2114 | |
@@ -2162,7 +2162,7 @@ discard block |
||
2162 | 2162 | * @param tx_crawler_cli_im $cliObj Command line object |
2163 | 2163 | * @return mixed Array of keys or null if no keys found |
2164 | 2164 | */ |
2165 | - protected function getConfigurationKeys(tx_crawler_cli_im &$cliObj) |
|
2165 | + protected function getConfigurationKeys(tx_crawler_cli_im & $cliObj) |
|
2166 | 2166 | { |
2167 | 2167 | $parameter = trim($cliObj->cli_argValue('-conf')); |
2168 | 2168 | return ($parameter != '' ? \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $parameter) : []); |
@@ -2189,7 +2189,7 @@ discard block |
||
2189 | 2189 | $purgeDate = $this->getCurrentTime() - 24 * 60 * 60 * intval($this->extensionSettings['purgeQueueDays']); |
2190 | 2190 | $del = $this->db->exec_DELETEquery( |
2191 | 2191 | 'tx_crawler_queue', |
2192 | - 'exec_time!=0 AND exec_time<' . $purgeDate |
|
2192 | + 'exec_time!=0 AND exec_time<'.$purgeDate |
|
2193 | 2193 | ); |
2194 | 2194 | } |
2195 | 2195 | |
@@ -2220,7 +2220,7 @@ discard block |
||
2220 | 2220 | //TODO make sure we're not taking assigned queue-entires |
2221 | 2221 | $this->db->exec_UPDATEquery( |
2222 | 2222 | 'tx_crawler_queue', |
2223 | - 'qid IN (' . implode(',', $quidList) . ')', |
|
2223 | + 'qid IN ('.implode(',', $quidList).')', |
|
2224 | 2224 | [ |
2225 | 2225 | 'process_scheduled' => intval($this->getCurrentTime()), |
2226 | 2226 | 'process_id' => $processId |
@@ -2231,7 +2231,7 @@ discard block |
||
2231 | 2231 | $numberOfAffectedRows = $this->db->sql_affected_rows(); |
2232 | 2232 | $this->db->exec_UPDATEquery( |
2233 | 2233 | 'tx_crawler_process', |
2234 | - "process_id = '" . $processId . "'", |
|
2234 | + "process_id = '".$processId."'", |
|
2235 | 2235 | [ |
2236 | 2236 | 'assigned_items_count' => intval($numberOfAffectedRows) |
2237 | 2237 | ] |
@@ -2241,7 +2241,7 @@ discard block |
||
2241 | 2241 | $this->db->sql_query('COMMIT'); |
2242 | 2242 | } else { |
2243 | 2243 | $this->db->sql_query('ROLLBACK'); |
2244 | - $this->CLI_debug("Nothing processed due to multi-process collision (" . $this->CLI_buildProcessId() . ")"); |
|
2244 | + $this->CLI_debug("Nothing processed due to multi-process collision (".$this->CLI_buildProcessId().")"); |
|
2245 | 2245 | return ($result | self::CLI_STATUS_ABORTED); |
2246 | 2246 | } |
2247 | 2247 | |
@@ -2258,7 +2258,7 @@ discard block |
||
2258 | 2258 | } |
2259 | 2259 | |
2260 | 2260 | if (!$this->CLI_checkIfProcessIsActive($this->CLI_buildProcessId())) { |
2261 | - $this->CLI_debug("conflict / timeout (" . $this->CLI_buildProcessId() . ")"); |
|
2261 | + $this->CLI_debug("conflict / timeout (".$this->CLI_buildProcessId().")"); |
|
2262 | 2262 | |
2263 | 2263 | //TODO might need an additional returncode |
2264 | 2264 | $result |= self::CLI_STATUS_ABORTED; |
@@ -2268,10 +2268,10 @@ discard block |
||
2268 | 2268 | |
2269 | 2269 | sleep(intval($sleepAfterFinish)); |
2270 | 2270 | |
2271 | - $msg = 'Rows: ' . $counter; |
|
2272 | - $this->CLI_debug($msg . " (" . $this->CLI_buildProcessId() . ")"); |
|
2271 | + $msg = 'Rows: '.$counter; |
|
2272 | + $this->CLI_debug($msg." (".$this->CLI_buildProcessId().")"); |
|
2273 | 2273 | } else { |
2274 | - $this->CLI_debug("Nothing within queue which needs to be processed (" . $this->CLI_buildProcessId() . ")"); |
|
2274 | + $this->CLI_debug("Nothing within queue which needs to be processed (".$this->CLI_buildProcessId().")"); |
|
2275 | 2275 | } |
2276 | 2276 | |
2277 | 2277 | if ($counter > 0) { |
@@ -2339,7 +2339,7 @@ discard block |
||
2339 | 2339 | |
2340 | 2340 | // if there are less than allowed active processes then add a new one |
2341 | 2341 | if ($processCount < intval($this->extensionSettings['processLimit'])) { |
2342 | - $this->CLI_debug("add process " . $this->CLI_buildProcessId() . " (" . ($processCount + 1) . "/" . intval($this->extensionSettings['processLimit']) . ")"); |
|
2342 | + $this->CLI_debug("add process ".$this->CLI_buildProcessId()." (".($processCount + 1)."/".intval($this->extensionSettings['processLimit']).")"); |
|
2343 | 2343 | |
2344 | 2344 | // create new process record |
2345 | 2345 | $this->db->exec_INSERTquery( |
@@ -2352,7 +2352,7 @@ discard block |
||
2352 | 2352 | ] |
2353 | 2353 | ); |
2354 | 2354 | } else { |
2355 | - $this->CLI_debug("Processlimit reached (" . ($processCount) . "/" . intval($this->extensionSettings['processLimit']) . ")"); |
|
2355 | + $this->CLI_debug("Processlimit reached (".($processCount)."/".intval($this->extensionSettings['processLimit']).")"); |
|
2356 | 2356 | $ret = false; |
2357 | 2357 | } |
2358 | 2358 | |
@@ -2378,7 +2378,7 @@ discard block |
||
2378 | 2378 | } |
2379 | 2379 | |
2380 | 2380 | if (!count($releaseIds) > 0) { |
2381 | - return false; //nothing to release |
|
2381 | + return false; //nothing to release |
|
2382 | 2382 | } |
2383 | 2383 | |
2384 | 2384 | if (!$withinLock) { |
@@ -2413,14 +2413,14 @@ discard block |
||
2413 | 2413 | // mark all requested processes as non-active |
2414 | 2414 | $this->db->exec_UPDATEquery( |
2415 | 2415 | 'tx_crawler_process', |
2416 | - 'process_id IN (\'' . implode('\',\'', $releaseIds) . '\') AND deleted=0', |
|
2416 | + 'process_id IN (\''.implode('\',\'', $releaseIds).'\') AND deleted=0', |
|
2417 | 2417 | [ |
2418 | 2418 | 'active' => '0' |
2419 | 2419 | ] |
2420 | 2420 | ); |
2421 | 2421 | $this->db->exec_UPDATEquery( |
2422 | 2422 | 'tx_crawler_queue', |
2423 | - 'exec_time=0 AND process_id IN ("' . implode('","', $releaseIds) . '")', |
|
2423 | + 'exec_time=0 AND process_id IN ("'.implode('","', $releaseIds).'")', |
|
2424 | 2424 | [ |
2425 | 2425 | 'process_scheduled' => 0, |
2426 | 2426 | 'process_id' => '' |
@@ -2460,7 +2460,7 @@ discard block |
||
2460 | 2460 | $res = $this->db->exec_SELECTquery( |
2461 | 2461 | 'process_id,active,ttl', |
2462 | 2462 | 'tx_crawler_process', |
2463 | - 'process_id = \'' . $pid . '\' AND deleted=0', |
|
2463 | + 'process_id = \''.$pid.'\' AND deleted=0', |
|
2464 | 2464 | '', |
2465 | 2465 | 'ttl', |
2466 | 2466 | '0,1' |
@@ -2504,7 +2504,7 @@ discard block |
||
2504 | 2504 | public function CLI_debug($msg) |
2505 | 2505 | { |
2506 | 2506 | if (intval($this->extensionSettings['processDebug'])) { |
2507 | - echo $msg . "\n"; |
|
2507 | + echo $msg."\n"; |
|
2508 | 2508 | flush(); |
2509 | 2509 | } |
2510 | 2510 | } |
@@ -2522,7 +2522,7 @@ discard block |
||
2522 | 2522 | |
2523 | 2523 | $cmd = escapeshellcmd($this->extensionSettings['phpPath']); |
2524 | 2524 | $cmd .= ' '; |
2525 | - $cmd .= escapeshellarg(\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath('crawler') . 'cli/bootstrap.php'); |
|
2525 | + $cmd .= escapeshellarg(\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath('crawler').'cli/bootstrap.php'); |
|
2526 | 2526 | $cmd .= ' '; |
2527 | 2527 | $cmd .= escapeshellarg($this->getFrontendBasePath()); |
2528 | 2528 | $cmd .= ' '; |
@@ -2532,10 +2532,10 @@ discard block |
||
2532 | 2532 | |
2533 | 2533 | $startTime = microtime(true); |
2534 | 2534 | $content = $this->executeShellCommand($cmd); |
2535 | - $this->log($url . ' ' . (microtime(true) - $startTime)); |
|
2535 | + $this->log($url.' '.(microtime(true) - $startTime)); |
|
2536 | 2536 | |
2537 | 2537 | $result = [ |
2538 | - 'request' => implode("\r\n", $requestHeaders) . "\r\n\r\n", |
|
2538 | + 'request' => implode("\r\n", $requestHeaders)."\r\n\r\n", |
|
2539 | 2539 | 'headers' => '', |
2540 | 2540 | 'content' => $content |
2541 | 2541 | ]; |
@@ -2556,7 +2556,7 @@ discard block |
||
2556 | 2556 | $scheduledAgeInSeconds = $this->extensionSettings['cleanUpScheduledAge'] * 86400; |
2557 | 2557 | |
2558 | 2558 | $now = time(); |
2559 | - $condition = '(exec_time<>0 AND exec_time<' . ($now - $processedAgeInSeconds) . ') OR scheduled<=' . ($now - $scheduledAgeInSeconds); |
|
2559 | + $condition = '(exec_time<>0 AND exec_time<'.($now - $processedAgeInSeconds).') OR scheduled<='.($now - $scheduledAgeInSeconds); |
|
2560 | 2560 | $this->flushQueue($condition); |
2561 | 2561 | } |
2562 | 2562 |