AOEpeople /
crawler
| Total Complexity | 124 |
| Total Lines | 1187 |
| Duplicated Lines | 0 % |
| Changes | 3 | ||
| Bugs | 1 | Features | 0 |
Complex classes like tx_crawler_modfunc1 often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use tx_crawler_modfunc1, and based on these observations, apply Extract Interface, too.
| 1 | <?php |
||
| 29 | class tx_crawler_modfunc1 extends \TYPO3\CMS\Backend\Module\AbstractFunctionModule |
||
|
|
|||
| 30 | { |
||
| 31 | // Internal, dynamic: |
||
| 32 | public $duplicateTrack = []; |
||
| 33 | public $submitCrawlUrls = false; |
||
| 34 | public $downloadCrawlUrls = false; |
||
| 35 | |||
| 36 | public $scheduledTime = 0; |
||
| 37 | public $reqMinute = 0; |
||
| 38 | |||
| 39 | /** |
||
| 40 | * @var array holds the selection of configuration from the configuration selector box |
||
| 41 | */ |
||
| 42 | public $incomingConfigurationSelection = []; |
||
| 43 | |||
| 44 | /** |
||
| 45 | * @var tx_crawler_lib |
||
| 46 | */ |
||
| 47 | public $crawlerObj; |
||
| 48 | |||
| 49 | public $CSVaccu = []; |
||
| 50 | |||
| 51 | /** |
||
| 52 | * If true the user requested a CSV export of the queue |
||
| 53 | * |
||
| 54 | * @var boolean |
||
| 55 | */ |
||
| 56 | public $CSVExport = false; |
||
| 57 | |||
| 58 | public $downloadUrls = []; |
||
| 59 | |||
| 60 | /** |
||
| 61 | * Holds the configuration from ext_conf_template loaded by loadExtensionSettings() |
||
| 62 | * |
||
| 63 | * @var array |
||
| 64 | */ |
||
| 65 | protected $extensionSettings = []; |
||
| 66 | |||
| 67 | /** |
||
| 68 | * Indicate that an flash message with an error is present. |
||
| 69 | * |
||
| 70 | * @var boolean |
||
| 71 | */ |
||
| 72 | protected $isErrorDetected = false; |
||
| 73 | |||
| 74 | /** |
||
| 75 | * the constructor |
||
| 76 | */ |
||
| 77 | public function __construct() |
||
| 78 | { |
||
| 79 | $this->processManager = new tx_crawler_domain_process_manager(); |
||
| 80 | } |
||
| 81 | |||
| 82 | /** |
||
| 83 | * Additions to the function menu array |
||
| 84 | * |
||
| 85 | * @return array Menu array |
||
| 86 | */ |
||
| 87 | public function modMenu() |
||
| 88 | { |
||
| 89 | global $LANG; |
||
| 90 | |||
| 91 | return [ |
||
| 92 | 'depth' => [ |
||
| 93 | 0 => $LANG->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_0'), |
||
| 94 | 1 => $LANG->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_1'), |
||
| 95 | 2 => $LANG->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_2'), |
||
| 96 | 3 => $LANG->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_3'), |
||
| 97 | 4 => $LANG->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_4'), |
||
| 98 | 99 => $LANG->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_infi'), |
||
| 99 | ], |
||
| 100 | 'crawlaction' => [ |
||
| 101 | 'start' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.start'), |
||
| 102 | 'log' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.log'), |
||
| 103 | 'multiprocess' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.multiprocess') |
||
| 104 | ], |
||
| 105 | 'log_resultLog' => '', |
||
| 106 | 'log_feVars' => '', |
||
| 107 | 'processListMode' => '', |
||
| 108 | 'log_display' => [ |
||
| 109 | 'all' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.all'), |
||
| 110 | 'pending' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.pending'), |
||
| 111 | 'finished' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.finished') |
||
| 112 | ], |
||
| 113 | 'itemsPerPage' => [ |
||
| 114 | '5' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.itemsPerPage.5'), |
||
| 115 | '10' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.itemsPerPage.10'), |
||
| 116 | '50' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.itemsPerPage.50'), |
||
| 117 | '0' => $LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.itemsPerPage.0') |
||
| 118 | ] |
||
| 119 | ]; |
||
| 120 | } |
||
| 121 | |||
| 122 | /** |
||
| 123 | * Load extension settings |
||
| 124 | * |
||
| 125 | * @param void |
||
| 126 | * @return void |
||
| 127 | */ |
||
| 128 | protected function loadExtensionSettings() |
||
| 129 | { |
||
| 130 | $this->extensionSettings = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['crawler']); |
||
| 131 | } |
||
| 132 | |||
| 133 | /** |
||
| 134 | * Main function |
||
| 135 | * |
||
| 136 | * @return string HTML output |
||
| 137 | */ |
||
| 138 | public function main() |
||
| 139 | { |
||
| 140 | global $LANG, $BACK_PATH; |
||
| 141 | |||
| 142 | $this->incLocalLang(); |
||
| 143 | |||
| 144 | $this->loadExtensionSettings(); |
||
| 145 | if (empty($this->pObj->MOD_SETTINGS['processListMode'])) { |
||
| 146 | $this->pObj->MOD_SETTINGS['processListMode'] = 'simple'; |
||
| 147 | } |
||
| 148 | |||
| 149 | // Set CSS styles specific for this document: |
||
| 150 | $this->pObj->content = str_replace('/*###POSTCSSMARKER###*/', ' |
||
| 151 | TABLE.c-list TR TD { white-space: nowrap; vertical-align: top; } |
||
| 152 | ', $this->pObj->content); |
||
| 153 | |||
| 154 | $this->pObj->content .= '<style type="text/css"><!-- |
||
| 155 | table.url-table, |
||
| 156 | table.param-expanded, |
||
| 157 | table.crawlerlog { |
||
| 158 | border-bottom: 1px solid grey; |
||
| 159 | border-spacing: 0; |
||
| 160 | border-collapse: collapse; |
||
| 161 | } |
||
| 162 | table.crawlerlog td, |
||
| 163 | table.url-table td { |
||
| 164 | border: 1px solid lightgrey; |
||
| 165 | border-bottom: 1px solid grey; |
||
| 166 | white-space: nowrap; vertical-align: top; |
||
| 167 | } |
||
| 168 | --></style> |
||
| 169 | <link rel="stylesheet" type="text/css" href="' . $BACK_PATH . '../typo3conf/ext/crawler/template/res.css" /> |
||
| 170 | '; |
||
| 171 | |||
| 172 | // Type function menu: |
||
| 173 | $h_func = \TYPO3\CMS\Backend\Utility\BackendUtility::getFuncMenu( |
||
| 174 | $this->pObj->id, |
||
| 175 | 'SET[crawlaction]', |
||
| 176 | $this->pObj->MOD_SETTINGS['crawlaction'], |
||
| 177 | $this->pObj->MOD_MENU['crawlaction'], |
||
| 178 | 'index.php' |
||
| 179 | ); |
||
| 180 | |||
| 181 | /* |
||
| 182 | // Showing depth-menu in certain cases: |
||
| 183 | if ($this->pObj->MOD_SETTINGS['crawlaction']!=='cli' && $this->pObj->MOD_SETTINGS['crawlaction']!== 'multiprocess' && ($this->pObj->MOD_SETTINGS['crawlaction']!=='log' || $this->pObj->id)) { |
||
| 184 | $h_func .= \TYPO3\CMS\Backend\Utility\BackendUtility::getFuncMenu( |
||
| 185 | $this->pObj->id, |
||
| 186 | 'SET[depth]', |
||
| 187 | $this->pObj->MOD_SETTINGS['depth'], |
||
| 188 | $this->pObj->MOD_MENU['depth'], |
||
| 189 | 'index.php' |
||
| 190 | ); |
||
| 191 | } |
||
| 192 | */ |
||
| 193 | |||
| 194 | // Additional menus for the log type: |
||
| 195 | if ($this->pObj->MOD_SETTINGS['crawlaction'] === 'log') { |
||
| 196 | $h_func .= \TYPO3\CMS\Backend\Utility\BackendUtility::getFuncMenu( |
||
| 197 | $this->pObj->id, |
||
| 198 | 'SET[depth]', |
||
| 199 | $this->pObj->MOD_SETTINGS['depth'], |
||
| 200 | $this->pObj->MOD_MENU['depth'], |
||
| 201 | 'index.php' |
||
| 202 | ); |
||
| 203 | |||
| 204 | $quiPart = \TYPO3\CMS\Core\Utility\GeneralUtility::_GP('qid_details') ? '&qid_details=' . intval(\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('qid_details')) : ''; |
||
| 205 | |||
| 206 | $setId = intval(\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('setID')); |
||
| 207 | |||
| 208 | $h_func .= '<hr/>' . |
||
| 209 | $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.display') . ': ' . \TYPO3\CMS\Backend\Utility\BackendUtility::getFuncMenu($this->pObj->id, 'SET[log_display]', $this->pObj->MOD_SETTINGS['log_display'], $this->pObj->MOD_MENU['log_display'], 'index.php', '&setID=' . $setId) . ' - ' . |
||
| 210 | $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.showresultlog') . ': ' . \TYPO3\CMS\Backend\Utility\BackendUtility::getFuncCheck($this->pObj->id, 'SET[log_resultLog]', $this->pObj->MOD_SETTINGS['log_resultLog'], 'index.php', '&setID=' . $setId . $quiPart) . ' - ' . |
||
| 211 | $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.showfevars') . ': ' . \TYPO3\CMS\Backend\Utility\BackendUtility::getFuncCheck($this->pObj->id, 'SET[log_feVars]', $this->pObj->MOD_SETTINGS['log_feVars'], 'index.php', '&setID=' . $setId . $quiPart) . ' - ' . |
||
| 212 | $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.itemsPerPage') . ': ' . |
||
| 213 | \TYPO3\CMS\Backend\Utility\BackendUtility::getFuncMenu( |
||
| 214 | $this->pObj->id, |
||
| 215 | 'SET[itemsPerPage]', |
||
| 216 | $this->pObj->MOD_SETTINGS['itemsPerPage'], |
||
| 217 | $this->pObj->MOD_MENU['itemsPerPage'], |
||
| 218 | 'index.php' |
||
| 219 | ); |
||
| 220 | } |
||
| 221 | |||
| 222 | $theOutput = $this->pObj->doc->section($LANG->getLL('title'), $h_func, 0, 1); |
||
| 223 | |||
| 224 | // Branch based on type: |
||
| 225 | switch ((string)$this->pObj->MOD_SETTINGS['crawlaction']) { |
||
| 226 | case 'start': |
||
| 227 | if (empty($this->pObj->id)) { |
||
| 228 | $this->addErrorMessage($GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.noPageSelected')); |
||
| 229 | } else { |
||
| 230 | $theOutput .= $this->pObj->doc->section('', $this->drawURLs(), 0, 1); |
||
| 231 | } |
||
| 232 | break; |
||
| 233 | case 'log': |
||
| 234 | if (empty($this->pObj->id)) { |
||
| 235 | $this->addErrorMessage($GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.noPageSelected')); |
||
| 236 | } else { |
||
| 237 | $theOutput .= $this->pObj->doc->section('', $this->drawLog(), 0, 1); |
||
| 238 | } |
||
| 239 | break; |
||
| 240 | case 'cli': |
||
| 241 | $theOutput .= $this->pObj->doc->section('', $this->drawCLIstatus(), 0, 1); |
||
| 242 | break; |
||
| 243 | case 'multiprocess': |
||
| 244 | $theOutput .= $this->pObj->doc->section('', $this->drawProcessOverviewAction(), 0, 1); |
||
| 245 | break; |
||
| 246 | } |
||
| 247 | |||
| 248 | return $theOutput; |
||
| 249 | } |
||
| 250 | |||
| 251 | /******************************* |
||
| 252 | * |
||
| 253 | * Generate URLs for crawling: |
||
| 254 | * |
||
| 255 | ******************************/ |
||
| 256 | |||
| 257 | /** |
||
| 258 | * Produces a table with overview of the URLs to be crawled for each page |
||
| 259 | * |
||
| 260 | * @return string HTML output |
||
| 261 | */ |
||
| 262 | public function drawURLs() |
||
| 263 | { |
||
| 264 | global $BACK_PATH, $BE_USER; |
||
| 265 | |||
| 266 | // Init: |
||
| 267 | $this->duplicateTrack = []; |
||
| 268 | $this->submitCrawlUrls = \TYPO3\CMS\Core\Utility\GeneralUtility::_GP('_crawl'); |
||
| 269 | $this->downloadCrawlUrls = \TYPO3\CMS\Core\Utility\GeneralUtility::_GP('_download'); |
||
| 270 | $this->makeCrawlerProcessableChecks(); |
||
| 271 | |||
| 272 | switch ((string)\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('tstamp')) { |
||
| 273 | case 'midnight': |
||
| 274 | $this->scheduledTime = mktime(0, 0, 0); |
||
| 275 | break; |
||
| 276 | case '04:00': |
||
| 277 | $this->scheduledTime = mktime(0, 0, 0) + 4 * 3600; |
||
| 278 | break; |
||
| 279 | case 'now': |
||
| 280 | default: |
||
| 281 | $this->scheduledTime = time(); |
||
| 282 | break; |
||
| 283 | } |
||
| 284 | // $this->reqMinute = \TYPO3\CMS\Core\Utility\GeneralUtility::intInRange(\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('perminute'),1,10000); |
||
| 285 | // TODO: check relevance |
||
| 286 | $this->reqMinute = 1000; |
||
| 287 | |||
| 288 | $this->incomingConfigurationSelection = \TYPO3\CMS\Core\Utility\GeneralUtility::_GP('configurationSelection'); |
||
| 289 | $this->incomingConfigurationSelection = is_array($this->incomingConfigurationSelection) ? $this->incomingConfigurationSelection : ['']; |
||
| 290 | |||
| 291 | $this->crawlerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_lib'); |
||
| 292 | $this->crawlerObj->setAccessMode('gui'); |
||
| 293 | $this->crawlerObj->setID = \TYPO3\CMS\Core\Utility\GeneralUtility::md5int(microtime()); |
||
| 294 | |||
| 295 | if (empty($this->incomingConfigurationSelection) |
||
| 296 | || (count($this->incomingConfigurationSelection) == 1 && empty($this->incomingConfigurationSelection[0])) |
||
| 297 | ) { |
||
| 298 | $code = ' |
||
| 299 | <tr> |
||
| 300 | <td colspan="7"><b>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.noConfigSelected') . '</b></td> |
||
| 301 | </tr>'; |
||
| 302 | } else { |
||
| 303 | if ($this->submitCrawlUrls) { |
||
| 304 | $reason = new tx_crawler_domain_reason(); |
||
| 305 | $reason->setReason(tx_crawler_domain_reason::REASON_GUI_SUBMIT); |
||
| 306 | |||
| 307 | if ($BE_USER instanceof \TYPO3\CMS\Core\Authentication\BackendUserAuthentication) { |
||
| 308 | $username = $BE_USER->user['username']; |
||
| 309 | } |
||
| 310 | $reason->setDetailText('The user ' . $username . ' added pages to the crawler queue manually '); |
||
| 311 | |||
| 312 | tx_crawler_domain_events_dispatcher::getInstance()->post( |
||
| 313 | 'invokeQueueChange', |
||
| 314 | $this->findCrawler()->setID, |
||
| 315 | [ 'reason' => $reason ] |
||
| 316 | ); |
||
| 317 | } |
||
| 318 | |||
| 319 | $code = $this->crawlerObj->getPageTreeAndUrls( |
||
| 320 | $this->pObj->id, |
||
| 321 | $this->pObj->MOD_SETTINGS['depth'], |
||
| 322 | $this->scheduledTime, |
||
| 323 | $this->reqMinute, |
||
| 324 | $this->submitCrawlUrls, |
||
| 325 | $this->downloadCrawlUrls, |
||
| 326 | [], // Do not filter any processing instructions |
||
| 327 | $this->incomingConfigurationSelection |
||
| 328 | ); |
||
| 329 | } |
||
| 330 | |||
| 331 | $this->downloadUrls = $this->crawlerObj->downloadUrls; |
||
| 332 | $this->duplicateTrack = $this->crawlerObj->duplicateTrack; |
||
| 333 | |||
| 334 | $output = ''; |
||
| 335 | if ($code) { |
||
| 336 | $output .= '<h3>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.configuration') . ':</h3>'; |
||
| 337 | $output .= '<input type="hidden" name="id" value="' . intval($this->pObj->id) . '" />'; |
||
| 338 | |||
| 339 | if (!$this->submitCrawlUrls) { |
||
| 340 | $output .= $this->drawURLs_cfgSelectors() . '<br />'; |
||
| 341 | $output .= '<input type="submit" name="_update" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.triggerUpdate') . '" /> '; |
||
| 342 | $output .= '<input type="submit" name="_crawl" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.triggerCrawl') . '" /> '; |
||
| 343 | $output .= '<input type="submit" name="_download" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.triggerDownload') . '" /><br /><br />'; |
||
| 344 | $output .= $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.count') . ': ' . count(array_keys($this->duplicateTrack)) . '<br />'; |
||
| 345 | $output .= $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.curtime') . ': ' . date('H:i:s', time()) . '<br />'; |
||
| 346 | $output .= '<br /> |
||
| 347 | <table class="lrPadding c-list url-table">' . |
||
| 348 | $this->drawURLs_printTableHeader() . |
||
| 349 | $code . |
||
| 350 | '</table>'; |
||
| 351 | } else { |
||
| 352 | $output .= count(array_keys($this->duplicateTrack)) . ' ' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.submitted') . '. <br /><br />'; |
||
| 353 | $output .= '<input type="submit" name="_" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.continue') . '" />'; |
||
| 354 | $output .= '<input type="submit" onclick="this.form.elements[\'SET[crawlaction]\'].value=\'log\';" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.continueinlog') . '" />'; |
||
| 355 | } |
||
| 356 | } |
||
| 357 | |||
| 358 | // Download Urls to crawl: |
||
| 359 | if ($this->downloadCrawlUrls) { |
||
| 360 | |||
| 361 | // Creating output header: |
||
| 362 | $mimeType = 'application/octet-stream'; |
||
| 363 | Header('Content-Type: ' . $mimeType); |
||
| 364 | Header('Content-Disposition: attachment; filename=CrawlerUrls.txt'); |
||
| 365 | |||
| 366 | // Printing the content of the CSV lines: |
||
| 367 | echo implode(chr(13) . chr(10), $this->downloadUrls); |
||
| 368 | |||
| 369 | // Exits: |
||
| 370 | exit; |
||
| 371 | } |
||
| 372 | |||
| 373 | // Return output: |
||
| 374 | return $output; |
||
| 375 | } |
||
| 376 | |||
| 377 | /** |
||
| 378 | * Draws the configuration selectors for compiling URLs: |
||
| 379 | * |
||
| 380 | * @return string HTML table |
||
| 381 | */ |
||
| 382 | public function drawURLs_cfgSelectors() |
||
| 383 | { |
||
| 384 | |||
| 385 | // depth |
||
| 386 | $cell[] = $this->selectorBox( |
||
| 387 | [ |
||
| 388 | 0 => $GLOBALS['LANG']->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_0'), |
||
| 389 | 1 => $GLOBALS['LANG']->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_1'), |
||
| 390 | 2 => $GLOBALS['LANG']->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_2'), |
||
| 391 | 3 => $GLOBALS['LANG']->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_3'), |
||
| 392 | 4 => $GLOBALS['LANG']->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_4'), |
||
| 393 | 99 => $GLOBALS['LANG']->sL('LLL:EXT:lang/Resources/Private/Language/locallang_core.xlf:labels.depth_infi'), |
||
| 394 | ], |
||
| 395 | 'SET[depth]', |
||
| 396 | $this->pObj->MOD_SETTINGS['depth'], |
||
| 397 | 0 |
||
| 398 | ); |
||
| 399 | $availableConfigurations = $this->crawlerObj->getConfigurationsForBranch($this->pObj->id, $this->pObj->MOD_SETTINGS['depth'] ? $this->pObj->MOD_SETTINGS['depth'] : 0); |
||
| 400 | |||
| 401 | // Configurations |
||
| 402 | $cell[] = $this->selectorBox( |
||
| 403 | empty($availableConfigurations) ? [] : array_combine($availableConfigurations, $availableConfigurations), |
||
| 404 | 'configurationSelection', |
||
| 405 | $this->incomingConfigurationSelection, |
||
| 406 | 1 |
||
| 407 | ); |
||
| 408 | |||
| 409 | // Scheduled time: |
||
| 410 | $cell[] = $this->selectorBox( |
||
| 411 | [ |
||
| 412 | 'now' => $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.time.now'), |
||
| 413 | 'midnight' => $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.time.midnight'), |
||
| 414 | '04:00' => $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.time.4am'), |
||
| 415 | ], |
||
| 416 | 'tstamp', |
||
| 417 | \TYPO3\CMS\Core\Utility\GeneralUtility::_POST('tstamp'), |
||
| 418 | 0 |
||
| 419 | ); |
||
| 420 | |||
| 421 | // TODO: check relevance |
||
| 422 | /* |
||
| 423 | // Requests per minute: |
||
| 424 | $cell[] = $this->selectorBox( |
||
| 425 | array( |
||
| 426 | 30 => '[Default]', |
||
| 427 | 1 => '1', |
||
| 428 | 5 => '5', |
||
| 429 | 10 => '10', |
||
| 430 | 20 => '20', |
||
| 431 | 30 => '30', |
||
| 432 | 50 => '50', |
||
| 433 | 100 => '100', |
||
| 434 | 200 => '200', |
||
| 435 | 1000 => '1000', |
||
| 436 | ), |
||
| 437 | 'SET[perminute]', |
||
| 438 | $this->pObj->MOD_SETTINGS['perminute'], |
||
| 439 | 0 |
||
| 440 | ); |
||
| 441 | */ |
||
| 442 | |||
| 443 | $output = ' |
||
| 444 | <table class="lrPadding c-list"> |
||
| 445 | <tr class="bgColor5 tableheader"> |
||
| 446 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.depth') . ':</td> |
||
| 447 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.configurations') . ':</td> |
||
| 448 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.scheduled') . ':</td> |
||
| 449 | </tr> |
||
| 450 | <tr class="bgColor4"> |
||
| 451 | <td valign="top">' . implode('</td> |
||
| 452 | <td valign="top">', $cell) . '</td> |
||
| 453 | </tr> |
||
| 454 | </table>'; |
||
| 455 | |||
| 456 | return $output; |
||
| 457 | } |
||
| 458 | |||
| 459 | /** |
||
| 460 | * Create Table header row for URL display |
||
| 461 | * |
||
| 462 | * @return string Table header |
||
| 463 | */ |
||
| 464 | public function drawURLs_printTableHeader() |
||
| 465 | { |
||
| 466 | $content = ' |
||
| 467 | <tr class="bgColor5 tableheader"> |
||
| 468 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.pagetitle') . ':</td> |
||
| 469 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.key') . ':</td> |
||
| 470 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.parametercfg') . ':</td> |
||
| 471 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.values') . ':</td> |
||
| 472 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.urls') . ':</td> |
||
| 473 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.options') . ':</td> |
||
| 474 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.parameters') . ':</td> |
||
| 475 | </tr>'; |
||
| 476 | |||
| 477 | return $content; |
||
| 478 | } |
||
| 479 | |||
| 480 | /******************************* |
||
| 481 | * |
||
| 482 | * Shows log of indexed URLs |
||
| 483 | * |
||
| 484 | ******************************/ |
||
| 485 | |||
| 486 | /** |
||
| 487 | * Shows the log of indexed URLs |
||
| 488 | * |
||
| 489 | * @return string HTML output |
||
| 490 | */ |
||
| 491 | public function drawLog() |
||
| 492 | { |
||
| 493 | global $BACK_PATH; |
||
| 494 | $output = ''; |
||
| 495 | |||
| 496 | // Init: |
||
| 497 | $this->crawlerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_lib'); |
||
| 498 | $this->crawlerObj->setAccessMode('gui'); |
||
| 499 | $this->crawlerObj->setID = \TYPO3\CMS\Core\Utility\GeneralUtility::md5int(microtime()); |
||
| 500 | |||
| 501 | $this->CSVExport = \TYPO3\CMS\Core\Utility\GeneralUtility::_POST('_csv'); |
||
| 502 | |||
| 503 | // Read URL: |
||
| 504 | if (\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('qid_read')) { |
||
| 505 | $this->crawlerObj->readUrl(intval(\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('qid_read')), true); |
||
| 506 | } |
||
| 507 | |||
| 508 | // Look for set ID sent - if it is, we will display contents of that set: |
||
| 509 | $showSetId = intval(\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('setID')); |
||
| 510 | |||
| 511 | // Show details: |
||
| 512 | if (\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('qid_details')) { |
||
| 513 | |||
| 514 | // Get entry record: |
||
| 515 | list($q_entry) = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows('*', 'tx_crawler_queue', 'qid=' . intval(\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('qid_details'))); |
||
| 516 | |||
| 517 | // Explode values: |
||
| 518 | $resStatus = $this->getResStatus($q_entry); |
||
| 519 | $q_entry['parameters'] = unserialize($q_entry['parameters']); |
||
| 520 | $q_entry['result_data'] = unserialize($q_entry['result_data']); |
||
| 521 | if (is_array($q_entry['result_data'])) { |
||
| 522 | $q_entry['result_data']['content'] = unserialize($q_entry['result_data']['content']); |
||
| 523 | } |
||
| 524 | |||
| 525 | if (!$this->pObj->MOD_SETTINGS['log_resultLog']) { |
||
| 526 | unset($q_entry['result_data']['content']['log']); |
||
| 527 | } |
||
| 528 | |||
| 529 | // Print rudimentary details: |
||
| 530 | $output .= ' |
||
| 531 | <br /><br /> |
||
| 532 | <input type="submit" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.back') . '" name="_back" /> |
||
| 533 | <input type="hidden" value="' . $this->pObj->id . '" name="id" /> |
||
| 534 | <input type="hidden" value="' . $showSetId . '" name="setID" /> |
||
| 535 | <br /> |
||
| 536 | Current server time: ' . date('H:i:s', time()) . '<br />' . |
||
| 537 | 'Status: ' . $resStatus . '<br />' . |
||
| 538 | \TYPO3\CMS\Core\Utility\DebugUtility::viewArray($q_entry); |
||
| 539 | } else { // Show list: |
||
| 540 | |||
| 541 | // If either id or set id, show list: |
||
| 542 | if ($this->pObj->id || $showSetId) { |
||
| 543 | if ($this->pObj->id) { |
||
| 544 | // Drawing tree: |
||
| 545 | $tree = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Backend\Tree\View\PageTreeView'); |
||
| 546 | $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1); |
||
| 547 | $tree->init('AND ' . $perms_clause); |
||
| 548 | |||
| 549 | // Set root row: |
||
| 550 | $HTML = \AOE\Crawler\Utility\IconUtility::getIconForRecord('pages', $this->pObj->pageinfo); |
||
| 551 | $tree->tree[] = [ |
||
| 552 | 'row' => $this->pObj->pageinfo, |
||
| 553 | 'HTML' => $HTML |
||
| 554 | ]; |
||
| 555 | |||
| 556 | // Get branch beneath: |
||
| 557 | if ($this->pObj->MOD_SETTINGS['depth']) { |
||
| 558 | $tree->getTree($this->pObj->id, $this->pObj->MOD_SETTINGS['depth'], ''); |
||
| 559 | } |
||
| 560 | |||
| 561 | // Traverse page tree: |
||
| 562 | $code = ''; |
||
| 563 | $count = 0; |
||
| 564 | foreach ($tree->tree as $data) { |
||
| 565 | $code .= $this->drawLog_addRows( |
||
| 566 | $data['row'], |
||
| 567 | $data['HTML'] . \TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages', $data['row'], true), |
||
| 568 | intval($this->pObj->MOD_SETTINGS['itemsPerPage']) |
||
| 569 | ); |
||
| 570 | if (++$count == 1000) { |
||
| 571 | break; |
||
| 572 | } |
||
| 573 | } |
||
| 574 | } else { |
||
| 575 | $code = ''; |
||
| 576 | $code .= $this->drawLog_addRows( |
||
| 577 | $showSetId, |
||
| 578 | 'Set ID: ' . $showSetId |
||
| 579 | ); |
||
| 580 | } |
||
| 581 | |||
| 582 | if ($code) { |
||
| 583 | $output .= ' |
||
| 584 | <br /><br /> |
||
| 585 | <input type="submit" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.reloadlist') . '" name="_reload" /> |
||
| 586 | <input type="submit" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.downloadcsv') . '" name="_csv" /> |
||
| 587 | <input type="submit" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.flushvisiblequeue') . '" name="_flush" onclick="return confirm(\'' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.confirmyouresure') . '\');" /> |
||
| 588 | <input type="submit" value="' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.flushfullqueue') . '" name="_flush_all" onclick="return confirm(\'' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.confirmyouresure') . '\');" /> |
||
| 589 | <input type="hidden" value="' . $this->pObj->id . '" name="id" /> |
||
| 590 | <input type="hidden" value="' . $showSetId . '" name="setID" /> |
||
| 591 | <br /> |
||
| 592 | ' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.curtime') . ': ' . date('H:i:s', time()) . ' |
||
| 593 | <br /><br /> |
||
| 594 | |||
| 595 | |||
| 596 | <table class="lrPadding c-list crawlerlog">' . |
||
| 597 | $this->drawLog_printTableHeader() . |
||
| 598 | $code . |
||
| 599 | '</table>'; |
||
| 600 | } |
||
| 601 | } else { // Otherwise show available sets: |
||
| 602 | $setList = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( |
||
| 603 | 'set_id, count(*) as count_value, scheduled', |
||
| 604 | 'tx_crawler_queue', |
||
| 605 | '', |
||
| 606 | 'set_id, scheduled', |
||
| 607 | 'scheduled DESC' |
||
| 608 | ); |
||
| 609 | |||
| 610 | $code = ' |
||
| 611 | <tr class="bgColor5 tableheader"> |
||
| 612 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.setid') . ':</td> |
||
| 613 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.count') . 't:</td> |
||
| 614 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.time') . ':</td> |
||
| 615 | </tr> |
||
| 616 | '; |
||
| 617 | |||
| 618 | $cc = 0; |
||
| 619 | foreach ($setList as $set) { |
||
| 620 | $code .= ' |
||
| 621 | <tr class="bgColor' . ($cc % 2 ? '-20' : '-10') . '"> |
||
| 622 | <td><a href="' . htmlspecialchars('index.php?setID=' . $set['set_id']) . '">' . $set['set_id'] . '</a></td> |
||
| 623 | <td>' . $set['count_value'] . '</td> |
||
| 624 | <td>' . \TYPO3\CMS\Backend\Utility\BackendUtility::dateTimeAge($set['scheduled']) . '</td> |
||
| 625 | </tr> |
||
| 626 | '; |
||
| 627 | |||
| 628 | $cc++; |
||
| 629 | } |
||
| 630 | |||
| 631 | $output .= ' |
||
| 632 | <br /><br /> |
||
| 633 | <table class="lrPadding c-list">' . |
||
| 634 | $code . |
||
| 635 | '</table>'; |
||
| 636 | } |
||
| 637 | } |
||
| 638 | |||
| 639 | if ($this->CSVExport) { |
||
| 640 | $this->outputCsvFile(); |
||
| 641 | } |
||
| 642 | |||
| 643 | // Return output |
||
| 644 | return $output; |
||
| 645 | } |
||
| 646 | |||
| 647 | /** |
||
| 648 | * Outputs the CSV file and sets the correct headers |
||
| 649 | */ |
||
| 650 | protected function outputCsvFile() |
||
| 651 | { |
||
| 652 | if (!count($this->CSVaccu)) { |
||
| 653 | $this->addWarningMessage($GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:message.canNotExportEmptyQueueToCsvText')); |
||
| 654 | return; |
||
| 655 | } |
||
| 656 | |||
| 657 | $csvLines = []; |
||
| 658 | |||
| 659 | // Field names: |
||
| 660 | reset($this->CSVaccu); |
||
| 661 | $fieldNames = array_keys(current($this->CSVaccu)); |
||
| 662 | $csvLines[] = \TYPO3\CMS\Core\Utility\GeneralUtility::csvValues($fieldNames); |
||
| 663 | |||
| 664 | // Data: |
||
| 665 | foreach ($this->CSVaccu as $row) { |
||
| 666 | $csvLines[] = \TYPO3\CMS\Core\Utility\GeneralUtility::csvValues($row); |
||
| 667 | } |
||
| 668 | |||
| 669 | // Creating output header: |
||
| 670 | $mimeType = 'application/octet-stream'; |
||
| 671 | Header('Content-Type: ' . $mimeType); |
||
| 672 | Header('Content-Disposition: attachment; filename=CrawlerLog.csv'); |
||
| 673 | |||
| 674 | // Printing the content of the CSV lines: |
||
| 675 | echo implode(chr(13) . chr(10), $csvLines); |
||
| 676 | |||
| 677 | // Exits: |
||
| 678 | exit; |
||
| 679 | } |
||
| 680 | |||
| 681 | /** |
||
| 682 | * Create the rows for display of the page tree |
||
| 683 | * For each page a number of rows are shown displaying GET variable configuration |
||
| 684 | * |
||
| 685 | * @param array $pageRow_setId Page row or set-id |
||
| 686 | * @param string $titleString Title string |
||
| 687 | * @param int $itemsPerPage Items per Page setting |
||
| 688 | * |
||
| 689 | * @return string HTML <tr> content (one or more) |
||
| 690 | */ |
||
| 691 | public function drawLog_addRows($pageRow_setId, $titleString, $itemsPerPage = 10) |
||
| 692 | { |
||
| 693 | |||
| 694 | // If Flush button is pressed, flush tables instead of selecting entries: |
||
| 695 | |||
| 696 | if (\TYPO3\CMS\Core\Utility\GeneralUtility::_POST('_flush')) { |
||
| 697 | $doFlush = true; |
||
| 698 | $doFullFlush = false; |
||
| 699 | } elseif (\TYPO3\CMS\Core\Utility\GeneralUtility::_POST('_flush_all')) { |
||
| 700 | $doFlush = true; |
||
| 701 | $doFullFlush = true; |
||
| 702 | } else { |
||
| 703 | $doFlush = false; |
||
| 704 | $doFullFlush = false; |
||
| 705 | } |
||
| 706 | |||
| 707 | // Get result: |
||
| 708 | if (is_array($pageRow_setId)) { |
||
| 709 | $res = $this->crawlerObj->getLogEntriesForPageId($pageRow_setId['uid'], $this->pObj->MOD_SETTINGS['log_display'], $doFlush, $doFullFlush, intval($itemsPerPage)); |
||
| 710 | } else { |
||
| 711 | $res = $this->crawlerObj->getLogEntriesForSetId($pageRow_setId, $this->pObj->MOD_SETTINGS['log_display'], $doFlush, $doFullFlush, intval($itemsPerPage)); |
||
| 712 | } |
||
| 713 | |||
| 714 | // Init var: |
||
| 715 | $colSpan = 9 |
||
| 716 | + ($this->pObj->MOD_SETTINGS['log_resultLog'] ? -1 : 0) |
||
| 717 | + ($this->pObj->MOD_SETTINGS['log_feVars'] ? 3 : 0); |
||
| 718 | |||
| 719 | if (count($res)) { |
||
| 720 | // Traverse parameter combinations: |
||
| 721 | $c = 0; |
||
| 722 | $content = ''; |
||
| 723 | foreach ($res as $kk => $vv) { |
||
| 724 | |||
| 725 | // Title column: |
||
| 726 | if (!$c) { |
||
| 727 | $titleClm = '<td rowspan="' . count($res) . '">' . $titleString . '</td>'; |
||
| 728 | } else { |
||
| 729 | $titleClm = ''; |
||
| 730 | } |
||
| 731 | |||
| 732 | // Result: |
||
| 733 | $resLog = $this->getResultLog($vv); |
||
| 734 | |||
| 735 | $resStatus = $this->getResStatus($vv); |
||
| 736 | $resFeVars = $this->getResFeVars($vv); |
||
| 737 | |||
| 738 | // Compile row: |
||
| 739 | $parameters = unserialize($vv['parameters']); |
||
| 740 | |||
| 741 | // Put data into array: |
||
| 742 | $rowData = []; |
||
| 743 | if ($this->pObj->MOD_SETTINGS['log_resultLog']) { |
||
| 744 | $rowData['result_log'] = $resLog; |
||
| 745 | } else { |
||
| 746 | $rowData['scheduled'] = ($vv['scheduled'] > 0) ? \TYPO3\CMS\Backend\Utility\BackendUtility::datetime($vv['scheduled']) : ' ' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.immediate'); |
||
| 747 | $rowData['exec_time'] = $vv['exec_time'] ? \TYPO3\CMS\Backend\Utility\BackendUtility::datetime($vv['exec_time']) : '-'; |
||
| 748 | } |
||
| 749 | $rowData['result_status'] = \TYPO3\CMS\Core\Utility\GeneralUtility::fixed_lgd_cs($resStatus, 50); |
||
| 750 | $rowData['url'] = '<a href="' . htmlspecialchars($parameters['url']) . '" target="_newWIndow">' . htmlspecialchars($parameters['url']) . '</a>'; |
||
| 751 | $rowData['feUserGroupList'] = $parameters['feUserGroupList']; |
||
| 752 | $rowData['procInstructions'] = is_array($parameters['procInstructions']) ? implode('; ', $parameters['procInstructions']) : ''; |
||
| 753 | $rowData['set_id'] = $vv['set_id']; |
||
| 754 | |||
| 755 | if ($this->pObj->MOD_SETTINGS['log_feVars']) { |
||
| 756 | $rowData['tsfe_id'] = $resFeVars['id']; |
||
| 757 | $rowData['tsfe_gr_list'] = $resFeVars['gr_list']; |
||
| 758 | $rowData['tsfe_no_cache'] = $resFeVars['no_cache']; |
||
| 759 | } |
||
| 760 | |||
| 761 | $setId = intval(\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('setID')); |
||
| 762 | $refreshIcon = $GLOBALS['BACK_PATH'] . 'sysext/t3skin/extjs/images/grid/refresh.gif'; |
||
| 763 | |||
| 764 | // Put rows together: |
||
| 765 | $content .= ' |
||
| 766 | <tr class="bgColor' . ($c % 2 ? '-20' : '-10') . '"> |
||
| 767 | ' . $titleClm . ' |
||
| 768 | <td><a href="' . $this->getModuleUrl(['qid_details' => $vv['qid'], 'setID' => $setId]) . '">' . htmlspecialchars($vv['qid']) . '</a></td> |
||
| 769 | <td><a href="' . $this->getModuleUrl(['qid_read' => $vv['qid'], 'setID' => $setId]) . '"><img src="' . $refreshIcon . '" width="14" hspace="1" vspace="2" height="14" border="0" title="' . htmlspecialchars('Read') . '" alt="" /></a></td>'; |
||
| 770 | foreach ($rowData as $fKey => $value) { |
||
| 771 | if (\TYPO3\CMS\Core\Utility\GeneralUtility::inList('url', $fKey)) { |
||
| 772 | $content .= ' |
||
| 773 | <td>' . $value . '</td>'; |
||
| 774 | } else { |
||
| 775 | $content .= ' |
||
| 776 | <td>' . nl2br(htmlspecialchars($value)) . '</td>'; |
||
| 777 | } |
||
| 778 | } |
||
| 779 | $content .= ' |
||
| 780 | </tr>'; |
||
| 781 | $c++; |
||
| 782 | |||
| 783 | if ($this->CSVExport) { |
||
| 784 | // Only for CSV (adding qid and scheduled/exec_time if needed): |
||
| 785 | $rowData['result_log'] = implode('// ', explode(chr(10), $resLog)); |
||
| 786 | $rowData['qid'] = $vv['qid']; |
||
| 787 | $rowData['scheduled'] = \TYPO3\CMS\Backend\Utility\BackendUtility::datetime($vv['scheduled']); |
||
| 788 | $rowData['exec_time'] = $vv['exec_time'] ? \TYPO3\CMS\Backend\Utility\BackendUtility::datetime($vv['exec_time']) : '-'; |
||
| 789 | $this->CSVaccu[] = $rowData; |
||
| 790 | } |
||
| 791 | } |
||
| 792 | } else { |
||
| 793 | |||
| 794 | // Compile row: |
||
| 795 | $content = ' |
||
| 796 | <tr class="bgColor-20"> |
||
| 797 | <td>' . $titleString . '</td> |
||
| 798 | <td colspan="' . $colSpan . '"><em>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.noentries') . '</em></td> |
||
| 799 | </tr>'; |
||
| 800 | } |
||
| 801 | |||
| 802 | return $content; |
||
| 803 | } |
||
| 804 | |||
| 805 | /** |
||
| 806 | * Find Fe vars |
||
| 807 | * |
||
| 808 | * @param array $row |
||
| 809 | * @return array |
||
| 810 | */ |
||
| 811 | public function getResFeVars($row) |
||
| 812 | { |
||
| 813 | $feVars = []; |
||
| 814 | |||
| 815 | if ($row['result_data']) { |
||
| 816 | $resultData = unserialize($row['result_data']); |
||
| 817 | $requestResult = unserialize($resultData['content']); |
||
| 818 | $feVars = $requestResult['vars']; |
||
| 819 | } |
||
| 820 | |||
| 821 | return $feVars; |
||
| 822 | } |
||
| 823 | |||
| 824 | /** |
||
| 825 | * Create Table header row (log) |
||
| 826 | * |
||
| 827 | * @return string Table header |
||
| 828 | */ |
||
| 829 | public function drawLog_printTableHeader() |
||
| 830 | { |
||
| 831 | $content = ' |
||
| 832 | <tr class="bgColor5 tableheader"> |
||
| 833 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.pagetitle') . ':</td> |
||
| 834 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.qid') . ':</td> |
||
| 835 | <td> </td>' . |
||
| 836 | ($this->pObj->MOD_SETTINGS['log_resultLog'] ? ' |
||
| 837 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.resultlog') . ':</td>' : ' |
||
| 838 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.scheduledtime') . ':</td> |
||
| 839 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.runtime') . ':</td>') . ' |
||
| 840 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.status') . ':</td> |
||
| 841 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.url') . ':</td> |
||
| 842 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.groups') . ':</td> |
||
| 843 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.procinstr') . ':</td> |
||
| 844 | <td>' . $GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.setid') . ':</td>' . |
||
| 845 | ($this->pObj->MOD_SETTINGS['log_feVars'] ? ' |
||
| 846 | <td>' . htmlspecialchars('TSFE->id') . '</td> |
||
| 847 | <td>' . htmlspecialchars('TSFE->gr_list') . '</td> |
||
| 848 | <td>' . htmlspecialchars('TSFE->no_cache') . '</td>' : '') . ' |
||
| 849 | </tr>'; |
||
| 850 | |||
| 851 | return $content; |
||
| 852 | } |
||
| 853 | |||
| 854 | /** |
||
| 855 | * Extract the log information from the current row and retrive it as formatted string. |
||
| 856 | * |
||
| 857 | * @param array $resultRow |
||
| 858 | * |
||
| 859 | * @return string |
||
| 860 | */ |
||
| 861 | protected function getResultLog($resultRow) |
||
| 862 | { |
||
| 863 | $content = ''; |
||
| 864 | |||
| 865 | if (is_array($resultRow) && array_key_exists('result_data', $resultRow)) { |
||
| 866 | $requestContent = unserialize($resultRow['result_data']); |
||
| 867 | $requestResult = unserialize($requestContent['content']); |
||
| 868 | |||
| 869 | if (is_array($requestResult) && array_key_exists('log', $requestResult)) { |
||
| 870 | $content = implode(chr(10), $requestResult['log']); |
||
| 871 | } |
||
| 872 | } |
||
| 873 | |||
| 874 | return $content; |
||
| 875 | } |
||
| 876 | |||
| 877 | public function getResStatus($vv) |
||
| 878 | { |
||
| 879 | if ($vv['result_data']) { |
||
| 880 | $requestContent = unserialize($vv['result_data']); |
||
| 881 | $requestResult = unserialize($requestContent['content']); |
||
| 882 | if (is_array($requestResult)) { |
||
| 883 | if (empty($requestResult['errorlog'])) { |
||
| 884 | $resStatus = 'OK'; |
||
| 885 | } else { |
||
| 886 | $resStatus = implode("\n", $requestResult['errorlog']); |
||
| 887 | } |
||
| 888 | $resLog = is_array($requestResult['log']) ? implode(chr(10), $requestResult['log']) : ''; |
||
| 889 | } else { |
||
| 890 | $resStatus = 'Error: ' . substr(preg_replace('/\s+/', ' ', strip_tags($requestContent['content'])), 0, 10000) . '...'; |
||
| 891 | } |
||
| 892 | } else { |
||
| 893 | $resStatus = '-'; |
||
| 894 | } |
||
| 895 | return $resStatus; |
||
| 896 | } |
||
| 897 | |||
| 898 | /***************************** |
||
| 899 | * |
||
| 900 | * CLI status display |
||
| 901 | * |
||
| 902 | *****************************/ |
||
| 903 | |||
| 904 | /** |
||
| 905 | * This method is used to show an overview about the active an the finished crawling processes |
||
| 906 | * |
||
| 907 | * @param void |
||
| 908 | * @return string |
||
| 909 | */ |
||
| 910 | protected function drawProcessOverviewAction() |
||
| 911 | { |
||
| 912 | $this->runRefreshHooks(); |
||
| 913 | |||
| 914 | global $BACK_PATH; |
||
| 915 | $this->makeCrawlerProcessableChecks(); |
||
| 916 | |||
| 917 | $crawler = $this->findCrawler(); |
||
| 918 | try { |
||
| 919 | $this->handleProcessOverviewActions(); |
||
| 920 | } catch (Exception $e) { |
||
| 921 | $this->addErrorMessage($e->getMessage()); |
||
| 922 | } |
||
| 923 | |||
| 924 | $offset = intval(\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('offset')); |
||
| 925 | $perpage = 20; |
||
| 926 | |||
| 927 | $processRepository = new tx_crawler_domain_process_repository(); |
||
| 928 | $queueRepository = new tx_crawler_domain_queue_repository(); |
||
| 929 | |||
| 930 | $mode = $this->pObj->MOD_SETTINGS['processListMode']; |
||
| 931 | if ($mode == 'detail') { |
||
| 932 | $where = ''; |
||
| 933 | } elseif ($mode == 'simple') { |
||
| 934 | $where = 'active = 1'; |
||
| 935 | } |
||
| 936 | |||
| 937 | $allProcesses = $processRepository->findAll('ttl', 'DESC', $perpage, $offset, $where); |
||
| 938 | $allCount = $processRepository->countAll($where); |
||
| 939 | |||
| 940 | $listView = new tx_crawler_view_process_list(); |
||
| 941 | $listView->setPageId($this->pObj->id); |
||
| 942 | $listView->setIconPath($BACK_PATH . '../typo3conf/ext/crawler/template/process/res/img/'); |
||
| 943 | $listView->setProcessCollection($allProcesses); |
||
| 944 | $listView->setCliPath($this->processManager->getCrawlerCliPath()); |
||
| 945 | $listView->setIsCrawlerEnabled(!$crawler->getDisabled() && !$this->isErrorDetected); |
||
| 946 | $listView->setTotalUnprocessedItemCount($queueRepository->countAllPendingItems()); |
||
| 947 | $listView->setAssignedUnprocessedItemCount($queueRepository->countAllAssignedPendingItems()); |
||
| 948 | $listView->setActiveProcessCount($processRepository->countActive()); |
||
| 949 | $listView->setMaxActiveProcessCount(\TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($this->extensionSettings['processLimit'], 1, 99, 1)); |
||
| 950 | $listView->setMode($mode); |
||
| 951 | |||
| 952 | $paginationView = new tx_crawler_view_pagination(); |
||
| 953 | $paginationView->setCurrentOffset($offset); |
||
| 954 | $paginationView->setPerPage($perpage); |
||
| 955 | $paginationView->setTotalItemCount($allCount); |
||
| 956 | |||
| 957 | $output = $listView->render(); |
||
| 958 | |||
| 959 | if ($paginationView->getTotalPagesCount() > 1) { |
||
| 960 | $output .= ' <br />' . $paginationView->render(); |
||
| 961 | } |
||
| 962 | |||
| 963 | return $output; |
||
| 964 | } |
||
| 965 | |||
| 966 | /** |
||
| 967 | * Verify that the crawler is exectuable. |
||
| 968 | * |
||
| 969 | * @return void |
||
| 970 | */ |
||
| 971 | protected function makeCrawlerProcessableChecks() |
||
| 972 | { |
||
| 973 | global $LANG; |
||
| 974 | |||
| 975 | if ($this->isCrawlerUserAvailable() === false) { |
||
| 976 | $this->addErrorMessage($LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:message.noBeUserAvailable')); |
||
| 977 | } elseif ($this->isCrawlerUserNotAdmin() === false) { |
||
| 978 | $this->addErrorMessage($LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:message.beUserIsAdmin')); |
||
| 979 | } |
||
| 980 | |||
| 981 | if ($this->isPhpForkAvailable() === false) { |
||
| 982 | $this->addErrorMessage($LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:message.noPhpForkAvailable')); |
||
| 983 | } |
||
| 984 | |||
| 985 | $exitCode = 0; |
||
| 986 | $out = []; |
||
| 987 | exec(escapeshellcmd($this->extensionSettings['phpPath'] . ' -v'), $out, $exitCode); |
||
| 988 | if ($exitCode > 0) { |
||
| 989 | $this->addErrorMessage(sprintf($LANG->sL('LLL:EXT:crawler/modfunc1/locallang.xml:message.phpBinaryNotFound'), htmlspecialchars($this->extensionSettings['phpPath']))); |
||
| 990 | } |
||
| 991 | } |
||
| 992 | |||
| 993 | /** |
||
| 994 | * Indicate that the required PHP method "popen" is |
||
| 995 | * available in the system. |
||
| 996 | * |
||
| 997 | * @return boolean |
||
| 998 | */ |
||
| 999 | protected function isPhpForkAvailable() |
||
| 1000 | { |
||
| 1001 | return function_exists('popen'); |
||
| 1002 | } |
||
| 1003 | |||
| 1004 | /** |
||
| 1005 | * Indicate that the required be_user "_cli_crawler" is |
||
| 1006 | * global available in the system. |
||
| 1007 | * |
||
| 1008 | * @return boolean |
||
| 1009 | */ |
||
| 1010 | protected function isCrawlerUserAvailable() |
||
| 1011 | { |
||
| 1012 | $isAvailable = false; |
||
| 1013 | $userArray = \TYPO3\CMS\Backend\Utility\BackendUtility::getRecordsByField('be_users', 'username', '_cli_crawler'); |
||
| 1014 | |||
| 1015 | if (is_array($userArray)) { |
||
| 1016 | $isAvailable = true; |
||
| 1017 | } |
||
| 1018 | |||
| 1019 | return $isAvailable; |
||
| 1020 | } |
||
| 1021 | |||
| 1022 | /** |
||
| 1023 | * Indicate that the required be_user "_cli_crawler" is |
||
| 1024 | * has no admin rights. |
||
| 1025 | * |
||
| 1026 | * @return boolean |
||
| 1027 | */ |
||
| 1028 | protected function isCrawlerUserNotAdmin() |
||
| 1029 | { |
||
| 1030 | $isAvailable = false; |
||
| 1031 | $userArray = \TYPO3\CMS\Backend\Utility\BackendUtility::getRecordsByField('be_users', 'username', '_cli_crawler'); |
||
| 1032 | |||
| 1033 | if (is_array($userArray) && $userArray[0]['admin'] == 0) { |
||
| 1034 | $isAvailable = true; |
||
| 1035 | } |
||
| 1036 | |||
| 1037 | return $isAvailable; |
||
| 1038 | } |
||
| 1039 | |||
| 1040 | /** |
||
| 1041 | * Method to handle incomming actions of the process overview |
||
| 1042 | * |
||
| 1043 | * @return void |
||
| 1044 | * @throws Exception |
||
| 1045 | */ |
||
| 1046 | protected function handleProcessOverviewActions() |
||
| 1047 | { |
||
| 1048 | $crawler = $this->findCrawler(); |
||
| 1049 | |||
| 1050 | switch (\TYPO3\CMS\Core\Utility\GeneralUtility::_GP('action')) { |
||
| 1051 | case 'stopCrawling': |
||
| 1052 | //set the cli status to disable (all processes will be terminated) |
||
| 1053 | $crawler->setDisabled(true); |
||
| 1054 | break; |
||
| 1055 | case 'resumeCrawling': |
||
| 1056 | //set the cli status to end (all processes will be terminated) |
||
| 1057 | $crawler->setDisabled(false); |
||
| 1058 | break; |
||
| 1059 | case 'addProcess': |
||
| 1060 | $handle = $this->processManager->startProcess(); |
||
| 1061 | if ($handle === false) { |
||
| 1062 | throw new Exception($GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.newprocesserror')); |
||
| 1063 | } |
||
| 1064 | $this->addNoticeMessage($GLOBALS['LANG']->sL('LLL:EXT:crawler/modfunc1/locallang.xml:labels.newprocess')); |
||
| 1065 | break; |
||
| 1066 | } |
||
| 1067 | } |
||
| 1068 | |||
| 1069 | /** |
||
| 1070 | * Returns the singleton instance of the crawler. |
||
| 1071 | * |
||
| 1072 | * @param void |
||
| 1073 | * @return tx_crawler_lib crawler object |
||
| 1074 | */ |
||
| 1075 | protected function findCrawler() |
||
| 1076 | { |
||
| 1077 | if (!$this->crawlerObj instanceof tx_crawler_lib) { |
||
| 1078 | $this->crawlerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_lib'); |
||
| 1079 | } |
||
| 1080 | return $this->crawlerObj; |
||
| 1081 | } |
||
| 1082 | |||
| 1083 | /***************************** |
||
| 1084 | * |
||
| 1085 | * General Helper Functions |
||
| 1086 | * |
||
| 1087 | *****************************/ |
||
| 1088 | |||
| 1089 | /** |
||
| 1090 | * This method is used to add a message to the internal queue |
||
| 1091 | * |
||
| 1092 | * NOTE: |
||
| 1093 | * This method is basesd on TYPO3 4.3 or higher! |
||
| 1094 | * |
||
| 1095 | * @param string the message itself |
||
| 1096 | * @param integer message level (-1 = success (default), 0 = info, 1 = notice, 2 = warning, 3 = error) |
||
| 1097 | * |
||
| 1098 | * @return void |
||
| 1099 | */ |
||
| 1100 | private function addMessage($message, $severity = \TYPO3\CMS\Core\Messaging\FlashMessage::OK) |
||
| 1101 | { |
||
| 1102 | $message = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( |
||
| 1103 | 'TYPO3\CMS\Core\Messaging\FlashMessage', |
||
| 1104 | $message, |
||
| 1105 | '', |
||
| 1106 | $severity |
||
| 1107 | ); |
||
| 1108 | |||
| 1109 | // TODO: |
||
| 1110 | /** @var \TYPO3\CMS\Core\Messaging\FlashMessageService $flashMessageService */ |
||
| 1111 | $flashMessageService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Core\Messaging\FlashMessageService::class); |
||
| 1112 | $flashMessageService->getMessageQueueByIdentifier()->addMessage($message); |
||
| 1113 | } |
||
| 1114 | |||
| 1115 | /** |
||
| 1116 | * Add notice message to the user interface. |
||
| 1117 | * |
||
| 1118 | * NOTE: |
||
| 1119 | * This method is basesd on TYPO3 4.3 or higher! |
||
| 1120 | * |
||
| 1121 | * @param string The message |
||
| 1122 | * |
||
| 1123 | * @return void |
||
| 1124 | */ |
||
| 1125 | protected function addNoticeMessage($message) |
||
| 1126 | { |
||
| 1127 | $this->addMessage($message, \TYPO3\CMS\Core\Messaging\FlashMessage::NOTICE); |
||
| 1128 | } |
||
| 1129 | |||
| 1130 | /** |
||
| 1131 | * Add error message to the user interface. |
||
| 1132 | * |
||
| 1133 | * NOTE: |
||
| 1134 | * This method is basesd on TYPO3 4.3 or higher! |
||
| 1135 | * |
||
| 1136 | * @param string The message |
||
| 1137 | * |
||
| 1138 | * @return void |
||
| 1139 | */ |
||
| 1140 | protected function addErrorMessage($message) |
||
| 1141 | { |
||
| 1142 | $this->isErrorDetected = true; |
||
| 1143 | $this->addMessage($message, \TYPO3\CMS\Core\Messaging\FlashMessage::ERROR); |
||
| 1144 | } |
||
| 1145 | |||
| 1146 | /** |
||
| 1147 | * Add error message to the user interface. |
||
| 1148 | * |
||
| 1149 | * NOTE: |
||
| 1150 | * This method is basesd on TYPO3 4.3 or higher! |
||
| 1151 | * |
||
| 1152 | * @param string The message |
||
| 1153 | * |
||
| 1154 | * @return void |
||
| 1155 | */ |
||
| 1156 | protected function addWarningMessage($message) |
||
| 1157 | { |
||
| 1158 | $this->addMessage($message, \TYPO3\CMS\Core\Messaging\FlashMessage::WARNING); |
||
| 1159 | } |
||
| 1160 | |||
| 1161 | /** |
||
| 1162 | * Create selector box |
||
| 1163 | * |
||
| 1164 | * @param array $optArray Options key(value) => label pairs |
||
| 1165 | * @param string $name Selector box name |
||
| 1166 | * @param string $value Selector box value (array for multiple...) |
||
| 1167 | * @param boolean $multiple If set, will draw multiple box. |
||
| 1168 | * |
||
| 1169 | * @return string HTML select element |
||
| 1170 | */ |
||
| 1171 | public function selectorBox($optArray, $name, $value, $multiple) |
||
| 1172 | { |
||
| 1173 | $options = []; |
||
| 1174 | foreach ($optArray as $key => $val) { |
||
| 1175 | $options[] = ' |
||
| 1176 | <option value="' . htmlspecialchars($key) . '"' . ((!$multiple && !strcmp($value, $key)) || ($multiple && in_array($key, (array)$value)) ? ' selected="selected"' : '') . '>' . htmlspecialchars($val) . '</option>'; |
||
| 1177 | } |
||
| 1178 | |||
| 1179 | $output = '<select name="' . htmlspecialchars($name . ($multiple ? '[]' : '')) . '"' . ($multiple ? ' multiple="multiple" size="' . count($options) . '"' : '') . '>' . implode('', $options) . '</select>'; |
||
| 1180 | |||
| 1181 | return $output; |
||
| 1182 | } |
||
| 1183 | |||
| 1184 | /** |
||
| 1185 | * Activate hooks |
||
| 1186 | * |
||
| 1187 | * @return void |
||
| 1188 | */ |
||
| 1189 | public function runRefreshHooks() |
||
| 1190 | { |
||
| 1191 | $crawlerLib = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_lib'); |
||
| 1192 | if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['refresh_hooks'])) { |
||
| 1193 | foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['refresh_hooks'] as $objRef) { |
||
| 1194 | $hookObj = &\TYPO3\CMS\Core\Utility\GeneralUtility::getUserObj($objRef); |
||
| 1195 | if (is_object($hookObj)) { |
||
| 1196 | $hookObj->crawler_init($crawlerLib); |
||
| 1197 | } |
||
| 1198 | } |
||
| 1199 | } |
||
| 1200 | } |
||
| 1201 | |||
| 1202 | /** |
||
| 1203 | * Returns the URL to the current module, including $_GET['id']. |
||
| 1204 | * |
||
| 1205 | * @param array $urlParameters optional parameters to add to the URL |
||
| 1206 | * @return string |
||
| 1207 | */ |
||
| 1208 | protected function getModuleUrl(array $urlParameters = []) |
||
| 1216 | } |
||
| 1217 | } |
||
| 1218 | |||
| 1219 | if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/modfunc1/class.tx_crawler_modfunc1.php']) { |
||
| 1220 | include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/modfunc1/class.tx_crawler_modfunc1.php']); |
||
| 1221 | } |
||
| 1222 |
Loading history...
The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g.
excluded_paths: ["lib/*"], you can move it to the dependency path list as follows:For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths