AOEpeople / crawler

cli/class.tx_crawler_cli_im.php

Indentation +5 added lines, -5 removed lines

@@ -49,11 +49,11 @@
 
         // Adding options to help archive:
         /**
-        * We removed the "proc" option as it seemd not to be working any more. But as the complete handling of the crawler has changed regarding the configuration
-        * this is completely ok. Since configuration records were introduced to configure "what should be done" additionally to page ts the way to setup jobs
-        * has drifted from selecting filtering processing instructions to selecting/filtering configuration keys (you can configure the processing instructions there).
-        * This is also reflected in the backend modules and allows you a much clearer and powerful way to work with the crawler extension.
-        */
+         * We removed the "proc" option as it seemd not to be working any more. But as the complete handling of the crawler has changed regarding the configuration
+         * this is completely ok. Since configuration records were introduced to configure "what should be done" additionally to page ts the way to setup jobs
+         * has drifted from selecting filtering processing instructions to selecting/filtering configuration keys (you can configure the processing instructions there).
+         * This is also reflected in the backend modules and allows you a much clearer and powerful way to work with the crawler extension.
+         */
         // $this->cli_options[] = array('-proc listOfProcInstr', 'Comma list of processing instructions. These are the "actions" carried out when crawling and you must specify at least one. Depends on third-party extensions. Examples are "tx_cachemgm_recache" from "cachemgm" extension (will recache pages), "tx_staticpub_publish" from "staticpub" (publishing pages to static files) or "tx_indexedsearch_reindex" from "indexed_search" (indexes pages).');
         // TODO: cleanup here!
         $this->cli_options[] = ['-d depth', 'Tree depth, 0-99', "How many levels under the 'page_id' to include."];

Spacing +2 added lines, -2 removed lines

@@ -59,7 +59,7 @@ 
         $this->cli_options[] = ['-d depth', 'Tree depth, 0-99', "How many levels under the 'page_id' to include."];
         $this->cli_options[] = ['-o mode', 'Output mode: "url", "exec", "queue"', "Specifies output modes\nurl : Will list URLs which wget could use as input.\nqueue: Will put entries in queue table.\nexec: Will execute all entries right away!"];
         $this->cli_options[] = ['-n number', 'Number of items per minute.', 'Specifies how many items are put in the queue per minute. Only valid for output mode "queue"'];
-        $this->cli_options[] = ['-conf configurationkeys','List of Configuration Keys','A commaseperated list of crawler configurations'];
+        $this->cli_options[] = ['-conf configurationkeys', 'List of Configuration Keys', 'A commaseperated list of crawler configurations'];
         #		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");
 
         // Setting help texts:
@@ -67,7 +67,7 @@ 
         $this->cli_help['synopsis'] = 'page_id ###OPTIONS###';
         $this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It can put entries in the queue from command line options, return the list of URLs and even execute all entries right away without having to queue them up - this can be useful for immediate re-cache, re-indexing or static publishing from command line.";
         $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_im 7 -d=2 -conf=<configurationKey> -o=exec\nWill re-cache pages from page 7 and two levels down, executed immediately.\n";
-        $this->cli_help['examples'].= "/.../cli_dispatch.phpsh crawler_im 7 -d=0 -conf=<configurationKey> -n=4 -o=queue\nWill put entries for re-caching pages from page 7 into queue, 4 every minute.\n";
+        $this->cli_help['examples'] .= "/.../cli_dispatch.phpsh crawler_im 7 -d=0 -conf=<configurationKey> -n=4 -o=queue\nWill put entries for re-caching pages from page 7 into queue, 4 every minute.\n";
         $this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';
     }
 }

cli/crawler_multiprocess.php

Spacing +1 added lines, -1 removed lines

@@ -9,5 +9,5 @@
 try {
     $processManager->multiProcess($timeout);
 } catch (Exception $e) {
-    echo PHP_EOL . $e->getMessage();
+    echo PHP_EOL.$e->getMessage();
 }

template/pagination.php

Spacing +2 added lines, -2 removed lines

@@ -3,12 +3,12 @@
 } ?>
 
 Page:
-<?php for ($currentPageOffset 	= 0; $currentPageOffset < $this->getTotalPagesCount(); $currentPageOffset++) {
+<?php for ($currentPageOffset = 0; $currentPageOffset < $this->getTotalPagesCount(); $currentPageOffset++) {
     ?>
 	<a href="index.php?offset=<?php echo htmlspecialchars($currentPageOffset * $this->getPerPage()); ?>">
 		<?php echo	htmlspecialchars($this->getLabelForPageOffset($currentPageOffset)); ?>
 	</a>
-	<?php if ($currentPageOffset+1 < $this->getTotalPagesCount()) {
+	<?php if ($currentPageOffset + 1 < $this->getTotalPagesCount()) {
         ?>
 	|
 	<?php

domain/process/class.tx_crawler_domain_process_collection.php

Spacing +6 added lines, -6 removed lines

@@ -50,8 +50,8 @@ 
      */
     public function offsetGet($index)
     {
-        if (! parent::offsetExists($index)) {
-            throw new Exception('Index "' . var_export($index, true) . '" for tx_crawler_domain_process are not available');
+        if (!parent::offsetExists($index)) {
+            throw new Exception('Index "'.var_export($index, true).'" for tx_crawler_domain_process are not available');
         }
         return parent::offsetGet($index);
     }
@@ -66,7 +66,7 @@ 
      */
     public function offsetSet($index, $subject)
     {
-        if (! $subject instanceof tx_crawler_domain_process) {
+        if (!$subject instanceof tx_crawler_domain_process) {
             throw new InvalidArgumentException('Wrong parameter type given, "tx_crawler_domain_process" expected!');
         }
         parent::offsetSet($index, $subject);
@@ -80,7 +80,7 @@ 
      */
     public function append($subject)
     {
-        if (! $subject instanceof tx_crawler_domain_process) {
+        if (!$subject instanceof tx_crawler_domain_process) {
             throw new InvalidArgumentException('Wrong parameter type given, "tx_crawler_domain_process" expected!');
         }
         parent::append($subject);
@@ -92,9 +92,9 @@ 
      */
     public function getProcessIds()
     {
-        $result=[];
+        $result = [];
         foreach ($this->getIterator() as $value) {
-            $result[]=$value->getProcess_id();
+            $result[] = $value->getProcess_id();
         }
         return $result;
     }

domain/process/class.tx_crawler_domain_process_manager.php

Spacing +17 added lines, -17 removed lines

@@ -67,8 +67,8 @@ 
      */
     public function __construct()
     {
-        $this->processRepository	= new tx_crawler_domain_process_repository();
-        $this->queueRepository	= new tx_crawler_domain_queue_repository();
+        $this->processRepository = new tx_crawler_domain_process_repository();
+        $this->queueRepository = new tx_crawler_domain_queue_repository();
         $this->crawlerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_lib');
         $this->timeToLive = intval($this->crawlerObj->extensionSettings['processMaxRunTime']);
         $this->countInARun = intval($this->crawlerObj->extensionSettings['countInARun']);
@@ -84,7 +84,7 @@ 
     public function multiProcess($timeout)
     {
         if ($this->processLimit <= 1) {
-            throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.' . PHP_EOL);
+            throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.'.PHP_EOL);
         }
 
         $pendingItemsStart = $this->queueRepository->countAllPendingItems();
@@ -102,7 +102,7 @@ 
             }
             if ($currentPendingItems == 0) {
                 if ($this->verbose) {
-                    echo 'Finished...' . chr(10);
+                    echo 'Finished...'.chr(10);
                 }
                 break;
             }
@@ -114,16 +114,16 @@ 
             }
             sleep(1);
             if ($nextTimeOut < time()) {
-                $timedOutProcesses = $this->processRepository->findAll('', 'DESC', null, 0, 'ttl >' . $nextTimeOut);
+                $timedOutProcesses = $this->processRepository->findAll('', 'DESC', null, 0, 'ttl >'.$nextTimeOut);
                 $nextTimeOut = time() + $this->timeToLive;
                 if ($this->verbose) {
-                    echo 'Cleanup' . implode(',', $timedOutProcesses->getProcessIds()) . chr(10);
+                    echo 'Cleanup'.implode(',', $timedOutProcesses->getProcessIds()).chr(10);
                 }
                 $this->crawlerObj->CLI_releaseProcesses($timedOutProcesses->getProcessIds(), true);
             }
         }
         if ($currentPendingItems > 0 && $this->verbose) {
-            echo 'Stop with timeout' . chr(10);
+            echo 'Stop with timeout'.chr(10);
         }
     }
 
@@ -143,17 +143,17 @@ 
     private function startRequiredProcesses()
     {
         $ret = false;
-        $currentProcesses= $this->processRepository->countActive();
-        $availableProcessesCount = $this->processLimit-$currentProcesses;
+        $currentProcesses = $this->processRepository->countActive();
+        $availableProcessesCount = $this->processLimit - $currentProcesses;
         $requiredProcessesCount = ceil($this->queueRepository->countAllUnassignedPendingItems() / $this->countInARun);
-        $startProcessCount = min([$availableProcessesCount,$requiredProcessesCount]);
+        $startProcessCount = min([$availableProcessesCount, $requiredProcessesCount]);
         if ($startProcessCount <= 0) {
             return $ret;
         }
         if ($startProcessCount && $this->verbose) {
             echo 'Start '.$startProcessCount.' new processes (Running:'.$currentProcesses.')';
         }
-        for ($i=0;$i<$startProcessCount;$i++) {
+        for ($i = 0; $i < $startProcessCount; $i++) {
             usleep(100);
             if ($this->startProcess()) {
                 if ($this->verbose) {
@@ -174,13 +174,13 @@ 
      */
     public function startProcess()
     {
-        $ttl = (time() + $this->timeToLive -1);
+        $ttl = (time() + $this->timeToLive - 1);
         $current = $this->processRepository->countNotTimeouted($ttl);
-        $completePath = '(' .escapeshellcmd($this->getCrawlerCliPath()) . ' &) > /dev/null';
+        $completePath = '('.escapeshellcmd($this->getCrawlerCliPath()).' &) > /dev/null';
         if (system($completePath) === false) {
             throw new Exception('could not start process!');
         } else {
-            for ($i=0;$i<10;$i++) {
+            for ($i = 0; $i < 10; $i++) {
                 if ($this->processRepository->countNotTimeouted($ttl) > $current) {
                     return true;
                 }
@@ -197,9 +197,9 @@ 
      */
     public function getCrawlerCliPath()
     {
-        $phpPath 		= $this->crawlerObj->extensionSettings['phpPath'] . ' ';
-        $pathToTypo3 	= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_DOCUMENT_ROOT'), '/');
-        $pathToTypo3 	.= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_PATH'), '/');
+        $phpPath 		= $this->crawlerObj->extensionSettings['phpPath'].' ';
+        $pathToTypo3 = rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_DOCUMENT_ROOT'), '/');
+        $pathToTypo3 .= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_PATH'), '/');
         $cliPart	 	= '/typo3/cli_dispatch.phpsh crawler';
         return $phpPath.$pathToTypo3.$cliPart;
     }

domain/events/class.tx_crawler_domain_events_dispatcher.php

Spacing +1 added lines, -1 removed lines

@@ -130,7 +130,7 @@
     {
         if (is_array($this->observers[$event])) {
             foreach ($this->observers[$event] as $eventObserver) {
-                call_user_func([$eventObserver['object'],$eventObserver['method']], $event, $group, $attachedData);
+                call_user_func([$eventObserver['object'], $eventObserver['method']], $event, $group, $attachedData);
             }
         }
     }

Classes/Utility/SchedulerUtility.php

Spacing +10 added lines, -10 removed lines

@@ -45,36 +45,36 @@
     {
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['scheduler']['tasks']['AOE\\Crawler\\Task\\CrawlerQueueTask'] = [
             'extension'        => $extKey,
-            'title'            => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_im.name',
-            'description'      => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_im.description',
+            'title'            => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_im.name',
+            'description'      => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_im.description',
             'additionalFields' => CrawlerQueueTaskAdditionalFieldProvider::class
         ];
 
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['scheduler']['tasks']['AOE\\Crawler\\Task\\CrawlerTask'] = [
             'extension'        => $extKey,
-            'title'            => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_crawl.name',
-            'description'      => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_crawl.description',
+            'title'            => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_crawl.name',
+            'description'      => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_crawl.description',
             'additionalFields' => CrawlerTaskAdditionalFieldProvider::class
         ];
 
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['scheduler']['tasks']['AOE\\Crawler\\Task\\CrawlMultiProcessTask'] = [
             'extension'        => $extKey,
-            'title'            => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_crawlMultiProcess.name',
-            'description'      => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_crawl.description',
+            'title'            => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_crawlMultiProcess.name',
+            'description'      => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_crawl.description',
             'additionalFields' => CrawlMultiProcessTaskAdditionalFieldProvider::class
         ];
 
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['scheduler']['tasks']['AOE\\Crawler\\Task\\FlushQueueTask'] = [
             'extension'        => $extKey,
-            'title'            => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_flush.name',
-            'description'      => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_flush.description',
+            'title'            => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_flush.name',
+            'description'      => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_flush.description',
             'additionalFields' => FlushQueueTaskAdditionalFieldProvider::class
         ];
 
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['scheduler']['tasks']['AOE\\Crawler\\Task\\ProcessCleanupTask'] = [
             'extension'        => $extKey,
-            'title'            => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_processCleanup.name',
-            'description'      => 'LLL:EXT:' . $extKey . '/Resources/Private/Language/Backend.xlf:crawler_processCleanup.description',
+            'title'            => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_processCleanup.name',
+            'description'      => 'LLL:EXT:'.$extKey.'/Resources/Private/Language/Backend.xlf:crawler_processCleanup.description',
         ];
     }
 }

Classes/Utility/HookUtility.php

Spacing +5 added lines, -5 removed lines

@@ -45,17 +45,17 @@
     public static function registerHooks($extKey)
     {
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['connectToDB']['tx_crawler'] =
-            TsfeHook::class . '->fe_init';
+            TsfeHook::class.'->fe_init';
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['initFEuser']['tx_crawler'] =
-            TsfeHook::class . '->fe_feuserInit';
+            TsfeHook::class.'->fe_feuserInit';
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['isOutputting']['tx_crawler'] =
-            TsfeHook::class . '->fe_isOutputting';
+            TsfeHook::class.'->fe_isOutputting';
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['tslib/class.tslib_fe.php']['hook_eofe']['tx_crawler'] =
-            TsfeHook::class . '->fe_eofe';
+            TsfeHook::class.'->fe_eofe';
 
         // Activating NC Static File Cache hook
         $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['nc_staticfilecache/class.tx_ncstaticfilecache.php']['createFile_initializeVariables']['tx_crawler'] =
-            StaticFileCacheCreateUriHook::class . '->initialize';
+            StaticFileCacheCreateUriHook::class.'->initialize';
 
         // Activating Crawler cli_hooks
         $GLOBALS['TYPO3_CONF_VARS']['EXTCONF'][$extKey]['cli_hooks'][] =

Classes/Hooks/ProcessCleanUpHook.php

Spacing +10 added lines, -10 removed lines

@@ -70,14 +70,14 @@ 
         $results = $this->getDatabaseConnection()->exec_SELECTgetRows(
             'process_id, system_process_id',
             'tx_crawler_process',
-            'ttl <= ' . intval(time() - $this->extensionSettings['processMaxRunTime'] - 3600) . ' AND active = 1'
+            'ttl <= '.intval(time() - $this->extensionSettings['processMaxRunTime'] - 3600).' AND active = 1'
         );
 
         if (!is_array($results)) {
             return;
         }
         foreach ($results as $result) {
-            $systemProcessId = (int)$result['system_process_id'];
+            $systemProcessId = (int) $result['system_process_id'];
             $processId = $result['process_id'];
             if ($systemProcessId > 1) {
                 if ($this->doProcessStillExists($systemProcessId)) {
@@ -98,7 +98,7 @@ 
         $results = $this->getDatabaseConnection()->exec_SELECTgetRows(
             'process_id, system_process_id',
             'tx_crawler_process',
-            'ttl <= ' . intval(time() - $this->extensionSettings['processMaxRunTime']) . ' AND active = 1'
+            'ttl <= '.intval(time() - $this->extensionSettings['processMaxRunTime']).' AND active = 1'
         );
 
         if (!is_array($results)) {
@@ -106,7 +106,7 @@ 
         }
         foreach ($results as $result) {
             $processExists = false;
-            $systemProcessId = (int)$result['system_process_id'];
+            $systemProcessId = (int) $result['system_process_id'];
             $processId = $result['process_id'];
             if ($systemProcessId > 1) {
                 $dispatcherProcesses = $this->findDispatcherProcesses();
@@ -116,7 +116,7 @@ 
                 }
                 foreach ($dispatcherProcesses as $process) {
                     $responseArray = $this->createResponseArray($process);
-                    if ($systemProcessId === (int)$responseArray[1]) {
+                    if ($systemProcessId === (int) $responseArray[1]) {
                         $processExists = true;
                     };
                 }
@@ -138,12 +138,12 @@ 
     {
         $this->getDatabaseConnection()->exec_DELETEquery(
             'tx_crawler_process',
-            'process_id = ' . $this->getDatabaseConnection()->fullQuoteStr($processId, 'tx_crawler_process')
+            'process_id = '.$this->getDatabaseConnection()->fullQuoteStr($processId, 'tx_crawler_process')
         );
 
         $this->getDatabaseConnection()->exec_UPDATEquery(
             'tx_crawler_queue',
-            'process_id = ' . $this->getDatabaseConnection()->fullQuoteStr($processId, 'tx_crawler_queue'),
+            'process_id = '.$this->getDatabaseConnection()->fullQuoteStr($processId, 'tx_crawler_queue'),
             ['process_id' => '']
         );
     }
@@ -177,12 +177,12 @@ 
         $doProcessStillExists = false;
         if (!$this->isOsWindows()) {
             // Not windows
-            if (file_exists('/proc/' . $pid)) {
+            if (file_exists('/proc/'.$pid)) {
                 $doProcessStillExists = true;
             }
         } else {
             // Windows
-            exec('tasklist | find "' . $pid . '"', $returnArray, $returnValue);
+            exec('tasklist | find "'.$pid.'"', $returnArray, $returnValue);
             if (count($returnArray) > 0 && preg_match('/php/i', $returnValue[0])) {
                 $doProcessStillExists = true;
             }
@@ -204,7 +204,7 @@ 
             posix_kill($pid, 9);
         } else {
             // Windows
-            exec('taskkill /PID ' . $pid);
+            exec('taskkill /PID '.$pid);
         }
     }