AOEpeople / crawler

Classes/Task/CrawlMultiProcessTask.php

Spacing +1 added lines, -1 removed lines

@@ -47,7 +47,7 @@
     public function execute()
     {
         $processManager = new \tx_crawler_domain_process_manager();
-        $timeout = is_int($this->timeOut) ? (int)$this->timeOut : 1800;
+        $timeout = is_int($this->timeOut) ? (int) $this->timeOut : 1800;
 
         try {
             $processManager->multiProcess($timeout);

cli/crawler_multiprocess.php

Indentation +3 added lines, -3 removed lines

@@ -1,13 +1,13 @@
 <?php
 if (!defined('TYPO3_cliMode')) {
-	die('You cannot run this script directly!');
+    die('You cannot run this script directly!');
 }
 
 $processManager = new tx_crawler_domain_process_manager();
 $timeout = isset($_SERVER['argv'][1] ) ? intval($_SERVER['argv'][1]) : 1800;
 
 try {
-	$processManager->multiProcess($timeout);
+    $processManager->multiProcess($timeout);
 } catch (Exception $e) {
-	echo PHP_EOL . $e->getMessage();
+    echo PHP_EOL . $e->getMessage();
 }

Spacing +2 added lines, -2 removed lines

@@ -4,10 +4,10 @@
 }
 
 $processManager = new tx_crawler_domain_process_manager();
-$timeout = isset($_SERVER['argv'][1] ) ? intval($_SERVER['argv'][1]) : 1800;
+$timeout = isset($_SERVER['argv'][1]) ? intval($_SERVER['argv'][1]) : 1800;
 
 try {
 	$processManager->multiProcess($timeout);
 } catch (Exception $e) {
-	echo PHP_EOL . $e->getMessage();
+	echo PHP_EOL.$e->getMessage();
 }

domain/process/class.tx_crawler_domain_process_manager.php

Indentation +158 added lines, -158 removed lines

@@ -27,175 +27,175 @@
  *
  */
 class tx_crawler_domain_process_manager  {
-	/**
-	 * @var $timeToLive integer
-	 */
-	private $timeToLive;
-	/**
-	 * @var integer
-	 */
-	private $countInARun;
+    /**
+     * @var $timeToLive integer
+     */
+    private $timeToLive;
+    /**
+     * @var integer
+     */
+    private $countInARun;
 
-	/**
-	 * @var integer
-	 */
-	private $processLimit;
+    /**
+     * @var integer
+     */
+    private $processLimit;
 
-	/**
-	 * @var $crawlerObj tx_crawler_lib
-	 */
-	private $crawlerObj;
+    /**
+     * @var $crawlerObj tx_crawler_lib
+     */
+    private $crawlerObj;
 
-	/**
-	 * @var $queueRepository tx_crawler_domain_queue_repository
-	 */
-	private $queueRepository;
+    /**
+     * @var $queueRepository tx_crawler_domain_queue_repository
+     */
+    private $queueRepository;
 
-	/**
-	 * @var tx_crawler_domain_process_repository
-	 */
-	private $processRepository;
+    /**
+     * @var tx_crawler_domain_process_repository
+     */
+    private $processRepository;
 
-	/**
-	 * @var $verbose boolean
-	 */
-	private $verbose;
+    /**
+     * @var $verbose boolean
+     */
+    private $verbose;
 
-	/**
-	 * the constructor
-	 */
-	public function __construct() {
-		$this->processRepository	= new tx_crawler_domain_process_repository();
-		$this->queueRepository	= new tx_crawler_domain_queue_repository();
-		$this->crawlerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_lib');
-		$this->timeToLive = intval($this->crawlerObj->extensionSettings['processMaxRunTime']);
-		$this->countInARun = intval($this->crawlerObj->extensionSettings['countInARun']);
-		$this->processLimit = intval($this->crawlerObj->extensionSettings['processLimit']);
-		$this->verbose = intval($this->crawlerObj->extensionSettings['processVerbose']);
-	}
+    /**
+     * the constructor
+     */
+    public function __construct() {
+        $this->processRepository	= new tx_crawler_domain_process_repository();
+        $this->queueRepository	= new tx_crawler_domain_queue_repository();
+        $this->crawlerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_lib');
+        $this->timeToLive = intval($this->crawlerObj->extensionSettings['processMaxRunTime']);
+        $this->countInARun = intval($this->crawlerObj->extensionSettings['countInARun']);
+        $this->processLimit = intval($this->crawlerObj->extensionSettings['processLimit']);
+        $this->verbose = intval($this->crawlerObj->extensionSettings['processVerbose']);
+    }
 
-	/**
-	 * starts multiple processes
-	 *
-	 * @param integer $timeout
-	 */
-	public function multiProcess($timeout) {
-		if ($this->processLimit <= 1) {
-			throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.' . PHP_EOL);
-		}
+    /**
+     * starts multiple processes
+     *
+     * @param integer $timeout
+     */
+    public function multiProcess($timeout) {
+        if ($this->processLimit <= 1) {
+            throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.' . PHP_EOL);
+        }
 
-		$pendingItemsStart = $this->queueRepository->countAllPendingItems();
-		$itemReportLimit = 20;
-		$reportItemCount = $pendingItemsStart - $itemReportLimit;
-		if ($this->verbose) {
-			$this->reportItemStatus();
-		}
-		$this->startRequiredProcesses();
-		$nextTimeOut = time() + $this->timeToLive;
-		for ($i = 0; $i < $timeout; $i++) {
-			$currentPendingItems = $this->queueRepository->countAllPendingItems();
-			if ($this->startRequiredProcesses($this->verbose)) {
-				$nextTimeOut = time() + $this->timeToLive;
-			}
-			if ($currentPendingItems == 0) {
-				if ($this->verbose) {
-					echo 'Finished...' . chr(10);
-				}
-				break;
-			}
-			if ($currentPendingItems < $reportItemCount) {
-				if ($this->verbose) {
-					$this->reportItemStatus();
-				}
-				$reportItemCount = $currentPendingItems - $itemReportLimit;
-			}
-			sleep(1);
-			if ($nextTimeOut < time()) {
-				$timedOutProcesses = $this->processRepository->findAll('', 'DESC', NULL, 0, 'ttl >' . $nextTimeOut);
-				$nextTimeOut = time() + $this->timeToLive;
-				if ($this->verbose) {
-					echo 'Cleanup' . implode(',', $timedOutProcesses->getProcessIds()) . chr(10);
-				}
-				$this->crawlerObj->CLI_releaseProcesses($timedOutProcesses->getProcessIds(), true);
-			}
-		}
-		if ($currentPendingItems > 0 && $this->verbose) {
-			echo 'Stop with timeout' . chr(10);
-		}
-	}
+        $pendingItemsStart = $this->queueRepository->countAllPendingItems();
+        $itemReportLimit = 20;
+        $reportItemCount = $pendingItemsStart - $itemReportLimit;
+        if ($this->verbose) {
+            $this->reportItemStatus();
+        }
+        $this->startRequiredProcesses();
+        $nextTimeOut = time() + $this->timeToLive;
+        for ($i = 0; $i < $timeout; $i++) {
+            $currentPendingItems = $this->queueRepository->countAllPendingItems();
+            if ($this->startRequiredProcesses($this->verbose)) {
+                $nextTimeOut = time() + $this->timeToLive;
+            }
+            if ($currentPendingItems == 0) {
+                if ($this->verbose) {
+                    echo 'Finished...' . chr(10);
+                }
+                break;
+            }
+            if ($currentPendingItems < $reportItemCount) {
+                if ($this->verbose) {
+                    $this->reportItemStatus();
+                }
+                $reportItemCount = $currentPendingItems - $itemReportLimit;
+            }
+            sleep(1);
+            if ($nextTimeOut < time()) {
+                $timedOutProcesses = $this->processRepository->findAll('', 'DESC', NULL, 0, 'ttl >' . $nextTimeOut);
+                $nextTimeOut = time() + $this->timeToLive;
+                if ($this->verbose) {
+                    echo 'Cleanup' . implode(',', $timedOutProcesses->getProcessIds()) . chr(10);
+                }
+                $this->crawlerObj->CLI_releaseProcesses($timedOutProcesses->getProcessIds(), true);
+            }
+        }
+        if ($currentPendingItems > 0 && $this->verbose) {
+            echo 'Stop with timeout' . chr(10);
+        }
+    }
 
-	/**
-	 * Reports curent Status of queue
-	 */
-	protected function reportItemStatus() {
-		echo 'Pending:'.$this->queueRepository->countAllPendingItems().' / Assigned:'.$this->queueRepository->countAllAssignedPendingItems().chr(10);
-	}
+    /**
+     * Reports curent Status of queue
+     */
+    protected function reportItemStatus() {
+        echo 'Pending:'.$this->queueRepository->countAllPendingItems().' / Assigned:'.$this->queueRepository->countAllAssignedPendingItems().chr(10);
+    }
 
-	/**
-	 * according to the given count of pending items and the countInARun Setting this method
-	 * starts more crawling processes
-	 * @return boolean if processes are started
-	 */
-	private function startRequiredProcesses() {
-		$ret = FALSE;
-		$currentProcesses= $this->processRepository->countActive();
-		$availableProcessesCount = $this->processLimit-$currentProcesses;
-		$requiredProcessesCount = ceil($this->queueRepository->countAllUnassignedPendingItems() / $this->countInARun);
-		$startProcessCount = min(array($availableProcessesCount,$requiredProcessesCount));
-		if ($startProcessCount <= 0) {
-			return $ret;
-		}
-		if ($startProcessCount && $this->verbose) {
-			echo 'Start '.$startProcessCount.' new processes (Running:'.$currentProcesses.')';
-		}
-		for($i=0;$i<$startProcessCount;$i++) {
-			usleep(100);
-			if ($this->startProcess()) {
-				if ($this->verbose) {
-					echo '.';
-					$ret = TRUE;
-				}
-			}
-		}
-		if ($this->verbose) {
-			echo chr(10);
-		}
-		return $ret;
-	}
+    /**
+     * according to the given count of pending items and the countInARun Setting this method
+     * starts more crawling processes
+     * @return boolean if processes are started
+     */
+    private function startRequiredProcesses() {
+        $ret = FALSE;
+        $currentProcesses= $this->processRepository->countActive();
+        $availableProcessesCount = $this->processLimit-$currentProcesses;
+        $requiredProcessesCount = ceil($this->queueRepository->countAllUnassignedPendingItems() / $this->countInARun);
+        $startProcessCount = min(array($availableProcessesCount,$requiredProcessesCount));
+        if ($startProcessCount <= 0) {
+            return $ret;
+        }
+        if ($startProcessCount && $this->verbose) {
+            echo 'Start '.$startProcessCount.' new processes (Running:'.$currentProcesses.')';
+        }
+        for($i=0;$i<$startProcessCount;$i++) {
+            usleep(100);
+            if ($this->startProcess()) {
+                if ($this->verbose) {
+                    echo '.';
+                    $ret = TRUE;
+                }
+            }
+        }
+        if ($this->verbose) {
+            echo chr(10);
+        }
+        return $ret;
+    }
 
-	/**
-	 * starts new process
-	 * @throws Exception if no crawlerprocess was started
-	 */
-	public function startProcess() {
-		$ttl = (time() + $this->timeToLive -1);
-		$current = $this->processRepository->countNotTimeouted($ttl);
-		$completePath = '(' .escapeshellcmd($this->getCrawlerCliPath()) . ' &) > /dev/null';
-		if (system($completePath) === FALSE) {
-			throw new Exception('could not start process!');
-		}
-		else {
-			for ($i=0;$i<10;$i++) {
-				if ($this->processRepository->countNotTimeouted($ttl) > $current) {
-					return true;
-				}
-				sleep(1);
-			}
-			throw new Exception('Something went wrong: process did not appear within 10 seconds.');
-		}
-	}
+    /**
+     * starts new process
+     * @throws Exception if no crawlerprocess was started
+     */
+    public function startProcess() {
+        $ttl = (time() + $this->timeToLive -1);
+        $current = $this->processRepository->countNotTimeouted($ttl);
+        $completePath = '(' .escapeshellcmd($this->getCrawlerCliPath()) . ' &) > /dev/null';
+        if (system($completePath) === FALSE) {
+            throw new Exception('could not start process!');
+        }
+        else {
+            for ($i=0;$i<10;$i++) {
+                if ($this->processRepository->countNotTimeouted($ttl) > $current) {
+                    return true;
+                }
+                sleep(1);
+            }
+            throw new Exception('Something went wrong: process did not appear within 10 seconds.');
+        }
+    }
 
-	/**
-	 * Returns the path to start the crawler from the command line
-	 *
-	 * @return string
-	 */
-	public function getCrawlerCliPath(){
-		$phpPath 		= $this->crawlerObj->extensionSettings['phpPath'] . ' ';
-		$pathToTypo3 	= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_DOCUMENT_ROOT'), '/');
-		$pathToTypo3 	.= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_PATH'), '/');
-		$cliPart	 	= '/typo3/cli_dispatch.phpsh crawler';
-		return $phpPath.$pathToTypo3.$cliPart;
-	}
+    /**
+     * Returns the path to start the crawler from the command line
+     *
+     * @return string
+     */
+    public function getCrawlerCliPath(){
+        $phpPath 		= $this->crawlerObj->extensionSettings['phpPath'] . ' ';
+        $pathToTypo3 	= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_DOCUMENT_ROOT'), '/');
+        $pathToTypo3 	.= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_PATH'), '/');
+        $cliPart	 	= '/typo3/cli_dispatch.phpsh crawler';
+        return $phpPath.$pathToTypo3.$cliPart;
+    }
 
 }

Spacing +19 added lines, -19 removed lines

@@ -26,7 +26,7 @@ 
  * Manages cralwer processes and can be used to start a new process or multiple processes
  *
  */
-class tx_crawler_domain_process_manager  {
+class tx_crawler_domain_process_manager {
 	/**
 	 * @var $timeToLive integer
 	 */
@@ -65,8 +65,8 @@ 
 	 * the constructor
 	 */
 	public function __construct() {
-		$this->processRepository	= new tx_crawler_domain_process_repository();
-		$this->queueRepository	= new tx_crawler_domain_queue_repository();
+		$this->processRepository = new tx_crawler_domain_process_repository();
+		$this->queueRepository = new tx_crawler_domain_queue_repository();
 		$this->crawlerObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_lib');
 		$this->timeToLive = intval($this->crawlerObj->extensionSettings['processMaxRunTime']);
 		$this->countInARun = intval($this->crawlerObj->extensionSettings['countInARun']);
@@ -81,7 +81,7 @@ 
 	 */
 	public function multiProcess($timeout) {
 		if ($this->processLimit <= 1) {
-			throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.' . PHP_EOL);
+			throw new RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.'.PHP_EOL);
 		}
 
 		$pendingItemsStart = $this->queueRepository->countAllPendingItems();
@@ -99,7 +99,7 @@ 
 			}
 			if ($currentPendingItems == 0) {
 				if ($this->verbose) {
-					echo 'Finished...' . chr(10);
+					echo 'Finished...'.chr(10);
 				}
 				break;
 			}
@@ -111,16 +111,16 @@ 
 			}
 			sleep(1);
 			if ($nextTimeOut < time()) {
-				$timedOutProcesses = $this->processRepository->findAll('', 'DESC', NULL, 0, 'ttl >' . $nextTimeOut);
+				$timedOutProcesses = $this->processRepository->findAll('', 'DESC', NULL, 0, 'ttl >'.$nextTimeOut);
 				$nextTimeOut = time() + $this->timeToLive;
 				if ($this->verbose) {
-					echo 'Cleanup' . implode(',', $timedOutProcesses->getProcessIds()) . chr(10);
+					echo 'Cleanup'.implode(',', $timedOutProcesses->getProcessIds()).chr(10);
 				}
 				$this->crawlerObj->CLI_releaseProcesses($timedOutProcesses->getProcessIds(), true);
 			}
 		}
 		if ($currentPendingItems > 0 && $this->verbose) {
-			echo 'Stop with timeout' . chr(10);
+			echo 'Stop with timeout'.chr(10);
 		}
 	}
 
@@ -138,17 +138,17 @@ 
 	 */
 	private function startRequiredProcesses() {
 		$ret = FALSE;
-		$currentProcesses= $this->processRepository->countActive();
-		$availableProcessesCount = $this->processLimit-$currentProcesses;
+		$currentProcesses = $this->processRepository->countActive();
+		$availableProcessesCount = $this->processLimit - $currentProcesses;
 		$requiredProcessesCount = ceil($this->queueRepository->countAllUnassignedPendingItems() / $this->countInARun);
-		$startProcessCount = min(array($availableProcessesCount,$requiredProcessesCount));
+		$startProcessCount = min(array($availableProcessesCount, $requiredProcessesCount));
 		if ($startProcessCount <= 0) {
 			return $ret;
 		}
 		if ($startProcessCount && $this->verbose) {
 			echo 'Start '.$startProcessCount.' new processes (Running:'.$currentProcesses.')';
 		}
-		for($i=0;$i<$startProcessCount;$i++) {
+		for ($i = 0; $i < $startProcessCount; $i++) {
 			usleep(100);
 			if ($this->startProcess()) {
 				if ($this->verbose) {
@@ -168,14 +168,14 @@ 
 	 * @throws Exception if no crawlerprocess was started
 	 */
 	public function startProcess() {
-		$ttl = (time() + $this->timeToLive -1);
+		$ttl = (time() + $this->timeToLive - 1);
 		$current = $this->processRepository->countNotTimeouted($ttl);
-		$completePath = '(' .escapeshellcmd($this->getCrawlerCliPath()) . ' &) > /dev/null';
+		$completePath = '('.escapeshellcmd($this->getCrawlerCliPath()).' &) > /dev/null';
 		if (system($completePath) === FALSE) {
 			throw new Exception('could not start process!');
 		}
 		else {
-			for ($i=0;$i<10;$i++) {
+			for ($i = 0; $i < 10; $i++) {
 				if ($this->processRepository->countNotTimeouted($ttl) > $current) {
 					return true;
 				}
@@ -190,10 +190,10 @@ 
 	 *
 	 * @return string
 	 */
-	public function getCrawlerCliPath(){
-		$phpPath 		= $this->crawlerObj->extensionSettings['phpPath'] . ' ';
-		$pathToTypo3 	= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_DOCUMENT_ROOT'), '/');
-		$pathToTypo3 	.= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_PATH'), '/');
+	public function getCrawlerCliPath() {
+		$phpPath 		= $this->crawlerObj->extensionSettings['phpPath'].' ';
+		$pathToTypo3 = rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_DOCUMENT_ROOT'), '/');
+		$pathToTypo3 .= rtrim(\TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_PATH'), '/');
 		$cliPart	 	= '/typo3/cli_dispatch.phpsh crawler';
 		return $phpPath.$pathToTypo3.$cliPart;
 	}