AOEpeople / crawler

Classes/Hooks/StaticFileCacheCreateUriHook.php

Doc Comments +1 added lines, -1 removed lines

@@ -53,7 +53,7 @@
      * @param string $uri
      * @param TypoScriptFrontendController $frontend
      *
-     * @return array
+     * @return string[]
      *
      * @throws \Exception
      *

Classes/Api/CrawlerApi.php

Doc Comments +1 added lines, -1 removed lines

@@ -209,7 +209,7 @@
     /**
      * Determines if a page is queued
      *
-     * @param $uid
+     * @param integer $uid
      * @param bool $unprocessed_only
      * @param bool $timed_only
      * @param bool $timestamp

template/process/list.php

Indentation +5 added lines, -5 removed lines

@@ -5,11 +5,11 @@
 	<?php echo $this->getRefreshLink(); ?>
 	<?php echo $this->getEnableDisableLink(); ?>
 	<?php
-		// Check if ActiveProcess is reached
-		if (\TYPO3\CMS\Core\Utility\MathUtility::convertToPositiveInteger($this->getActiveProcessCount()) < \TYPO3\CMS\Core\Utility\MathUtility::convertToPositiveInteger($this->getMaxActiveProcessCount())) {
-			echo $this->getAddLink();
-		}
-	?>
+        // Check if ActiveProcess is reached
+        if (\TYPO3\CMS\Core\Utility\MathUtility::convertToPositiveInteger($this->getActiveProcessCount()) < \TYPO3\CMS\Core\Utility\MathUtility::convertToPositiveInteger($this->getMaxActiveProcessCount())) {
+            echo $this->getAddLink();
+        }
+    ?>
 	<?php echo $this->getModeLink(); ?>
 </div>
 

cli/crawler_multiprocess.php

Indentation +3 added lines, -3 removed lines

@@ -1,13 +1,13 @@
 <?php
 if (!defined('TYPO3_cliMode')) {
-	die('You cannot run this script directly!');
+    die('You cannot run this script directly!');
 }
 
 $processManager = new tx_crawler_domain_process_manager();
 $timeout = isset($_SERVER['argv'][1] ) ? intval($_SERVER['argv'][1]) : 10000;
 
 try {
-	$processManager->multiProcess($timeout);
+    $processManager->multiProcess($timeout);
 } catch (Exception $e) {
-	echo PHP_EOL . $e->getMessage();
+    echo PHP_EOL . $e->getMessage();
 }

cli/class.tx_crawler_cli.php

Indentation +20 added lines, -20 removed lines

@@ -37,29 +37,29 @@
  */

 class tx_crawler_cli extends \TYPO3\CMS\Core\Controller\CommandLineController {

 

-	/**

-	 * Constructor

-	 *

-	 * @return	void

-	 */

-	function __construct() {

-		parent::__construct();

+    /**

+     * Constructor

+     *

+     * @return	void

+     */

+    function __construct() {

+        parent::__construct();

 

-		$this->cli_options[] = array('-h', 'Show the help', '');

-		$this->cli_options[] = array('--help', 'Same as -h', '');

-		$this->cli_options[] = array('--countInARun count', 'Amount of pages', 'How many pages should be crawled during that run.');

-		$this->cli_options[] = array('--sleepTime milliseconds', 'Millisecounds to relax system during crawls', 'Amount of millisecounds which the system should use to relax between crawls.');

-		$this->cli_options[] = array('--sleepAfterFinish seconds', 'Secounds to relax system after all crawls.', 'Amount of secounds which the system should use to relax after all crawls are done.');

+        $this->cli_options[] = array('-h', 'Show the help', '');

+        $this->cli_options[] = array('--help', 'Same as -h', '');

+        $this->cli_options[] = array('--countInARun count', 'Amount of pages', 'How many pages should be crawled during that run.');

+        $this->cli_options[] = array('--sleepTime milliseconds', 'Millisecounds to relax system during crawls', 'Amount of millisecounds which the system should use to relax between crawls.');

+        $this->cli_options[] = array('--sleepAfterFinish seconds', 'Secounds to relax system after all crawls.', 'Amount of secounds which the system should use to relax after all crawls are done.');

 

-		// Setting help texts:

-		$this->cli_help['name'] = 'crawler CLI interface -- Crawling the URLs from the queue';

-		$this->cli_help['synopsis'] = '###OPTIONS###';

-		$this->cli_help['description'] = "";

-		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler\nWill trigger the crawler which starts to process the queue entires\n";

-		$this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2010';

-	}

+        // Setting help texts:

+        $this->cli_help['name'] = 'crawler CLI interface -- Crawling the URLs from the queue';

+        $this->cli_help['synopsis'] = '###OPTIONS###';

+        $this->cli_help['description'] = "";

+        $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler\nWill trigger the crawler which starts to process the queue entires\n";

+        $this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2010';

+    }

 }

 

 if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli.php']) {

-	include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli.php']);

+    include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli.php']);

 }

cli/class.tx_crawler_cli_im.php

Indentation +30 added lines, -30 removed lines

@@ -37,41 +37,41 @@
  */

 class tx_crawler_cli_im extends \TYPO3\CMS\Core\Controller\CommandLineController {

 

-	/**

-	 * Constructor

-	 *

-	 * @return	void

-	 */

-	function __construct() {

-		parent::__construct();

+    /**

+     * Constructor

+     *

+     * @return	void

+     */

+    function __construct() {

+        parent::__construct();

 

-		// Adding options to help archive:

-		/**

-		* We removed the "proc" option as it seemd not to be working any more. But as the complete handling of the crawler has changed regarding the configuration

-		* this is completely ok. Since configuration records were introduced to configure "what should be done" additionally to page ts the way to setup jobs

-		* has drifted from selecting filtering processing instructions to selecting/filtering configuration keys (you can configure the processing instructions there).

-		* This is also reflected in the backend modules and allows you a much clearer and powerful way to work with the crawler extension.

-		*/

-		// $this->cli_options[] = array('-proc listOfProcInstr', 'Comma list of processing instructions. These are the "actions" carried out when crawling and you must specify at least one. Depends on third-party extensions. Examples are "tx_cachemgm_recache" from "cachemgm" extension (will recache pages), "tx_staticpub_publish" from "staticpub" (publishing pages to static files) or "tx_indexedsearch_reindex" from "indexed_search" (indexes pages).');

-		// TODO: cleanup here!

-		$this->cli_options[] = array('-d depth', 'Tree depth, 0-99', "How many levels under the 'page_id' to include.");

-		$this->cli_options[] = array('-o mode', 'Output mode: "url", "exec", "queue"', "Specifies output modes\nurl : Will list URLs which wget could use as input.\nqueue: Will put entries in queue table.\nexec: Will execute all entries right away!");

-		$this->cli_options[] = array('-n number', 'Number of items per minute.', 'Specifies how many items are put in the queue per minute. Only valid for output mode "queue"');

-		$this->cli_options[] = array('-conf configurationkeys','List of Configuration Keys','A commaseperated list of crawler configurations');

-		#		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");

+        // Adding options to help archive:

+        /**

+         * We removed the "proc" option as it seemd not to be working any more. But as the complete handling of the crawler has changed regarding the configuration

+         * this is completely ok. Since configuration records were introduced to configure "what should be done" additionally to page ts the way to setup jobs

+         * has drifted from selecting filtering processing instructions to selecting/filtering configuration keys (you can configure the processing instructions there).

+         * This is also reflected in the backend modules and allows you a much clearer and powerful way to work with the crawler extension.

+         */

+        // $this->cli_options[] = array('-proc listOfProcInstr', 'Comma list of processing instructions. These are the "actions" carried out when crawling and you must specify at least one. Depends on third-party extensions. Examples are "tx_cachemgm_recache" from "cachemgm" extension (will recache pages), "tx_staticpub_publish" from "staticpub" (publishing pages to static files) or "tx_indexedsearch_reindex" from "indexed_search" (indexes pages).');

+        // TODO: cleanup here!

+        $this->cli_options[] = array('-d depth', 'Tree depth, 0-99', "How many levels under the 'page_id' to include.");

+        $this->cli_options[] = array('-o mode', 'Output mode: "url", "exec", "queue"', "Specifies output modes\nurl : Will list URLs which wget could use as input.\nqueue: Will put entries in queue table.\nexec: Will execute all entries right away!");

+        $this->cli_options[] = array('-n number', 'Number of items per minute.', 'Specifies how many items are put in the queue per minute. Only valid for output mode "queue"');

+        $this->cli_options[] = array('-conf configurationkeys','List of Configuration Keys','A commaseperated list of crawler configurations');

+        #		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");

 

-		// Setting help texts:

-		$this->cli_help['name'] = 'crawler CLI interface -- Submitting URLs to be crawled via CLI interface.';

-		$this->cli_help['synopsis'] = 'page_id ###OPTIONS###';

-		$this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It can put entries in the queue from command line options, return the list of URLs and even execute all entries right away without having to queue them up - this can be useful for immediate re-cache, re-indexing or static publishing from command line.";

-		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_im 7 -d=2 -conf=<configurationKey> -o=exec\nWill re-cache pages from page 7 and two levels down, executed immediately.\n";

-		$this->cli_help['examples'].= "/.../cli_dispatch.phpsh crawler_im 7 -d=0 -conf=<configurationKey> -n=4 -o=queue\nWill put entries for re-caching pages from page 7 into queue, 4 every minute.\n";

-		$this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';

-	}

+        // Setting help texts:

+        $this->cli_help['name'] = 'crawler CLI interface -- Submitting URLs to be crawled via CLI interface.';

+        $this->cli_help['synopsis'] = 'page_id ###OPTIONS###';

+        $this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It can put entries in the queue from command line options, return the list of URLs and even execute all entries right away without having to queue them up - this can be useful for immediate re-cache, re-indexing or static publishing from command line.";

+        $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_im 7 -d=2 -conf=<configurationKey> -o=exec\nWill re-cache pages from page 7 and two levels down, executed immediately.\n";

+        $this->cli_help['examples'].= "/.../cli_dispatch.phpsh crawler_im 7 -d=0 -conf=<configurationKey> -n=4 -o=queue\nWill put entries for re-caching pages from page 7 into queue, 4 every minute.\n";

+        $this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';

+    }

 }

 

 if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_im.php']) {

-	include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_im.php']);

+    include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_im.php']);

 }

 

 ?>

cli/bootstrap.php

Indentation +38 added lines, -38 removed lines

@@ -5,20 +5,20 @@ 
  * Retrieve path (taken from cli_dispatch.phpsh)
  */
 
-	// Get path to this script
+    // Get path to this script
 $tempPathThisScript = isset($_SERVER['argv'][0]) ? $_SERVER['argv'][0] : (isset($_ENV['_']) ? $_ENV['_'] : $_SERVER['_']);
 
-	// Resolve path
+    // Resolve path
 if (!isAbsPath($tempPathThisScript)) {
-	$workingDirectory = $_SERVER['PWD'] ? $_SERVER['PWD'] : getcwd();
-	if ($workingDirectory) {
-		$tempPathThisScript = $workingDirectory . '/' . preg_replace('/\.\//', '', $tempPathThisScript);
-		if (!@is_file($tempPathThisScript)) {
-			die('Relative path found, but an error occured during resolving of the absolute path: ' . $tempPathThisScript . PHP_EOL);
-		}
-	} else {
-		die('Relative path found, but resolving absolute path is not supported on this platform.' . PHP_EOL);
-	}
+    $workingDirectory = $_SERVER['PWD'] ? $_SERVER['PWD'] : getcwd();
+    if ($workingDirectory) {
+        $tempPathThisScript = $workingDirectory . '/' . preg_replace('/\.\//', '', $tempPathThisScript);
+        if (!@is_file($tempPathThisScript)) {
+            die('Relative path found, but an error occured during resolving of the absolute path: ' . $tempPathThisScript . PHP_EOL);
+        }
+    } else {
+        die('Relative path found, but resolving absolute path is not supported on this platform.' . PHP_EOL);
+    }
 }
 
 $typo3Root = preg_replace('#typo3conf/ext/crawler/cli/bootstrap.php$#', '', $tempPathThisScript);
@@ -30,33 +30,33 @@ 
  */
 $additionalHeaders = unserialize(base64_decode($_SERVER['argv'][3]));
 if (is_array($additionalHeaders)) {
-	foreach ($additionalHeaders as $additionalHeader) {
-		if (strpos($additionalHeader, ':') !== FALSE) {
-			list($key, $value) = explode(':', $additionalHeader, 2);
-			$key = str_replace('-', '_', strtoupper(trim($key)));
-			if ($key != 'HOST') {
-				$_SERVER['HTTP_' . $key] = $value;
-			}
-		}
-	}
+    foreach ($additionalHeaders as $additionalHeader) {
+        if (strpos($additionalHeader, ':') !== FALSE) {
+            list($key, $value) = explode(':', $additionalHeader, 2);
+            $key = str_replace('-', '_', strtoupper(trim($key)));
+            if ($key != 'HOST') {
+                $_SERVER['HTTP_' . $key] = $value;
+            }
+        }
+    }
 }
 
 
-	// put parsed query parts into $_GET array
+    // put parsed query parts into $_GET array
 $urlParts = parse_url($_SERVER['argv'][2]);
-	// Populating $_GET
+    // Populating $_GET
 parse_str($urlParts['query'], $_GET);
-	// Populating $_REQUEST
+    // Populating $_REQUEST
 parse_str($urlParts['query'], $_REQUEST);
-	// Populating $_POST
+    // Populating $_POST
 $_POST = array();
-	// Populating $_COOKIE
+    // Populating $_COOKIE
 $_COOKIE = array();
 
-	// Get the TYPO3_SITE_PATH of the website frontend:
+    // Get the TYPO3_SITE_PATH of the website frontend:
 $typo3SitePath = $_SERVER['argv'][1];
 
-	// faking the environment
+    // faking the environment
 $_SERVER['DOCUMENT_ROOT'] = preg_replace('#' . preg_quote($typo3SitePath, '#') . '$#', '', $typo3Root);
 $_SERVER['HTTP_USER_AGENT'] = 'CLI Mode';
 $_SERVER['HTTP_HOST'] = $_SERVER['SERVER_NAME'] = $urlParts['host'];
@@ -66,15 +66,15 @@ 
 $_SERVER['REQUEST_URI'] = $urlParts['path'] . (isset($urlParts['query']) ? '?' . $urlParts['query'] : '');
 $_SERVER['REQUEST_METHOD'] = 'GET';
 
-	// Define a port if used in the URL:
+    // Define a port if used in the URL:
 if (isset($urlParts['port'])) {
-	$_SERVER['HTTP_HOST'] .= ':' . $urlParts['port'];
-	$_SERVER['SERVER_PORT'] = $urlParts['port'];
+    $_SERVER['HTTP_HOST'] .= ':' . $urlParts['port'];
+    $_SERVER['SERVER_PORT'] = $urlParts['port'];
 }
 
-	// Define HTTPS disposal:
+    // Define HTTPS disposal:
 if ($urlParts['scheme'] === 'https') {
-	$_SERVER['HTTPS'] = 'on';
+    $_SERVER['HTTPS'] = 'on';
 }
 
 chdir($typo3Root);
@@ -88,11 +88,11 @@ 
  * @return boolean
  */
 function isAbsPath($path) {
-		// on Windows also a path starting with a drive letter is absolute: X:/
-	if (stristr(PHP_OS, 'win') && substr($path, 1, 2) === ':/') {
-		return TRUE;
-	}
+        // on Windows also a path starting with a drive letter is absolute: X:/
+    if (stristr(PHP_OS, 'win') && substr($path, 1, 2) === ':/') {
+        return TRUE;
+    }
 
-		// path starting with a / is always absolute, on every system
-	return (substr($path, 0, 1) === '/');
+        // path starting with a / is always absolute, on every system
+    return (substr($path, 0, 1) === '/');
 }

cli/class.tx_crawler_cli_flush.php

Indentation +19 added lines, -19 removed lines

@@ -37,30 +37,30 @@
  */
 class tx_crawler_cli_flush extends \TYPO3\CMS\Core\Controller\CommandLineController {
 
-	/**
-	 * Constructor
-	 *
-	 * @return	void
-	 */
-	function __construct() {
-		parent::__construct();
+    /**
+     * Constructor
+     *
+     * @return	void
+     */
+    function __construct() {
+        parent::__construct();
 
-			// Adding options to help archive:
-		$this->cli_options[] = array('-o mode', 'Output mode: "finished", "all", "pending"', "Specifies the type queue entries which is flushed in the process.");
-		#		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");
+            // Adding options to help archive:
+        $this->cli_options[] = array('-o mode', 'Output mode: "finished", "all", "pending"', "Specifies the type queue entries which is flushed in the process.");
+        #		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");
 
-			// Setting help texts:
-		$this->cli_help['name'] = 'crawler CLI interface -- Cleaning up the queue.';
-		$this->cli_help['synopsis'] = 'page_id ###OPTIONS###';
-		$this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It will remove queue entires and perform a cleanup.";
-		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_flush 5 -o=finished\nWill remove all finished queue-entries in the sub-branch of page 5\n";
-		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_flush 0 -o=all\nWill remove all queue-entries for every page\n";
-		$this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';
-	}
+            // Setting help texts:
+        $this->cli_help['name'] = 'crawler CLI interface -- Cleaning up the queue.';
+        $this->cli_help['synopsis'] = 'page_id ###OPTIONS###';
+        $this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It will remove queue entires and perform a cleanup.";
+        $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_flush 5 -o=finished\nWill remove all finished queue-entries in the sub-branch of page 5\n";
+        $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_flush 0 -o=all\nWill remove all queue-entries for every page\n";
+        $this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';
+    }
 }
 
 if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_flush.php']) {
-	include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_flush.php']);
+    include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_flush.php']);
 }
 
 ?>

domain/lib/class.tx_crawler_domain_lib_abstract_dbobject.php

Indentation +33 added lines, -33 removed lines

@@ -24,43 +24,43 @@
 
 abstract class tx_crawler_domain_lib_abstract_dbobject {
 
-	/**
-	 * @var array
-	 */
-	protected $row;
+    /**
+     * @var array
+     */
+    protected $row;
 
-	/**
-	 * @var string table name
-	 */
-	protected static $tableName;
+    /**
+     * @var string table name
+     */
+    protected static $tableName;
 
-	/**
-	 * Constructor
-	 *
-	 * @param array $row optional array with propertys
-	 */
-	public function __construct($row = array()) {
-		$this->row = $row;
-	}
+    /**
+     * Constructor
+     *
+     * @param array $row optional array with propertys
+     */
+    public function __construct($row = array()) {
+        $this->row = $row;
+    }
 
-	/**
-	 * Get table name
-	 *
-	 * @param void
-	 * @return string table name
-	 */
-	public static function getTableName(){
-		return self::$tableName;
-	}
+    /**
+     * Get table name
+     *
+     * @param void
+     * @return string table name
+     */
+    public static function getTableName(){
+        return self::$tableName;
+    }
 
-	/**
-	 * Returns the propertys of the object as array
-	 *
-	 * @return array
-	 */
-	public function getRow() {
-		return $this->row;
-	}
+    /**
+     * Returns the propertys of the object as array
+     *
+     * @return array
+     */
+    public function getRow() {
+        return $this->row;
+    }
 
 
 }