AOEpeople / crawler

template/pagination.php

Spacing +2 added lines, -2 removed lines

@@ -1,11 +1,11 @@
 <?php if (!defined('TYPO3_MODE')) die ('Access denied.'); ?>
 
 Page:
-<?php for($currentPageOffset 	= 0; $currentPageOffset < $this->getTotalPagesCount(); $currentPageOffset++ ){  ?>
+<?php for ($currentPageOffset = 0; $currentPageOffset < $this->getTotalPagesCount(); $currentPageOffset++) {  ?>
 	<a href="index.php?offset=<?php echo htmlspecialchars($currentPageOffset * $this->getPerPage()); ?>">
 		<?php echo	htmlspecialchars($this->getLabelForPageOffset($currentPageOffset)); ?>
 	</a>
-	<?php if($currentPageOffset+1 < $this->getTotalPagesCount()){ ?>
+	<?php if ($currentPageOffset + 1 < $this->getTotalPagesCount()) { ?>
 	|
 	<?php } ?>
 

template/process/list.php

Indentation +5 added lines, -5 removed lines

@@ -5,11 +5,11 @@
 	<?php echo $this->getRefreshLink(); ?>
 	<?php echo $this->getEnableDisableLink(); ?>
 	<?php
-		// Check if ActiveProcess is reached
-		if (\TYPO3\CMS\Core\Utility\MathUtility::convertToPositiveInteger($this->getActiveProcessCount()) < \TYPO3\CMS\Core\Utility\MathUtility::convertToPositiveInteger($this->getMaxActiveProcessCount())) {
-			echo $this->getAddLink();
-		}
-	?>
+        // Check if ActiveProcess is reached
+        if (\TYPO3\CMS\Core\Utility\MathUtility::convertToPositiveInteger($this->getActiveProcessCount()) < \TYPO3\CMS\Core\Utility\MathUtility::convertToPositiveInteger($this->getMaxActiveProcessCount())) {
+            echo $this->getAddLink();
+        }
+    ?>
 	<?php echo $this->getModeLink(); ?>
 </div>
 

Spacing +4 added lines, -4 removed lines

@@ -50,17 +50,17 @@
 		</tr>
 	</thead>
 	<tbody>
-		<?php foreach($this->getProcessCollection() as $process): /* @var $process tx_crawler_domain_process */ ?>
-			<tr class="<?php echo (++$count % 2 == 0) ? 'odd': 'even' ?>">
+		<?php foreach ($this->getProcessCollection() as $process): /* @var $process tx_crawler_domain_process */ ?>
+			<tr class="<?php echo (++$count % 2 == 0) ? 'odd' : 'even' ?>">
 				<td><?php echo $this->getIconForState(htmlspecialchars($process->getState())); ?></td>
 				<td><?php echo htmlspecialchars($process->getProcess_id()); ?></td>
 				<td><?php echo htmlspecialchars($this->asDate($process->getTimeForFirstItem())); ?></td>
 				<td><?php echo htmlspecialchars($this->asDate($process->getTimeForLastItem())); ?></td>
-				<td><?php echo htmlspecialchars(floor($process->getRuntime()/ 60)); ?> min. <?php echo htmlspecialchars($process->getRuntime()) % 60 ?> sec.</td>
+				<td><?php echo htmlspecialchars(floor($process->getRuntime() / 60)); ?> min. <?php echo htmlspecialchars($process->getRuntime()) % 60 ?> sec.</td>
 				<td><?php echo htmlspecialchars($this->asDate($process->getTTL())); ?></td>
 				<td><?php echo htmlspecialchars($process->countItemsProcessed()); ?></td>
 				<td><?php echo htmlspecialchars($process->countItemsAssigned()); ?></td>
-				<td><?php echo htmlspecialchars($process->countItemsToProcess()+$process->countItemsProcessed()); ?></td>
+				<td><?php echo htmlspecialchars($process->countItemsToProcess() + $process->countItemsProcessed()); ?></td>
 				<td>
 				<?php if ($process->getState() == 'running'): ?>
 					<div class="crawlerprocessprogress" style="width: 200px;">

ext_autoload.php

Indentation +20 added lines, -20 removed lines

@@ -1,24 +1,24 @@
 <?php
 $extensionPath = \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath('crawler');
 return array(
-	'tx_crawler_lib' => $extensionPath . 'class.tx_crawler_lib.php',
-	'tx_crawler_cli_flush' => $extensionPath . 'cli/class.tx_crawler_cli_flush.php',
-	'tx_crawler_cli' => $extensionPath . 'cli/class.tx_crawler_cli.php',
-	'tx_crawler_cli_im' => $extensionPath . 'cli/class.tx_crawler_cli_im.php',
-	'tx_crawler_domain_events_dispatcher' => $extensionPath . 'domain/events/class.tx_crawler_domain_events_dispatcher.php',
-	'tx_crawler_domain_events_observer' => $extensionPath . 'domain/events/interface.tx_crawler_domain_events_observer.php',
-	'tx_crawler_domain_lib_abstract_dbobject' => $extensionPath . 'domain/lib/class.tx_crawler_domain_lib_abstract_dbobject.php',
-	'tx_crawler_domain_process_manager' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_manager.php',
-	'tx_crawler_domain_process' => $extensionPath . 'domain/process/class.tx_crawler_domain_process.php',
-	'tx_crawler_domain_process_collection' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_collection.php',
-	'tx_crawler_domain_process_repository' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_repository.php',
-	'tx_crawler_domain_queue_entry' => $extensionPath . 'domain/queue/class.tx_crawler_domain_queue_entry.php',
-	'tx_crawler_domain_queue_repository' => $extensionPath . 'domain/queue/class.tx_crawler_domain_queue_repository.php',
-	'tx_crawler_domain_reason' => $extensionPath . 'domain/reason/class.tx_crawler_domain_reason.php',
-	'tx_crawler_hooks_tsfe' => $extensionPath . 'hooks/class.tx_crawler_hooks_tsfe.php',
-	'tx_crawler_hooks_staticFileCacheCreateUri' => $extensionPath . 'hooks/class.tx_crawler_hooks_staticFileCacheCreateUri.php',
-	'tx_crawler_hooks_processCleanUp' => $extensionPath . 'hooks/class.tx_crawler_hooks_processCleanUp.php',
-	'tx_crawler_modfunc1' => $extensionPath . 'modfunc1/class.tx_crawler_modfunc1.php',
-	'tx_crawler_view_pagination' => $extensionPath . 'view/class.tx_crawler_view_pagination.php',
-	'tx_crawler_view_process_list' => $extensionPath . 'view/process/class.tx_crawler_view_process_list.php',
+    'tx_crawler_lib' => $extensionPath . 'class.tx_crawler_lib.php',
+    'tx_crawler_cli_flush' => $extensionPath . 'cli/class.tx_crawler_cli_flush.php',
+    'tx_crawler_cli' => $extensionPath . 'cli/class.tx_crawler_cli.php',
+    'tx_crawler_cli_im' => $extensionPath . 'cli/class.tx_crawler_cli_im.php',
+    'tx_crawler_domain_events_dispatcher' => $extensionPath . 'domain/events/class.tx_crawler_domain_events_dispatcher.php',
+    'tx_crawler_domain_events_observer' => $extensionPath . 'domain/events/interface.tx_crawler_domain_events_observer.php',
+    'tx_crawler_domain_lib_abstract_dbobject' => $extensionPath . 'domain/lib/class.tx_crawler_domain_lib_abstract_dbobject.php',
+    'tx_crawler_domain_process_manager' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_manager.php',
+    'tx_crawler_domain_process' => $extensionPath . 'domain/process/class.tx_crawler_domain_process.php',
+    'tx_crawler_domain_process_collection' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_collection.php',
+    'tx_crawler_domain_process_repository' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_repository.php',
+    'tx_crawler_domain_queue_entry' => $extensionPath . 'domain/queue/class.tx_crawler_domain_queue_entry.php',
+    'tx_crawler_domain_queue_repository' => $extensionPath . 'domain/queue/class.tx_crawler_domain_queue_repository.php',
+    'tx_crawler_domain_reason' => $extensionPath . 'domain/reason/class.tx_crawler_domain_reason.php',
+    'tx_crawler_hooks_tsfe' => $extensionPath . 'hooks/class.tx_crawler_hooks_tsfe.php',
+    'tx_crawler_hooks_staticFileCacheCreateUri' => $extensionPath . 'hooks/class.tx_crawler_hooks_staticFileCacheCreateUri.php',
+    'tx_crawler_hooks_processCleanUp' => $extensionPath . 'hooks/class.tx_crawler_hooks_processCleanUp.php',
+    'tx_crawler_modfunc1' => $extensionPath . 'modfunc1/class.tx_crawler_modfunc1.php',
+    'tx_crawler_view_pagination' => $extensionPath . 'view/class.tx_crawler_view_pagination.php',
+    'tx_crawler_view_process_list' => $extensionPath . 'view/process/class.tx_crawler_view_process_list.php',
 );

Spacing +20 added lines, -20 removed lines

@@ -1,24 +1,24 @@
 <?php
 $extensionPath = \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath('crawler');
 return array(
-	'tx_crawler_lib' => $extensionPath . 'class.tx_crawler_lib.php',
-	'tx_crawler_cli_flush' => $extensionPath . 'cli/class.tx_crawler_cli_flush.php',
-	'tx_crawler_cli' => $extensionPath . 'cli/class.tx_crawler_cli.php',
-	'tx_crawler_cli_im' => $extensionPath . 'cli/class.tx_crawler_cli_im.php',
-	'tx_crawler_domain_events_dispatcher' => $extensionPath . 'domain/events/class.tx_crawler_domain_events_dispatcher.php',
-	'tx_crawler_domain_events_observer' => $extensionPath . 'domain/events/interface.tx_crawler_domain_events_observer.php',
-	'tx_crawler_domain_lib_abstract_dbobject' => $extensionPath . 'domain/lib/class.tx_crawler_domain_lib_abstract_dbobject.php',
-	'tx_crawler_domain_process_manager' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_manager.php',
-	'tx_crawler_domain_process' => $extensionPath . 'domain/process/class.tx_crawler_domain_process.php',
-	'tx_crawler_domain_process_collection' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_collection.php',
-	'tx_crawler_domain_process_repository' => $extensionPath . 'domain/process/class.tx_crawler_domain_process_repository.php',
-	'tx_crawler_domain_queue_entry' => $extensionPath . 'domain/queue/class.tx_crawler_domain_queue_entry.php',
-	'tx_crawler_domain_queue_repository' => $extensionPath . 'domain/queue/class.tx_crawler_domain_queue_repository.php',
-	'tx_crawler_domain_reason' => $extensionPath . 'domain/reason/class.tx_crawler_domain_reason.php',
-	'tx_crawler_hooks_tsfe' => $extensionPath . 'hooks/class.tx_crawler_hooks_tsfe.php',
-	'tx_crawler_hooks_staticFileCacheCreateUri' => $extensionPath . 'hooks/class.tx_crawler_hooks_staticFileCacheCreateUri.php',
-	'tx_crawler_hooks_processCleanUp' => $extensionPath . 'hooks/class.tx_crawler_hooks_processCleanUp.php',
-	'tx_crawler_modfunc1' => $extensionPath . 'modfunc1/class.tx_crawler_modfunc1.php',
-	'tx_crawler_view_pagination' => $extensionPath . 'view/class.tx_crawler_view_pagination.php',
-	'tx_crawler_view_process_list' => $extensionPath . 'view/process/class.tx_crawler_view_process_list.php',
+	'tx_crawler_lib' => $extensionPath.'class.tx_crawler_lib.php',
+	'tx_crawler_cli_flush' => $extensionPath.'cli/class.tx_crawler_cli_flush.php',
+	'tx_crawler_cli' => $extensionPath.'cli/class.tx_crawler_cli.php',
+	'tx_crawler_cli_im' => $extensionPath.'cli/class.tx_crawler_cli_im.php',
+	'tx_crawler_domain_events_dispatcher' => $extensionPath.'domain/events/class.tx_crawler_domain_events_dispatcher.php',
+	'tx_crawler_domain_events_observer' => $extensionPath.'domain/events/interface.tx_crawler_domain_events_observer.php',
+	'tx_crawler_domain_lib_abstract_dbobject' => $extensionPath.'domain/lib/class.tx_crawler_domain_lib_abstract_dbobject.php',
+	'tx_crawler_domain_process_manager' => $extensionPath.'domain/process/class.tx_crawler_domain_process_manager.php',
+	'tx_crawler_domain_process' => $extensionPath.'domain/process/class.tx_crawler_domain_process.php',
+	'tx_crawler_domain_process_collection' => $extensionPath.'domain/process/class.tx_crawler_domain_process_collection.php',
+	'tx_crawler_domain_process_repository' => $extensionPath.'domain/process/class.tx_crawler_domain_process_repository.php',
+	'tx_crawler_domain_queue_entry' => $extensionPath.'domain/queue/class.tx_crawler_domain_queue_entry.php',
+	'tx_crawler_domain_queue_repository' => $extensionPath.'domain/queue/class.tx_crawler_domain_queue_repository.php',
+	'tx_crawler_domain_reason' => $extensionPath.'domain/reason/class.tx_crawler_domain_reason.php',
+	'tx_crawler_hooks_tsfe' => $extensionPath.'hooks/class.tx_crawler_hooks_tsfe.php',
+	'tx_crawler_hooks_staticFileCacheCreateUri' => $extensionPath.'hooks/class.tx_crawler_hooks_staticFileCacheCreateUri.php',
+	'tx_crawler_hooks_processCleanUp' => $extensionPath.'hooks/class.tx_crawler_hooks_processCleanUp.php',
+	'tx_crawler_modfunc1' => $extensionPath.'modfunc1/class.tx_crawler_modfunc1.php',
+	'tx_crawler_view_pagination' => $extensionPath.'view/class.tx_crawler_view_pagination.php',
+	'tx_crawler_view_process_list' => $extensionPath.'view/process/class.tx_crawler_view_process_list.php',
 );

cli/crawler_multiprocess.php

Indentation +3 added lines, -3 removed lines

@@ -1,13 +1,13 @@
 <?php
 if (!defined('TYPO3_cliMode')) {
-	die('You cannot run this script directly!');
+    die('You cannot run this script directly!');
 }
 
 $processManager = new tx_crawler_domain_process_manager();
 $timeout = isset($_SERVER['argv'][1] ) ? intval($_SERVER['argv'][1]) : 10000;
 
 try {
-	$processManager->multiProcess($timeout);
+    $processManager->multiProcess($timeout);
 } catch (Exception $e) {
-	echo PHP_EOL . $e->getMessage();
+    echo PHP_EOL . $e->getMessage();
 }

Spacing +2 added lines, -2 removed lines

@@ -4,10 +4,10 @@
 }
 
 $processManager = new tx_crawler_domain_process_manager();
-$timeout = isset($_SERVER['argv'][1] ) ? intval($_SERVER['argv'][1]) : 10000;
+$timeout = isset($_SERVER['argv'][1]) ? intval($_SERVER['argv'][1]) : 10000;
 
 try {
 	$processManager->multiProcess($timeout);
 } catch (Exception $e) {
-	echo PHP_EOL . $e->getMessage();
+	echo PHP_EOL.$e->getMessage();
 }

cli/class.tx_crawler_cli.php

Indentation +20 added lines, -20 removed lines

@@ -37,29 +37,29 @@
  */

 class tx_crawler_cli extends \TYPO3\CMS\Core\Controller\CommandLineController {

 

-	/**

-	 * Constructor

-	 *

-	 * @return	void

-	 */

-	function __construct() {

-		parent::__construct();

+    /**

+     * Constructor

+     *

+     * @return	void

+     */

+    function __construct() {

+        parent::__construct();

 

-		$this->cli_options[] = array('-h', 'Show the help', '');

-		$this->cli_options[] = array('--help', 'Same as -h', '');

-		$this->cli_options[] = array('--countInARun count', 'Amount of pages', 'How many pages should be crawled during that run.');

-		$this->cli_options[] = array('--sleepTime milliseconds', 'Millisecounds to relax system during crawls', 'Amount of millisecounds which the system should use to relax between crawls.');

-		$this->cli_options[] = array('--sleepAfterFinish seconds', 'Secounds to relax system after all crawls.', 'Amount of secounds which the system should use to relax after all crawls are done.');

+        $this->cli_options[] = array('-h', 'Show the help', '');

+        $this->cli_options[] = array('--help', 'Same as -h', '');

+        $this->cli_options[] = array('--countInARun count', 'Amount of pages', 'How many pages should be crawled during that run.');

+        $this->cli_options[] = array('--sleepTime milliseconds', 'Millisecounds to relax system during crawls', 'Amount of millisecounds which the system should use to relax between crawls.');

+        $this->cli_options[] = array('--sleepAfterFinish seconds', 'Secounds to relax system after all crawls.', 'Amount of secounds which the system should use to relax after all crawls are done.');

 

-		// Setting help texts:

-		$this->cli_help['name'] = 'crawler CLI interface -- Crawling the URLs from the queue';

-		$this->cli_help['synopsis'] = '###OPTIONS###';

-		$this->cli_help['description'] = "";

-		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler\nWill trigger the crawler which starts to process the queue entires\n";

-		$this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2010';

-	}

+        // Setting help texts:

+        $this->cli_help['name'] = 'crawler CLI interface -- Crawling the URLs from the queue';

+        $this->cli_help['synopsis'] = '###OPTIONS###';

+        $this->cli_help['description'] = "";

+        $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler\nWill trigger the crawler which starts to process the queue entires\n";

+        $this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2010';

+    }

 }

 

 if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli.php']) {

-	include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli.php']);

+    include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli.php']);

 }

cli/class.tx_crawler_cli_im.php

Indentation +30 added lines, -30 removed lines

@@ -37,41 +37,41 @@
  */

 class tx_crawler_cli_im extends \TYPO3\CMS\Core\Controller\CommandLineController {

 

-	/**

-	 * Constructor

-	 *

-	 * @return	void

-	 */

-	function __construct() {

-		parent::__construct();

+    /**

+     * Constructor

+     *

+     * @return	void

+     */

+    function __construct() {

+        parent::__construct();

 

-		// Adding options to help archive:

-		/**

-		* We removed the "proc" option as it seemd not to be working any more. But as the complete handling of the crawler has changed regarding the configuration

-		* this is completely ok. Since configuration records were introduced to configure "what should be done" additionally to page ts the way to setup jobs

-		* has drifted from selecting filtering processing instructions to selecting/filtering configuration keys (you can configure the processing instructions there).

-		* This is also reflected in the backend modules and allows you a much clearer and powerful way to work with the crawler extension.

-		*/

-		// $this->cli_options[] = array('-proc listOfProcInstr', 'Comma list of processing instructions. These are the "actions" carried out when crawling and you must specify at least one. Depends on third-party extensions. Examples are "tx_cachemgm_recache" from "cachemgm" extension (will recache pages), "tx_staticpub_publish" from "staticpub" (publishing pages to static files) or "tx_indexedsearch_reindex" from "indexed_search" (indexes pages).');

-		// TODO: cleanup here!

-		$this->cli_options[] = array('-d depth', 'Tree depth, 0-99', "How many levels under the 'page_id' to include.");

-		$this->cli_options[] = array('-o mode', 'Output mode: "url", "exec", "queue"', "Specifies output modes\nurl : Will list URLs which wget could use as input.\nqueue: Will put entries in queue table.\nexec: Will execute all entries right away!");

-		$this->cli_options[] = array('-n number', 'Number of items per minute.', 'Specifies how many items are put in the queue per minute. Only valid for output mode "queue"');

-		$this->cli_options[] = array('-conf configurationkeys','List of Configuration Keys','A commaseperated list of crawler configurations');

-		#		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");

+        // Adding options to help archive:

+        /**

+         * We removed the "proc" option as it seemd not to be working any more. But as the complete handling of the crawler has changed regarding the configuration

+         * this is completely ok. Since configuration records were introduced to configure "what should be done" additionally to page ts the way to setup jobs

+         * has drifted from selecting filtering processing instructions to selecting/filtering configuration keys (you can configure the processing instructions there).

+         * This is also reflected in the backend modules and allows you a much clearer and powerful way to work with the crawler extension.

+         */

+        // $this->cli_options[] = array('-proc listOfProcInstr', 'Comma list of processing instructions. These are the "actions" carried out when crawling and you must specify at least one. Depends on third-party extensions. Examples are "tx_cachemgm_recache" from "cachemgm" extension (will recache pages), "tx_staticpub_publish" from "staticpub" (publishing pages to static files) or "tx_indexedsearch_reindex" from "indexed_search" (indexes pages).');

+        // TODO: cleanup here!

+        $this->cli_options[] = array('-d depth', 'Tree depth, 0-99', "How many levels under the 'page_id' to include.");

+        $this->cli_options[] = array('-o mode', 'Output mode: "url", "exec", "queue"', "Specifies output modes\nurl : Will list URLs which wget could use as input.\nqueue: Will put entries in queue table.\nexec: Will execute all entries right away!");

+        $this->cli_options[] = array('-n number', 'Number of items per minute.', 'Specifies how many items are put in the queue per minute. Only valid for output mode "queue"');

+        $this->cli_options[] = array('-conf configurationkeys','List of Configuration Keys','A commaseperated list of crawler configurations');

+        #		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");

 

-		// Setting help texts:

-		$this->cli_help['name'] = 'crawler CLI interface -- Submitting URLs to be crawled via CLI interface.';

-		$this->cli_help['synopsis'] = 'page_id ###OPTIONS###';

-		$this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It can put entries in the queue from command line options, return the list of URLs and even execute all entries right away without having to queue them up - this can be useful for immediate re-cache, re-indexing or static publishing from command line.";

-		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_im 7 -d=2 -conf=<configurationKey> -o=exec\nWill re-cache pages from page 7 and two levels down, executed immediately.\n";

-		$this->cli_help['examples'].= "/.../cli_dispatch.phpsh crawler_im 7 -d=0 -conf=<configurationKey> -n=4 -o=queue\nWill put entries for re-caching pages from page 7 into queue, 4 every minute.\n";

-		$this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';

-	}

+        // Setting help texts:

+        $this->cli_help['name'] = 'crawler CLI interface -- Submitting URLs to be crawled via CLI interface.';

+        $this->cli_help['synopsis'] = 'page_id ###OPTIONS###';

+        $this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It can put entries in the queue from command line options, return the list of URLs and even execute all entries right away without having to queue them up - this can be useful for immediate re-cache, re-indexing or static publishing from command line.";

+        $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_im 7 -d=2 -conf=<configurationKey> -o=exec\nWill re-cache pages from page 7 and two levels down, executed immediately.\n";

+        $this->cli_help['examples'].= "/.../cli_dispatch.phpsh crawler_im 7 -d=0 -conf=<configurationKey> -n=4 -o=queue\nWill put entries for re-caching pages from page 7 into queue, 4 every minute.\n";

+        $this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';

+    }

 }

 

 if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_im.php']) {

-	include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_im.php']);

+    include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_im.php']);

 }

 

 ?>

Spacing +2 added lines, -2 removed lines

@@ -57,7 +57,7 @@ 
 		$this->cli_options[] = array('-d depth', 'Tree depth, 0-99', "How many levels under the 'page_id' to include.");

 		$this->cli_options[] = array('-o mode', 'Output mode: "url", "exec", "queue"', "Specifies output modes\nurl : Will list URLs which wget could use as input.\nqueue: Will put entries in queue table.\nexec: Will execute all entries right away!");

 		$this->cli_options[] = array('-n number', 'Number of items per minute.', 'Specifies how many items are put in the queue per minute. Only valid for output mode "queue"');

-		$this->cli_options[] = array('-conf configurationkeys','List of Configuration Keys','A commaseperated list of crawler configurations');

+		$this->cli_options[] = array('-conf configurationkeys', 'List of Configuration Keys', 'A commaseperated list of crawler configurations');

 		#		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");

 

 		// Setting help texts:

@@ -65,7 +65,7 @@ 
 		$this->cli_help['synopsis'] = 'page_id ###OPTIONS###';

 		$this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It can put entries in the queue from command line options, return the list of URLs and even execute all entries right away without having to queue them up - this can be useful for immediate re-cache, re-indexing or static publishing from command line.";

 		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_im 7 -d=2 -conf=<configurationKey> -o=exec\nWill re-cache pages from page 7 and two levels down, executed immediately.\n";

-		$this->cli_help['examples'].= "/.../cli_dispatch.phpsh crawler_im 7 -d=0 -conf=<configurationKey> -n=4 -o=queue\nWill put entries for re-caching pages from page 7 into queue, 4 every minute.\n";

+		$this->cli_help['examples'] .= "/.../cli_dispatch.phpsh crawler_im 7 -d=0 -conf=<configurationKey> -n=4 -o=queue\nWill put entries for re-caching pages from page 7 into queue, 4 every minute.\n";

 		$this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';

 	}

 }

cli/bootstrap.php

Indentation +38 added lines, -38 removed lines

@@ -5,20 +5,20 @@ 
  * Retrieve path (taken from cli_dispatch.phpsh)
  */
 
-	// Get path to this script
+    // Get path to this script
 $tempPathThisScript = isset($_SERVER['argv'][0]) ? $_SERVER['argv'][0] : (isset($_ENV['_']) ? $_ENV['_'] : $_SERVER['_']);
 
-	// Resolve path
+    // Resolve path
 if (!isAbsPath($tempPathThisScript)) {
-	$workingDirectory = $_SERVER['PWD'] ? $_SERVER['PWD'] : getcwd();
-	if ($workingDirectory) {
-		$tempPathThisScript = $workingDirectory . '/' . preg_replace('/\.\//', '', $tempPathThisScript);
-		if (!@is_file($tempPathThisScript)) {
-			die('Relative path found, but an error occured during resolving of the absolute path: ' . $tempPathThisScript . PHP_EOL);
-		}
-	} else {
-		die('Relative path found, but resolving absolute path is not supported on this platform.' . PHP_EOL);
-	}
+    $workingDirectory = $_SERVER['PWD'] ? $_SERVER['PWD'] : getcwd();
+    if ($workingDirectory) {
+        $tempPathThisScript = $workingDirectory . '/' . preg_replace('/\.\//', '', $tempPathThisScript);
+        if (!@is_file($tempPathThisScript)) {
+            die('Relative path found, but an error occured during resolving of the absolute path: ' . $tempPathThisScript . PHP_EOL);
+        }
+    } else {
+        die('Relative path found, but resolving absolute path is not supported on this platform.' . PHP_EOL);
+    }
 }
 
 $typo3Root = preg_replace('#typo3conf/ext/crawler/cli/bootstrap.php$#', '', $tempPathThisScript);
@@ -30,33 +30,33 @@ 
  */
 $additionalHeaders = unserialize(base64_decode($_SERVER['argv'][3]));
 if (is_array($additionalHeaders)) {
-	foreach ($additionalHeaders as $additionalHeader) {
-		if (strpos($additionalHeader, ':') !== FALSE) {
-			list($key, $value) = explode(':', $additionalHeader, 2);
-			$key = str_replace('-', '_', strtoupper(trim($key)));
-			if ($key != 'HOST') {
-				$_SERVER['HTTP_' . $key] = $value;
-			}
-		}
-	}
+    foreach ($additionalHeaders as $additionalHeader) {
+        if (strpos($additionalHeader, ':') !== FALSE) {
+            list($key, $value) = explode(':', $additionalHeader, 2);
+            $key = str_replace('-', '_', strtoupper(trim($key)));
+            if ($key != 'HOST') {
+                $_SERVER['HTTP_' . $key] = $value;
+            }
+        }
+    }
 }
 
 
-	// put parsed query parts into $_GET array
+    // put parsed query parts into $_GET array
 $urlParts = parse_url($_SERVER['argv'][2]);
-	// Populating $_GET
+    // Populating $_GET
 parse_str($urlParts['query'], $_GET);
-	// Populating $_REQUEST
+    // Populating $_REQUEST
 parse_str($urlParts['query'], $_REQUEST);
-	// Populating $_POST
+    // Populating $_POST
 $_POST = array();
-	// Populating $_COOKIE
+    // Populating $_COOKIE
 $_COOKIE = array();
 
-	// Get the TYPO3_SITE_PATH of the website frontend:
+    // Get the TYPO3_SITE_PATH of the website frontend:
 $typo3SitePath = $_SERVER['argv'][1];
 
-	// faking the environment
+    // faking the environment
 $_SERVER['DOCUMENT_ROOT'] = preg_replace('#' . preg_quote($typo3SitePath, '#') . '$#', '', $typo3Root);
 $_SERVER['HTTP_USER_AGENT'] = 'CLI Mode';
 $_SERVER['HTTP_HOST'] = $_SERVER['SERVER_NAME'] = $urlParts['host'];
@@ -66,15 +66,15 @@ 
 $_SERVER['REQUEST_URI'] = $urlParts['path'] . (isset($urlParts['query']) ? '?' . $urlParts['query'] : '');
 $_SERVER['REQUEST_METHOD'] = 'GET';
 
-	// Define a port if used in the URL:
+    // Define a port if used in the URL:
 if (isset($urlParts['port'])) {
-	$_SERVER['HTTP_HOST'] .= ':' . $urlParts['port'];
-	$_SERVER['SERVER_PORT'] = $urlParts['port'];
+    $_SERVER['HTTP_HOST'] .= ':' . $urlParts['port'];
+    $_SERVER['SERVER_PORT'] = $urlParts['port'];
 }
 
-	// Define HTTPS disposal:
+    // Define HTTPS disposal:
 if ($urlParts['scheme'] === 'https') {
-	$_SERVER['HTTPS'] = 'on';
+    $_SERVER['HTTPS'] = 'on';
 }
 
 chdir($typo3Root);
@@ -88,11 +88,11 @@ 
  * @return boolean
  */
 function isAbsPath($path) {
-		// on Windows also a path starting with a drive letter is absolute: X:/
-	if (stristr(PHP_OS, 'win') && substr($path, 1, 2) === ':/') {
-		return TRUE;
-	}
+        // on Windows also a path starting with a drive letter is absolute: X:/
+    if (stristr(PHP_OS, 'win') && substr($path, 1, 2) === ':/') {
+        return TRUE;
+    }
 
-		// path starting with a / is always absolute, on every system
-	return (substr($path, 0, 1) === '/');
+        // path starting with a / is always absolute, on every system
+    return (substr($path, 0, 1) === '/');
 }

Spacing +10 added lines, -10 removed lines

@@ -12,12 +12,12 @@ 
 if (!isAbsPath($tempPathThisScript)) {
 	$workingDirectory = $_SERVER['PWD'] ? $_SERVER['PWD'] : getcwd();
 	if ($workingDirectory) {
-		$tempPathThisScript = $workingDirectory . '/' . preg_replace('/\.\//', '', $tempPathThisScript);
+		$tempPathThisScript = $workingDirectory.'/'.preg_replace('/\.\//', '', $tempPathThisScript);
 		if (!@is_file($tempPathThisScript)) {
-			die('Relative path found, but an error occured during resolving of the absolute path: ' . $tempPathThisScript . PHP_EOL);
+			die('Relative path found, but an error occured during resolving of the absolute path: '.$tempPathThisScript.PHP_EOL);
 		}
 	} else {
-		die('Relative path found, but resolving absolute path is not supported on this platform.' . PHP_EOL);
+		die('Relative path found, but resolving absolute path is not supported on this platform.'.PHP_EOL);
 	}
 }
 
@@ -35,7 +35,7 @@ 
 			list($key, $value) = explode(':', $additionalHeader, 2);
 			$key = str_replace('-', '_', strtoupper(trim($key)));
 			if ($key != 'HOST') {
-				$_SERVER['HTTP_' . $key] = $value;
+				$_SERVER['HTTP_'.$key] = $value;
 			}
 		}
 	}
@@ -57,18 +57,18 @@ 
 $typo3SitePath = $_SERVER['argv'][1];
 
 	// faking the environment
-$_SERVER['DOCUMENT_ROOT'] = preg_replace('#' . preg_quote($typo3SitePath, '#') . '$#', '', $typo3Root);
+$_SERVER['DOCUMENT_ROOT'] = preg_replace('#'.preg_quote($typo3SitePath, '#').'$#', '', $typo3Root);
 $_SERVER['HTTP_USER_AGENT'] = 'CLI Mode';
 $_SERVER['HTTP_HOST'] = $_SERVER['SERVER_NAME'] = $urlParts['host'];
-$_SERVER['SCRIPT_NAME'] = $_SERVER['PHP_SELF'] = $typo3SitePath . 'index.php';
-$_SERVER['SCRIPT_FILENAME'] = $_SERVER['PATH_TRANSLATED'] = $typo3Root . 'index.php';
+$_SERVER['SCRIPT_NAME'] = $_SERVER['PHP_SELF'] = $typo3SitePath.'index.php';
+$_SERVER['SCRIPT_FILENAME'] = $_SERVER['PATH_TRANSLATED'] = $typo3Root.'index.php';
 $_SERVER['QUERY_STRING'] = (isset($urlParts['query']) ? $urlParts['query'] : '');
-$_SERVER['REQUEST_URI'] = $urlParts['path'] . (isset($urlParts['query']) ? '?' . $urlParts['query'] : '');
+$_SERVER['REQUEST_URI'] = $urlParts['path'].(isset($urlParts['query']) ? '?'.$urlParts['query'] : '');
 $_SERVER['REQUEST_METHOD'] = 'GET';
 
 	// Define a port if used in the URL:
 if (isset($urlParts['port'])) {
-	$_SERVER['HTTP_HOST'] .= ':' . $urlParts['port'];
+	$_SERVER['HTTP_HOST'] .= ':'.$urlParts['port'];
 	$_SERVER['SERVER_PORT'] = $urlParts['port'];
 }
 
@@ -78,7 +78,7 @@ 
 }
 
 chdir($typo3Root);
-include($typo3Root . '/index.php');
+include($typo3Root.'/index.php');
 
 
 /**

cli/class.tx_crawler_cli_flush.php

Indentation +19 added lines, -19 removed lines

@@ -37,30 +37,30 @@
  */
 class tx_crawler_cli_flush extends \TYPO3\CMS\Core\Controller\CommandLineController {
 
-	/**
-	 * Constructor
-	 *
-	 * @return	void
-	 */
-	function __construct() {
-		parent::__construct();
+    /**
+     * Constructor
+     *
+     * @return	void
+     */
+    function __construct() {
+        parent::__construct();
 
-			// Adding options to help archive:
-		$this->cli_options[] = array('-o mode', 'Output mode: "finished", "all", "pending"', "Specifies the type queue entries which is flushed in the process.");
-		#		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");
+            // Adding options to help archive:
+        $this->cli_options[] = array('-o mode', 'Output mode: "finished", "all", "pending"', "Specifies the type queue entries which is flushed in the process.");
+        #		$this->cli_options[] = array('-v level', 'Verbosity level 0-3', "The value of level can be:\n  0 = all output\n  1 = info and greater (default)\n  2 = warnings and greater\n  3 = errors");
 
-			// Setting help texts:
-		$this->cli_help['name'] = 'crawler CLI interface -- Cleaning up the queue.';
-		$this->cli_help['synopsis'] = 'page_id ###OPTIONS###';
-		$this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It will remove queue entires and perform a cleanup.";
-		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_flush 5 -o=finished\nWill remove all finished queue-entries in the sub-branch of page 5\n";
-		$this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_flush 0 -o=all\nWill remove all queue-entries for every page\n";
-		$this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';
-	}
+            // Setting help texts:
+        $this->cli_help['name'] = 'crawler CLI interface -- Cleaning up the queue.';
+        $this->cli_help['synopsis'] = 'page_id ###OPTIONS###';
+        $this->cli_help['description'] = "Works as a CLI interface to some functionality from the Web > Info > Site Crawler module; It will remove queue entires and perform a cleanup.";
+        $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_flush 5 -o=finished\nWill remove all finished queue-entries in the sub-branch of page 5\n";
+        $this->cli_help['examples'] = "/.../cli_dispatch.phpsh crawler_flush 0 -o=all\nWill remove all queue-entries for every page\n";
+        $this->cli_help['author'] = 'Kasper Skaarhoj, Daniel Poetzinger, Fabrizio Branca, Tolleiv Nietsch, Timo Schmidt - AOE media 2009';
+    }
 }
 
 if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_flush.php']) {
-	include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_flush.php']);
+    include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/cli/class.tx_crawler_cli_flush.php']);
 }
 
 ?>

class.tx_crawler_lib.php

Indentation +3 added lines, -3 removed lines

@@ -1149,7 +1149,7 @@ 
         return time();
     }
 
-     /************************************
+        /************************************
      *
      * URL reading
      *
@@ -1316,7 +1316,7 @@ 
             return FALSE;
         }
 
- 	    // direct request
+            // direct request
         if ($this->extensionSettings['makeDirectRequests']) {
             $result = $this->sendDirectRequest($originalUrl, $crawlerId);
             return $result;
@@ -2328,7 +2328,7 @@ 
      *
      * @return void
      */
-     public function CLI_deleteProcessesMarkedDeleted() {
+        public function CLI_deleteProcessesMarkedDeleted() {
         $this->db->exec_DELETEquery('tx_crawler_process', 'deleted = 1');
     }
 

Spacing +286 added lines, -286 removed lines

@@ -29,8 +29,8 @@ 
 class tx_crawler_lib {
 
     var $setID = 0;
-    var $processID ='';
-    var $max_CLI_exec_time = 3600;    // One hour is max stalled time for the CLI (If the process has had the status "start" for 3600 seconds it will be regarded stalled and a new process is started.
+    var $processID = '';
+    var $max_CLI_exec_time = 3600; // One hour is max stalled time for the CLI (If the process has had the status "start" for 3600 seconds it will be regarded stalled and a new process is started.
 
     var $duplicateTrack = array();
     var $downloadUrls = array();
@@ -43,9 +43,9 @@ 
     var $queueEntries = array();
     var $urlList = array();
 
-    var $debugMode=FALSE;
+    var $debugMode = FALSE;
 
-    var $extensionSettings=array();
+    var $extensionSettings = array();
 
     var $MP = false; // mount point
 
@@ -69,9 +69,9 @@ 
     private $backendUser;
 
     const CLI_STATUS_NOTHING_PROCCESSED = 0;
-    const CLI_STATUS_REMAIN = 1;    //queue not empty
-    const CLI_STATUS_PROCESSED = 2;    //(some) queue items where processed
-    const CLI_STATUS_ABORTED = 4;    //instance didn't finish
+    const CLI_STATUS_REMAIN = 1; //queue not empty
+    const CLI_STATUS_PROCESSED = 2; //(some) queue items where processed
+    const CLI_STATUS_ABORTED = 4; //instance didn't finish
     const CLI_STATUS_POLLABLE_PROCESSED = 8;
 
     /**
@@ -162,7 +162,7 @@ 
             $this->extensionSettings['countInARun'] = 100;
         }
 
-        $this->extensionSettings['processLimit'] = \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($this->extensionSettings['processLimit'],1,99,1);
+        $this->extensionSettings['processLimit'] = \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($this->extensionSettings['processLimit'], 1, 99, 1);
     }
 
     /**
@@ -195,7 +195,7 @@ 
         }
 
         if (!$skipPage) {
-            if (\TYPO3\CMS\Core\Utility\GeneralUtility::inList('3,4', $pageRow['doktype']) || $pageRow['doktype']>=199)    {
+            if (\TYPO3\CMS\Core\Utility\GeneralUtility::inList('3,4', $pageRow['doktype']) || $pageRow['doktype'] >= 199) {
                 $skipPage = true;
                 $skipMessage = 'Because doktype is not allowed';
             }
@@ -216,13 +216,13 @@ 
         if (!$skipPage) {
                 // veto hook
             if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['pageVeto'])) {
-                foreach($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['pageVeto'] as $key => $func)    {
+                foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['pageVeto'] as $key => $func) {
                     $params = array(
                         'pageRow' => $pageRow
                     );
                     // expects "false" if page is ok and "true" or a skipMessage if this page should _not_ be crawled
                     $veto = \TYPO3\CMS\Core\Utility\GeneralUtility::callUserFunction($func, $params, $this);
-                    if ($veto !== false)    {
+                    if ($veto !== false) {
                         $skipPage = true;
                         if (is_string($veto)) {
                             $skipMessage = $veto;
@@ -271,9 +271,9 @@ 
      * @param  string $configurationHash
      * @return boolean
      */
-    protected function noUnprocessedQueueEntriesForPageWithConfigurationHashExist($uid,$configurationHash) {
-        $configurationHash = $this->db->fullQuoteStr($configurationHash,'tx_crawler_queue');
-        $res = $this->db->exec_SELECTquery('count(*) as anz','tx_crawler_queue',"page_id=".intval($uid)." AND configuration_hash=".$configurationHash." AND exec_time=0");
+    protected function noUnprocessedQueueEntriesForPageWithConfigurationHashExist($uid, $configurationHash) {
+        $configurationHash = $this->db->fullQuoteStr($configurationHash, 'tx_crawler_queue');
+        $res = $this->db->exec_SELECTquery('count(*) as anz', 'tx_crawler_queue', "page_id=".intval($uid)." AND configuration_hash=".$configurationHash." AND exec_time=0");
         $row = $this->db->sql_fetch_assoc($res);
 
         return ($row['anz'] == 0);
@@ -338,26 +338,26 @@ 
             }
         }
 
-        if (is_array($vv['URLs']))    {
-            $configurationHash     =    md5(serialize($vv));
-            $skipInnerCheck     =    $this->noUnprocessedQueueEntriesForPageWithConfigurationHashExist($pageRow['uid'],$configurationHash);
+        if (is_array($vv['URLs'])) {
+            $configurationHash = md5(serialize($vv));
+            $skipInnerCheck = $this->noUnprocessedQueueEntriesForPageWithConfigurationHashExist($pageRow['uid'], $configurationHash);
 
-            foreach($vv['URLs'] as $urlQuery)    {
+            foreach ($vv['URLs'] as $urlQuery) {
 
-                if ($this->drawURLs_PIfilter($vv['subCfg']['procInstrFilter'], $incomingProcInstructions))    {
+                if ($this->drawURLs_PIfilter($vv['subCfg']['procInstrFilter'], $incomingProcInstructions)) {
 
                     // Calculate cHash:
-                    if ($vv['subCfg']['cHash'])    {
+                    if ($vv['subCfg']['cHash']) {
                         /* @var $cacheHash \TYPO3\CMS\Frontend\Page\CacheHashCalculator */
                         $cacheHash = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Frontend\Page\CacheHashCalculator');
-                        $urlQuery .= '&cHash=' . $cacheHash->generateForParameters($urlQuery);
+                        $urlQuery .= '&cHash='.$cacheHash->generateForParameters($urlQuery);
                     }
 
                     // Create key by which to determine unique-ness:
                     $uKey = $urlQuery.'|'.$vv['subCfg']['userGroups'].'|'.$vv['subCfg']['baseUrl'].'|'.$vv['subCfg']['procInstrFilter'];
 
                     // realurl support (thanks to Ingo Renner)
-                    $urlQuery = 'index.php' . $urlQuery;
+                    $urlQuery = 'index.php'.$urlQuery;
                     if (\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::isLoaded('realurl') && $vv['subCfg']['realurl']) {
                         $params = array(
                             'LD' => array(
@@ -370,8 +370,8 @@ 
                     }
 
                     // Scheduled time:
-                    $schTime = $scheduledTime + round(count($duplicateTrack)*(60/$reqMinute));
-                    $schTime = floor($schTime/60)*60;
+                    $schTime = $scheduledTime + round(count($duplicateTrack) * (60 / $reqMinute));
+                    $schTime = floor($schTime / 60) * 60;
 
                     if (isset($duplicateTrack[$uKey])) {
 
@@ -383,10 +383,10 @@ 
                         $urlList = '['.date('d.m.y H:i', $schTime).'] '.htmlspecialchars($urlQuery);
                         $this->urlList[] = '['.date('d.m.y H:i', $schTime).'] '.$urlQuery;
 
-                        $theUrl = ($vv['subCfg']['baseUrl'] ? $vv['subCfg']['baseUrl'] : \TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_URL')) . $urlQuery;
+                        $theUrl = ($vv['subCfg']['baseUrl'] ? $vv['subCfg']['baseUrl'] : \TYPO3\CMS\Core\Utility\GeneralUtility::getIndpEnv('TYPO3_SITE_URL')).$urlQuery;
 
                         // Submit for crawling!
-                        if ($submitCrawlUrls)    {
+                        if ($submitCrawlUrls) {
                             $added = $this->addUrl(
                             $pageRow['uid'],
                             $theUrl,
@@ -398,7 +398,7 @@ 
                             if ($added === false) {
                                 $urlList .= ' (Url already existed)';
                             }
-                        } elseif ($downloadCrawlUrls)    {
+                        } elseif ($downloadCrawlUrls) {
                             $downloadUrls[$theUrl] = $theUrl;
                         }
 
@@ -427,7 +427,7 @@ 
             return TRUE;
         }
 
-        foreach($incomingProcInstructions as $pi) {
+        foreach ($incomingProcInstructions as $pi) {
             if (\TYPO3\CMS\Core\Utility\GeneralUtility::inList($piString, $pi)) {
                 return TRUE;
             }
@@ -440,7 +440,7 @@ 
      * @return array
      */
     public function getPageTSconfigForId($id) {
-        if(!$this->MP){
+        if (!$this->MP) {
             $pageTSconfig = \TYPO3\CMS\Backend\Utility\BackendUtility::getPagesTSconfig($id);
         } else {
             list(,$mountPointId) = explode('-', $this->MP);
@@ -468,7 +468,7 @@ 
      * @param  integer $id  Page ID
      * @return array        Configurations from pages and configuration records
      */
-    protected function getUrlsForPageId($id)    {
+    protected function getUrlsForPageId($id) {
 
         /**
          * Get configuration from tsConfig
@@ -479,24 +479,24 @@ 
 
         $res = array();
 
-        if (is_array($pageTSconfig) && is_array($pageTSconfig['tx_crawler.']['crawlerCfg.']))    {
+        if (is_array($pageTSconfig) && is_array($pageTSconfig['tx_crawler.']['crawlerCfg.'])) {
             $crawlerCfg = $pageTSconfig['tx_crawler.']['crawlerCfg.'];
 
-            if (is_array($crawlerCfg['paramSets.']))    {
-                foreach($crawlerCfg['paramSets.'] as $key => $values)    {
-                    if (!is_array($values))    {
+            if (is_array($crawlerCfg['paramSets.'])) {
+                foreach ($crawlerCfg['paramSets.'] as $key => $values) {
+                    if (!is_array($values)) {
 
                         // Sub configuration for a single configuration string:
-                        $subCfg = (array)$crawlerCfg['paramSets.'][$key.'.'];
+                        $subCfg = (array) $crawlerCfg['paramSets.'][$key.'.'];
                         $subCfg['key'] = $key;
 
-                        if (strcmp($subCfg['procInstrFilter'],''))    {
-                            $subCfg['procInstrFilter'] = implode(',',\TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',',$subCfg['procInstrFilter']));
+                        if (strcmp($subCfg['procInstrFilter'], '')) {
+                            $subCfg['procInstrFilter'] = implode(',', \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $subCfg['procInstrFilter']));
                         }
-                        $pidOnlyList = implode(',',\TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',',$subCfg['pidsOnly'],1));
+                        $pidOnlyList = implode(',', \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $subCfg['pidsOnly'], 1));
 
                             // process configuration if it is not page-specific or if the specific page is the current page:
-                        if (!strcmp($subCfg['pidsOnly'],'') || \TYPO3\CMS\Core\Utility\GeneralUtility::inList($pidOnlyList,$id))    {
+                        if (!strcmp($subCfg['pidsOnly'], '') || \TYPO3\CMS\Core\Utility\GeneralUtility::inList($pidOnlyList, $id)) {
 
                                 // add trailing slash if not present
                             if (!empty($subCfg['baseUrl']) && substr($subCfg['baseUrl'], -1) != '/') {
@@ -507,14 +507,14 @@ 
                             $res[$key] = array();
                             $res[$key]['subCfg'] = $subCfg;
                             $res[$key]['paramParsed'] = $this->parseParams($values);
-                            $res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'],$id);
+                            $res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $id);
                             $res[$key]['origin'] = 'pagets';
 
                                 // recognize MP value
-                            if(!$this->MP){
-                                $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'],array('?id='.$id));
+                            if (!$this->MP) {
+                                $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], array('?id='.$id));
                             } else {
-                                $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'],array('?id='.$id.'&MP='.$this->MP));
+                                $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], array('?id='.$id.'&MP='.$this->MP));
                             }
                         }
                     }
@@ -535,7 +535,7 @@ 
                 'tx_crawler_configuration',
                 'pid',
                 intval($page['uid']),
-                \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('tx_crawler_configuration') . \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('tx_crawler_configuration')
+                \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('tx_crawler_configuration').\TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('tx_crawler_configuration')
             );
 
             if (is_array($configurationRecordsForCurrentPage)) {
@@ -544,10 +544,10 @@ 
                         // check access to the configuration record
                     if (empty($configurationRecord['begroups']) || $GLOBALS['BE_USER']->isAdmin() || $this->hasGroupAccess($GLOBALS['BE_USER']->user['usergroup_cached_list'], $configurationRecord['begroups'])) {
 
-                        $pidOnlyList = implode(',',\TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',',$configurationRecord['pidsonly'],1));
+                        $pidOnlyList = implode(',', \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $configurationRecord['pidsonly'], 1));
 
                             // process configuration if it is not page-specific or if the specific page is the current page:
-                        if (!strcmp($configurationRecord['pidsonly'],'') || \TYPO3\CMS\Core\Utility\GeneralUtility::inList($pidOnlyList,$id)) {
+                        if (!strcmp($configurationRecord['pidsonly'], '') || \TYPO3\CMS\Core\Utility\GeneralUtility::inList($pidOnlyList, $id)) {
                             $key = $configurationRecord['name'];
 
                                 // don't overwrite previously defined paramSets
@@ -577,7 +577,7 @@ 
                                     $res[$key]['subCfg'] = $subCfg;
                                     $res[$key]['paramParsed'] = $this->parseParams($configurationRecord['configuration']);
                                     $res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $id);
-                                    $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], array('?id=' . $id));
+                                    $res[$key]['URLs'] = $this->compileUrls($res[$key]['paramExpanded'], array('?id='.$id));
                                     $res[$key]['origin'] = 'tx_crawler_configuration_'.$configurationRecord['uid'];
                                 }
                             }
@@ -587,8 +587,8 @@ 
             }
         }
 
-        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['processUrls']))    {
-            foreach($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['processUrls'] as $func)    {
+        if (is_array($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['processUrls'])) {
+            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['processUrls'] as $func) {
                 $params = array(
                     'res' => &$res,
                 );
@@ -613,8 +613,8 @@ 
             $res = $this->db->exec_SELECTquery(
                 '*',
                 'sys_domain',
-                'uid = '.$sysDomainUid .
-                \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_domain') .
+                'uid = '.$sysDomainUid.
+                \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('sys_domain').
                 \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('sys_domain')
             );
             $row = $this->db->sql_fetch_assoc($res);
@@ -638,24 +638,24 @@ 
         $pageTSconfig = $this->getPageTSconfigForId($rootId);
         if (is_array($pageTSconfig) && is_array($pageTSconfig['tx_crawler.']['crawlerCfg.']) && is_array($pageTSconfig['tx_crawler.']['crawlerCfg.']['paramSets.'])) {
             $sets = $pageTSconfig['tx_crawler.']['crawlerCfg.']['paramSets.'];
-            if(is_array($sets)) {
-                foreach($sets as $key=>$value) {
-                    if(!is_array($value)) continue;
-                    $configurationsForBranch[] = substr($key,-1)=='.'?substr($key,0,-1):$key;
+            if (is_array($sets)) {
+                foreach ($sets as $key=>$value) {
+                    if (!is_array($value)) continue;
+                    $configurationsForBranch[] = substr($key, -1) == '.' ?substr($key, 0, -1) : $key;
                 }
             }
         }
         $pids = array();
         $rootLine = \TYPO3\CMS\Backend\Utility\BackendUtility::BEgetRootLine($rootId);
-        foreach($rootLine as $node) {
+        foreach ($rootLine as $node) {
             $pids[] = $node['uid'];
         }
         /* @var \TYPO3\CMS\Backend\Tree\View\PageTreeView */
         $tree = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Backend\Tree\View\PageTreeView');
         $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1);
-        $tree->init('AND ' . $perms_clause);
+        $tree->init('AND '.$perms_clause);
         $tree->getTree($rootId, $depth, '');
-        foreach($tree->tree as $node) {
+        foreach ($tree->tree as $node) {
             $pids[] = $node['row']['uid'];
         }
 
@@ -663,12 +663,12 @@ 
             '*',
             'tx_crawler_configuration',
             'pid IN ('.implode(',', $pids).') '.
-            \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('tx_crawler_configuration') .
+            \TYPO3\CMS\Backend\Utility\BackendUtility::BEenableFields('tx_crawler_configuration').
             \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause('tx_crawler_configuration').' '.
             \TYPO3\CMS\Backend\Utility\BackendUtility::versioningPlaceholderClause('tx_crawler_configuration').' '
         );
 
-        while($row = $this->db->sql_fetch_assoc($res)) {
+        while ($row = $this->db->sql_fetch_assoc($res)) {
             $configurationsForBranch[] = $row['name'];
         }
         $this->db->sql_free_result($res);
@@ -690,7 +690,7 @@ 
         if (empty($accessList)) {
             return true;
         }
-        foreach(\TYPO3\CMS\Core\Utility\GeneralUtility::intExplode(',', $groupList) as $groupUid) {
+        foreach (\TYPO3\CMS\Core\Utility\GeneralUtility::intExplode(',', $groupList) as $groupUid) {
             if (\TYPO3\CMS\Core\Utility\GeneralUtility::inList($accessList, $groupUid)) {
                 return true;
             }
@@ -709,9 +709,9 @@ 
         $paramKeyValues = array();
         $GETparams = explode('&', $inputQuery);
 
-        foreach($GETparams as $paramAndValue)    {
-            list($p,$v) = explode('=', $paramAndValue, 2);
-            if (strlen($p))        {
+        foreach ($GETparams as $paramAndValue) {
+            list($p, $v) = explode('=', $paramAndValue, 2);
+            if (strlen($p)) {
                 $paramKeyValues[rawurldecode($p)] = rawurldecode($v);
             }
         }
@@ -734,84 +734,84 @@ 
      * @param    integer        Current page ID
      * @return    array        Array with key (GET var name) with the value being an array of all possible values for that key.
      */
-    protected function expandParameters($paramArray, $pid)    {
+    protected function expandParameters($paramArray, $pid) {
         global $TCA;
 
             // Traverse parameter names:
-        foreach($paramArray as $p => $v)    {
+        foreach ($paramArray as $p => $v) {
             $v = trim($v);
 
                 // If value is encapsulated in square brackets it means there are some ranges of values to find, otherwise the value is literal
-            if (substr($v,0,1)==='[' && substr($v,-1)===']')    {
+            if (substr($v, 0, 1) === '[' && substr($v, -1) === ']') {
                     // So, find the value inside brackets and reset the paramArray value as an array.
-                $v = substr($v,1,-1);
+                $v = substr($v, 1, -1);
                 $paramArray[$p] = array();
 
                     // Explode parts and traverse them:
-                $parts = explode('|',$v);
-                foreach($parts as $pV)    {
+                $parts = explode('|', $v);
+                foreach ($parts as $pV) {
 
                         // Look for integer range: (fx. 1-34 or -40--30 // reads minus 40 to minus 30)
-                    if (preg_match('/^(-?[0-9]+)\s*-\s*(-?[0-9]+)$/',trim($pV),$reg))    {    // Integer range:
+                    if (preg_match('/^(-?[0-9]+)\s*-\s*(-?[0-9]+)$/', trim($pV), $reg)) {    // Integer range:
 
                             // Swap if first is larger than last:
-                        if ($reg[1] > $reg[2])    {
+                        if ($reg[1] > $reg[2]) {
                             $temp = $reg[2];
                             $reg[2] = $reg[1];
                             $reg[1] = $temp;
                         }
 
                             // Traverse range, add values:
-                        $runAwayBrake = 1000;    // Limit to size of range!
-                        for($a=$reg[1]; $a<=$reg[2];$a++)    {
+                        $runAwayBrake = 1000; // Limit to size of range!
+                        for ($a = $reg[1]; $a <= $reg[2]; $a++) {
                             $paramArray[$p][] = $a;
                             $runAwayBrake--;
-                            if ($runAwayBrake<=0)    {
+                            if ($runAwayBrake <= 0) {
                                 break;
                             }
                         }
-                    } elseif (substr(trim($pV),0,7)=='_TABLE:')    {
+                    } elseif (substr(trim($pV), 0, 7) == '_TABLE:') {
 
                             // Parse parameters:
-                        $subparts = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(';',$pV);
+                        $subparts = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(';', $pV);
                         $subpartParams = array();
-                        foreach($subparts as $spV)    {
-                            list($pKey,$pVal) = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(':',$spV);
+                        foreach ($subparts as $spV) {
+                            list($pKey, $pVal) = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(':', $spV);
                             $subpartParams[$pKey] = $pVal;
                         }
 
                             // Table exists:
-                        if (isset($TCA[$subpartParams['_TABLE']]))    {
+                        if (isset($TCA[$subpartParams['_TABLE']])) {
                             $lookUpPid = isset($subpartParams['_PID']) ? intval($subpartParams['_PID']) : $pid;
                             $pidField = isset($subpartParams['_PIDFIELD']) ? trim($subpartParams['_PIDFIELD']) : 'pid';
                             $where = isset($subpartParams['_WHERE']) ? $subpartParams['_WHERE'] : '';
                             $addTable = isset($subpartParams['_ADDTABLE']) ? $subpartParams['_ADDTABLE'] : '';
 
                             $fieldName = $subpartParams['_FIELD'] ? $subpartParams['_FIELD'] : 'uid';
-                            if ($fieldName==='uid' || $TCA[$subpartParams['_TABLE']]['columns'][$fieldName]) {
+                            if ($fieldName === 'uid' || $TCA[$subpartParams['_TABLE']]['columns'][$fieldName]) {
 
                                 $andWhereLanguage = '';
                                 $transOrigPointerField = $TCA[$subpartParams['_TABLE']]['ctrl']['transOrigPointerField'];
 
                                 if ($subpartParams['_ENABLELANG'] && $transOrigPointerField) {
-                                    $andWhereLanguage = ' AND ' . $this->db->quoteStr($transOrigPointerField, $subpartParams['_TABLE']) .' <= 0 ';
+                                    $andWhereLanguage = ' AND '.$this->db->quoteStr($transOrigPointerField, $subpartParams['_TABLE']).' <= 0 ';
                                 }
 
-                                $where = $this->db->quoteStr($pidField, $subpartParams['_TABLE']) .'='.intval($lookUpPid) . ' ' .
-                                    $andWhereLanguage . $where;
+                                $where = $this->db->quoteStr($pidField, $subpartParams['_TABLE']).'='.intval($lookUpPid).' '.
+                                    $andWhereLanguage.$where;
 
                                 $rows = $this->db->exec_SELECTgetRows(
                                     $fieldName,
-                                    $subpartParams['_TABLE'] . $addTable,
-                                    $where . \TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause($subpartParams['_TABLE']),
+                                    $subpartParams['_TABLE'].$addTable,
+                                    $where.\TYPO3\CMS\Backend\Utility\BackendUtility::deleteClause($subpartParams['_TABLE']),
                                     '',
                                     '',
                                     '',
                                     $fieldName
                                 );
 
-                                if (is_array($rows))    {
-                                    $paramArray[$p] = array_merge($paramArray[$p],array_keys($rows));
+                                if (is_array($rows)) {
+                                    $paramArray[$p] = array_merge($paramArray[$p], array_keys($rows));
                                 }
                             }
                         }
@@ -827,7 +827,7 @@ 
                             'currentValue' => $pV,
                             'pid' => $pid
                         );
-                        foreach($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['crawler/class.tx_crawler_lib.php']['expandParameters'] as $key => $_funcRef)    {
+                        foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['crawler/class.tx_crawler_lib.php']['expandParameters'] as $key => $_funcRef) {
                             \TYPO3\CMS\Core\Utility\GeneralUtility::callUserFunction($_funcRef, $_params, $this);
                         }
                     }
@@ -863,11 +863,11 @@ 
 
                 // Traverse value set:
             $newUrls = array();
-            foreach($urls as $url) {
-                foreach($valueSet as $val) {
-                    $newUrls[] = $url.(strcmp($val,'') ? '&'.rawurlencode($varName).'='.rawurlencode($val) : '');
+            foreach ($urls as $url) {
+                foreach ($valueSet as $val) {
+                    $newUrls[] = $url.(strcmp($val, '') ? '&'.rawurlencode($varName).'='.rawurlencode($val) : '');
 
-                    if (count($newUrls) >  \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($this->extensionSettings['maxCompileUrls'], 1, 1000000000, 10000)) {
+                    if (count($newUrls) > \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($this->extensionSettings['maxCompileUrls'], 1, 1000000000, 10000)) {
                         break;
                     }
                 }
@@ -897,7 +897,7 @@ 
      */
     public function getLogEntriesForPageId($id, $filter = '', $doFlush = FALSE, $doFullFlush = FALSE, $itemsPerPage = 10) {
         // FIXME: Write Unit tests for Filters
-        switch($filter) {
+        switch ($filter) {
             case 'pending':
                 $addWhere = ' AND exec_time=0';
                 break;
@@ -911,13 +911,13 @@ 
 
         // FIXME: Write unit test that ensures that the right records are deleted.
         if ($doFlush) {
-            $this->flushQueue( ($doFullFlush?'1=1':('page_id='.intval($id))) .$addWhere);
+            $this->flushQueue(($doFullFlush ? '1=1' : ('page_id='.intval($id))).$addWhere);
             return array();
         } else {
             return $this->db->exec_SELECTgetRows('*',
                 'tx_crawler_queue',
-                'page_id=' . intval($id) . $addWhere, '', 'scheduled DESC',
-                (intval($itemsPerPage)>0 ? intval($itemsPerPage) : ''));
+                'page_id='.intval($id).$addWhere, '', 'scheduled DESC',
+                (intval($itemsPerPage) > 0 ? intval($itemsPerPage) : ''));
         }
     }
 
@@ -930,9 +930,9 @@ 
      * @param    integer        Limit the amount of entires per page default is 10
      * @return    array
      */
-    public function getLogEntriesForSetId($set_id,$filter='',$doFlush=FALSE, $doFullFlush=FALSE, $itemsPerPage=10)    {
+    public function getLogEntriesForSetId($set_id, $filter = '', $doFlush = FALSE, $doFullFlush = FALSE, $itemsPerPage = 10) {
         // FIXME: Write Unit tests for Filters
-        switch($filter)    {
+        switch ($filter) {
             case 'pending':
                 $addWhere = ' AND exec_time=0';
                 break;
@@ -944,14 +944,14 @@ 
                 break;
         }
         // FIXME: Write unit test that ensures that the right records are deleted.
-        if ($doFlush)    {
-            $this->flushQueue($doFullFlush?'':('set_id='.intval($set_id).$addWhere));
+        if ($doFlush) {
+            $this->flushQueue($doFullFlush ? '' : ('set_id='.intval($set_id).$addWhere));
             return array();
         } else {
             return $this->db->exec_SELECTgetRows('*',
                 'tx_crawler_queue',
-                'set_id='.intval($set_id).$addWhere,'','scheduled DESC',
-                (intval($itemsPerPage)>0 ? intval($itemsPerPage) : ''));
+                'set_id='.intval($set_id).$addWhere, '', 'scheduled DESC',
+                (intval($itemsPerPage) > 0 ? intval($itemsPerPage) : ''));
         }
     }
 
@@ -961,14 +961,14 @@ 
      * @param $where    SQL related filter for the entries which should be removed
      * @return void
      */
-    protected function flushQueue($where='') {
+    protected function flushQueue($where = '') {
 
-        $realWhere = strlen($where)>0?$where:'1=1';
+        $realWhere = strlen($where) > 0 ? $where : '1=1';
 
-        if(tx_crawler_domain_events_dispatcher::getInstance()->hasObserver('queueEntryFlush')) {
-            $groups = $this->db->exec_SELECTgetRows('DISTINCT set_id','tx_crawler_queue',$realWhere);
-            foreach($groups as $group) {
-                tx_crawler_domain_events_dispatcher::getInstance()->post('queueEntryFlush',$group['set_id'], $this->db->exec_SELECTgetRows('uid, set_id','tx_crawler_queue',$realWhere.' AND set_id="'.$group['set_id'].'"'));
+        if (tx_crawler_domain_events_dispatcher::getInstance()->hasObserver('queueEntryFlush')) {
+            $groups = $this->db->exec_SELECTgetRows('DISTINCT set_id', 'tx_crawler_queue', $realWhere);
+            foreach ($groups as $group) {
+                tx_crawler_domain_events_dispatcher::getInstance()->post('queueEntryFlush', $group['set_id'], $this->db->exec_SELECTgetRows('uid, set_id', 'tx_crawler_queue', $realWhere.' AND set_id="'.$group['set_id'].'"'));
             }
         }
 
@@ -985,7 +985,7 @@ 
      * @param    integer        Time at which to activate
      * @return    void
      */
-    public function addQueueEntry_callBack($setId,$params,$callBack,$page_id=0,$schedule=0) {
+    public function addQueueEntry_callBack($setId, $params, $callBack, $page_id = 0, $schedule = 0) {
 
         if (!is_array($params))    $params = array();
         $params['_CALLBACKOBJ'] = $callBack;
@@ -1000,7 +1000,7 @@ 
             'result_data' => '',
         );
 
-        $this->db->exec_INSERTquery('tx_crawler_queue',$fieldArray);
+        $this->db->exec_INSERTquery('tx_crawler_queue', $fieldArray);
     }
 
     /************************************
@@ -1020,13 +1020,13 @@ 
      * @param     bool        (optional) skip inner duplication check
      * @return    bool        true if the url was added, false if it already existed
      */
-    protected function addUrl (
+    protected function addUrl(
         $id,
         $url,
         array $subCfg,
         $tstamp,
-        $configurationHash='',
-        $skipInnerDuplicationCheck=false
+        $configurationHash = '',
+        $skipInnerDuplicationCheck = false
     ) {
 
         $urlAdded = false;
@@ -1037,14 +1037,14 @@ 
         );
 
             // fe user group simulation:
-        $uGs = implode(',',array_unique(\TYPO3\CMS\Core\Utility\GeneralUtility::intExplode(',',$subCfg['userGroups'],1)));
-        if ($uGs)    {
+        $uGs = implode(',', array_unique(\TYPO3\CMS\Core\Utility\GeneralUtility::intExplode(',', $subCfg['userGroups'], 1)));
+        if ($uGs) {
             $parameters['feUserGroupList'] = $uGs;
         }
 
             // Setting processing instructions
-        $parameters['procInstructions'] = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',',$subCfg['procInstrFilter']);
-        if (is_array($subCfg['procInstrParams.']))    {
+        $parameters['procInstructions'] = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $subCfg['procInstrFilter']);
+        if (is_array($subCfg['procInstrParams.'])) {
             $parameters['procInstrParams'] = $subCfg['procInstrParams.'];
         }
 
@@ -1063,14 +1063,14 @@ 
             'configuration' => $subCfg['key'],
         );
 
-        if ($this->registerQueueEntriesInternallyOnly)    {
+        if ($this->registerQueueEntriesInternallyOnly) {
                 //the entries will only be registered and not stored to the database
             $this->queueEntries[] = $fieldArray;
         } else {
 
-            if(!$skipInnerDuplicationCheck){
+            if (!$skipInnerDuplicationCheck) {
                     // check if there is already an equal entry
-                $rows = $this->getDuplicateRowsIfExist($tstamp,$fieldArray);
+                $rows = $this->getDuplicateRowsIfExist($tstamp, $fieldArray);
             }
 
             if (count($rows) == 0) {
@@ -1078,9 +1078,9 @@ 
                 $uid = $this->db->sql_insert_id();
                 $rows[] = $uid;
                 $urlAdded = true;
-                tx_crawler_domain_events_dispatcher::getInstance()->post('urlAddedToQueue',$this->setID,array('uid' => $uid, 'fieldArray' => $fieldArray));
-            }else{
-                tx_crawler_domain_events_dispatcher::getInstance()->post('duplicateUrlInQueue',$this->setID,array('rows' => $rows, 'fieldArray' => $fieldArray));
+                tx_crawler_domain_events_dispatcher::getInstance()->post('urlAddedToQueue', $this->setID, array('uid' => $uid, 'fieldArray' => $fieldArray));
+            } else {
+                tx_crawler_domain_events_dispatcher::getInstance()->post('duplicateUrlInQueue', $this->setID, array('rows' => $rows, 'fieldArray' => $fieldArray));
             }
         }
 
@@ -1097,34 +1097,34 @@ 
      *
      * @return array;
      */
-    protected function getDuplicateRowsIfExist($tstamp,$fieldArray){
+    protected function getDuplicateRowsIfExist($tstamp, $fieldArray) {
         $rows = array();
 
         $currentTime = $this->getCurrentTime();
 
             //if this entry is scheduled with "now"
         if ($tstamp <= $currentTime) {
-            if($this->extensionSettings['enableTimeslot']){
+            if ($this->extensionSettings['enableTimeslot']) {
                 $timeBegin     = $currentTime - 100;
-                $timeEnd     = $currentTime + 100;
-                $where         = ' ((scheduled BETWEEN '.$timeBegin.' AND '.$timeEnd.' ) OR scheduled <= '. $currentTime.') ';
-            }else{
-                $where = 'scheduled <= ' . $currentTime;
+                $timeEnd = $currentTime + 100;
+                $where         = ' ((scheduled BETWEEN '.$timeBegin.' AND '.$timeEnd.' ) OR scheduled <= '.$currentTime.') ';
+            } else {
+                $where = 'scheduled <= '.$currentTime;
             }
         } elseif ($tstamp > $currentTime) {
                 //entry with a timestamp in the future need to have the same schedule time
-            $where = 'scheduled = ' . $tstamp ;
+            $where = 'scheduled = '.$tstamp;
         }
 
-        if(!empty($where)){
+        if (!empty($where)) {
             $result = $this->db->exec_SELECTgetRows(
                 'qid',
                 'tx_crawler_queue',
                 $where.
-                ' AND NOT exec_time' .
+                ' AND NOT exec_time'.
                 ' AND NOT process_id '.
                 ' AND page_id='.intval($fieldArray['page_id']).
-                ' AND parameters_hash = ' . $this->db->fullQuoteStr($fieldArray['parameters_hash'], 'tx_crawler_queue')
+                ' AND parameters_hash = '.$this->db->fullQuoteStr($fieldArray['parameters_hash'], 'tx_crawler_queue')
             );
 
             if (is_array($result)) {
@@ -1145,7 +1145,7 @@ 
      *
      * @codeCoverageIgnore
      */
-    public function getCurrentTime(){
+    public function getCurrentTime() {
         return time();
     }
 
@@ -1166,18 +1166,18 @@ 
     public function readUrl($queueId, $force = FALSE) {
         $ret = 0;
         if ($this->debugMode) {
-            \TYPO3\CMS\Core\Utility\GeneralUtility::devlog('crawler-readurl start ' . microtime(true), __FUNCTION__);
+            \TYPO3\CMS\Core\Utility\GeneralUtility::devlog('crawler-readurl start '.microtime(true), __FUNCTION__);
         }
         // Get entry:
         list($queueRec) = $this->db->exec_SELECTgetRows('*', 'tx_crawler_queue',
-            'qid=' . intval($queueId) . ($force ? '' : ' AND exec_time=0 AND process_scheduled > 0'));
+            'qid='.intval($queueId).($force ? '' : ' AND exec_time=0 AND process_scheduled > 0'));
 
         if (!is_array($queueRec)) {
             return;
         }
 
-        $pageUidRootTypoScript = \AOE\Crawler\Utility\TypoScriptUtility::getPageUidForTypoScriptRootTemplateInRootLine((int)$queueRec['page_id']);
-        $this->initTSFE((int)$pageUidRootTypoScript);
+        $pageUidRootTypoScript = \AOE\Crawler\Utility\TypoScriptUtility::getPageUidForTypoScriptRootTemplateInRootLine((int) $queueRec['page_id']);
+        $this->initTSFE((int) $pageUidRootTypoScript);
 
         \AOE\Crawler\Utility\SignalSlotUtility::emitSignal(
             __CLASS__,
@@ -1192,7 +1192,7 @@ 
             //if mulitprocessing is used we need to store the id of the process which has handled this entry
             $field_array['process_id_completed'] = $this->processID;
         }
-        $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid=' . intval($queueId), $field_array);
+        $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid='.intval($queueId), $field_array);
 
         $result = $this->readUrl_exec($queueRec);
         $resultData = unserialize($result['content']);
@@ -1221,11 +1221,11 @@ 
             array($queueId, &$field_array)
         );
 
-        $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid=' . intval($queueId), $field_array);
+        $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid='.intval($queueId), $field_array);
 
 
         if ($this->debugMode) {
-            \TYPO3\CMS\Core\Utility\GeneralUtility::devlog('crawler-readurl stop ' . microtime(true), __FUNCTION__);
+            \TYPO3\CMS\Core\Utility\GeneralUtility::devlog('crawler-readurl stop '.microtime(true), __FUNCTION__);
         }
 
         return $ret;
@@ -1238,7 +1238,7 @@ 
      *
      * @return string
      */
-    protected function readUrlFromArray($field_array)    {
+    protected function readUrlFromArray($field_array) {
 
             // Set exec_time to lock record:
         $field_array['exec_time'] = $this->getCurrentTime();
@@ -1256,7 +1256,7 @@ 
             array($queueId, &$field_array)
         );
 
-        $this->db->exec_UPDATEquery('tx_crawler_queue','qid='.intval($queueId), $field_array);
+        $this->db->exec_UPDATEquery('tx_crawler_queue', 'qid='.intval($queueId), $field_array);
 
         return $result;
     }
@@ -1267,17 +1267,17 @@ 
      * @param array $queueRec Queue record
      * @return string Result output.
      */
-    protected function readUrl_exec($queueRec)    {
+    protected function readUrl_exec($queueRec) {
             // Decode parameters:
         $parameters = unserialize($queueRec['parameters']);
         $result = 'ERROR';
-        if (is_array($parameters))    {
-            if ($parameters['_CALLBACKOBJ'])    {    // Calling object:
+        if (is_array($parameters)) {
+            if ($parameters['_CALLBACKOBJ']) {    // Calling object:
                 $objRef = $parameters['_CALLBACKOBJ'];
                 $callBackObj = &\TYPO3\CMS\Core\Utility\GeneralUtility::getUserObj($objRef);
-                if (is_object($callBackObj))    {
+                if (is_object($callBackObj)) {
                     unset($parameters['_CALLBACKOBJ']);
-                    $result = array('content' => serialize($callBackObj->crawler_execute($parameters,$this)));
+                    $result = array('content' => serialize($callBackObj->crawler_execute($parameters, $this)));
                 } else {
                     $result = array('content' => 'No object: '.$objRef);
                 }
@@ -1287,9 +1287,9 @@ 
                 $crawlerId = $queueRec['qid'].':'.md5($queueRec['qid'].'|'.$queueRec['set_id'].'|'.$GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey']);
 
                     // Get result:
-                $result = $this->requestUrl($parameters['url'],$crawlerId);
+                $result = $this->requestUrl($parameters['url'], $crawlerId);
 
-                tx_crawler_domain_events_dispatcher::getInstance()->post('urlCrawled',$queueRec['set_id'],array('url' => $parameters['url'], 'result' => $result));
+                tx_crawler_domain_events_dispatcher::getInstance()->post('urlCrawled', $queueRec['set_id'], array('url' => $parameters['url'], 'result' => $result));
             }
         }
 
@@ -1306,7 +1306,7 @@ 
      * @param  integer  $recursion      Recursion limiter for 302 redirects
      * @return array                    Array with content
      */
-    public function requestUrl($originalUrl, $crawlerId, $timeout=2, $recursion=10) {
+    public function requestUrl($originalUrl, $crawlerId, $timeout = 2, $recursion = 10) {
 
         if (!$recursion) return false;
 
@@ -1318,7 +1318,7 @@ 
             return FALSE;
         }
 
-        if (!in_array($url['scheme'], array('','http','https'))) {
+        if (!in_array($url['scheme'], array('', 'http', 'https'))) {
             if (TYPO3_DLOG) \TYPO3\CMS\Core\Utility\GeneralUtility::devLog(sprintf('Scheme does not match for url "%s"', $url), 'crawler', 4, array('crawlerId' => $crawlerId));
             return FALSE;
         }
@@ -1336,14 +1336,14 @@ 
 
         if ($GLOBALS['TYPO3_CONF_VARS']['SYS']['curlUse'] && $GLOBALS['TYPO3_CONF_VARS']['SYS']['curlProxyServer']) {
             $rurl = parse_url($GLOBALS['TYPO3_CONF_VARS']['SYS']['curlProxyServer']);
-            $url['path'] = $url['scheme'] . '://' . $url['host'] . ($url['port'] > 0 ? ':' . $url['port'] : '') . $url['path'];
+            $url['path'] = $url['scheme'].'://'.$url['host'].($url['port'] > 0 ? ':'.$url['port'] : '').$url['path'];
             $reqHeaders = $this->buildRequestHeaderArray($url, $crawlerId);
         }
 
         $host = $rurl['host'];
 
         if ($url['scheme'] == 'https') {
-            $host = 'ssl://' . $host;
+            $host = 'ssl://'.$host;
             $port = ($rurl['port'] > 0) ? $rurl['port'] : 443;
         } else {
             $port = ($rurl['port'] > 0) ? $rurl['port'] : 80;
@@ -1357,24 +1357,24 @@ 
             return FALSE;
         } else {
                 // Request message:
-            $msg = implode("\r\n",$reqHeaders)."\r\n\r\n";
-            fputs ($fp, $msg);
+            $msg = implode("\r\n", $reqHeaders)."\r\n\r\n";
+            fputs($fp, $msg);
 
                 // Read response:
             $d = $this->getHttpResponseFromStream($fp);
-            fclose ($fp);
+            fclose($fp);
 
             $time = microtime(true) - $startTime;
-            $this->log($originalUrl .' '.$time);
+            $this->log($originalUrl.' '.$time);
 
                 // Implode content and headers:
             $result = array(
                 'request' => $msg,
                 'headers' => implode('', $d['headers']),
-                'content' => implode('', (array)$d['content'])
+                'content' => implode('', (array) $d['content'])
             );
 
-            if (($this->extensionSettings['follow30x']) && ($newUrl = $this->getRequestUrlFrom302Header($d['headers'],$url['user'],$url['pass']))) {
+            if (($this->extensionSettings['follow30x']) && ($newUrl = $this->getRequestUrlFrom302Header($d['headers'], $url['user'], $url['pass']))) {
                 $result = array_merge(array('parentRequest'=>$result), $this->requestUrl($newUrl, $crawlerId, $recursion--));
                 $newRequestUrl = $this->requestUrl($newUrl, $crawlerId, $timeout, --$recursion);
 
@@ -1413,8 +1413,8 @@ 
 
         // Base path must be '/<pathSegements>/':
         if ($frontendBasePath != '/') {
-            $frontendBasePath = '/' . ltrim($frontendBasePath, '/');
-            $frontendBasePath = rtrim($frontendBasePath, '/') . '/';
+            $frontendBasePath = '/'.ltrim($frontendBasePath, '/');
+            $frontendBasePath = rtrim($frontendBasePath, '/').'/';
         }
 
         return $frontendBasePath;
@@ -1447,7 +1447,7 @@ 
 
         if (is_resource($streamPointer)) {
                 // read headers
-            while($line = fgets($streamPointer, '2048')) {
+            while ($line = fgets($streamPointer, '2048')) {
                 $line = trim($line);
                 if ($line !== '') {
                     $response['headers'][] = $line;
@@ -1457,7 +1457,7 @@ 
             }
 
                 // read content
-            while($line = fgets($streamPointer, '2048')) {
+            while ($line = fgets($streamPointer, '2048')) {
                 $response['content'][] = $line;
             }
         }
@@ -1470,7 +1470,7 @@ 
      */
     protected function log($message) {
         if (!empty($this->extensionSettings['logFileName'])) {
-            @file_put_contents($this->extensionSettings['logFileName'], date('Ymd His') . $message . "\n", FILE_APPEND);
+            @file_put_contents($this->extensionSettings['logFileName'], date('Ymd His').$message."\n", FILE_APPEND);
         }
     }
 
@@ -1486,12 +1486,12 @@ 
         $reqHeaders = array();
         $reqHeaders[] = 'GET '.$url['path'].($url['query'] ? '?'.$url['query'] : '').' HTTP/1.0';
         $reqHeaders[] = 'Host: '.$url['host'];
-        if (stristr($url['query'],'ADMCMD_previewWS')) {
+        if (stristr($url['query'], 'ADMCMD_previewWS')) {
             $reqHeaders[] = 'Cookie: $Version="1"; be_typo_user="1"; $Path=/';
         }
         $reqHeaders[] = 'Connection: close';
-        if ($url['user']!='') {
-            $reqHeaders[] = 'Authorization: Basic '. base64_encode($url['user'].':'.$url['pass']);
+        if ($url['user'] != '') {
+            $reqHeaders[] = 'Authorization: Basic '.base64_encode($url['user'].':'.$url['pass']);
         }
         $reqHeaders[] = 'X-T3crawler: '.$crawlerId;
         $reqHeaders[] = 'User-Agent: TYPO3 crawler';
@@ -1506,21 +1506,21 @@ 
      * @param    string        HTTP Auth. Password
      * @return    string        URL from redirection
      */
-    protected function getRequestUrlFrom302Header($headers,$user='',$pass='') {
-        if(!is_array($headers)) return false;
-        if(!(stristr($headers[0],'301 Moved') || stristr($headers[0],'302 Found') || stristr($headers[0],'302 Moved'))) return false;
+    protected function getRequestUrlFrom302Header($headers, $user = '', $pass = '') {
+        if (!is_array($headers)) return false;
+        if (!(stristr($headers[0], '301 Moved') || stristr($headers[0], '302 Found') || stristr($headers[0], '302 Moved'))) return false;
 
-        foreach($headers as $hl) {
-            $tmp = explode(": ",$hl);
+        foreach ($headers as $hl) {
+            $tmp = explode(": ", $hl);
             $header[trim($tmp[0])] = trim($tmp[1]);
-            if(trim($tmp[0])=='Location') break;
+            if (trim($tmp[0]) == 'Location') break;
         }
-        if(!array_key_exists('Location',$header)) return false;
+        if (!array_key_exists('Location', $header)) return false;
 
-        if($user!='') {
-            if(!($tmp = parse_url($header['Location']))) return false;
-            $newUrl = $tmp['scheme'] . '://' . $user . ':' . $pass . '@' . $tmp['host'] . $tmp['path'];
-            if($tmp['query']!='') $newUrl .= '?' . $tmp['query'];
+        if ($user != '') {
+            if (!($tmp = parse_url($header['Location']))) return false;
+            $newUrl = $tmp['scheme'].'://'.$user.':'.$pass.'@'.$tmp['host'].$tmp['path'];
+            if ($tmp['query'] != '') $newUrl .= '?'.$tmp['query'];
         } else {
             $newUrl = $header['Location'];
         }
@@ -1541,15 +1541,15 @@ 
      * @param    object        TSFE object (reference under PHP5)
      * @return    void
      */
-    function fe_init(&$params, $ref)    {
+    function fe_init(&$params, $ref) {
 
             // Authenticate crawler request:
-        if (isset($_SERVER['HTTP_X_T3CRAWLER']))    {
-            list($queueId,$hash) = explode(':', $_SERVER['HTTP_X_T3CRAWLER']);
-            list($queueRec) = $this->db->exec_SELECTgetRows('*','tx_crawler_queue','qid='.intval($queueId));
+        if (isset($_SERVER['HTTP_X_T3CRAWLER'])) {
+            list($queueId, $hash) = explode(':', $_SERVER['HTTP_X_T3CRAWLER']);
+            list($queueRec) = $this->db->exec_SELECTgetRows('*', 'tx_crawler_queue', 'qid='.intval($queueId));
 
                 // If a crawler record was found and hash was matching, set it up:
-            if (is_array($queueRec) && $hash === md5($queueRec['qid'].'|'.$queueRec['set_id'].'|'.$GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey']))    {
+            if (is_array($queueRec) && $hash === md5($queueRec['qid'].'|'.$queueRec['set_id'].'|'.$GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey'])) {
                 $params['pObj']->applicationData['tx_crawler']['running'] = TRUE;
                 $params['pObj']->applicationData['tx_crawler']['parameters'] = unserialize($queueRec['parameters']);
                 $params['pObj']->applicationData['tx_crawler']['log'] = array();
@@ -1607,7 +1607,7 @@ 
             /* @var $tree \TYPO3\CMS\Backend\Tree\View\PageTreeView */
         $tree = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Backend\Tree\View\PageTreeView');
         $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1);
-        $tree->init('AND ' . $perms_clause);
+        $tree->init('AND '.$perms_clause);
 
         $pageinfo = \TYPO3\CMS\Backend\Utility\BackendUtility::readPageAccess($id, $perms_clause);
 
@@ -1618,7 +1618,7 @@ 
         );
 
             // Get branch beneath:
-        if ($depth)    {
+        if ($depth) {
             $tree->getTree($id, $depth, '');
         }
 
@@ -1630,7 +1630,7 @@ 
             $this->MP = false;
 
                 // recognize mount points
-            if($data['row']['doktype'] == 7){
+            if ($data['row']['doktype'] == 7) {
                 $mountpage = $this->db->exec_SELECTgetRows('*', 'pages', 'uid = '.$data['row']['uid']);
 
                     // fetch mounted pages
@@ -1640,15 +1640,15 @@ 
                 $mountTree->init('AND '.$perms_clause);
                 $mountTree->getTree($mountpage[0]['mount_pid'], $depth, '');
 
-                foreach($mountTree->tree as $mountData)    {
+                foreach ($mountTree->tree as $mountData) {
                     $code .= $this->drawURLs_addRowsForPage(
                         $mountData['row'],
-                        $mountData['HTML'].\TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages',$mountData['row'],TRUE)
+                        $mountData['HTML'].\TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages', $mountData['row'], TRUE)
                     );
                 }
 
                     // replace page when mount_pid_ol is enabled
-                if($mountpage[0]['mount_pid_ol']){
+                if ($mountpage[0]['mount_pid_ol']) {
                     $data['row']['uid'] = $mountpage[0]['mount_pid'];
                 } else {
                         // if the mount_pid_ol is not set the MP must not be used for the mountpoint page
@@ -1658,7 +1658,7 @@ 
 
             $code .= $this->drawURLs_addRowsForPage(
                 $data['row'],
-                $data['HTML'] . \TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages', $data['row'], TRUE)
+                $data['HTML'].\TYPO3\CMS\Backend\Utility\BackendUtility::getRecordTitle('pages', $data['row'], TRUE)
             );
         }
 
@@ -1682,7 +1682,7 @@ 
             if (!empty($excludeString)) {
                 /* @var $tree \TYPO3\CMS\Backend\Tree\View\PageTreeView */
                 $tree = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Backend\Tree\View\PageTreeView');
-                $tree->init('AND ' . $this->backendUser->getPagePermsClause(1));
+                $tree->init('AND '.$this->backendUser->getPagePermsClause(1));
 
                 $excludeParts = \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $excludeString);
 
@@ -1691,7 +1691,7 @@ 
 
                         // default is "page only" = "depth=0"
                     if (empty($depth)) {
-                        $depth = ( stristr($excludePart,'+')) ? 99 : 0;
+                        $depth = (stristr($excludePart, '+')) ? 99 : 0;
                     }
 
                     $pidList[] = $pid;
@@ -1724,7 +1724,7 @@ 
      * @param    string        Page icon and title for row
      * @return    string        HTML <tr> content (one or more)
      */
-    public function drawURLs_addRowsForPage(array $pageRow, $pageTitleAndIcon)    {
+    public function drawURLs_addRowsForPage(array $pageRow, $pageTitleAndIcon) {
 
         $skipMessage = '';
 
@@ -1745,7 +1745,7 @@ 
         $cc = 0;
         $content = '';
         if (count($configurations)) {
-            foreach($configurations as $confKey => $confArray)    {
+            foreach ($configurations as $confKey => $confArray) {
 
                     // Title column:
                 if (!$c) {
@@ -1774,47 +1774,47 @@ 
                     $paramExpanded = '';
                     $calcAccu = array();
                     $calcRes = 1;
-                    foreach($confArray['paramExpanded'] as $gVar => $gVal)    {
-                        $paramExpanded.= '
+                    foreach ($confArray['paramExpanded'] as $gVar => $gVal) {
+                        $paramExpanded .= '
                             <tr>
                                 <td class="bgColor4-20">'.htmlspecialchars('&'.$gVar.'=').'<br/>'.
                                                 '('.count($gVal).')'.
                                                 '</td>
-                                <td class="bgColor4" nowrap="nowrap">'.nl2br(htmlspecialchars(implode(chr(10),$gVal))).'</td>
+                                <td class="bgColor4" nowrap="nowrap">'.nl2br(htmlspecialchars(implode(chr(10), $gVal))).'</td>
                             </tr>
                         ';
-                        $calcRes*= count($gVal);
+                        $calcRes *= count($gVal);
                         $calcAccu[] = count($gVal);
                     }
                     $paramExpanded = '<table class="lrPadding c-list param-expanded">'.$paramExpanded.'</table>';
-                    $paramExpanded.= 'Comb: '.implode('*',$calcAccu).'='.$calcRes;
+                    $paramExpanded .= 'Comb: '.implode('*', $calcAccu).'='.$calcRes;
 
                         // Options
                     $optionValues = '';
-                    if ($confArray['subCfg']['userGroups'])    {
-                        $optionValues.='User Groups: '.$confArray['subCfg']['userGroups'].'<br/>';
+                    if ($confArray['subCfg']['userGroups']) {
+                        $optionValues .= 'User Groups: '.$confArray['subCfg']['userGroups'].'<br/>';
                     }
-                    if ($confArray['subCfg']['baseUrl'])    {
-                        $optionValues.='Base Url: '.$confArray['subCfg']['baseUrl'].'<br/>';
+                    if ($confArray['subCfg']['baseUrl']) {
+                        $optionValues .= 'Base Url: '.$confArray['subCfg']['baseUrl'].'<br/>';
                     }
-                    if ($confArray['subCfg']['procInstrFilter'])    {
-                        $optionValues.='ProcInstr: '.$confArray['subCfg']['procInstrFilter'].'<br/>';
+                    if ($confArray['subCfg']['procInstrFilter']) {
+                        $optionValues .= 'ProcInstr: '.$confArray['subCfg']['procInstrFilter'].'<br/>';
                     }
 
                         // Compile row:
                     $content .= '
-                        <tr class="bgColor' . ($c%2 ? '-20':'-10') . '">
-                            ' . $titleClm . '
-                            <td>' . htmlspecialchars($confKey) . '</td>
-                            <td>' . nl2br(htmlspecialchars(rawurldecode(trim(str_replace('&', chr(10) . '&', \TYPO3\CMS\Core\Utility\GeneralUtility::implodeArrayForUrl('', $confArray['paramParsed'])))))) . '</td>
+                        <tr class="bgColor' . ($c % 2 ? '-20' : '-10').'">
+                            ' . $titleClm.'
+                            <td>' . htmlspecialchars($confKey).'</td>
+                            <td>' . nl2br(htmlspecialchars(rawurldecode(trim(str_replace('&', chr(10).'&', \TYPO3\CMS\Core\Utility\GeneralUtility::implodeArrayForUrl('', $confArray['paramParsed'])))))).'</td>
                             <td>'.$paramExpanded.'</td>
-                            <td nowrap="nowrap">' . $urlList . '</td>
-                            <td nowrap="nowrap">' . $optionValues . '</td>
-                            <td nowrap="nowrap">' . \TYPO3\CMS\Core\Utility\DebugUtility::viewArray($confArray['subCfg']['procInstrParams.']) . '</td>
+                            <td nowrap="nowrap">' . $urlList.'</td>
+                            <td nowrap="nowrap">' . $optionValues.'</td>
+                            <td nowrap="nowrap">' . \TYPO3\CMS\Core\Utility\DebugUtility::viewArray($confArray['subCfg']['procInstrParams.']).'</td>
                         </tr>';
                 } else {
 
-                    $content .= '<tr class="bgColor'.($c%2 ? '-20':'-10') . '">
+                    $content .= '<tr class="bgColor'.($c % 2 ? '-20' : '-10').'">
                             '.$titleClm.'
                             <td>'.htmlspecialchars($confKey).'</td>
                             <td colspan="5"><em>No entries</em> (Page is excluded in this configuration)</td>
@@ -1829,7 +1829,7 @@ 
             $message = !empty($skipMessage) ? ' ('.$skipMessage.')' : '';
 
                 // Compile row:
-            $content.= '
+            $content .= '
                 <tr class="bgColor-20" style="border-bottom: 1px solid black;">
                     <td>'.$pageTitleAndIcon.'</td>
                     <td colspan="6"><em>No entries</em>'.$message.'</td>
@@ -1890,7 +1890,7 @@ 
                     // Run process:
                 $result = $this->CLI_run($countInARun, $sleepTime, $sleepAfterFinish);
             } catch (Exception $e) {
-                $this->CLI_debug(get_class($e) . ': ' . $e->getMessage());
+                $this->CLI_debug(get_class($e).': '.$e->getMessage());
                 $result = self::CLI_STATUS_ABORTED;
             }
 
@@ -1901,7 +1901,7 @@ 
             $releaseStatus = $this->CLI_releaseProcesses($this->CLI_buildProcessId());
 
             $this->CLI_debug("Unprocessed Items remaining:".$this->getUnprocessedItemsCount()." (".$this->CLI_buildProcessId().")");
-            $result |= ( $this->getUnprocessedItemsCount() > 0 ? self::CLI_STATUS_REMAIN : self::CLI_STATUS_NOTHING_PROCCESSED );
+            $result |= ($this->getUnprocessedItemsCount() > 0 ? self::CLI_STATUS_REMAIN : self::CLI_STATUS_NOTHING_PROCCESSED);
         } else {
             $result |= self::CLI_STATUS_ABORTED;
         }
@@ -1914,7 +1914,7 @@ 
      *
      * @return    void
      */
-    function CLI_main_im()    {
+    function CLI_main_im() {
         $this->setAccessMode('cli_im');
 
         $cliObj = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('tx_crawler_cli_im');
@@ -1924,7 +1924,7 @@ 
         $this->backendUser->setWorkspace(0);
 
             // Print help
-        if (!isset($cliObj->cli_args['_DEFAULT'][1]))    {
+        if (!isset($cliObj->cli_args['_DEFAULT'][1])) {
             $cliObj->cli_validateArgs();
             $cliObj->cli_help();
             exit;
@@ -1932,8 +1932,8 @@ 
 
         $cliObj->cli_validateArgs();
 
-        if ($cliObj->cli_argValue('-o')==='exec')    {
-            $this->registerQueueEntriesInternallyOnly=TRUE;
+        if ($cliObj->cli_argValue('-o') === 'exec') {
+            $this->registerQueueEntriesInternallyOnly = TRUE;
         }
 
         if (isset($cliObj->cli_args['_DEFAULT'][2])) {
@@ -1946,16 +1946,16 @@ 
 
         $configurationKeys  = $this->getConfigurationKeys($cliObj);
 
-        if(!is_array($configurationKeys)){
+        if (!is_array($configurationKeys)) {
             $configurations = $this->getUrlsForPageId($pageId);
-            if(is_array($configurations)){
+            if (is_array($configurations)) {
                 $configurationKeys = array_keys($configurations);
-            }else{
+            } else {
                 $configurationKeys = array();
             }
         }
 
-        if($cliObj->cli_argValue('-o')==='queue' || $cliObj->cli_argValue('-o')==='exec'){
+        if ($cliObj->cli_argValue('-o') === 'queue' || $cliObj->cli_argValue('-o') === 'exec') {
 
             $reason = new tx_crawler_domain_reason();
             $reason->setReason(tx_crawler_domain_reason::REASON_GUI_SUBMIT);
@@ -1963,7 +1963,7 @@ 
             tx_crawler_domain_events_dispatcher::getInstance()->post(
                 'invokeQueueChange',
                 $this->setID,
-                array(    'reason' => $reason )
+                array('reason' => $reason)
             );
         }
 
@@ -1974,42 +1974,42 @@ 
         $this->setID = \TYPO3\CMS\Core\Utility\GeneralUtility::md5int(microtime());
         $this->getPageTreeAndUrls(
             $pageId,
-            \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($cliObj->cli_argValue('-d'),0,99),
+            \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($cliObj->cli_argValue('-d'), 0, 99),
             $this->getCurrentTime(),
-            \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($cliObj->cli_isArg('-n') ? $cliObj->cli_argValue('-n') : 30,1,1000),
-            $cliObj->cli_argValue('-o')==='queue' || $cliObj->cli_argValue('-o')==='exec',
-            $cliObj->cli_argValue('-o')==='url',
-            \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',',$cliObj->cli_argValue('-proc'),1),
+            \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($cliObj->cli_isArg('-n') ? $cliObj->cli_argValue('-n') : 30, 1, 1000),
+            $cliObj->cli_argValue('-o') === 'queue' || $cliObj->cli_argValue('-o') === 'exec',
+            $cliObj->cli_argValue('-o') === 'url',
+            \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $cliObj->cli_argValue('-proc'), 1),
             $configurationKeys
         );
 
-        if ($cliObj->cli_argValue('-o')==='url') {
-            $cliObj->cli_echo(implode(chr(10),$this->downloadUrls).chr(10),1);
-        } elseif ($cliObj->cli_argValue('-o')==='exec')    {
+        if ($cliObj->cli_argValue('-o') === 'url') {
+            $cliObj->cli_echo(implode(chr(10), $this->downloadUrls).chr(10), 1);
+        } elseif ($cliObj->cli_argValue('-o') === 'exec') {
             $cliObj->cli_echo("Executing ".count($this->urlList)." requests right away:\n\n");
-            $cliObj->cli_echo(implode(chr(10),$this->urlList).chr(10));
+            $cliObj->cli_echo(implode(chr(10), $this->urlList).chr(10));
             $cliObj->cli_echo("\nProcessing:\n");
 
-            foreach($this->queueEntries as $queueRec)    {
+            foreach ($this->queueEntries as $queueRec) {
                 $p = unserialize($queueRec['parameters']);
-                $cliObj->cli_echo($p['url'].' ('.implode(',',$p['procInstructions']).') => ');
+                $cliObj->cli_echo($p['url'].' ('.implode(',', $p['procInstructions']).') => ');
 
                 $result = $this->readUrlFromArray($queueRec);
 
                 $requestResult = unserialize($result['content']);
-                if (is_array($requestResult))    {
-                    $resLog = is_array($requestResult['log']) ?  chr(10).chr(9).chr(9).implode(chr(10).chr(9).chr(9),$requestResult['log']) : '';
+                if (is_array($requestResult)) {
+                    $resLog = is_array($requestResult['log']) ?  chr(10).chr(9).chr(9).implode(chr(10).chr(9).chr(9), $requestResult['log']) : '';
                     $cliObj->cli_echo('OK: '.$resLog.chr(10));
                 } else {
-                    $cliObj->cli_echo('Error checking Crawler Result: '.substr(preg_replace('/\s+/',' ',strip_tags($result['content'])),0,30000).'...'.chr(10));
+                    $cliObj->cli_echo('Error checking Crawler Result: '.substr(preg_replace('/\s+/', ' ', strip_tags($result['content'])), 0, 30000).'...'.chr(10));
                 }
             }
-        } elseif ($cliObj->cli_argValue('-o')==='queue')    {
+        } elseif ($cliObj->cli_argValue('-o') === 'queue') {
             $cliObj->cli_echo("Putting ".count($this->urlList)." entries in queue:\n\n");
-            $cliObj->cli_echo(implode(chr(10),$this->urlList).chr(10));
+            $cliObj->cli_echo(implode(chr(10), $this->urlList).chr(10));
         } else {
-            $cliObj->cli_echo(count($this->urlList)." entries found for processing. (Use -o to decide action):\n\n",1);
-            $cliObj->cli_echo(implode(chr(10),$this->urlList).chr(10),1);
+            $cliObj->cli_echo(count($this->urlList)." entries found for processing. (Use -o to decide action):\n\n", 1);
+            $cliObj->cli_echo(implode(chr(10), $this->urlList).chr(10), 1);
         }
     }
 
@@ -2034,12 +2034,12 @@ 
         }
 
         $cliObj->cli_validateArgs();
-        $pageId = \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($cliObj->cli_args['_DEFAULT'][1],0);
+        $pageId = \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($cliObj->cli_args['_DEFAULT'][1], 0);
         $fullFlush = ($pageId == 0);
 
         $mode = $cliObj->cli_argValue('-o');
 
-        switch($mode) {
+        switch ($mode) {
             case 'all':
                 $result = $this->getLogEntriesForPageId($pageId, '', true, $fullFlush);
                 break;
@@ -2062,7 +2062,7 @@ 
      * @param  tx_crawler_cli_im $cliObj    Command line object
      * @return mixed                        Array of keys or null if no keys found
      */
-    protected function getConfigurationKeys(tx_crawler_cli_im &$cliObj) {
+    protected function getConfigurationKeys(tx_crawler_cli_im & $cliObj) {
         $parameter = trim($cliObj->cli_argValue('-conf'));
         return ($parameter != '' ? \TYPO3\CMS\Core\Utility\GeneralUtility::trimExplode(',', $parameter) : array());
     }
@@ -2087,7 +2087,7 @@ 
             $purgeDate = $this->getCurrentTime() - 24 * 60 * 60 * intval($this->extensionSettings['purgeQueueDays']);
             $del = $this->db->exec_DELETEquery(
                 'tx_crawler_queue',
-                'exec_time!=0 AND exec_time<' . $purgeDate
+                'exec_time!=0 AND exec_time<'.$purgeDate
             );
         }
 
@@ -2104,10 +2104,10 @@ 
         intval($countInARun)
         );
 
-        if (count($rows)>0) {
+        if (count($rows) > 0) {
             $quidList = array();
 
-            foreach($rows as $r) {
+            foreach ($rows as $r) {
                 $quidList[] = $r['qid'];
             }
 
@@ -2118,7 +2118,7 @@ 
                 //TODO make sure we're not taking assigned queue-entires
             $this->db->exec_UPDATEquery(
                 'tx_crawler_queue',
-                'qid IN ('.implode(',',$quidList).')',
+                'qid IN ('.implode(',', $quidList).')',
                 array(
                     'process_scheduled' => intval($this->getCurrentTime()),
                     'process_id' => $processId
@@ -2129,32 +2129,32 @@ 
             $numberOfAffectedRows = $this->db->sql_affected_rows();
             $this->db->exec_UPDATEquery(
                 'tx_crawler_process',
-                "process_id = '".$processId."'" ,
+                "process_id = '".$processId."'",
                 array(
                     'assigned_items_count' => intval($numberOfAffectedRows)
                 )
             );
 
-            if($numberOfAffectedRows == count($quidList)) {
+            if ($numberOfAffectedRows == count($quidList)) {
                 $this->db->sql_query('COMMIT');
-            } else  {
+            } else {
                 $this->db->sql_query('ROLLBACK');
                 $this->CLI_debug("Nothing processed due to multi-process collision (".$this->CLI_buildProcessId().")");
-                return ( $result | self::CLI_STATUS_ABORTED );
+                return ($result | self::CLI_STATUS_ABORTED);
             }
 
 
 
-            foreach($rows as $r)    {
+            foreach ($rows as $r) {
                 $result |= $this->readUrl($r['qid']);
 
                 $counter++;
-                usleep(intval($sleepTime));    // Just to relax the system
+                usleep(intval($sleepTime)); // Just to relax the system
 
                     // if during the start and the current read url the cli has been disable we need to return from the function
                     // mark the process NOT as ended.
                 if ($this->getDisabled()) {
-                    return ( $result | self::CLI_STATUS_ABORTED );
+                    return ($result | self::CLI_STATUS_ABORTED);
                 }
 
                 if (!$this->CLI_checkIfProcessIsActive($this->CLI_buildProcessId())) {
@@ -2162,7 +2162,7 @@ 
 
                         //TODO might need an additional returncode
                     $result |= self::CLI_STATUS_ABORTED;
-                    break;        //possible timeout
+                    break; //possible timeout
                 }
             }
 
@@ -2175,7 +2175,7 @@ 
             $this->CLI_debug("Nothing within queue which needs to be processed (".$this->CLI_buildProcessId().")");
         }
 
-        if($counter > 0) {
+        if ($counter > 0) {
             $result |= self::CLI_STATUS_PROCESSED;
         }
 
@@ -2187,12 +2187,12 @@ 
      *
      * @return    void
      */
-    function CLI_runHooks()    {
+    function CLI_runHooks() {
         global $TYPO3_CONF_VARS;
-        if (is_array($TYPO3_CONF_VARS['EXTCONF']['crawler']['cli_hooks']))    {
-            foreach($TYPO3_CONF_VARS['EXTCONF']['crawler']['cli_hooks'] as $objRef)    {
+        if (is_array($TYPO3_CONF_VARS['EXTCONF']['crawler']['cli_hooks'])) {
+            foreach ($TYPO3_CONF_VARS['EXTCONF']['crawler']['cli_hooks'] as $objRef) {
                 $hookObj = &\TYPO3\CMS\Core\Utility\GeneralUtility::getUserObj($objRef);
-                if (is_object($hookObj))    {
+                if (is_object($hookObj)) {
                     $hookObj->crawler_init($this);
                 }
             }
@@ -2229,7 +2229,7 @@ 
 
             $currentTime = $this->getCurrentTime();
 
-            while($row = $this->db->sql_fetch_assoc($res))    {
+            while ($row = $this->db->sql_fetch_assoc($res)) {
                 if ($row['ttl'] < $currentTime) {
                     $orphanProcesses[] = $row['process_id'];
                 } else {
@@ -2239,7 +2239,7 @@ 
 
                 // if there are less than allowed active processes then add a new one
             if ($processCount < intval($this->extensionSettings['processLimit'])) {
-                $this->CLI_debug("add process ".$this->CLI_buildProcessId()." (".($processCount+1)."/".intval($this->extensionSettings['processLimit']).")");
+                $this->CLI_debug("add process ".$this->CLI_buildProcessId()." (".($processCount + 1)."/".intval($this->extensionSettings['processLimit']).")");
 
                     // create new process record
                 $this->db->exec_INSERTquery(
@@ -2272,17 +2272,17 @@ 
      * @param  boolean  $withinLock   show whether the DB-actions are included within an existing lock
      * @return boolean
      */
-    function CLI_releaseProcesses($releaseIds, $withinLock=false) {
+    function CLI_releaseProcesses($releaseIds, $withinLock = false) {
 
         if (!is_array($releaseIds)) {
             $releaseIds = array($releaseIds);
         }
 
         if (!count($releaseIds) > 0) {
-            return false;   //nothing to release
+            return false; //nothing to release
         }
 
-        if(!$withinLock) $this->db->sql_query('BEGIN');
+        if (!$withinLock) $this->db->sql_query('BEGIN');
 
             // some kind of 2nd chance algo - this way you need at least 2 processes to have a real cleanup
             // this ensures that a single process can't mess up the entire process table
@@ -2312,21 +2312,21 @@ 
                 // mark all requested processes as non-active
         $this->db->exec_UPDATEquery(
             'tx_crawler_process',
-            'process_id IN (\''.implode('\',\'',$releaseIds).'\') AND deleted=0',
+            'process_id IN (\''.implode('\',\'', $releaseIds).'\') AND deleted=0',
             array(
                 'active'=>'0'
             )
         );
         $this->db->exec_UPDATEquery(
             'tx_crawler_queue',
-            'exec_time=0 AND process_id IN ("'.implode('","',$releaseIds).'")',
+            'exec_time=0 AND process_id IN ("'.implode('","', $releaseIds).'")',
             array(
                 'process_scheduled'=>0,
                 'process_id'=>''
             )
         );
 
-        if(!$withinLock) $this->db->sql_query('COMMIT');
+        if (!$withinLock) $this->db->sql_query('COMMIT');
 
         return true;
     }
@@ -2354,13 +2354,13 @@ 
         $this->db->sql_query('BEGIN');
         $res = $this->db->exec_SELECTquery(
             'process_id,active,ttl',
-            'tx_crawler_process','process_id = \''.$pid.'\'  AND deleted=0',
+            'tx_crawler_process', 'process_id = \''.$pid.'\'  AND deleted=0',
             '',
             'ttl',
             '0,1'
         );
-        if($row = $this->db->sql_fetch_assoc($res))    {
-            $ret = intVal($row['active'])==1;
+        if ($row = $this->db->sql_fetch_assoc($res)) {
+            $ret = intVal($row['active']) == 1;
         }
         $this->db->sql_query('COMMIT');
 
@@ -2373,8 +2373,8 @@ 
      * @return string  the ID
      */
     protected function CLI_buildProcessId() {
-        if(!$this->processID) {
-            $this->processID= \TYPO3\CMS\Core\Utility\GeneralUtility::shortMD5($this->microtime(true));
+        if (!$this->processID) {
+            $this->processID = \TYPO3\CMS\Core\Utility\GeneralUtility::shortMD5($this->microtime(true));
         }
         return $this->processID;
     }
@@ -2386,7 +2386,7 @@ 
      *
      * @codeCoverageIgnore
      */
-    protected function microtime($get_as_float = false )
+    protected function microtime($get_as_float = false)
     {
         return microtime($get_as_float);
     }
@@ -2399,7 +2399,7 @@ 
      * @codeCoverageIgnore
      */
     function CLI_debug($msg) {
-        if(intval($this->extensionSettings['processDebug'])) {
+        if (intval($this->extensionSettings['processDebug'])) {
             echo $msg."\n"; flush();
         }
     }
@@ -2418,7 +2418,7 @@ 
 
         $cmd  = escapeshellcmd($this->extensionSettings['phpPath']);
         $cmd .= ' ';
-        $cmd .= escapeshellarg(\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath('crawler') . 'cli/bootstrap.php');
+        $cmd .= escapeshellarg(\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extPath('crawler').'cli/bootstrap.php');
         $cmd .= ' ';
         $cmd .= escapeshellarg($this->getFrontendBasePath());
         $cmd .= ' ';
@@ -2428,10 +2428,10 @@ 
 
         $startTime = microtime(true);
         $content = $this->executeShellCommand($cmd);
-        $this->log($url . (microtime(true) - $startTime));
+        $this->log($url.(microtime(true) - $startTime));
 
         $result = array(
-            'request' => implode("\r\n", $requestHeaders) . "\r\n\r\n",
+            'request' => implode("\r\n", $requestHeaders)."\r\n\r\n",
             'headers' => '',
             'content' => $content
         );
@@ -2451,7 +2451,7 @@ 
         $scheduledAgeInSeconds = $this->extensionSettings['cleanUpScheduledAge'] * 86400;
 
         $now = time();
-        $condition = '(exec_time<>0 AND exec_time<' . ($now - $processedAgeInSeconds) . ') OR scheduled<=' . ($now - $scheduledAgeInSeconds);
+        $condition = '(exec_time<>0 AND exec_time<'.($now - $processedAgeInSeconds).') OR scheduled<='.($now - $scheduledAgeInSeconds);
         $this->flushQueue($condition);
     }
 
@@ -2472,7 +2472,7 @@ 
             $GLOBALS['TT']->start();
         }
 
-        $GLOBALS['TSFE'] = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\\CMS\\Frontend\\Controller\\TypoScriptFrontendController',  $GLOBALS['TYPO3_CONF_VARS'], $id, $typeNum);
+        $GLOBALS['TSFE'] = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\\CMS\\Frontend\\Controller\\TypoScriptFrontendController', $GLOBALS['TYPO3_CONF_VARS'], $id, $typeNum);
         $GLOBALS['TSFE']->sys_page = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\\CMS\\Frontend\\Page\\PageRepository');
         $GLOBALS['TSFE']->sys_page->init(TRUE);
         $GLOBALS['TSFE']->connectToDB();
@@ -2485,6 +2485,6 @@ 
     }
 }
 
-if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/class.tx_crawler_lib.php'])    {
+if (defined('TYPO3_MODE') && $TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/class.tx_crawler_lib.php']) {
     include_once($TYPO3_CONF_VARS[TYPO3_MODE]['XCLASS']['ext/crawler/class.tx_crawler_lib.php']);
 }