jenskooij / cloudcontrol

src/search/indexer/InverseDocumentFrequency.php

Indentation +39 added lines, -39 removed lines

@@ -10,51 +10,51 @@
 
 class InverseDocumentFrequency
 {
-	/**
-	 * @var \PDO
-	 */
-	protected $dbHandle;
-	protected $documentCount;
+    /**
+     * @var \PDO
+     */
+    protected $dbHandle;
+    protected $documentCount;
 
-	/**
-	 * InverseDocumentFrequency constructor.
-	 *
-	 * @param \PDO 	$dbHandle
-	 * @param int   $documentCount
-	 */
-	public function __construct($dbHandle, $documentCount)
-	{
-		$this->dbHandle = $dbHandle;
-		$this->documentCount = $documentCount;
-	}
+    /**
+     * InverseDocumentFrequency constructor.
+     *
+     * @param \PDO 	$dbHandle
+     * @param int   $documentCount
+     */
+    public function __construct($dbHandle, $documentCount)
+    {
+        $this->dbHandle = $dbHandle;
+        $this->documentCount = $documentCount;
+    }
 
-	/**
-	 * Formula to calculate:
-	 * 		idf(t) = 1 + log ( totalDocuments / (documentsThatContainTheTerm + 1))
-	 * @throws \Exception
-	 */
-	public function execute()
-	{
-		$db = $this->dbHandle;
-		$db->sqliteCreateFunction('log', 'log', 1);
-		$sql = '
+    /**
+     * Formula to calculate:
+     * 		idf(t) = 1 + log ( totalDocuments / (documentsThatContainTheTerm + 1))
+     * @throws \Exception
+     */
+    public function execute()
+    {
+        $db = $this->dbHandle;
+        $db->sqliteCreateFunction('log', 'log', 1);
+        $sql = '
 		INSERT INTO inverse_document_frequency (term, inverseDocumentFrequency)
 		SELECT DISTINCT term, (1+(log(:documentCount / COUNT(documentPath) + 1))) as inverseDocumentFrequency
 					  FROM term_count
 				  GROUP BY term
 		';
 
-		if (!$stmt = $db->prepare($sql)) {
-			$errorInfo = $db->errorInfo();
-			$errorMsg = $errorInfo[2];
-			throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
-		}
-		$stmt->bindValue(':documentCount', $this->documentCount);
-		$result = $stmt->execute();
-		if ($result === false) {
-			$errorInfo = $db->errorInfo();
-			$errorMsg = $errorInfo[2];
-			throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
-		}
-	}
+        if (!$stmt = $db->prepare($sql)) {
+            $errorInfo = $db->errorInfo();
+            $errorMsg = $errorInfo[2];
+            throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
+        }
+        $stmt->bindValue(':documentCount', $this->documentCount);
+        $result = $stmt->execute();
+        if ($result === false) {
+            $errorInfo = $db->errorInfo();
+            $errorMsg = $errorInfo[2];
+            throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
+        }
+    }
 }
\ No newline at end of file

src/search/indexer/TermCount.php

Indentation +100 added lines, -100 removed lines

@@ -14,113 +14,113 @@
 
 class TermCount
 {
-	/**
-	 * @var \PDO
-	 */
-	protected $dbHandle;
-	protected $documents;
-	protected $filters;
-	protected $storage;
+    /**
+     * @var \PDO
+     */
+    protected $dbHandle;
+    protected $documents;
+    protected $filters;
+    protected $storage;
 
-	/**
-	 * TermCount constructor.
-	 *
-	 * @param \PDO    $dbHandle
-	 * @param array   $documents
-	 * @param array   $filters
-	 * @param Storage $jsonStorage
-	 */
-	public function __construct($dbHandle, $documents, $filters, $jsonStorage)
-	{
-		$this->dbHandle = $dbHandle;
-		$this->documents = $documents;
-		$this->filters = $filters;
-		$this->storage = $jsonStorage;
-	}
+    /**
+     * TermCount constructor.
+     *
+     * @param \PDO    $dbHandle
+     * @param array   $documents
+     * @param array   $filters
+     * @param Storage $jsonStorage
+     */
+    public function __construct($dbHandle, $documents, $filters, $jsonStorage)
+    {
+        $this->dbHandle = $dbHandle;
+        $this->documents = $documents;
+        $this->filters = $filters;
+        $this->storage = $jsonStorage;
+    }
 
-	public function execute()
-	{
-		$this->iterateDocumentsAndCreateTermCount($this->documents);
-	}
+    public function execute()
+    {
+        $this->iterateDocumentsAndCreateTermCount($this->documents);
+    }
 
-	protected function applyFilters($tokens)
-	{
-		foreach ($this->filters as $filterName) {
-			$filterClassName = '\CloudControl\Cms\search\filters\\' . $filterName;
-			$filter = new $filterClassName($tokens);
-			$tokens = $filter->getFilterResults();
-		}
-		return $tokens;
-	}
+    protected function applyFilters($tokens)
+    {
+        foreach ($this->filters as $filterName) {
+            $filterClassName = '\CloudControl\Cms\search\filters\\' . $filterName;
+            $filter = new $filterClassName($tokens);
+            $tokens = $filter->getFilterResults();
+        }
+        return $tokens;
+    }
 
-	protected function storeDocumentTermCount($document, $documentTermCount)
-	{
-		$db = $this->dbHandle;
-		$sqlStart = '
+    protected function storeDocumentTermCount($document, $documentTermCount)
+    {
+        $db = $this->dbHandle;
+        $sqlStart = '
 			INSERT INTO `term_count` (`documentPath`, `term`, `count`, `field`)
 				 VALUES ';
-		$sql = $sqlStart;
-		$values = array();
-		$quotedDocumentPath = $db->quote($document->path);
-		$i = 0;
-		foreach ($documentTermCount as $field => $countArray) {
-			$quotedField = $db->quote($field);
-			foreach ($countArray as $term => $count) {
-				$values[] = $quotedDocumentPath . ', ' . $db->quote($term) . ', ' . $db->quote($count) . ', ' . $quotedField;
-				$i += 1;
-				if ($i >= Indexer::SQLITE_MAX_COMPOUND_SELECT) {
-					$this->executeStoreDocumentTermCount($values, $sql, $db);
-					$values = array();
-					$sql = $sqlStart;
-					$i = 0;
-				}
-			}
-		}
-		if (count($values) != 0) {
-			$this->executeStoreDocumentTermCount($values, $sql, $db);
-		}
-	}
+        $sql = $sqlStart;
+        $values = array();
+        $quotedDocumentPath = $db->quote($document->path);
+        $i = 0;
+        foreach ($documentTermCount as $field => $countArray) {
+            $quotedField = $db->quote($field);
+            foreach ($countArray as $term => $count) {
+                $values[] = $quotedDocumentPath . ', ' . $db->quote($term) . ', ' . $db->quote($count) . ', ' . $quotedField;
+                $i += 1;
+                if ($i >= Indexer::SQLITE_MAX_COMPOUND_SELECT) {
+                    $this->executeStoreDocumentTermCount($values, $sql, $db);
+                    $values = array();
+                    $sql = $sqlStart;
+                    $i = 0;
+                }
+            }
+        }
+        if (count($values) != 0) {
+            $this->executeStoreDocumentTermCount($values, $sql, $db);
+        }
+    }
 
-	/**
-	 * @param $values
-	 * @param $sql
-	 * @param $db
-	 *
-	 * @throws \Exception
-	 */
-	protected function executeStoreDocumentTermCount($values, $sql, $db)
-	{
-		$sql .= '(' . implode('),' . PHP_EOL . '(', $values) . ');';
-		$stmt = $db->prepare($sql);
-		if ($stmt === false || !$stmt->execute()) {
-			$errorInfo = $db->errorInfo();
-			$errorMsg = $errorInfo[2];
-			throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
-		}
-	}
+    /**
+     * @param $values
+     * @param $sql
+     * @param $db
+     *
+     * @throws \Exception
+     */
+    protected function executeStoreDocumentTermCount($values, $sql, $db)
+    {
+        $sql .= '(' . implode('),' . PHP_EOL . '(', $values) . ');';
+        $stmt = $db->prepare($sql);
+        if ($stmt === false || !$stmt->execute()) {
+            $errorInfo = $db->errorInfo();
+            $errorMsg = $errorInfo[2];
+            throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
+        }
+    }
 
-	/**
-	 * @param $document
-	 */
-	private function createTermCountForDocument($document)
-	{
-		$tokenizer = new DocumentTokenizer($document, $this->storage);
-		$tokens = $tokenizer->getTokens();
-		$documentTermCount = $this->applyFilters($tokens);
-		$this->storeDocumentTermCount($document, $documentTermCount);
-	}
+    /**
+     * @param $document
+     */
+    private function createTermCountForDocument($document)
+    {
+        $tokenizer = new DocumentTokenizer($document, $this->storage);
+        $tokens = $tokenizer->getTokens();
+        $documentTermCount = $this->applyFilters($tokens);
+        $this->storeDocumentTermCount($document, $documentTermCount);
+    }
 
-	/**
-	 * @param $documents
-	 */
-	private function iterateDocumentsAndCreateTermCount($documents)
-	{
-		foreach ($documents as $document) {
-			if ($document->type === 'folder') {
-				$this->iterateDocumentsAndCreateTermCount($document->content);
-			} else {
-				$this->createTermCountForDocument($document);
-			}
-		}
-	}
+    /**
+     * @param $documents
+     */
+    private function iterateDocumentsAndCreateTermCount($documents)
+    {
+        foreach ($documents as $document) {
+            if ($document->type === 'folder') {
+                $this->iterateDocumentsAndCreateTermCount($document->content);
+            } else {
+                $this->createTermCountForDocument($document);
+            }
+        }
+    }
 }
\ No newline at end of file

src/search/indexer/TermFieldLengthNorm.php

Indentation +61 added lines, -61 removed lines

@@ -17,72 +17,72 @@
  */
 class TermFieldLengthNorm
 {
-	/**
-	 * @var \PDO
-	 */
-	protected $dbHandle;
+    /**
+     * @var \PDO
+     */
+    protected $dbHandle;
 
-	/**
-	 * TermFieldLengthNorm constructor.
-	 *
-	 * @param \PDO $dbHandle
-	 */
-	public function __construct($dbHandle)
-	{
-		$this->dbHandle = $dbHandle;
-	}
+    /**
+     * TermFieldLengthNorm constructor.
+     *
+     * @param \PDO $dbHandle
+     */
+    public function __construct($dbHandle)
+    {
+        $this->dbHandle = $dbHandle;
+    }
 
-	public function execute()
-	{
-		$db = $this->dbHandle;
-		$db->sqliteCreateFunction('sqrt', 'sqrt', 1);
-		$sql = '
+    public function execute()
+    {
+        $db = $this->dbHandle;
+        $db->sqliteCreateFunction('sqrt', 'sqrt', 1);
+        $sql = '
 		SELECT documentPath, field, COUNT(`count`) as termCount
 		  FROM term_count
 	  GROUP BY documentPath, field
 		';
-		$stmt = $db->prepare($sql);
-		if ($stmt === false) {
-			$errorInfo = $db->errorInfo();
-			$errorMsg = $errorInfo[2];
-			throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
-		}
-		if (($stmt->execute()) === false) {
-			$errorInfo = $db->errorInfo();
-			$errorMsg = $errorInfo[2];
-			throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
-		}
-		$uniqueFieldsPerDocument = $stmt->fetchAll(\PDO::FETCH_OBJ);
-		$values = array();
-		$i = 0;
-		foreach ($uniqueFieldsPerDocument as $fieldRow) {
-			$values[] = 'UPDATE term_frequency SET termNorm = 1/sqrt(' . intval($fieldRow->termCount) . ') WHERE documentPath = ' . $db->quote($fieldRow->documentPath) . ' AND field = ' . $db->quote($fieldRow->field) . ';';
-			$i += 1;
-			if ($i >= Indexer::SQLITE_MAX_COMPOUND_SELECT) {
-				$this->executeUpdateTermNorm($values, $db);
-				$values = array();
-				$i = 0;
-			}
-		}
-		if (count($values) != 0) {
-			$this->executeUpdateTermNorm($values, $db);
-		}
-	}
+        $stmt = $db->prepare($sql);
+        if ($stmt === false) {
+            $errorInfo = $db->errorInfo();
+            $errorMsg = $errorInfo[2];
+            throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
+        }
+        if (($stmt->execute()) === false) {
+            $errorInfo = $db->errorInfo();
+            $errorMsg = $errorInfo[2];
+            throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
+        }
+        $uniqueFieldsPerDocument = $stmt->fetchAll(\PDO::FETCH_OBJ);
+        $values = array();
+        $i = 0;
+        foreach ($uniqueFieldsPerDocument as $fieldRow) {
+            $values[] = 'UPDATE term_frequency SET termNorm = 1/sqrt(' . intval($fieldRow->termCount) . ') WHERE documentPath = ' . $db->quote($fieldRow->documentPath) . ' AND field = ' . $db->quote($fieldRow->field) . ';';
+            $i += 1;
+            if ($i >= Indexer::SQLITE_MAX_COMPOUND_SELECT) {
+                $this->executeUpdateTermNorm($values, $db);
+                $values = array();
+                $i = 0;
+            }
+        }
+        if (count($values) != 0) {
+            $this->executeUpdateTermNorm($values, $db);
+        }
+    }
 
-	/**
-	 * @param array $values
-	 * @param \PDO $db
-	 * @throws \Exception
-	 */
-	private function executeUpdateTermNorm($values, $db)
-	{
-		$sql  = 'BEGIN TRANSACTION;' . PHP_EOL;
-		$sql .= implode(PHP_EOL, $values) . PHP_EOL;
-		$sql .= 'COMMIT;';
-		if (($db->exec($sql)) === false) {
-			$errorInfo = $db->errorInfo();
-			$errorMsg = $errorInfo[2];
-			throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
-		}
-	}
+    /**
+     * @param array $values
+     * @param \PDO $db
+     * @throws \Exception
+     */
+    private function executeUpdateTermNorm($values, $db)
+    {
+        $sql  = 'BEGIN TRANSACTION;' . PHP_EOL;
+        $sql .= implode(PHP_EOL, $values) . PHP_EOL;
+        $sql .= 'COMMIT;';
+        if (($db->exec($sql)) === false) {
+            $errorInfo = $db->errorInfo();
+            $errorMsg = $errorInfo[2];
+            throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
+        }
+    }
 }
\ No newline at end of file

src/search/Search.php

Indentation +225 added lines, -225 removed lines

@@ -26,117 +26,117 @@ 
  */
 class Search extends SearchDbConnected
 {
-	/**
-	 * @var Tokenizer
-	 */
-	protected $tokenizer;
-	protected $results = array();
+    /**
+     * @var Tokenizer
+     */
+    protected $tokenizer;
+    protected $results = array();
 
-	/**
-	 * An array containing classes implementing \CloudControl\Cms\search\Filters
-	 * These will be applied to all tokenizers
-	 * @var array
-	 */
-	protected $filters = array(
-		'DutchStopWords',
-		'EnglishStopWords'
-	);
+    /**
+     * An array containing classes implementing \CloudControl\Cms\search\Filters
+     * These will be applied to all tokenizers
+     * @var array
+     */
+    protected $filters = array(
+        'DutchStopWords',
+        'EnglishStopWords'
+    );
 
-	/**
-	 * Returns an array of SeachResult and / or SearchSuggestion objects,
-	 * based on the tokens in the Tokenizer
-	 * @param Tokenizer $tokenizer
-	 *
-	 * @return array
-	 */
-	public function getDocumentsForTokenizer(Tokenizer $tokenizer)
-	{
-		$this->tokenizer = $tokenizer;
-		$resultsPerTokens = $this->queryTokens();
+    /**
+     * Returns an array of SeachResult and / or SearchSuggestion objects,
+     * based on the tokens in the Tokenizer
+     * @param Tokenizer $tokenizer
+     *
+     * @return array
+     */
+    public function getDocumentsForTokenizer(Tokenizer $tokenizer)
+    {
+        $this->tokenizer = $tokenizer;
+        $resultsPerTokens = $this->queryTokens();
 
-		$flatResults = $this->flattenResults($resultsPerTokens);
-		$flatResults = $this->applyQueryCoordination($flatResults);
-		usort($flatResults, array($this, "scoreCompare"));
+        $flatResults = $this->flattenResults($resultsPerTokens);
+        $flatResults = $this->applyQueryCoordination($flatResults);
+        usort($flatResults, array($this, "scoreCompare"));
 
-		$flatResults = array_merge($this->getSearchSuggestions(), $flatResults);
+        $flatResults = array_merge($this->getSearchSuggestions(), $flatResults);
 
-		return $flatResults;
-	}
+        return $flatResults;
+    }
 
-	/**
-	 * Returns the amount of distinct documents
-	 * that are currently in the search index.
-	 * @return int
-	 * @throws \Exception
-	 */
-	public function getIndexedDocuments()
-	{
-		$db = $this->getSearchDbHandle();
-		$sql = '
+    /**
+     * Returns the amount of distinct documents
+     * that are currently in the search index.
+     * @return int
+     * @throws \Exception
+     */
+    public function getIndexedDocuments()
+    {
+        $db = $this->getSearchDbHandle();
+        $sql = '
 			SELECT count(DISTINCT documentPath) as indexedDocuments
 			  FROM term_frequency
 		';
-		if (!$stmt = $db->query($sql)) {
-			$errorInfo = $db->errorInfo();
-			$errorMsg = $errorInfo[2];
-			throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
-		}
-		$result = $stmt->fetch(\PDO::FETCH_COLUMN);
-		if (false === $result) {
-			$errorInfo = $db->errorInfo();
-			$errorMsg = $errorInfo[2];
-			throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
-		}
-		return intval($result);
-	}
+        if (!$stmt = $db->query($sql)) {
+            $errorInfo = $db->errorInfo();
+            $errorMsg = $errorInfo[2];
+            throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
+        }
+        $result = $stmt->fetch(\PDO::FETCH_COLUMN);
+        if (false === $result) {
+            $errorInfo = $db->errorInfo();
+            $errorMsg = $errorInfo[2];
+            throw new \Exception('SQLite Exception: ' . $errorMsg . ' in SQL: <br /><pre>' . $sql . '</pre>');
+        }
+        return intval($result);
+    }
 
-	/**
-	 * Queries each token present in the Tokenizer
-	 * and returns SearchResult objects for the found
-	 * documents
-	 * @return array
-	 */
-	private function queryTokens()
-	{
-		$tokens = $this->getTokens();
+    /**
+     * Queries each token present in the Tokenizer
+     * and returns SearchResult objects for the found
+     * documents
+     * @return array
+     */
+    private function queryTokens()
+    {
+        $tokens = $this->getTokens();
 
-		$queryNorm = $this->getQueryNorm($tokens);
-		$results = array();
-		foreach ($tokens as $token) {
-			$results[$token] = $this->getResultsForToken($token, $queryNorm);
-		}
-		return $results;
-	}
+        $queryNorm = $this->getQueryNorm($tokens);
+        $results = array();
+        foreach ($tokens as $token) {
+            $results[$token] = $this->getResultsForToken($token, $queryNorm);
+        }
+        return $results;
+    }
 
-	/**
-	 * Applies the Filter objects in the the filter array to the
-	 * tokens in the Tokenizer
-	 * @param $tokens
-	 *
-	 * @return mixed
-	 */
-	protected function applyFilters($tokens)
-	{
-		foreach ($this->filters as $filterName) {
-			$filterClassName = '\CloudControl\Cms\search\filters\\' . $filterName;
-			$filter = new $filterClassName($tokens);
-			$tokens = $filter->getFilterResults();
-		}
-		return $tokens;
-	}
+    /**
+     * Applies the Filter objects in the the filter array to the
+     * tokens in the Tokenizer
+     * @param $tokens
+     *
+     * @return mixed
+     */
+    protected function applyFilters($tokens)
+    {
+        foreach ($this->filters as $filterName) {
+            $filterClassName = '\CloudControl\Cms\search\filters\\' . $filterName;
+            $filter = new $filterClassName($tokens);
+            $tokens = $filter->getFilterResults();
+        }
+        return $tokens;
+    }
 
-	/**
-	 * Queries the search index for a given token
-	 * and the query norm.
-	 * @param $token
-	 * @param $queryNorm
-	 *
-	 * @return array
-	 * @throws \Exception
-	 */
-	public function getResultsForToken($token, $queryNorm) {
-		$db = $this->getSearchDbHandle();
-		$sql = '
+    /**
+     * Queries the search index for a given token
+     * and the query norm.
+     * @param $token
+     * @param $queryNorm
+     *
+     * @return array
+     * @throws \Exception
+     */
+    public function getResultsForToken($token, $queryNorm) {
+        $db = $this->getSearchDbHandle();
+        $sql = '
 			SELECT (:queryNorm * 
 						(SUM(term_frequency.frequency) --TF
 						* inverse_document_frequency.inverseDocumentFrequency -- IDF
@@ -154,113 +154,113 @@ 
 		  GROUP BY term_frequency.documentPath, term_frequency.term
 		  ORDER BY score DESC
 		';
-		if(!$stmt = $db->prepare($sql)) {
-			throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
-		}
-		$stmt->bindValue(':query', $token);
-		$stmt->bindValue(':queryNorm', $queryNorm);
-		if (!$stmt->execute()) {
-			throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
-		}
-		return $stmt->fetchAll(\PDO::FETCH_CLASS, '\CloudControl\Cms\search\results\SearchResult');
-	}
+        if(!$stmt = $db->prepare($sql)) {
+            throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
+        }
+        $stmt->bindValue(':query', $token);
+        $stmt->bindValue(':queryNorm', $queryNorm);
+        if (!$stmt->execute()) {
+            throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
+        }
+        return $stmt->fetchAll(\PDO::FETCH_CLASS, '\CloudControl\Cms\search\results\SearchResult');
+    }
 
-	/**
-	 * @param $resultsPerTokens
-	 *
-	 * @return array
-	 */
-	private function flattenResults($resultsPerTokens)
-	{
-		$finalResults = array();
-		foreach ($resultsPerTokens as $token => $resultPerToken) {
-			foreach ($resultPerToken as $result) {
-				if (isset($finalResults[$result->documentPath])) {
-					$finalResults[$result->documentPath]->score += $result->score;
-					$finalResults[$result->documentPath]->matchingTokens[] = $token;
-				} else {
-					$resultObj = new SearchResult();
-					$resultObj->documentPath = $result->documentPath;
-					$resultObj->matchingTokens = array($token);
-					$resultObj->score = floatval($result->score);
-					$resultObj->setStorage($this->storage);
-					$finalResults[$result->documentPath] = $resultObj;
-				}
-			}
-		}
-		return $finalResults;
-	}
+    /**
+     * @param $resultsPerTokens
+     *
+     * @return array
+     */
+    private function flattenResults($resultsPerTokens)
+    {
+        $finalResults = array();
+        foreach ($resultsPerTokens as $token => $resultPerToken) {
+            foreach ($resultPerToken as $result) {
+                if (isset($finalResults[$result->documentPath])) {
+                    $finalResults[$result->documentPath]->score += $result->score;
+                    $finalResults[$result->documentPath]->matchingTokens[] = $token;
+                } else {
+                    $resultObj = new SearchResult();
+                    $resultObj->documentPath = $result->documentPath;
+                    $resultObj->matchingTokens = array($token);
+                    $resultObj->score = floatval($result->score);
+                    $resultObj->setStorage($this->storage);
+                    $finalResults[$result->documentPath] = $resultObj;
+                }
+            }
+        }
+        return $finalResults;
+    }
 
-	private function scoreCompare($a, $b) {
-		if ($a->score == $b->score) {
-			return 0;
-		}
-		return ($a->score > $b->score) ? -1 : 1;
-	}
+    private function scoreCompare($a, $b) {
+        if ($a->score == $b->score) {
+            return 0;
+        }
+        return ($a->score > $b->score) ? -1 : 1;
+    }
 
-	/**
-	 * Calculates the query norm for all tokens in the Tokenizer
-	 * @param $tokens
-	 *
-	 * @return int
-	 * @throws \Exception
-	 */
-	private function getQueryNorm($tokens)
-	{
-		$db = $this->getSearchDbHandle();
-		$db->sqliteCreateFunction('sqrt', 'sqrt', 1);
-		foreach ($tokens as $key => $token) {
-			$tokens[$key] = $db->quote($token);
-		}
-		$terms = implode(',', $tokens);
-		$sql = '
+    /**
+     * Calculates the query norm for all tokens in the Tokenizer
+     * @param $tokens
+     *
+     * @return int
+     * @throws \Exception
+     */
+    private function getQueryNorm($tokens)
+    {
+        $db = $this->getSearchDbHandle();
+        $db->sqliteCreateFunction('sqrt', 'sqrt', 1);
+        foreach ($tokens as $key => $token) {
+            $tokens[$key] = $db->quote($token);
+        }
+        $terms = implode(',', $tokens);
+        $sql = '
 			SELECT (1 / sqrt(SUM(inverseDocumentFrequency))) as queryNorm
 			  FROM inverse_document_frequency
 			 WHERE term IN (' . $terms . ') 
 		';
-		if(!$stmt = $db->prepare($sql)) {
-			throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
-		}
-		if (!$stmt->execute()) {
-			throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
-		}
-		$result = $stmt->fetch(\PDO::FETCH_OBJ);
-		return $result->queryNorm == null ? 1 : $result->queryNorm;
-	}
+        if(!$stmt = $db->prepare($sql)) {
+            throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
+        }
+        if (!$stmt->execute()) {
+            throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
+        }
+        $result = $stmt->fetch(\PDO::FETCH_OBJ);
+        return $result->queryNorm == null ? 1 : $result->queryNorm;
+    }
 
-	/**
-	 * Applies query coordination to all results
-	 * @param $flatResults
-	 *
-	 * @return mixed
-	 */
-	private function applyQueryCoordination($flatResults)
-	{
-		$tokenVector = $this->tokenizer->getTokenVector();
-		$tokens = array_keys($tokenVector);
-		$tokenCount = count($tokens);
-		foreach ($flatResults as $key => $result) {
-			$matchCount = count($result->matchingTokens);
-			$result->score = ($matchCount / $tokenCount) * $result->score;
-			$flatResults[$key] = $result;
-		}
-		return $flatResults;
-	}
+    /**
+     * Applies query coordination to all results
+     * @param $flatResults
+     *
+     * @return mixed
+     */
+    private function applyQueryCoordination($flatResults)
+    {
+        $tokenVector = $this->tokenizer->getTokenVector();
+        $tokens = array_keys($tokenVector);
+        $tokenCount = count($tokens);
+        foreach ($flatResults as $key => $result) {
+            $matchCount = count($result->matchingTokens);
+            $result->score = ($matchCount / $tokenCount) * $result->score;
+            $flatResults[$key] = $result;
+        }
+        return $flatResults;
+    }
 
-	/**
-	 * Uses the levenshtein algorithm to determine the term that is
-	 * closest to the token that was input for the search
-	 * @return array
-	 * @throws \Exception
-	 */
-	private function getSearchSuggestions()
-	{
-		$tokens = $this->getTokens();
-		$allResults = array();
-		foreach ($tokens as $token) {
-			$db = $this->getSearchDbHandle();
-			$db->sqliteCreateFunction('levenshtein', 'levenshtein', 2);
-			$sql = '
+    /**
+     * Uses the levenshtein algorithm to determine the term that is
+     * closest to the token that was input for the search
+     * @return array
+     * @throws \Exception
+     */
+    private function getSearchSuggestions()
+    {
+        $tokens = $this->getTokens();
+        $allResults = array();
+        foreach ($tokens as $token) {
+            $db = $this->getSearchDbHandle();
+            $db->sqliteCreateFunction('levenshtein', 'levenshtein', 2);
+            $sql = '
 				SELECT *
 				  FROM (
 				  	SELECT :token as original, term, levenshtein(term, :token) as editDistance
@@ -270,35 +270,35 @@ 
 			  	     )
 			  	   WHERE editDistance > 0
 			';
-			$stmt = $db->prepare($sql);
-			if ($stmt === false) {
-				throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
-			}
-			$stmt->bindValue(':token', $token);
-			if (($stmt === false) | (!$stmt->execute())) {
-				throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
-			}
-			$result = $stmt->fetchAll(\PDO::FETCH_CLASS, '\CloudControl\Cms\search\results\SearchSuggestion');
-			$allResults = array_merge($result, $allResults);
-		}
-		return $allResults;
-	}
+            $stmt = $db->prepare($sql);
+            if ($stmt === false) {
+                throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
+            }
+            $stmt->bindValue(':token', $token);
+            if (($stmt === false) | (!$stmt->execute())) {
+                throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
+            }
+            $result = $stmt->fetchAll(\PDO::FETCH_CLASS, '\CloudControl\Cms\search\results\SearchSuggestion');
+            $allResults = array_merge($result, $allResults);
+        }
+        return $allResults;
+    }
 
-	/**
-	 * Retrieves all tokens from the tokenizer
-	 * @return array
-	 */
-	private function getTokens()
-	{
-		$tokenVector = array(
-			'query' => array(),
-		);
-		$tokenVector['query'] = $this->tokenizer->getTokenVector();
-		$tokens = $this->applyFilters($tokenVector);
-		if (!empty($tokens)) {
-			$tokens = array_keys($tokens['query']);
-		}
+    /**
+     * Retrieves all tokens from the tokenizer
+     * @return array
+     */
+    private function getTokens()
+    {
+        $tokenVector = array(
+            'query' => array(),
+        );
+        $tokenVector['query'] = $this->tokenizer->getTokenVector();
+        $tokens = $this->applyFilters($tokenVector);
+        if (!empty($tokens)) {
+            $tokens = array_keys($tokens['query']);
+        }
 
-		return $tokens;
-	}
+        return $tokens;
+    }
 }
\ No newline at end of file

Spacing +2 added lines, -2 removed lines

@@ -154,7 +154,7 @@ 
 		  GROUP BY term_frequency.documentPath, term_frequency.term
 		  ORDER BY score DESC
 		';
-		if(!$stmt = $db->prepare($sql)) {
+		if (!$stmt = $db->prepare($sql)) {
 			throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
 		}
 		$stmt->bindValue(':query', $token);
@@ -218,7 +218,7 @@ 
 			  FROM inverse_document_frequency
 			 WHERE term IN (' . $terms . ') 
 		';
-		if(!$stmt = $db->prepare($sql)) {
+		if (!$stmt = $db->prepare($sql)) {
 			throw new \Exception('SQLite exception: <pre>' . print_r($db->errorInfo(), true) . '</pre> for SQL:<pre>' . $sql . '</pre>');
 		}
 		if (!$stmt->execute()) {

src/search/Tokenizer.php

Indentation +41 added lines, -41 removed lines

@@ -13,47 +13,47 @@
  */
 class Tokenizer
 {
-	protected $inputString;
-	protected $tokenVector = array();
-
-	/**
-	 * Tokenizer constructor.
-	 *
-	 * @param string $string Should preferably be parsed wit \CloudControl\Cms\search\CharacterFilter
-	 * @see \CloudControl\Cms\search\CharacterFilter
-	 */
-	public function __construct($string)
-	{
-		$this->inputString = $string;
-		$this->tokenize();
-	}
-
-	protected function tokenize()
-	{
-		$tokens = explode(' ', $this->inputString);
-		foreach ($tokens as $token) {
-			$this->addTokenToVector($token);
-		}
-	}
-
-	protected function addTokenToVector($token)
-	{
-		if (!empty($token)) {
-			if (isset($this->tokenVector[$token])) {
-				$this->tokenVector[$token] += 1;
-			} else {
-				$this->tokenVector[$token] = 1;
-			}
-		}
-	}
-
-	/**
-	 * @return array
-	 */
-	public function getTokenVector()
-	{
-		return $this->tokenVector;
-	}
+    protected $inputString;
+    protected $tokenVector = array();
+
+    /**
+     * Tokenizer constructor.
+     *
+     * @param string $string Should preferably be parsed wit \CloudControl\Cms\search\CharacterFilter
+     * @see \CloudControl\Cms\search\CharacterFilter
+     */
+    public function __construct($string)
+    {
+        $this->inputString = $string;
+        $this->tokenize();
+    }
+
+    protected function tokenize()
+    {
+        $tokens = explode(' ', $this->inputString);
+        foreach ($tokens as $token) {
+            $this->addTokenToVector($token);
+        }
+    }
+
+    protected function addTokenToVector($token)
+    {
+        if (!empty($token)) {
+            if (isset($this->tokenVector[$token])) {
+                $this->tokenVector[$token] += 1;
+            } else {
+                $this->tokenVector[$token] = 1;
+            }
+        }
+    }
+
+    /**
+     * @return array
+     */
+    public function getTokenVector()
+    {
+        return $this->tokenVector;
+    }
 
 
 }
\ No newline at end of file

src/search/Filter.php

Indentation +10 added lines, -10 removed lines

@@ -11,15 +11,15 @@
 
 interface Filter
 {
-	/**
-	 * Filter constructor.
-	 *
-	 * @param array $tokens
-	 */
-	public function __construct($tokens);
+    /**
+     * Filter constructor.
+     *
+     * @param array $tokens
+     */
+    public function __construct($tokens);
 
-	/**
-	 * @return array
-	 */
-	public function getFilterResults();
+    /**
+     * @return array
+     */
+    public function getFilterResults();
 }
\ No newline at end of file

src/search/DocumentTokenizer.php

Indentation +145 added lines, -145 removed lines

@@ -13,149 +13,149 @@
 
 class DocumentTokenizer
 {
-	/**
-	 * @var Document
-	 */
-	protected $document;
-
-	/**
-	 * @var array
-	 */
-	protected $tokenVector = array();
-	protected $storage;
-
-	/**
-	 * Tokenizer constructor.
-	 *
-	 * @param \CloudControl\Cms\storage\Document $document
-	 * @param Storage                   $storage
-	 */
-	public function __construct(Document $document, Storage $storage)
-	{
-		$this->document = $document;
-		$this->storage = $storage;
-		$this->tokenize();
-	}
-
-	/**
-	 * Execute tokenization of all document fields
-	 */
-	private function tokenize()
-	{
-		$this->tokenizeTitle();
-		$this->tokenizeFields();
-		$this->tokenizeBricks();
-		$this->tokenizeDynamicBricks();
-		$this->tokenVector = array_filter($this->tokenVector);
-		arsort($this->tokenVector);
-	}
-
-	private function tokenizeTitle()
-	{
-		$filteredString = new CharacterFilter($this->document->title);
-		$tokenizer = new Tokenizer($filteredString);
-		$this->addTokenVectorToVector($tokenizer->getTokenVector(), 'title');
-	}
-
-	private function tokenizeFields()
-	{
-		$fields = $this->document->fields;
-		$documentDefinition = $this->storage->getDocumentTypes()->getDocumentTypeBySlug($this->document->documentTypeSlug);
-		foreach ($fields as $fieldName => $field) {
-			$fieldType = $this->getFieldType($fieldName, $documentDefinition);
-			$this->tokenizeField($field, $fieldName, $fieldType);
-		}
-	}
-
-	private function tokenizeField($field, $fieldName, $fieldType)
-	{
-		foreach ($field as $value) {
-			// Only index fields that contain text
-			if (in_array($fieldType, array('String', 'Text', 'Rich Text'))) {
-				$filteredString = new CharacterFilter($value);
-				$tokenizer = new Tokenizer($filteredString);
-				$this->addTokenVectorToVector($tokenizer->getTokenVector(), $fieldName);
-			}
-		}
-	}
-
-	private function tokenizeBricks()
-	{
-		$bricks = $this->document->bricks;
-		foreach ($bricks as $brickSlug => $bricks) {
-			foreach ($bricks as $brick) {
-				$this->tokenizeBrick($brick, $brickSlug);
-			}
-		}
-	}
-
-	private function tokenizeBrick($brick, $brickSlug)
-	{
-		$fields  = $brick->fields;
-		$brickDefinition = $this->storage->getBricks()->getBrickBySlug($brick->type);
-		foreach ($fields as $fieldName => $field) {
-			$fieldType = $this->getFieldType($fieldName, $brickDefinition);
-			$this->tokenizeField($field, $brickSlug . '__' . $fieldName, $fieldType);
-		}
-	}
-
-	private function tokenizeDynamicBricks()
-	{
-		$dynamicBricks = $this->document->dynamicBricks;
-		foreach ($dynamicBricks as $key => $brick) {
-			$this->tokenizeBrick($brick, 'dynamicBricks__' . $brick->type . $key);
-		}
-	}
-
-	public function getTokens()
-	{
-		return $this->tokenVector;
-	}
-
-	/**
-	 * Add a token to the existing tokenvector
-	 * @param     		$token
-	 * @param string    $field
-	 * @param int 		$count
-	 */
-	private function addTokenToVector($token, $field, $count = 1)
-	{
-		if (!empty($token)) {
-			if (isset($this->tokenVector[$field][$token])) {
-				$this->tokenVector[$field][$token] += $count;
-			} else {
-				$this->tokenVector[$field][$token] = $count;
-			}
-		}
-	}
-
-	/**
-	 * Add a complete token vector to the existing one.
-	 * @param $tokenVector
-	 * @param $field
-	 */
-	private function addTokenVectorToVector($tokenVector, $field)
-	{
-		foreach ($tokenVector as $token => $count) {
-			$this->addTokenToVector($token, $field, $count);
-		}
-	}
-
-	/**
-	 * Get the type for a field
-	 * @param $fieldName
-	 * @param $documentDefinition
-	 * @return mixed
-	 * @throws \Exception
-	 */
-	private function getFieldType($fieldName, $documentDefinition)
-	{
-		foreach ($documentDefinition->fields as $fieldTypeDefinition) {
-			if ($fieldTypeDefinition->slug === $fieldName) {
-				return $fieldTypeDefinition->type;
-			}
-		}
-
-		throw new \Exception('Unknown field type for field' . $fieldName . ' in document ' . $this->document->path);
-	}
+    /**
+     * @var Document
+     */
+    protected $document;
+
+    /**
+     * @var array
+     */
+    protected $tokenVector = array();
+    protected $storage;
+
+    /**
+     * Tokenizer constructor.
+     *
+     * @param \CloudControl\Cms\storage\Document $document
+     * @param Storage                   $storage
+     */
+    public function __construct(Document $document, Storage $storage)
+    {
+        $this->document = $document;
+        $this->storage = $storage;
+        $this->tokenize();
+    }
+
+    /**
+     * Execute tokenization of all document fields
+     */
+    private function tokenize()
+    {
+        $this->tokenizeTitle();
+        $this->tokenizeFields();
+        $this->tokenizeBricks();
+        $this->tokenizeDynamicBricks();
+        $this->tokenVector = array_filter($this->tokenVector);
+        arsort($this->tokenVector);
+    }
+
+    private function tokenizeTitle()
+    {
+        $filteredString = new CharacterFilter($this->document->title);
+        $tokenizer = new Tokenizer($filteredString);
+        $this->addTokenVectorToVector($tokenizer->getTokenVector(), 'title');
+    }
+
+    private function tokenizeFields()
+    {
+        $fields = $this->document->fields;
+        $documentDefinition = $this->storage->getDocumentTypes()->getDocumentTypeBySlug($this->document->documentTypeSlug);
+        foreach ($fields as $fieldName => $field) {
+            $fieldType = $this->getFieldType($fieldName, $documentDefinition);
+            $this->tokenizeField($field, $fieldName, $fieldType);
+        }
+    }
+
+    private function tokenizeField($field, $fieldName, $fieldType)
+    {
+        foreach ($field as $value) {
+            // Only index fields that contain text
+            if (in_array($fieldType, array('String', 'Text', 'Rich Text'))) {
+                $filteredString = new CharacterFilter($value);
+                $tokenizer = new Tokenizer($filteredString);
+                $this->addTokenVectorToVector($tokenizer->getTokenVector(), $fieldName);
+            }
+        }
+    }
+
+    private function tokenizeBricks()
+    {
+        $bricks = $this->document->bricks;
+        foreach ($bricks as $brickSlug => $bricks) {
+            foreach ($bricks as $brick) {
+                $this->tokenizeBrick($brick, $brickSlug);
+            }
+        }
+    }
+
+    private function tokenizeBrick($brick, $brickSlug)
+    {
+        $fields  = $brick->fields;
+        $brickDefinition = $this->storage->getBricks()->getBrickBySlug($brick->type);
+        foreach ($fields as $fieldName => $field) {
+            $fieldType = $this->getFieldType($fieldName, $brickDefinition);
+            $this->tokenizeField($field, $brickSlug . '__' . $fieldName, $fieldType);
+        }
+    }
+
+    private function tokenizeDynamicBricks()
+    {
+        $dynamicBricks = $this->document->dynamicBricks;
+        foreach ($dynamicBricks as $key => $brick) {
+            $this->tokenizeBrick($brick, 'dynamicBricks__' . $brick->type . $key);
+        }
+    }
+
+    public function getTokens()
+    {
+        return $this->tokenVector;
+    }
+
+    /**
+     * Add a token to the existing tokenvector
+     * @param     		$token
+     * @param string    $field
+     * @param int 		$count
+     */
+    private function addTokenToVector($token, $field, $count = 1)
+    {
+        if (!empty($token)) {
+            if (isset($this->tokenVector[$field][$token])) {
+                $this->tokenVector[$field][$token] += $count;
+            } else {
+                $this->tokenVector[$field][$token] = $count;
+            }
+        }
+    }
+
+    /**
+     * Add a complete token vector to the existing one.
+     * @param $tokenVector
+     * @param $field
+     */
+    private function addTokenVectorToVector($tokenVector, $field)
+    {
+        foreach ($tokenVector as $token => $count) {
+            $this->addTokenToVector($token, $field, $count);
+        }
+    }
+
+    /**
+     * Get the type for a field
+     * @param $fieldName
+     * @param $documentDefinition
+     * @return mixed
+     * @throws \Exception
+     */
+    private function getFieldType($fieldName, $documentDefinition)
+    {
+        foreach ($documentDefinition->fields as $fieldTypeDefinition) {
+            if ($fieldTypeDefinition->slug === $fieldName) {
+                return $fieldTypeDefinition->type;
+            }
+        }
+
+        throw new \Exception('Unknown field type for field' . $fieldName . ' in document ' . $this->document->path);
+    }
 }
\ No newline at end of file

Spacing +1 added lines, -1 removed lines

@@ -91,7 +91,7 @@
 
 	private function tokenizeBrick($brick, $brickSlug)
 	{
-		$fields  = $brick->fields;
+		$fields = $brick->fields;
 		$brickDefinition = $this->storage->getBricks()->getBrickBySlug($brick->type);
 		foreach ($fields as $fieldName => $field) {
 			$fieldType = $this->getFieldType($fieldName, $brickDefinition);

src/search/filters/EnglishStopWords.php

Indentation +1 added lines, -1 removed lines

@@ -10,5 +10,5 @@
 
 class EnglishStopWords extends StopWordsFilter
 {
-	protected $stopWords = array('a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'arent', 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'cant', 'cannot', 'could', 'couldnt', 'did', 'didnt', 'do', 'does', 'doesnt', 'doing', 'dont', 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'hadnt', 'has', 'hasnt', 'have', 'havent', 'having', 'he', 'hed', 'hell', 'hes', 'her', 'here', 'heres', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'hows', 'i', 'id', 'ill', 'im', 'ive', 'if', 'in', 'into', 'is', 'isnt', 'it', 'its', 'its', 'itself', 'lets', 'me', 'more', 'most', 'mustnt', 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours	ourselves', 'out', 'over', 'own', 'same', 'shant', 'she', 'shed', 'shell', 'shes', 'should', 'shouldnt', 'so', 'some', 'such', 'than', 'that', 'thats', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'theres', 'these', 'they', 'theyd', 'theyll', 'theyre', 'theyve', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', 'wasnt', 'we', 'wed', 'well', 'were', 'weve', 'were', 'werent', 'what', 'whats', 'when', 'whens', 'where', 'wheres', 'which', 'while', 'who', 'whos', 'whom', 'why', 'whys', 'with', 'wont', 'would', 'wouldnt', 'you', 'youd', 'youll', 'youre', 'youve', 'your', 'yours', 'yourself', 'yourselves');
+    protected $stopWords = array('a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'arent', 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'cant', 'cannot', 'could', 'couldnt', 'did', 'didnt', 'do', 'does', 'doesnt', 'doing', 'dont', 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'hadnt', 'has', 'hasnt', 'have', 'havent', 'having', 'he', 'hed', 'hell', 'hes', 'her', 'here', 'heres', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'hows', 'i', 'id', 'ill', 'im', 'ive', 'if', 'in', 'into', 'is', 'isnt', 'it', 'its', 'its', 'itself', 'lets', 'me', 'more', 'most', 'mustnt', 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours	ourselves', 'out', 'over', 'own', 'same', 'shant', 'she', 'shed', 'shell', 'shes', 'should', 'shouldnt', 'so', 'some', 'such', 'than', 'that', 'thats', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'theres', 'these', 'they', 'theyd', 'theyll', 'theyre', 'theyve', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', 'wasnt', 'we', 'wed', 'well', 'were', 'weve', 'were', 'werent', 'what', 'whats', 'when', 'whens', 'where', 'wheres', 'which', 'while', 'who', 'whos', 'whom', 'why', 'whys', 'with', 'wont', 'would', 'wouldnt', 'you', 'youd', 'youll', 'youre', 'youve', 'your', 'yours', 'yourself', 'yourselves');
 }
\ No newline at end of file

src/search/Indexer.php

Indentation +159 added lines, -159 removed lines

@@ -22,87 +22,87 @@ 
  */
 class Indexer extends SearchDbConnected
 {
-	const SQLITE_MAX_COMPOUND_SELECT = 100;
-	protected $filters = array(
-		'DutchStopWords',
-		'EnglishStopWords'
-	);
-	protected $storageDir;
-	/**
-	 * @var double
-	 */
-	protected $loggingStart;
-	/**
-	 * @var string
-	 */
-	protected $log;
-	/**
-	 * @var double
-	 */
-	protected $lastLog;
-
-	const SEARCH_TEMP_DB = 'search_tmp.db';
-
-	/**
-	 * Creates a new temporary search db, cleans it if it exists
-	 * then calculates and stores the search index in this db
-	 * and finally if indexing completed replaces the current search
-	 * db with the temporary one. Returns the log in string format.
-	 * @return string
-	 */
-	public function updateIndex()
-	{
-		$this->startLogging();
-		$this->addLog('Indexing start.');
-		$this->addLog('Clearing index.');
-		$this->resetIndex();
-		$this->addLog('Cleaning Published Deleted Documents');
-		$this->storage->getDocuments()->cleanPublishedDeletedDocuments();
-		$this->addLog('Retrieving documents to be indexed.');
-		$documents = $this->storage->getDocuments()->getPublishedDocumentsNoFolders();
-		$this->addLog('Start Document Term Count for ' . count($documents) . ' documents');
-		$this->createDocumentTermCount($documents);
-		$this->addLog('Start Document Term Frequency.');
-		$this->createDocumentTermFrequency();
-		$this->addLog('Start Term Field Length Norm.');
-		$this->createTermFieldLengthNorm();
-		$this->addLog('Start Inverse Document Frequency.');
-		$this->createInverseDocumentFrequency();
-		$this->addLog('Replacing old index.');
-		$this->replaceOldIndex();
-		$this->addLog('Indexing complete.');
-		return $this->log;
-	}
-
-	/**
-	 * Count how often a term is used in a document
-	 *
-	 * @param $documents
-	 */
-	public function createDocumentTermCount($documents)
-	{
-		$termCount = new TermCount($this->getSearchDbHandle(), $documents, $this->filters, $this->storage);
-		$termCount->execute();
-	}
-
-	/**
-	 * Calculate the frequency index for a term with
-	 * a field
-	 */
-	public function createDocumentTermFrequency()
-	{
-		$termFrequency = new TermFrequency($this->getSearchDbHandle());
-		$termFrequency->execute();
-	}
-
-
-	/**
-	 * Resets the entire index
-	 */
-	public function resetIndex()
-	{
-		$db = $this->getSearchDbHandle();
-		$sql = '
+    const SQLITE_MAX_COMPOUND_SELECT = 100;
+    protected $filters = array(
+        'DutchStopWords',
+        'EnglishStopWords'
+    );
+    protected $storageDir;
+    /**
+     * @var double
+     */
+    protected $loggingStart;
+    /**
+     * @var string
+     */
+    protected $log;
+    /**
+     * @var double
+     */
+    protected $lastLog;
+
+    const SEARCH_TEMP_DB = 'search_tmp.db';
+
+    /**
+     * Creates a new temporary search db, cleans it if it exists
+     * then calculates and stores the search index in this db
+     * and finally if indexing completed replaces the current search
+     * db with the temporary one. Returns the log in string format.
+     * @return string
+     */
+    public function updateIndex()
+    {
+        $this->startLogging();
+        $this->addLog('Indexing start.');
+        $this->addLog('Clearing index.');
+        $this->resetIndex();
+        $this->addLog('Cleaning Published Deleted Documents');
+        $this->storage->getDocuments()->cleanPublishedDeletedDocuments();
+        $this->addLog('Retrieving documents to be indexed.');
+        $documents = $this->storage->getDocuments()->getPublishedDocumentsNoFolders();
+        $this->addLog('Start Document Term Count for ' . count($documents) . ' documents');
+        $this->createDocumentTermCount($documents);
+        $this->addLog('Start Document Term Frequency.');
+        $this->createDocumentTermFrequency();
+        $this->addLog('Start Term Field Length Norm.');
+        $this->createTermFieldLengthNorm();
+        $this->addLog('Start Inverse Document Frequency.');
+        $this->createInverseDocumentFrequency();
+        $this->addLog('Replacing old index.');
+        $this->replaceOldIndex();
+        $this->addLog('Indexing complete.');
+        return $this->log;
+    }
+
+    /**
+     * Count how often a term is used in a document
+     *
+     * @param $documents
+     */
+    public function createDocumentTermCount($documents)
+    {
+        $termCount = new TermCount($this->getSearchDbHandle(), $documents, $this->filters, $this->storage);
+        $termCount->execute();
+    }
+
+    /**
+     * Calculate the frequency index for a term with
+     * a field
+     */
+    public function createDocumentTermFrequency()
+    {
+        $termFrequency = new TermFrequency($this->getSearchDbHandle());
+        $termFrequency->execute();
+    }
+
+
+    /**
+     * Resets the entire index
+     */
+    public function resetIndex()
+    {
+        $db = $this->getSearchDbHandle();
+        $sql = '
 			DELETE FROM term_count;
 			DELETE FROM term_frequency;
 			DELETE FROM inverse_document_frequency;
@@ -110,82 +110,82 @@ 
 			UPDATE `sqlite_sequence` SET `seq`= 0 WHERE `name`=\'term_frequency\';
 			UPDATE `sqlite_sequence` SET `seq`= 0 WHERE `name`=\'inverse_document_frequency\';
 		';
-		$db->exec($sql);
-	}
-
-	/**
-	 * Calculates the inverse document frequency for each
-	 * term. This is a representation of how often a certain
-	 * term is used in comparison to all terms.
-	 */
-	public function createInverseDocumentFrequency()
-	{
-		$documentCount = $this->getTotalDocumentCount();
-		$inverseDocumentFrequency = new InverseDocumentFrequency($this->getSearchDbHandle(), $documentCount);
-		$inverseDocumentFrequency->execute();
-	}
-
-	/**
-	 * @return int|mixed
-	 */
-	private function getTotalDocumentCount()
-	{
-		return $this->storage->getDocuments()->getTotalDocumentCount();
-	}
-
-	/**
-	 * Calculates the Term Field Length Norm.
-	 * This is an index determining how important a
-	 * term is, based on the total length of the field
-	 * it comes from.
-	 */
-	public function createTermFieldLengthNorm()
-	{
-		$termFieldLengthNorm = new TermFieldLengthNorm($this->getSearchDbHandle());
-		$termFieldLengthNorm->execute();
-	}
-
-	/**
-	 * Stores the time the indexing started in memory
-	 */
-	private function startLogging()
-	{
-		$this->loggingStart = round(microtime(true) * 1000);
-		$this->lastLog = $this->loggingStart;
-	}
-
-	/**
-	 * Adds a logline with the time since last log
-	 * @param $string
-	 */
-	private function addLog($string)
-	{
-		$currentTime = round(microtime(true) * 1000);
-		$this->log .= date('d-m-Y H:i:s - ') . str_pad($string, 50, " ", STR_PAD_RIGHT) . "\t" . ($currentTime - $this->lastLog) . 'ms since last log. ' . "\t" . ($currentTime - $this->loggingStart) . 'ms since start.' . PHP_EOL;
-		$this->lastLog = round(microtime(true) * 1000);
-	}
-
-	/**
-	 * Creates the SQLite \PDO object if it doesnt
-	 * exist and returns it.
-	 * @return \PDO
-	 */
-	protected function getSearchDbHandle()
-	{
-		if ($this->searchDbHandle === null) {
-			$path = $this->storageDir . DIRECTORY_SEPARATOR;
-			$this->searchDbHandle = new \PDO('sqlite:' . $path . self::SEARCH_TEMP_DB);
-		}
-		return $this->searchDbHandle;
-	}
-
-	/**
-	 * Replaces the old search index database with the new one.
-	 */
-	public function replaceOldIndex()
-	{
-		$this->searchDbHandle = null;
-		$path = $this->storageDir . DIRECTORY_SEPARATOR;
-		rename($path . self::SEARCH_TEMP_DB, $path . 'search.db');
-	}
+        $db->exec($sql);
+    }
+
+    /**
+     * Calculates the inverse document frequency for each
+     * term. This is a representation of how often a certain
+     * term is used in comparison to all terms.
+     */
+    public function createInverseDocumentFrequency()
+    {
+        $documentCount = $this->getTotalDocumentCount();
+        $inverseDocumentFrequency = new InverseDocumentFrequency($this->getSearchDbHandle(), $documentCount);
+        $inverseDocumentFrequency->execute();
+    }
+
+    /**
+     * @return int|mixed
+     */
+    private function getTotalDocumentCount()
+    {
+        return $this->storage->getDocuments()->getTotalDocumentCount();
+    }
+
+    /**
+     * Calculates the Term Field Length Norm.
+     * This is an index determining how important a
+     * term is, based on the total length of the field
+     * it comes from.
+     */
+    public function createTermFieldLengthNorm()
+    {
+        $termFieldLengthNorm = new TermFieldLengthNorm($this->getSearchDbHandle());
+        $termFieldLengthNorm->execute();
+    }
+
+    /**
+     * Stores the time the indexing started in memory
+     */
+    private function startLogging()
+    {
+        $this->loggingStart = round(microtime(true) * 1000);
+        $this->lastLog = $this->loggingStart;
+    }
+
+    /**
+     * Adds a logline with the time since last log
+     * @param $string
+     */
+    private function addLog($string)
+    {
+        $currentTime = round(microtime(true) * 1000);
+        $this->log .= date('d-m-Y H:i:s - ') . str_pad($string, 50, " ", STR_PAD_RIGHT) . "\t" . ($currentTime - $this->lastLog) . 'ms since last log. ' . "\t" . ($currentTime - $this->loggingStart) . 'ms since start.' . PHP_EOL;
+        $this->lastLog = round(microtime(true) * 1000);
+    }
+
+    /**
+     * Creates the SQLite \PDO object if it doesnt
+     * exist and returns it.
+     * @return \PDO
+     */
+    protected function getSearchDbHandle()
+    {
+        if ($this->searchDbHandle === null) {
+            $path = $this->storageDir . DIRECTORY_SEPARATOR;
+            $this->searchDbHandle = new \PDO('sqlite:' . $path . self::SEARCH_TEMP_DB);
+        }
+        return $this->searchDbHandle;
+    }
+
+    /**
+     * Replaces the old search index database with the new one.
+     */
+    public function replaceOldIndex()
+    {
+        $this->searchDbHandle = null;
+        $path = $this->storageDir . DIRECTORY_SEPARATOR;
+        rename($path . self::SEARCH_TEMP_DB, $path . 'search.db');
+    }
 }
\ No newline at end of file