jenskooij / cloudcontrol

src/search/Tokenizer.php

Indentation +41 added lines, -41 removed lines

@@ -13,47 +13,47 @@
  */
 class Tokenizer
 {
-	protected $inputString;
-	protected $tokenVector = array();
-
-	/**
-	 * Tokenizer constructor.
-	 *
-	 * @param string $string Should preferably be parsed wit \CloudControl\Cms\search\CharacterFilter
-	 * @see \CloudControl\Cms\search\CharacterFilter
-	 */
-	public function __construct($string)
-	{
-		$this->inputString = $string;
-		$this->tokenize();
-	}
-
-	protected function tokenize()
-	{
-		$tokens = explode(' ', $this->inputString);
-		foreach ($tokens as $token) {
-			$this->addTokenToVector($token);
-		}
-	}
-
-	protected function addTokenToVector($token)
-	{
-		if (!empty($token)) {
-			if (isset($this->tokenVector[$token])) {
-				$this->tokenVector[$token] += 1;
-			} else {
-				$this->tokenVector[$token] = 1;
-			}
-		}
-	}
-
-	/**
-	 * @return array
-	 */
-	public function getTokenVector()
-	{
-		return $this->tokenVector;
-	}
+    protected $inputString;
+    protected $tokenVector = array();
+
+    /**
+     * Tokenizer constructor.
+     *
+     * @param string $string Should preferably be parsed wit \CloudControl\Cms\search\CharacterFilter
+     * @see \CloudControl\Cms\search\CharacterFilter
+     */
+    public function __construct($string)
+    {
+        $this->inputString = $string;
+        $this->tokenize();
+    }
+
+    protected function tokenize()
+    {
+        $tokens = explode(' ', $this->inputString);
+        foreach ($tokens as $token) {
+            $this->addTokenToVector($token);
+        }
+    }
+
+    protected function addTokenToVector($token)
+    {
+        if (!empty($token)) {
+            if (isset($this->tokenVector[$token])) {
+                $this->tokenVector[$token] += 1;
+            } else {
+                $this->tokenVector[$token] = 1;
+            }
+        }
+    }
+
+    /**
+     * @return array
+     */
+    public function getTokenVector()
+    {
+        return $this->tokenVector;
+    }
 
 
 }
\ No newline at end of file

src/search/Filter.php

Indentation +10 added lines, -10 removed lines

@@ -11,15 +11,15 @@
 
 interface Filter
 {
-	/**
-	 * Filter constructor.
-	 *
-	 * @param array $tokens
-	 */
-	public function __construct($tokens);
+    /**
+     * Filter constructor.
+     *
+     * @param array $tokens
+     */
+    public function __construct($tokens);
 
-	/**
-	 * @return array
-	 */
-	public function getFilterResults();
+    /**
+     * @return array
+     */
+    public function getFilterResults();
 }
\ No newline at end of file

src/search/DocumentTokenizer.php

Indentation +145 added lines, -145 removed lines

@@ -13,149 +13,149 @@
 
 class DocumentTokenizer
 {
-	/**
-	 * @var Document
-	 */
-	protected $document;
-
-	/**
-	 * @var array
-	 */
-	protected $tokenVector = array();
-	protected $storage;
-
-	/**
-	 * Tokenizer constructor.
-	 *
-	 * @param \CloudControl\Cms\storage\Document $document
-	 * @param Storage                   $storage
-	 */
-	public function __construct(Document $document, Storage $storage)
-	{
-		$this->document = $document;
-		$this->storage = $storage;
-		$this->tokenize();
-	}
-
-	/**
-	 * Execute tokenization of all document fields
-	 */
-	private function tokenize()
-	{
-		$this->tokenizeTitle();
-		$this->tokenizeFields();
-		$this->tokenizeBricks();
-		$this->tokenizeDynamicBricks();
-		$this->tokenVector = array_filter($this->tokenVector);
-		arsort($this->tokenVector);
-	}
-
-	private function tokenizeTitle()
-	{
-		$filteredString = new CharacterFilter($this->document->title);
-		$tokenizer = new Tokenizer($filteredString);
-		$this->addTokenVectorToVector($tokenizer->getTokenVector(), 'title');
-	}
-
-	private function tokenizeFields()
-	{
-		$fields = $this->document->fields;
-		$documentDefinition = $this->storage->getDocumentTypes()->getDocumentTypeBySlug($this->document->documentTypeSlug);
-		foreach ($fields as $fieldName => $field) {
-			$fieldType = $this->getFieldType($fieldName, $documentDefinition);
-			$this->tokenizeField($field, $fieldName, $fieldType);
-		}
-	}
-
-	private function tokenizeField($field, $fieldName, $fieldType)
-	{
-		foreach ($field as $value) {
-			// Only index fields that contain text
-			if (in_array($fieldType, array('String', 'Text', 'Rich Text'))) {
-				$filteredString = new CharacterFilter($value);
-				$tokenizer = new Tokenizer($filteredString);
-				$this->addTokenVectorToVector($tokenizer->getTokenVector(), $fieldName);
-			}
-		}
-	}
-
-	private function tokenizeBricks()
-	{
-		$bricks = $this->document->bricks;
-		foreach ($bricks as $brickSlug => $bricks) {
-			foreach ($bricks as $brick) {
-				$this->tokenizeBrick($brick, $brickSlug);
-			}
-		}
-	}
-
-	private function tokenizeBrick($brick, $brickSlug)
-	{
-		$fields  = $brick->fields;
-		$brickDefinition = $this->storage->getBricks()->getBrickBySlug($brick->type);
-		foreach ($fields as $fieldName => $field) {
-			$fieldType = $this->getFieldType($fieldName, $brickDefinition);
-			$this->tokenizeField($field, $brickSlug . '__' . $fieldName, $fieldType);
-		}
-	}
-
-	private function tokenizeDynamicBricks()
-	{
-		$dynamicBricks = $this->document->dynamicBricks;
-		foreach ($dynamicBricks as $key => $brick) {
-			$this->tokenizeBrick($brick, 'dynamicBricks__' . $brick->type . $key);
-		}
-	}
-
-	public function getTokens()
-	{
-		return $this->tokenVector;
-	}
-
-	/**
-	 * Add a token to the existing tokenvector
-	 * @param     		$token
-	 * @param string    $field
-	 * @param int 		$count
-	 */
-	private function addTokenToVector($token, $field, $count = 1)
-	{
-		if (!empty($token)) {
-			if (isset($this->tokenVector[$field][$token])) {
-				$this->tokenVector[$field][$token] += $count;
-			} else {
-				$this->tokenVector[$field][$token] = $count;
-			}
-		}
-	}
-
-	/**
-	 * Add a complete token vector to the existing one.
-	 * @param $tokenVector
-	 * @param $field
-	 */
-	private function addTokenVectorToVector($tokenVector, $field)
-	{
-		foreach ($tokenVector as $token => $count) {
-			$this->addTokenToVector($token, $field, $count);
-		}
-	}
-
-	/**
-	 * Get the type for a field
-	 * @param $fieldName
-	 * @param $documentDefinition
-	 * @return mixed
-	 * @throws \Exception
-	 */
-	private function getFieldType($fieldName, $documentDefinition)
-	{
-		foreach ($documentDefinition->fields as $fieldTypeDefinition) {
-			if ($fieldTypeDefinition->slug === $fieldName) {
-				return $fieldTypeDefinition->type;
-			}
-		}
-
-		throw new \Exception('Unknown field type for field' . $fieldName . ' in document ' . $this->document->path);
-	}
+    /**
+     * @var Document
+     */
+    protected $document;
+
+    /**
+     * @var array
+     */
+    protected $tokenVector = array();
+    protected $storage;
+
+    /**
+     * Tokenizer constructor.
+     *
+     * @param \CloudControl\Cms\storage\Document $document
+     * @param Storage                   $storage
+     */
+    public function __construct(Document $document, Storage $storage)
+    {
+        $this->document = $document;
+        $this->storage = $storage;
+        $this->tokenize();
+    }
+
+    /**
+     * Execute tokenization of all document fields
+     */
+    private function tokenize()
+    {
+        $this->tokenizeTitle();
+        $this->tokenizeFields();
+        $this->tokenizeBricks();
+        $this->tokenizeDynamicBricks();
+        $this->tokenVector = array_filter($this->tokenVector);
+        arsort($this->tokenVector);
+    }
+
+    private function tokenizeTitle()
+    {
+        $filteredString = new CharacterFilter($this->document->title);
+        $tokenizer = new Tokenizer($filteredString);
+        $this->addTokenVectorToVector($tokenizer->getTokenVector(), 'title');
+    }
+
+    private function tokenizeFields()
+    {
+        $fields = $this->document->fields;
+        $documentDefinition = $this->storage->getDocumentTypes()->getDocumentTypeBySlug($this->document->documentTypeSlug);
+        foreach ($fields as $fieldName => $field) {
+            $fieldType = $this->getFieldType($fieldName, $documentDefinition);
+            $this->tokenizeField($field, $fieldName, $fieldType);
+        }
+    }
+
+    private function tokenizeField($field, $fieldName, $fieldType)
+    {
+        foreach ($field as $value) {
+            // Only index fields that contain text
+            if (in_array($fieldType, array('String', 'Text', 'Rich Text'))) {
+                $filteredString = new CharacterFilter($value);
+                $tokenizer = new Tokenizer($filteredString);
+                $this->addTokenVectorToVector($tokenizer->getTokenVector(), $fieldName);
+            }
+        }
+    }
+
+    private function tokenizeBricks()
+    {
+        $bricks = $this->document->bricks;
+        foreach ($bricks as $brickSlug => $bricks) {
+            foreach ($bricks as $brick) {
+                $this->tokenizeBrick($brick, $brickSlug);
+            }
+        }
+    }
+
+    private function tokenizeBrick($brick, $brickSlug)
+    {
+        $fields  = $brick->fields;
+        $brickDefinition = $this->storage->getBricks()->getBrickBySlug($brick->type);
+        foreach ($fields as $fieldName => $field) {
+            $fieldType = $this->getFieldType($fieldName, $brickDefinition);
+            $this->tokenizeField($field, $brickSlug . '__' . $fieldName, $fieldType);
+        }
+    }
+
+    private function tokenizeDynamicBricks()
+    {
+        $dynamicBricks = $this->document->dynamicBricks;
+        foreach ($dynamicBricks as $key => $brick) {
+            $this->tokenizeBrick($brick, 'dynamicBricks__' . $brick->type . $key);
+        }
+    }
+
+    public function getTokens()
+    {
+        return $this->tokenVector;
+    }
+
+    /**
+     * Add a token to the existing tokenvector
+     * @param     		$token
+     * @param string    $field
+     * @param int 		$count
+     */
+    private function addTokenToVector($token, $field, $count = 1)
+    {
+        if (!empty($token)) {
+            if (isset($this->tokenVector[$field][$token])) {
+                $this->tokenVector[$field][$token] += $count;
+            } else {
+                $this->tokenVector[$field][$token] = $count;
+            }
+        }
+    }
+
+    /**
+     * Add a complete token vector to the existing one.
+     * @param $tokenVector
+     * @param $field
+     */
+    private function addTokenVectorToVector($tokenVector, $field)
+    {
+        foreach ($tokenVector as $token => $count) {
+            $this->addTokenToVector($token, $field, $count);
+        }
+    }
+
+    /**
+     * Get the type for a field
+     * @param $fieldName
+     * @param $documentDefinition
+     * @return mixed
+     * @throws \Exception
+     */
+    private function getFieldType($fieldName, $documentDefinition)
+    {
+        foreach ($documentDefinition->fields as $fieldTypeDefinition) {
+            if ($fieldTypeDefinition->slug === $fieldName) {
+                return $fieldTypeDefinition->type;
+            }
+        }
+
+        throw new \Exception('Unknown field type for field' . $fieldName . ' in document ' . $this->document->path);
+    }
 }
\ No newline at end of file

src/search/CharacterFilter.php

Indentation +65 added lines, -65 removed lines

@@ -9,76 +9,76 @@
 
 class CharacterFilter
 {
-	protected $originalString;
-	protected $filteredString = '';
+    protected $originalString;
+    protected $filteredString = '';
 
-	/**
-	 * CharacterFilter constructor.
-	 *
-	 * @param $string
-	 */
-	public function __construct($string)
-	{
-		$this->originalString = $string;
-		$string = $this->convertToUTF8($string);
-		$string = mb_strtolower($string);
-		$string = $this->filterSpecialCharacters($string);
-		$this->filteredString = $string;
-	}
+    /**
+     * CharacterFilter constructor.
+     *
+     * @param $string
+     */
+    public function __construct($string)
+    {
+        $this->originalString = $string;
+        $string = $this->convertToUTF8($string);
+        $string = mb_strtolower($string);
+        $string = $this->filterSpecialCharacters($string);
+        $this->filteredString = $string;
+    }
 
-	/**
-	 * Returns the filtered string
-	 * @return string|void
-	 */
-	public function __toString()
-	{
-		return $this->filteredString;
-	}
+    /**
+     * Returns the filtered string
+     * @return string|void
+     */
+    public function __toString()
+    {
+        return $this->filteredString;
+    }
 
-	/**
-	 * Filter out all special characters, like punctuation and characters with accents
-	 *
-	 * @param $string
-	 *
-	 * @return mixed|string
-	 */
-	private function filterSpecialCharacters($string)
-	{
-		$string = str_replace('<', ' <', $string); // This is need, otherwise this: <h1>something</h1><h2>something</h2> will result in somethingsomething
-		$string = strip_tags($string);
-		$string = trim($string);
-		$string = iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $string); // Remove special alphanumeric characters
-		$string = str_replace(array('+', '=', '!', ',', '.',';', ':', '?'), ' ', $string); // Replace sentence breaking charaters with spaces
-		$string = preg_replace("/[\r\n]+/", " ", $string); // Replace multiple newlines with a single space.
-		$string = preg_replace("/[\t]+/", " ", $string); // Replace multiple tabs with a single space.
-		$string = preg_replace("/[^a-zA-Z0-9 ]/", '', $string); // Filter out everything that is not alphanumeric or a space
-		$string = preg_replace('!\s+!', ' ', $string); // Replace multiple spaces with a single space
-		return $string;
-	}
+    /**
+     * Filter out all special characters, like punctuation and characters with accents
+     *
+     * @param $string
+     *
+     * @return mixed|string
+     */
+    private function filterSpecialCharacters($string)
+    {
+        $string = str_replace('<', ' <', $string); // This is need, otherwise this: <h1>something</h1><h2>something</h2> will result in somethingsomething
+        $string = strip_tags($string);
+        $string = trim($string);
+        $string = iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $string); // Remove special alphanumeric characters
+        $string = str_replace(array('+', '=', '!', ',', '.',';', ':', '?'), ' ', $string); // Replace sentence breaking charaters with spaces
+        $string = preg_replace("/[\r\n]+/", " ", $string); // Replace multiple newlines with a single space.
+        $string = preg_replace("/[\t]+/", " ", $string); // Replace multiple tabs with a single space.
+        $string = preg_replace("/[^a-zA-Z0-9 ]/", '', $string); // Filter out everything that is not alphanumeric or a space
+        $string = preg_replace('!\s+!', ' ', $string); // Replace multiple spaces with a single space
+        return $string;
+    }
 
-	/**
-	 * Convert the string to UTF-8 encoding
-	 * @param $string
-	 *
-	 * @return string
-	 */
-	private function convertToUTF8($string)
-	{
-		$encoding = mb_detect_encoding($string, mb_detect_order(), false);
+    /**
+     * Convert the string to UTF-8 encoding
+     * @param $string
+     *
+     * @return string
+     */
+    private function convertToUTF8($string)
+    {
+        $encoding = mb_detect_encoding($string, mb_detect_order(), false);
 
-		if($encoding == "UTF-8") {
-			$string = mb_convert_encoding($string, 'UTF-8', 'UTF-8');
-		}
+        if($encoding == "UTF-8") {
+            $string = mb_convert_encoding($string, 'UTF-8', 'UTF-8');
+        }
 
-		$out = iconv(mb_detect_encoding($string, mb_detect_order(), false), "UTF-8//IGNORE", $string);
-		return $out;
-	}
+        $out = iconv(mb_detect_encoding($string, mb_detect_order(), false), "UTF-8//IGNORE", $string);
+        return $out;
+    }
 
-	/**
-	 * @return mixed|string
-	 */
-	public function getFilteredString()
-	{
-		return $this->filteredString;
-	}
+    /**
+     * @return mixed|string
+     */
+    public function getFilteredString()
+    {
+        return $this->filteredString;
+    }
 }
\ No newline at end of file

src/search/filters/EnglishStopWords.php

Indentation +1 added lines, -1 removed lines

@@ -10,5 +10,5 @@
 
 class EnglishStopWords extends StopWordsFilter
 {
-	protected $stopWords = array('a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'arent', 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'cant', 'cannot', 'could', 'couldnt', 'did', 'didnt', 'do', 'does', 'doesnt', 'doing', 'dont', 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'hadnt', 'has', 'hasnt', 'have', 'havent', 'having', 'he', 'hed', 'hell', 'hes', 'her', 'here', 'heres', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'hows', 'i', 'id', 'ill', 'im', 'ive', 'if', 'in', 'into', 'is', 'isnt', 'it', 'its', 'its', 'itself', 'lets', 'me', 'more', 'most', 'mustnt', 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours	ourselves', 'out', 'over', 'own', 'same', 'shant', 'she', 'shed', 'shell', 'shes', 'should', 'shouldnt', 'so', 'some', 'such', 'than', 'that', 'thats', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'theres', 'these', 'they', 'theyd', 'theyll', 'theyre', 'theyve', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', 'wasnt', 'we', 'wed', 'well', 'were', 'weve', 'were', 'werent', 'what', 'whats', 'when', 'whens', 'where', 'wheres', 'which', 'while', 'who', 'whos', 'whom', 'why', 'whys', 'with', 'wont', 'would', 'wouldnt', 'you', 'youd', 'youll', 'youre', 'youve', 'your', 'yours', 'yourself', 'yourselves');
+    protected $stopWords = array('a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'arent', 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'cant', 'cannot', 'could', 'couldnt', 'did', 'didnt', 'do', 'does', 'doesnt', 'doing', 'dont', 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'hadnt', 'has', 'hasnt', 'have', 'havent', 'having', 'he', 'hed', 'hell', 'hes', 'her', 'here', 'heres', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'hows', 'i', 'id', 'ill', 'im', 'ive', 'if', 'in', 'into', 'is', 'isnt', 'it', 'its', 'its', 'itself', 'lets', 'me', 'more', 'most', 'mustnt', 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours	ourselves', 'out', 'over', 'own', 'same', 'shant', 'she', 'shed', 'shell', 'shes', 'should', 'shouldnt', 'so', 'some', 'such', 'than', 'that', 'thats', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'theres', 'these', 'they', 'theyd', 'theyll', 'theyre', 'theyve', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', 'wasnt', 'we', 'wed', 'well', 'were', 'weve', 'were', 'werent', 'what', 'whats', 'when', 'whens', 'where', 'wheres', 'which', 'while', 'who', 'whos', 'whom', 'why', 'whys', 'with', 'wont', 'would', 'wouldnt', 'you', 'youd', 'youll', 'youre', 'youve', 'your', 'yours', 'yourself', 'yourselves');
 }
\ No newline at end of file

src/search/filters/DutchStopWords.php

Indentation +1 added lines, -1 removed lines

@@ -10,5 +10,5 @@
 
 class DutchStopWords extends StopWordsFilter
 {
-	protected $stopWords = array('aan','af','al','alles','als','altijd','andere','ben','bij','daar','dan','dat','de','der','deze','die','dit','doch','doen','door','doorgaans','dus','een','eens','en','er','ge','geen','geweest','haar','had','heb','hebben','heeft','hem','het','hier','hij','hoe','hun','iemand','iets','ik','in','is','ja','je','kan','kon','kunnen','maar','me','meer','men','met','mij','mijn','moet','na','naar','niet','niets','nog','nu','of','om','omdat','ons','ook','op','over','reeds','te','tegen','toch','toen','tot','u','uit','uw','van','veel','voor','want','waren','was','wat','we','wel','werd','wezen','wie','wij','wil','worden','zal','ze','zei','zelf','zich','zij','zijn','zo','zodat','zonder','zou');
+    protected $stopWords = array('aan','af','al','alles','als','altijd','andere','ben','bij','daar','dan','dat','de','der','deze','die','dit','doch','doen','door','doorgaans','dus','een','eens','en','er','ge','geen','geweest','haar','had','heb','hebben','heeft','hem','het','hier','hij','hoe','hun','iemand','iets','ik','in','is','ja','je','kan','kon','kunnen','maar','me','meer','men','met','mij','mijn','moet','na','naar','niet','niets','nog','nu','of','om','omdat','ons','ook','op','over','reeds','te','tegen','toch','toen','tot','u','uit','uw','van','veel','voor','want','waren','was','wat','we','wel','werd','wezen','wie','wij','wil','worden','zal','ze','zei','zelf','zich','zij','zijn','zo','zodat','zonder','zou');
 }
\ No newline at end of file

src/search/Indexer.php

Indentation +159 added lines, -159 removed lines

@@ -22,87 +22,87 @@ 
  */
 class Indexer extends SearchDbConnected
 {
-	const SQLITE_MAX_COMPOUND_SELECT = 100;
-	protected $filters = array(
-		'DutchStopWords',
-		'EnglishStopWords'
-	);
-	protected $storageDir;
-	/**
-	 * @var double
-	 */
-	protected $loggingStart;
-	/**
-	 * @var string
-	 */
-	protected $log;
-	/**
-	 * @var double
-	 */
-	protected $lastLog;
-
-	const SEARCH_TEMP_DB = 'search_tmp.db';
-
-	/**
-	 * Creates a new temporary search db, cleans it if it exists
-	 * then calculates and stores the search index in this db
-	 * and finally if indexing completed replaces the current search
-	 * db with the temporary one. Returns the log in string format.
-	 * @return string
-	 */
-	public function updateIndex()
-	{
-		$this->startLogging();
-		$this->addLog('Indexing start.');
-		$this->addLog('Clearing index.');
-		$this->resetIndex();
-		$this->addLog('Cleaning Published Deleted Documents');
-		$this->storage->getDocuments()->cleanPublishedDeletedDocuments();
-		$this->addLog('Retrieving documents to be indexed.');
-		$documents = $this->storage->getDocuments()->getPublishedDocumentsNoFolders();
-		$this->addLog('Start Document Term Count for ' . count($documents) . ' documents');
-		$this->createDocumentTermCount($documents);
-		$this->addLog('Start Document Term Frequency.');
-		$this->createDocumentTermFrequency();
-		$this->addLog('Start Term Field Length Norm.');
-		$this->createTermFieldLengthNorm();
-		$this->addLog('Start Inverse Document Frequency.');
-		$this->createInverseDocumentFrequency();
-		$this->addLog('Replacing old index.');
-		$this->replaceOldIndex();
-		$this->addLog('Indexing complete.');
-		return $this->log;
-	}
-
-	/**
-	 * Count how often a term is used in a document
-	 *
-	 * @param $documents
-	 */
-	public function createDocumentTermCount($documents)
-	{
-		$termCount = new TermCount($this->getSearchDbHandle(), $documents, $this->filters, $this->storage);
-		$termCount->execute();
-	}
-
-	/**
-	 * Calculate the frequency index for a term with
-	 * a field
-	 */
-	public function createDocumentTermFrequency()
-	{
-		$termFrequency = new TermFrequency($this->getSearchDbHandle());
-		$termFrequency->execute();
-	}
-
-
-	/**
-	 * Resets the entire index
-	 */
-	public function resetIndex()
-	{
-		$db = $this->getSearchDbHandle();
-		$sql = '
+    const SQLITE_MAX_COMPOUND_SELECT = 100;
+    protected $filters = array(
+        'DutchStopWords',
+        'EnglishStopWords'
+    );
+    protected $storageDir;
+    /**
+     * @var double
+     */
+    protected $loggingStart;
+    /**
+     * @var string
+     */
+    protected $log;
+    /**
+     * @var double
+     */
+    protected $lastLog;
+
+    const SEARCH_TEMP_DB = 'search_tmp.db';
+
+    /**
+     * Creates a new temporary search db, cleans it if it exists
+     * then calculates and stores the search index in this db
+     * and finally if indexing completed replaces the current search
+     * db with the temporary one. Returns the log in string format.
+     * @return string
+     */
+    public function updateIndex()
+    {
+        $this->startLogging();
+        $this->addLog('Indexing start.');
+        $this->addLog('Clearing index.');
+        $this->resetIndex();
+        $this->addLog('Cleaning Published Deleted Documents');
+        $this->storage->getDocuments()->cleanPublishedDeletedDocuments();
+        $this->addLog('Retrieving documents to be indexed.');
+        $documents = $this->storage->getDocuments()->getPublishedDocumentsNoFolders();
+        $this->addLog('Start Document Term Count for ' . count($documents) . ' documents');
+        $this->createDocumentTermCount($documents);
+        $this->addLog('Start Document Term Frequency.');
+        $this->createDocumentTermFrequency();
+        $this->addLog('Start Term Field Length Norm.');
+        $this->createTermFieldLengthNorm();
+        $this->addLog('Start Inverse Document Frequency.');
+        $this->createInverseDocumentFrequency();
+        $this->addLog('Replacing old index.');
+        $this->replaceOldIndex();
+        $this->addLog('Indexing complete.');
+        return $this->log;
+    }
+
+    /**
+     * Count how often a term is used in a document
+     *
+     * @param $documents
+     */
+    public function createDocumentTermCount($documents)
+    {
+        $termCount = new TermCount($this->getSearchDbHandle(), $documents, $this->filters, $this->storage);
+        $termCount->execute();
+    }
+
+    /**
+     * Calculate the frequency index for a term with
+     * a field
+     */
+    public function createDocumentTermFrequency()
+    {
+        $termFrequency = new TermFrequency($this->getSearchDbHandle());
+        $termFrequency->execute();
+    }
+
+
+    /**
+     * Resets the entire index
+     */
+    public function resetIndex()
+    {
+        $db = $this->getSearchDbHandle();
+        $sql = '
 			DELETE FROM term_count;
 			DELETE FROM term_frequency;
 			DELETE FROM inverse_document_frequency;
@@ -110,82 +110,82 @@ 
 			UPDATE `sqlite_sequence` SET `seq`= 0 WHERE `name`=\'term_frequency\';
 			UPDATE `sqlite_sequence` SET `seq`= 0 WHERE `name`=\'inverse_document_frequency\';
 		';
-		$db->exec($sql);
-	}
-
-	/**
-	 * Calculates the inverse document frequency for each
-	 * term. This is a representation of how often a certain
-	 * term is used in comparison to all terms.
-	 */
-	public function createInverseDocumentFrequency()
-	{
-		$documentCount = $this->getTotalDocumentCount();
-		$inverseDocumentFrequency = new InverseDocumentFrequency($this->getSearchDbHandle(), $documentCount);
-		$inverseDocumentFrequency->execute();
-	}
-
-	/**
-	 * @return int|mixed
-	 */
-	private function getTotalDocumentCount()
-	{
-		return $this->storage->getDocuments()->getTotalDocumentCount();
-	}
-
-	/**
-	 * Calculates the Term Field Length Norm.
-	 * This is an index determining how important a
-	 * term is, based on the total length of the field
-	 * it comes from.
-	 */
-	public function createTermFieldLengthNorm()
-	{
-		$termFieldLengthNorm = new TermFieldLengthNorm($this->getSearchDbHandle());
-		$termFieldLengthNorm->execute();
-	}
-
-	/**
-	 * Stores the time the indexing started in memory
-	 */
-	private function startLogging()
-	{
-		$this->loggingStart = round(microtime(true) * 1000);
-		$this->lastLog = $this->loggingStart;
-	}
-
-	/**
-	 * Adds a logline with the time since last log
-	 * @param $string
-	 */
-	private function addLog($string)
-	{
-		$currentTime = round(microtime(true) * 1000);
-		$this->log .= date('d-m-Y H:i:s - ') . str_pad($string, 50, " ", STR_PAD_RIGHT) . "\t" . ($currentTime - $this->lastLog) . 'ms since last log. ' . "\t" . ($currentTime - $this->loggingStart) . 'ms since start.' . PHP_EOL;
-		$this->lastLog = round(microtime(true) * 1000);
-	}
-
-	/**
-	 * Creates the SQLite \PDO object if it doesnt
-	 * exist and returns it.
-	 * @return \PDO
-	 */
-	protected function getSearchDbHandle()
-	{
-		if ($this->searchDbHandle === null) {
-			$path = $this->storageDir . DIRECTORY_SEPARATOR;
-			$this->searchDbHandle = new \PDO('sqlite:' . $path . self::SEARCH_TEMP_DB);
-		}
-		return $this->searchDbHandle;
-	}
-
-	/**
-	 * Replaces the old search index database with the new one.
-	 */
-	public function replaceOldIndex()
-	{
-		$this->searchDbHandle = null;
-		$path = $this->storageDir . DIRECTORY_SEPARATOR;
-		rename($path . self::SEARCH_TEMP_DB, $path . 'search.db');
-	}
+        $db->exec($sql);
+    }
+
+    /**
+     * Calculates the inverse document frequency for each
+     * term. This is a representation of how often a certain
+     * term is used in comparison to all terms.
+     */
+    public function createInverseDocumentFrequency()
+    {
+        $documentCount = $this->getTotalDocumentCount();
+        $inverseDocumentFrequency = new InverseDocumentFrequency($this->getSearchDbHandle(), $documentCount);
+        $inverseDocumentFrequency->execute();
+    }
+
+    /**
+     * @return int|mixed
+     */
+    private function getTotalDocumentCount()
+    {
+        return $this->storage->getDocuments()->getTotalDocumentCount();
+    }
+
+    /**
+     * Calculates the Term Field Length Norm.
+     * This is an index determining how important a
+     * term is, based on the total length of the field
+     * it comes from.
+     */
+    public function createTermFieldLengthNorm()
+    {
+        $termFieldLengthNorm = new TermFieldLengthNorm($this->getSearchDbHandle());
+        $termFieldLengthNorm->execute();
+    }
+
+    /**
+     * Stores the time the indexing started in memory
+     */
+    private function startLogging()
+    {
+        $this->loggingStart = round(microtime(true) * 1000);
+        $this->lastLog = $this->loggingStart;
+    }
+
+    /**
+     * Adds a logline with the time since last log
+     * @param $string
+     */
+    private function addLog($string)
+    {
+        $currentTime = round(microtime(true) * 1000);
+        $this->log .= date('d-m-Y H:i:s - ') . str_pad($string, 50, " ", STR_PAD_RIGHT) . "\t" . ($currentTime - $this->lastLog) . 'ms since last log. ' . "\t" . ($currentTime - $this->loggingStart) . 'ms since start.' . PHP_EOL;
+        $this->lastLog = round(microtime(true) * 1000);
+    }
+
+    /**
+     * Creates the SQLite \PDO object if it doesnt
+     * exist and returns it.
+     * @return \PDO
+     */
+    protected function getSearchDbHandle()
+    {
+        if ($this->searchDbHandle === null) {
+            $path = $this->storageDir . DIRECTORY_SEPARATOR;
+            $this->searchDbHandle = new \PDO('sqlite:' . $path . self::SEARCH_TEMP_DB);
+        }
+        return $this->searchDbHandle;
+    }
+
+    /**
+     * Replaces the old search index database with the new one.
+     */
+    public function replaceOldIndex()
+    {
+        $this->searchDbHandle = null;
+        $path = $this->storageDir . DIRECTORY_SEPARATOR;
+        rename($path . self::SEARCH_TEMP_DB, $path . 'search.db');
+    }
 }
\ No newline at end of file

src/search/results/SearchResult.php

Indentation +39 added lines, -39 removed lines

@@ -13,43 +13,43 @@
 
 class SearchResult
 {
-	/**
-	 * @var string
-	 */
-	public $documentPath;
-	/**
-	 * @var array
-	 */
-	public $matchingTokens;
-	/**
-	 * @var float
-	 */
-	public $score;
-
-	protected $document;
-	/**
-	 * @var Storage
-	 */
-	protected $storage;
-
-	/**
-	 * @return Document
-	 */
-	public function getDocument()
-	{
-		if ($this->document instanceof Document) {
-			return $this->document;
-		} else {
-			$this->document = $this->storage->getDocuments()->getDocumentBySlug(substr($this->documentPath, 1));
-			$this->document->dbHandle = $this->storage->getContentDbHandle();
-			$this->document->documentStorage = $this->storage->getRepository();
-
-			return $this->document;
-		}
-	}
-
-	public function setStorage($storage)
-	{
-		$this->storage = $storage;
-	}
+    /**
+     * @var string
+     */
+    public $documentPath;
+    /**
+     * @var array
+     */
+    public $matchingTokens;
+    /**
+     * @var float
+     */
+    public $score;
+
+    protected $document;
+    /**
+     * @var Storage
+     */
+    protected $storage;
+
+    /**
+     * @return Document
+     */
+    public function getDocument()
+    {
+        if ($this->document instanceof Document) {
+            return $this->document;
+        } else {
+            $this->document = $this->storage->getDocuments()->getDocumentBySlug(substr($this->documentPath, 1));
+            $this->document->dbHandle = $this->storage->getContentDbHandle();
+            $this->document->documentStorage = $this->storage->getRepository();
+
+            return $this->document;
+        }
+    }
+
+    public function setStorage($storage)
+    {
+        $this->storage = $storage;
+    }
 }
\ No newline at end of file

src/search/results/SearchSuggestion.php

Indentation +3 added lines, -3 removed lines

@@ -10,7 +10,7 @@
 
 class SearchSuggestion
 {
-	public $original;
-	public $term;
-	public $editDistance;
+    public $original;
+    public $term;
+    public $editDistance;
 }
\ No newline at end of file