ElasticaService::refreshRecords() - Code Metrics - Inspection of "WIP: Next test, SearchableClassTest" - gordonbanderson/silverstripe-elastica - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — dev2 ( d5d1ea...bf3a9f )

by Gordon

created 2015-12-30 05:55 UTC

ElasticaService::refreshRecords() A

↳ Parent: ElasticaService

Complexity

Conditions	3
Paths	3

Size

Total Lines	7
Code Lines	4

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	4
CRAP Score	3.7086

Importance

Changes	1
Bugs	1	Features	0

Metric	Value
c	1
b	1
f	0
dl	0
loc	7
ccs	4
cts	7
cp	0.5714
rs	9.4286
cc	3
eloc	4
nc	3
nop	1
crap	3.7086

<?php

namespace SilverStripe\Elastica;

use Elastica\Client;
use Elastica\Query;
use Elastica\Search;
use SilverStripe\Elastica\ElasticaUtil;

/**
 * A service used to interact with elastic search.
 */
class ElasticaService {

	/**
	 * @var \Elastica\Document[]
	 */
	protected $buffer = array();


	/**
	 * @var bool controls whether indexing operations are buffered or not
	 */
	protected $buffered = false;


	/**
	 * @var \Elastica\Client Elastica Client object
	 */
	private $client;


	/**
	 * @var string index name
	 */
	private $indexName;


	/**
	 * The code of the locale being indexed or searched
	 * @var string e.g. th_TH, en_US
	 */
	private $locale;


	/**
	 * Mapping of DataObject ClassName and whether it is in the SiteTree or not
	 * @var array $site_tree_classes;
	 */
	private static $site_tree_classes = array();


	/**
	 * Counter used to for testing, records indexing requests
	 * @var integer
	 */
	public static $indexing_request_ctr = 0;


	/**
	 * Array of highlighted fields, e.g. Title, Title.standard.  If this is empty then the
	 * ShowHighlight field of SearchableField is used to determine which fields to highlight
	 * @var array
	 */
	private $highlightedFields = array();


	/**
	 * The number of documents to index currently for this locale
	 * @var integer The number of documents left to index
	 */
	private $nDocumentsToIndexForLocale = 0;


	/*
	Set the highlight fields for subsequent searches
	 */
	public function setHighlightedFields($newHighlightedFields) {
		$this->highlightedFields = $newHighlightedFields;
	}


	/*
	Enable this to allow test classes not to be ignored when indexing
	 */
	public $test_mode = false;


	/**
	 * @param \Elastica\Client $client
	 * @param string $newIndexName Name of the new index
	 */
	public function __construct(Client $client, $newIndexName) {
		$this->client = $client;
		$this->indexName = $newIndexName;
		$this->locale = \i18n::default_locale();
	}


	public function setTestMode($newTestMode) {
		$this->test_mode = $newTestMode;
	}


	/**
	 * @return \Elastica\Client
	 */
	public function getClient() {
		return $this->client;
	}


	/**
	 * @return \Elastica\Index
	 */
	public function getIndex() {
		$index = $this->getClient()->getIndex($this->getLocaleIndexName());
		return $index;
	}


	public function setLocale($newLocale) {
		$this->locale = $newLocale;
	}

	public function getIndexName() {
		return $this->indexName;
	}

	private function getLocaleIndexName() {
		$name = $this->indexName . '-' . $this->locale;
		$name = strtolower($name);
		$name = str_replace('-', '_', $name);
		return $name;
	}


	/**
	 * Performs a search query and returns a result list.
	 *
	 * @param \Elastica\Query|string|array $query
	 * @param string|array $types List of comma separated SilverStripe classes to search, or blank for all
	 * @return \Elastica\ResultList
	 */
	public function search($query, $types = '') {
		$query = Query::create($query); // may be a string
		if(is_string($types)) {
			$types = explode(',', $types);
		}

		$data = $query->toArray();
		$query->MoreLikeThis = isset($data['query']['more_like_this']);

		$search = new Search(new Client());

		// get results from all shards, this makes test repeatable
		if($this->test_mode) {
			$search->setOption('search_type', Search::OPTION_SEARCH_TYPE_DFS_QUERY_THEN_FETCH);
		}

		$search->addIndex($this->getLocaleIndexName());
		$this->addTypesToSearch($search, $types, $query);

		$highlights = $this->getHighlightingConfig();
		$this->addExtractedQueryTermsForMoreLikeThis($query, $highlights);
		$query->setHighlight($highlights);

		$search->addIndex($this->getLocaleIndexName());
		if(!empty($types)) {
			foreach($types as $type) {
				$search->addType($type);
			}
		}

		$params = $search->getOptions();
		$searchResults = $search->search($query, $params);
		if(isset($this->MoreLikeThisTerms)) {
			$searchResults->MoreLikeThisTerms = $this->MoreLikeThisTerms;
		}

		return $searchResults;
	}


	/**
	 * @param Query $query
	 */
	private function addExtractedQueryTermsForMoreLikeThis($query, &$highlights) {
		if($query->MoreLikeThis) {
			$termsMatchingQuery = array();
			foreach($this->MoreLikeThisTerms as $field => $terms) {
				$termQuery = array('multi_match' => array(
					'query' => implode(' ', $terms),
					'type' => 'most_fields',
					'fields' => array($field)
				));
				$termsMatchingQuery[$field] = array('highlight_query' => $termQuery);
			}
			$highlights['fields'] = $termsMatchingQuery;
		}
	}


	/**
	 * @param Search $search
	 * @param Query $query
	 */
	private function addTypesToSearch(&$search, $types, $query) {
		// If the query is a 'more like this' we can get the terms used for searching by performing
		// an extra query, in this case a query validation with explain and rewrite turned on
		$this->checkForTermsMoreLikeThis($query, $search);

		if(!empty($types)) {
			foreach($types as $type) {
				$search->addType($type);
			}
		}
	}


	private function getHighlightingConfig() {
		$highlightsCfg = \Config::inst()->get('Elastica', 'Highlights');
		$preTags = $highlightsCfg['PreTags'];
		$postTags = $highlightsCfg['PostTags'];
		$fragmentSize = $highlightsCfg['Phrase']['FragmentSize'];
		$nFragments = $highlightsCfg['Phrase']['NumberOfFragments'];

		$stringFields = $this->highlightedFields;
		$usingProvidedHighlightFields = true;

		if(sizeof($stringFields) == 0) {
			$filter = array('Type' => 'string', 'ShowHighlights' => true);
			$stringFields = \SearchableField::get()->filter($filter)->map('Name')->toArray();
			$usingProvidedHighlightFields = false;
		}


		$highlightFields = array();
		foreach($stringFields as $name) {
			// Add the stemmed and the unstemmed for now
			$fieldName = $name;
			if(!$usingProvidedHighlightFields) {
				$fieldName .= '.standard';
			}
			$highlightFields[$fieldName] = array(
				'fragment_size' => $fragmentSize,
				'number_of_fragments' => $nFragments,
				'no_match_size'=> 200
			);
		}

		$highlights = array(
			'pre_tags' => array($preTags),
			'post_tags' => array($postTags),
			'fields' => $highlightFields
		);

		return $highlights;
	}


	private function checkForTermsMoreLikeThis($elasticaQuery, $search) {
		if($elasticaQuery->MoreLikeThis) {


			$path = $search->getPath();

			$termData = array();
			$data = $elasticaQuery->toArray();
			$termData['query'] = $data['query'];

			$path = str_replace('_search', '_validate/query', $path);
			$params = array('explain' => true, 'rewrite' => true);

			if($this->test_mode) {
				$params['search_type'] = Search::OPTION_SEARCH_TYPE_DFS_QUERY_THEN_FETCH;
			}

			$response = $this->getClient()->request(
				$path,
				\Elastica\Request::GET,
				$termData,
				$params
			);

			$rData = $response->getData();
			$terms = null; // keep in scope

			if(isset($rData['explanations'])) {
				$explanation = $rData['explanations'][0]['explanation'];
				$terms = ElasticaUtil::parseSuggestionExplanation($explanation);
			}

			if(isset($terms)) {
				$this->MoreLikeThisTerms = $terms;
			}
		}
	}


	/**
	 * Ensure that the index is present
	 */
	protected function ensureIndex() {
		$index = $this->getIndex();
		if(!$index->exists()) {
			$this->createIndex();
		}
	}


	/**
	 * Ensure that there is a mapping present
	 *
	 * @param \Elastica\Type Type object
	 * @param SilverStripe\Elastica\Searchable DataObject that implements Searchable
	 * @return \Elastica\Mapping Mapping object
	 */
	protected function ensureMapping(\Elastica\Type $type, \DataObject $record) {
		$mapping = $type->getMapping();
		if($mapping == array()) {
			$this->ensureIndex();
			$mapping = $record->getElasticaMapping();
			$type->setMapping($mapping);
			$mapping = $mapping->toArray();
		}
		return $mapping;
	}


	/**
	 * Either creates or updates a record in the index.
	 *
	 * @param Searchable $record
	 */
	public function index($record) {
		$document = $record->getElasticaDocument();
		$typeName = $record->getElasticaType();

		if($this->buffered) {
			if(array_key_exists($typeName, $this->buffer)) {
				$this->buffer[$typeName][] = $document;
			} else {
				$this->buffer[$typeName] = array($document);
			}
		} else {
			$index = $this->getIndex();
			$type = $index->getType($typeName);

			$this->ensureMapping($type, $record);

			$type->addDocument($document);
			$index->refresh();
			self::$indexing_request_ctr++;
		}
	}


	/**
	 * Begins a bulk indexing operation where documents are buffered rather than
	 * indexed immediately.
	 */
	public function startBulkIndex() {
		$this->buffered = true;
	}


	public function listIndexes($trace) {
		$command = "curl 'localhost:9200/_cat/indices?v'";
		exec($command, $op);
		ElasticaUtil::message("\n++++ $trace ++++\n");
		ElasticaUtil::message(print_r($op, 1));
		ElasticaUtil::message("++++ /{$trace} ++++\n\n");
		return $op;
	}


	/**
	 * Ends the current bulk index operation and indexes the buffered documents.
	 */
	public function endBulkIndex() {
		$index = $this->getIndex();
		foreach($this->buffer as $type => $documents) {
			$amount = 0;

			foreach(array_keys($this->buffer) as $key) {
				$amount += sizeof($this->buffer[$key]);
			}
			$index->getType($type)->addDocuments($documents);
			$index->refresh();

			ElasticaUtil::message("\tAdding $amount documents to the index\n");
			if(isset($this->StartTime)) {
				$elapsed = microtime(true) - $this->StartTime;
				$timePerDoc = ($elapsed) / ($this->nDocumentsIndexed);
				$documentsRemaining = $this->nDocumentsToIndexForLocale - $this->nDocumentsIndexed;
				$eta = ($documentsRemaining) * $timePerDoc;
				$hours = (int)($eta / 3600);
				$minutes = (int)(($eta - $hours * 3600) / 60);
				$seconds = (int)(0.5 + $eta - $minutes * 60 - $hours * 3600);
				$etaHR = "{$hours}h {$minutes}m {$seconds}s";
				ElasticaUtil::message("ETA to completion of indexing $this->locale ($documentsRemaining documents): $etaHR");
			}
			self::$indexing_request_ctr++;
		}

		$this->buffered = false;
		$this->buffer = array();
	}


	/**
	 * Deletes a record from the index.
	 *
	 * @param Searchable $record
	 */
	public function remove($record) {
		$index = $this->getIndex();
		$type = $index->getType($record->getElasticaType());
		$type->deleteDocument($record->getElasticaDocument());
		$index->refresh();
	}


	/**
	 * Creates the index and the type mappings.
	 */
	public function define() {
		$index = $this->getIndex();

		# Recreate the index
		if($index->exists()) {
			$index->delete();
		}
		$this->createIndex();

		foreach($this->getIndexedClasses() as $class) {
			$sng = singleton($class);
			$mapping = $sng->getElasticaMapping();
			$mapping->setType($index->getType($sng->getElasticaType()));
			$mapping->send();
		}
	}


	/**
	 * Refresh an array of records in the index
	 *
	 * @param array $records
	 */
	protected function refreshRecords($records) {
		foreach($records as $record) {
			if($record->showRecordInSearch()) {
				$this->index($record);
			}
		}
	}


	/**
	 * Get a List of all records by class. Get the "Live data" If the class has the "Versioned" extension
	 *
	 * @param string $class Class Name
	 * @param  int $pageSize Optional page size, only a max of this number of records returned
	 * @param  int $page Page number to return
	 * @return \DataList $records
	 */
	protected function recordsByClassConsiderVersioned($class, $pageSize = 0, $page = 0) {
		$offset = $page * $pageSize;

		if($class::has_extension("Versioned")) {
			if($pageSize > 0) {
				$records = \Versioned::get_by_stage($class, 'Live')->limit($pageSize, $offset);
			} else {
				$records = \Versioned::get_by_stage($class, 'Live');
			}
		} else {
			if($pageSize > 0) {
				$records = $class::get()->limit($pageSize, $offset);
			} else {
				$records = $class::get();
			}

		}
		return $records;
	}


	/**
	 * Refresh the records of a given class within the search index
	 *
	 * @param string $class Class Name
	 */
	protected function refreshClass($class) {
		$nRecords = $this->recordsByClassConsiderVersioned($class)->count();
		$batchSize = 500;
		$pages = $nRecords / $batchSize + 1;

		for($i = 0; $i < $pages; $i++) {
			$this->startBulkIndex();
			$pagedRecords = $this->recordsByClassConsiderVersioned($class, $batchSize, $i);
			$this->nDocumentsIndexed += $pagedRecords->count();
			$batch = $pagedRecords->toArray();
			$this->refreshRecords($batch);
			$this->endBulkIndex();
		}
	}


	/**
	 * Re-indexes each record in the index.
	 */
	public function refresh() {
		$this->StartTime = microtime(true);

		$classes = $this->getIndexedClasses();

		//Count the number of documents for this locale
		$amount = 0;
		foreach($classes as $class) {
			$amount += $this->recordsByClassConsiderVersioned($class)->count();
		}

		$this->nDocumentsToIndexForLocale = $amount;
		$this->nDocumentsIndexed = 0;

		foreach($this->getIndexedClasses() as $classname) {
			ElasticaUtil::message("Indexing class $classname");

			$inSiteTree = null;
			if(isset(self::$site_tree_classes[$classname])) {
				$inSiteTree = self::$site_tree_classes[$classname];
			} else {
				$inSiteTree = SearchableHelper::isInSiteTree($classname);
				self::$site_tree_classes[$classname] = $inSiteTree;
			}

			if($inSiteTree) {
				// this prevents the same item being indexed twice due to class inheritance
				if($classname === 'SiteTree') {
					$this->refreshClass($classname);
				}
			// Data objects
			} else {
				$this->refreshClass($classname);
			}

		}

		ElasticaUtil::message("Completed indexing documents for locale $this->locale\n");

	}


	/**
	 * Reset the current index
	 */
	public function reset() {
		$index = $this->getIndex();
		$index->delete();
		$this->createIndex();
	}


	private function createIndex() {
		$index = $this->getIndex();
		$settings = $this->getIndexSettingsForCurrentLocale()->generateConfig();
		$index->create($settings, true);
	}


	/**
	 * Get the index settings for the current locale
	 * @return IndexSettings index settings for the current locale
	 */
	public function getIndexSettingsForCurrentLocale() {
		$result = null;
		$indexSettings = \Config::inst()->get('Elastica', 'indexsettings');
		if(isset($indexSettings[$this->locale])) {
			$settingsClassName = $indexSettings[$this->locale];
			$result = \Injector::inst()->create($settingsClassName);
		} else {
			throw new \Exception('ERROR: No index settings are provided for locale ' . $this->locale . "\n");

		}
		return $result;
	}


	/**
	 * Gets the classes which are indexed (i.e. have the extension applied).
	 *
	 * @return array
	 */
	public function getIndexedClasses() {
		$classes = array();

		$whitelist = array('SearchableTestPage', 'SearchableTestFatherPage', 'SearchableTestGrandFatherPage',
			'FlickrPhotoTO', 'FlickrTagTO', 'FlickrPhotoTO', 'FlickrAuthorTO', 'FlickrSetTO');

		foreach(\ClassInfo::subclassesFor('DataObject') as $candidate) {
			$instance = singleton($candidate);

			$interfaces = class_implements($candidate);
			// Only allow test classes in testing mode
			if(isset($interfaces['TestOnly'])) {
				if(in_array($candidate, $whitelist)) {
					if(!$this->test_mode) {
						continue;
					}
				} else {
					// If it's not in the test whitelist we definitely do not want to know
					continue;
				}
			}

			if($instance->hasExtension('SilverStripe\\Elastica\\Searchable')) {
				$classes[] = $candidate;
			}
		}

		return $classes;
	}


	/**
	 * Get the number of indexing requests made.  Used for testing bulk indexing
	 * @return integer indexing request counter
	 */
	public function getIndexingRequestCtr() {
		return self::$indexing_request_ctr;
	}


	/**
	 * Get the term vectors in the index for the provided  Searchable is_object
	 * @param  Searchable $searchable An object that implements Searchable
	 * @return array             array of field name to terms indexed
	 */
	public function getTermVectors($searchable) {
		$params = array();

		$fieldMappings = $searchable->getElasticaMapping()->getProperties();
		$fields = array_keys($fieldMappings);
		$allFields = array();
		foreach($fields as $field) {
			array_push($allFields, $field);

			$mapping = $fieldMappings[$field];


			if(isset($mapping['fields'])) {
				$subFields = array_keys($mapping['fields']);
				foreach($subFields as $subField) {
					$name = $field . '.' . $subField;
					array_push($allFields, $name);
				}
			}
		}
		sort($allFields);
		$data = array(
			'fields' => $allFields,
			'offsets' => true,
			'payloads' => true,
			'positions' => true,
			'term_statistics' => true,
			'field_statistics' => true
		);

		$path = $this->getIndex()->getName() . '/' . $searchable->ClassName . '/' . $searchable->ID . '/_termvector';
		$response = $this->getClient()->request(
				$path,
				\Elastica\Request::GET,
				$data,
				$params
		);

		$data = $response->getData();
		return isset($data['term_vectors']) ? $data['term_vectors'] : array();
	}
}


1		<?php
2
3		namespace SilverStripe\Elastica;
4
5		use Elastica\Client;
6		use Elastica\Query;
7		use Elastica\Search;
8		use SilverStripe\Elastica\ElasticaUtil;
9
10		/**
11		* A service used to interact with elastic search.
12		*/
13		class ElasticaService {
14
15		/**
16		* @var \Elastica\Document[]
17		*/
18		protected $buffer = array();
19
20
21		/**
22		* @var bool controls whether indexing operations are buffered or not
23		*/
24		protected $buffered = false;
25
26
27		/**
28		* @var \Elastica\Client Elastica Client object
29		*/
30		private $client;
31
32
33		/**
34		* @var string index name
35		*/
36		private $indexName;
37
38
39		/**
40		* The code of the locale being indexed or searched
41		* @var string e.g. th_TH, en_US
42		*/
43		private $locale;
44
45
46		/**
47		* Mapping of DataObject ClassName and whether it is in the SiteTree or not
48		* @var array $site_tree_classes;
49		*/
50		private static $site_tree_classes = array();
51
52
53		/**
54		* Counter used to for testing, records indexing requests
55		* @var integer
56		*/
57		public static $indexing_request_ctr = 0;
58
59
60		/**
61		* Array of highlighted fields, e.g. Title, Title.standard. If this is empty then the
62		* ShowHighlight field of SearchableField is used to determine which fields to highlight
63		* @var array
64		*/
65		private $highlightedFields = array();
66
67
68		/**
69		* The number of documents to index currently for this locale
70		* @var integer The number of documents left to index
71		*/
72		private $nDocumentsToIndexForLocale = 0;
73
74
75		/*
76		Set the highlight fields for subsequent searches
77		*/
78		public function setHighlightedFields($newHighlightedFields) {
79		$this->highlightedFields = $newHighlightedFields;
80		}
81
82
83		/*
84		Enable this to allow test classes not to be ignored when indexing
85		*/
86		public $test_mode = false;
87
88
89		/**
90		* @param \Elastica\Client $client
91		* @param string $newIndexName Name of the new index
92		*/
93	2	public function __construct(Client $client, $newIndexName) {
94	2	$this->client = $client;
95	2	$this->indexName = $newIndexName;
96	2	$this->locale = \i18n::default_locale();
97	2	}
98
99
100	2	public function setTestMode($newTestMode) {
101	2	$this->test_mode = $newTestMode;
102	2	}
103
104
105		/**
106		* @return \Elastica\Client
107		*/
108	2	public function getClient() {
109	2	return $this->client;
110		}
111
112
113		/**
114		* @return \Elastica\Index
115		*/
116	2	public function getIndex() {
117	2	$index = $this->getClient()->getIndex($this->getLocaleIndexName());
118	2	return $index;
119		}
120
121
122	2	public function setLocale($newLocale) {
123	2	$this->locale = $newLocale;
124	2	}
125
126		public function getIndexName() {
127		return $this->indexName;
128		}
129
130	2	private function getLocaleIndexName() {
131	2	$name = $this->indexName . '-' . $this->locale;
132	2	$name = strtolower($name);
133	2	$name = str_replace('-', '_', $name);
134	2	return $name;
135		}
136
137
138		/**
139		* Performs a search query and returns a result list.
140		*
141		* @param \Elastica\Query\|string\|array $query
142		* @param string\|array $types List of comma separated SilverStripe classes to search, or blank for all
143		* @return \Elastica\ResultList
144		*/
145		public function search($query, $types = '') {
146		$query = Query::create($query); // may be a string
147		if(is_string($types)) {
148		$types = explode(',', $types);
149		}
150
151		$data = $query->toArray();
152		$query->MoreLikeThis = isset($data['query']['more_like_this']);
153
154		$search = new Search(new Client());
155
156		// get results from all shards, this makes test repeatable
157		if($this->test_mode) {
158		$search->setOption('search_type', Search::OPTION_SEARCH_TYPE_DFS_QUERY_THEN_FETCH);
159		}
160
161		$search->addIndex($this->getLocaleIndexName());
162		$this->addTypesToSearch($search, $types, $query);
163
164		$highlights = $this->getHighlightingConfig();
165		$this->addExtractedQueryTermsForMoreLikeThis($query, $highlights);
166		$query->setHighlight($highlights);
167
168		$search->addIndex($this->getLocaleIndexName());
169		if(!empty($types)) {
170		foreach($types as $type) {
171		$search->addType($type);
172		}
173		}
174
175		$params = $search->getOptions();
176		$searchResults = $search->search($query, $params);
177		if(isset($this->MoreLikeThisTerms)) {
178		$searchResults->MoreLikeThisTerms = $this->MoreLikeThisTerms;
179		}
180
181		return $searchResults;
182		}
183
184
185		/**
186		* @param Query $query
187		*/
188		private function addExtractedQueryTermsForMoreLikeThis($query, &$highlights) {
189		if($query->MoreLikeThis) {
190		$termsMatchingQuery = array();
191		foreach($this->MoreLikeThisTerms as $field => $terms) {
192		$termQuery = array('multi_match' => array(
193		'query' => implode(' ', $terms),
194		'type' => 'most_fields',
195		'fields' => array($field)
196		));
197		$termsMatchingQuery[$field] = array('highlight_query' => $termQuery);
198		}
199		$highlights['fields'] = $termsMatchingQuery;
200		}
201		}
202
203
204		/**
205		* @param Search $search
206		* @param Query $query
207		*/
208	1	private function addTypesToSearch(&$search, $types, $query) {
209		// If the query is a 'more like this' we can get the terms used for searching by performing
210		// an extra query, in this case a query validation with explain and rewrite turned on
211		$this->checkForTermsMoreLikeThis($query, $search);
212
213		if(!empty($types)) {
214		foreach($types as $type) {
215	1	$search->addType($type);
216		}
217		}
218		}
219
220
221		private function getHighlightingConfig() {
222		$highlightsCfg = \Config::inst()->get('Elastica', 'Highlights');
223		$preTags = $highlightsCfg['PreTags'];
224		$postTags = $highlightsCfg['PostTags'];
225		$fragmentSize = $highlightsCfg['Phrase']['FragmentSize'];
226		$nFragments = $highlightsCfg['Phrase']['NumberOfFragments'];
227
228		$stringFields = $this->highlightedFields;
229		$usingProvidedHighlightFields = true;
230
231		if(sizeof($stringFields) == 0) {
232		$filter = array('Type' => 'string', 'ShowHighlights' => true);
233		$stringFields = \SearchableField::get()->filter($filter)->map('Name')->toArray();
234		$usingProvidedHighlightFields = false;
235		}
236
237
238		$highlightFields = array();
239		foreach($stringFields as $name) {
240		// Add the stemmed and the unstemmed for now
241		$fieldName = $name;
242		if(!$usingProvidedHighlightFields) {
243		$fieldName .= '.standard';
244		}
245		$highlightFields[$fieldName] = array(
246		'fragment_size' => $fragmentSize,
247		'number_of_fragments' => $nFragments,
248		'no_match_size'=> 200
249		);
250		}
251
252		$highlights = array(
253		'pre_tags' => array($preTags),
254		'post_tags' => array($postTags),
255		'fields' => $highlightFields
256		);
257
258		return $highlights;
259		}
260
261
262		private function checkForTermsMoreLikeThis($elasticaQuery, $search) {
263		if($elasticaQuery->MoreLikeThis) {
264
265
266		$path = $search->getPath();
267
268		$termData = array();
269		$data = $elasticaQuery->toArray();
270		$termData['query'] = $data['query'];
271
272		$path = str_replace('_search', '_validate/query', $path);
273		$params = array('explain' => true, 'rewrite' => true);
274
275		if($this->test_mode) {
276		$params['search_type'] = Search::OPTION_SEARCH_TYPE_DFS_QUERY_THEN_FETCH;
277		}
278
279		$response = $this->getClient()->request(
280		$path,
281		\Elastica\Request::GET,
282		$termData,
283		$params
284		);
285
286		$rData = $response->getData();
287		$terms = null; // keep in scope
288
289		if(isset($rData['explanations'])) {
290		$explanation = $rData['explanations'][0]['explanation'];
291		$terms = ElasticaUtil::parseSuggestionExplanation($explanation);
292		}
293
294		if(isset($terms)) {
295		$this->MoreLikeThisTerms = $terms;
296		}
297		}
298		}
299
300
301		/**
302		* Ensure that the index is present
303		*/
304		protected function ensureIndex() {
305		$index = $this->getIndex();
306		if(!$index->exists()) {
307		$this->createIndex();
308		}
309		}
310
311
312		/**
313		* Ensure that there is a mapping present
314		*
315		* @param \Elastica\Type Type object
316		* @param SilverStripe\Elastica\Searchable DataObject that implements Searchable
317		* @return \Elastica\Mapping Mapping object
318		*/
319		protected function ensureMapping(\Elastica\Type $type, \DataObject $record) {
320		$mapping = $type->getMapping();
321		if($mapping == array()) {
322		$this->ensureIndex();
323		$mapping = $record->getElasticaMapping();
324		$type->setMapping($mapping);
325		$mapping = $mapping->toArray();
326		}
327		return $mapping;
328		}
329
330
331		/**
332		* Either creates or updates a record in the index.
333		*
334		* @param Searchable $record
335		*/
336		public function index($record) {
337		$document = $record->getElasticaDocument();
338		$typeName = $record->getElasticaType();
339
340		if($this->buffered) {
341		if(array_key_exists($typeName, $this->buffer)) {
342		$this->buffer[$typeName][] = $document;
343		} else {
344		$this->buffer[$typeName] = array($document);
345		}
346		} else {
347		$index = $this->getIndex();
348		$type = $index->getType($typeName);
349
350		$this->ensureMapping($type, $record);
351
352		$type->addDocument($document);
353		$index->refresh();
354		self::$indexing_request_ctr++;
355		}
356		}
357
358
359		/**
360		* Begins a bulk indexing operation where documents are buffered rather than
361		* indexed immediately.
362		*/
363	2	public function startBulkIndex() {
364	2	$this->buffered = true;
365	2	}
366
367
368		public function listIndexes($trace) {
369		$command = "curl 'localhost:9200/_cat/indices?v'";
370		exec($command, $op);
371		ElasticaUtil::message("\n++++ $trace ++++\n");
372		ElasticaUtil::message(print_r($op, 1));
373		ElasticaUtil::message("++++ /{$trace} ++++\n\n");
374		return $op;
375		}
376
377
378		/**
379		* Ends the current bulk index operation and indexes the buffered documents.
380		*/
381	2	public function endBulkIndex() {
382	2	$index = $this->getIndex();
383	2	foreach($this->buffer as $type => $documents) {
384		$amount = 0;
385
386		foreach(array_keys($this->buffer) as $key) {
387		$amount += sizeof($this->buffer[$key]);
388		}
389		$index->getType($type)->addDocuments($documents);
390		$index->refresh();
391
392		ElasticaUtil::message("\tAdding $amount documents to the index\n");
393		if(isset($this->StartTime)) {
394		$elapsed = microtime(true) - $this->StartTime;
395		$timePerDoc = ($elapsed) / ($this->nDocumentsIndexed);
396		$documentsRemaining = $this->nDocumentsToIndexForLocale - $this->nDocumentsIndexed;
397		$eta = ($documentsRemaining) * $timePerDoc;
398		$hours = (int)($eta / 3600);
399		$minutes = (int)(($eta - $hours * 3600) / 60);
400		$seconds = (int)(0.5 + $eta - $minutes * 60 - $hours * 3600);
401		$etaHR = "{$hours}h {$minutes}m {$seconds}s";
402		ElasticaUtil::message("ETA to completion of indexing $this->locale ($documentsRemaining documents): $etaHR");
403		}
404		self::$indexing_request_ctr++;
405	2	}
406
407	2	$this->buffered = false;
408	2	$this->buffer = array();
409	2	}
410
411
412		/**
413		* Deletes a record from the index.
414		*
415		* @param Searchable $record
416		*/
417		public function remove($record) {
418		$index = $this->getIndex();
419		$type = $index->getType($record->getElasticaType());
420		$type->deleteDocument($record->getElasticaDocument());
421		$index->refresh();
422		}
423
424
425		/**
426		* Creates the index and the type mappings.
427		*/
428	2	public function define() {
429	2	$index = $this->getIndex();
430
431		# Recreate the index
432	2	if($index->exists()) {
433	2	$index->delete();
434	2	}
435	2	$this->createIndex();
436
437	2	foreach($this->getIndexedClasses() as $class) {
438	2	$sng = singleton($class);
439	2	$mapping = $sng->getElasticaMapping();
440	2	$mapping->setType($index->getType($sng->getElasticaType()));
441	2	$mapping->send();
442	2	}
443	2	}
444
445
446		/**
447		* Refresh an array of records in the index
448		*
449		* @param array $records
450		*/
451	2	protected function refreshRecords($records) {
452	2	foreach($records as $record) {
453		if($record->showRecordInSearch()) {
454		$this->index($record);
455		}
456	2	}
457	2	}
458
459
460		/**
461		* Get a List of all records by class. Get the "Live data" If the class has the "Versioned" extension
462		*
463		* @param string $class Class Name
464		* @param int $pageSize Optional page size, only a max of this number of records returned
465		* @param int $page Page number to return
466		* @return \DataList $records
467		*/
468	2	protected function recordsByClassConsiderVersioned($class, $pageSize = 0, $page = 0) {
469	2	$offset = $page * $pageSize;
470
471	2	if($class::has_extension("Versioned")) {
472	2	if($pageSize > 0) {
473	2	$records = \Versioned::get_by_stage($class, 'Live')->limit($pageSize, $offset);
474	2	} else {
475	2	$records = \Versioned::get_by_stage($class, 'Live');
476		}
477	2	} else {
478	2	if($pageSize > 0) {
479	2	$records = $class::get()->limit($pageSize, $offset);
480	2	} else {
481	2	$records = $class::get();
482		}
483
484		}
485	2	return $records;
486		}
487
488
489		/**
490		* Refresh the records of a given class within the search index
491		*
492		* @param string $class Class Name
493		*/
494	2	protected function refreshClass($class) {
495	2	$nRecords = $this->recordsByClassConsiderVersioned($class)->count();
496	2	$batchSize = 500;
497	2	$pages = $nRecords / $batchSize + 1;
498
499	2	for($i = 0; $i < $pages; $i++) {
500	2	$this->startBulkIndex();
501	2	$pagedRecords = $this->recordsByClassConsiderVersioned($class, $batchSize, $i);
502	2	$this->nDocumentsIndexed += $pagedRecords->count();
503	2	$batch = $pagedRecords->toArray();
504	2	$this->refreshRecords($batch);
505	2	$this->endBulkIndex();
506	2	}
507	2	}
508
509
510		/**
511		* Re-indexes each record in the index.
512		*/
513	2	public function refresh() {
514	2	$this->StartTime = microtime(true);
515
516	2	$classes = $this->getIndexedClasses();
517
518		//Count the number of documents for this locale
519	2	$amount = 0;
520	2	foreach($classes as $class) {
521	2	$amount += $this->recordsByClassConsiderVersioned($class)->count();
522	2	}
523
524	2	$this->nDocumentsToIndexForLocale = $amount;
525	2	$this->nDocumentsIndexed = 0;
526
527	2	foreach($this->getIndexedClasses() as $classname) {
528	2	ElasticaUtil::message("Indexing class $classname");
529
530	2	$inSiteTree = null;
531	2	if(isset(self::$site_tree_classes[$classname])) {
532	1	$inSiteTree = self::$site_tree_classes[$classname];
533	1	} else {
534	1	$inSiteTree = SearchableHelper::isInSiteTree($classname);
535	1	self::$site_tree_classes[$classname] = $inSiteTree;
536		}
537
538	2	if($inSiteTree) {
539		// this prevents the same item being indexed twice due to class inheritance
540	2	if($classname === 'SiteTree') {
541	2	$this->refreshClass($classname);
542	2	}
543		// Data objects
544	2	} else {
545	2	$this->refreshClass($classname);
546		}
547
548	2	}
549
550	2	ElasticaUtil::message("Completed indexing documents for locale $this->locale\n");
551
552	2	}
553
554
555		/**
556		* Reset the current index
557		*/
558	2	public function reset() {
559	2	$index = $this->getIndex();
560	2	$index->delete();
561	2	$this->createIndex();
562	2	}
563
564
565	2	private function createIndex() {
566	2	$index = $this->getIndex();
567	2	$settings = $this->getIndexSettingsForCurrentLocale()->generateConfig();
568	2	$index->create($settings, true);
569	2	}
570
571
572		/**
573		* Get the index settings for the current locale
574		* @return IndexSettings index settings for the current locale
575		*/
576	2	public function getIndexSettingsForCurrentLocale() {
577	2	$result = null;
578	2	$indexSettings = \Config::inst()->get('Elastica', 'indexsettings');
579	2	if(isset($indexSettings[$this->locale])) {
580	2	$settingsClassName = $indexSettings[$this->locale];
581	2	$result = \Injector::inst()->create($settingsClassName);
582	2	} else {
583		throw new \Exception('ERROR: No index settings are provided for locale ' . $this->locale . "\n");
584
585		}
586	2	return $result;
587		}
588
589
590		/**
591		* Gets the classes which are indexed (i.e. have the extension applied).
592		*
593		* @return array
594		*/
595	2	public function getIndexedClasses() {
596	2	$classes = array();
597
598	2	$whitelist = array('SearchableTestPage', 'SearchableTestFatherPage', 'SearchableTestGrandFatherPage',
599	2	'FlickrPhotoTO', 'FlickrTagTO', 'FlickrPhotoTO', 'FlickrAuthorTO', 'FlickrSetTO');
600
601	2	foreach(\ClassInfo::subclassesFor('DataObject') as $candidate) {
602	2	$instance = singleton($candidate);
603
604	2	$interfaces = class_implements($candidate);
605		// Only allow test classes in testing mode
606	2	if(isset($interfaces['TestOnly'])) {
607	2	if(in_array($candidate, $whitelist)) {
608	2	if(!$this->test_mode) {
609		continue;
610		}
611	2	} else {
612		// If it's not in the test whitelist we definitely do not want to know
613	2	continue;
614		}
615	2	}
616
617	2	if($instance->hasExtension('SilverStripe\\Elastica\\Searchable')) {
618	2	$classes[] = $candidate;
619	2	}
620	2	}
621
622	2	return $classes;
623		}
624
625
626		/**
627		* Get the number of indexing requests made. Used for testing bulk indexing
628		* @return integer indexing request counter
629		*/
630		public function getIndexingRequestCtr() {
631		return self::$indexing_request_ctr;
632		}
633
634
635		/**
636		* Get the term vectors in the index for the provided Searchable is_object
637		* @param Searchable $searchable An object that implements Searchable
638		* @return array array of field name to terms indexed
639		*/
640		public function getTermVectors($searchable) {
641		$params = array();
642
643		$fieldMappings = $searchable->getElasticaMapping()->getProperties();
644		$fields = array_keys($fieldMappings);
645		$allFields = array();
646		foreach($fields as $field) {
647		array_push($allFields, $field);
648
649		$mapping = $fieldMappings[$field];
650
651
652		if(isset($mapping['fields'])) {
653		$subFields = array_keys($mapping['fields']);
654		foreach($subFields as $subField) {
655		$name = $field . '.' . $subField;
656		array_push($allFields, $name);
657		}
658		}
659		}
660		sort($allFields);
661		$data = array(
662		'fields' => $allFields,
663		'offsets' => true,
664		'payloads' => true,
665		'positions' => true,
666		'term_statistics' => true,
667		'field_statistics' => true
668		);
669
670		$path = $this->getIndex()->getName() . '/' . $searchable->ClassName . '/' . $searchable->ID . '/_termvector';
671		$response = $this->getClient()->request(
672		$path,
673		\Elastica\Request::GET,
674		$data,
675		$params
676		);
677
678		$data = $response->getData();
679		return isset($data['term_vectors']) ? $data['term_vectors'] : array();
680		}
681		}
682

gordonbanderson / silverstripe-elastica

Push — dev2 ( d5d1ea...bf3a9f )

ElasticaService::refreshRecords() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like