ElasticSearchPlatform::indexDocuments() - Code Metrics - Inspection of "Merge branch 'compat-nc15'" - nextcloud/fulltextsearch_elasticsearch - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 0cdbd8...fb0474 )

by Maxence

created 2018-11-01 21:17 UTC

ElasticSearchPlatform::indexDocuments() A

↳ Parent: ElasticSearchPlatform::indexDocument()

Complexity

Conditions	1
Paths	1

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	3
rs	10
c	0
b	0
f	0
cc	1
nc	1
nop	2

<?php
declare(strict_types=1);


/**
 * FullTextSearch_ElasticSearch - Use Elasticsearch to index the content of your nextcloud
 *
 * This file is licensed under the Affero General Public License version 3 or
 * later. See the COPYING file.
 *
 * @author Maxence Lange <[email protected]>
 * @copyright 2018
 * @license GNU AGPL version 3 or any later version
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */


namespace OCA\FullTextSearch_ElasticSearch\Platform;


use daita\MySmallPhpTools\Traits\TPathTools;
use Elasticsearch\Client;
use Elasticsearch\ClientBuilder;
use Elasticsearch\Common\Exceptions\BadRequest400Exception;
use Exception;
use OCA\FullTextSearch_ElasticSearch\Exceptions\AccessIsEmptyException;
use OCA\FullTextSearch_ElasticSearch\Exceptions\ConfigurationException;
use OCA\FullTextSearch_ElasticSearch\Service\ConfigService;
use OCA\FullTextSearch_ElasticSearch\Service\IndexService;
use OCA\FullTextSearch_ElasticSearch\Service\MiscService;
use OCA\FullTextSearch_ElasticSearch\Service\SearchService;
use OCP\FullTextSearch\IFullTextSearchPlatform;
use OCP\FullTextSearch\Model\DocumentAccess;
use OCP\FullTextSearch\Model\IIndex;
use OCP\FullTextSearch\Model\IndexDocument;
use OCP\FullTextSearch\Model\IRunner;
use OCP\FullTextSearch\Model\ISearchResult;


/**
 * Class ElasticSearchPlatform
 *
 * @package OCA\FullTextSearch_ElasticSearch\Platform
 */
class ElasticSearchPlatform implements IFullTextSearchPlatform {


	use TPathTools;


	/** @var ConfigService */
	private $configService;

	/** @var IndexService */
	private $indexService;

	/** @var SearchService */
	private $searchService;

	/** @var MiscService */
	private $miscService;

	/** @var Client */
	private $client;

	/** @var IRunner */
	private $runner;


	/**
	 * ElasticSearchPlatform constructor.
	 *
	 * @param ConfigService $configService
	 * @param IndexService $indexService
	 * @param SearchService $searchService
	 * @param MiscService $miscService
	 */
	public function __construct(
		ConfigService $configService, IndexService $indexService, SearchService $searchService,
		MiscService $miscService
	) {
		$this->configService = $configService;
		$this->indexService = $indexService;
		$this->searchService = $searchService;
		$this->miscService = $miscService;
	}


	/**
	 * return a unique Id of the platform.
	 */
	public function getId(): string {
		return 'elastic_search';
	}


	/**
	 * return a unique Id of the platform.
	 */
	public function getName(): string {
		return 'Elasticsearch';
	}


	/**
	 * @return array
	 * @throws ConfigurationException
	 */
	public function getConfiguration(): array {

		$result = [];
		$hosts = $this->configService->getElasticHost();

		foreach ($hosts as $host) {
			$parsedHost = parse_url($host);
			$safeHost = $parsedHost['scheme'] . '://';
			if (array_key_exists('user', $parsedHost)) {
				$safeHost .= $parsedHost['user'] . ':' . '********' . '@';
			}
			$safeHost .= $parsedHost['host'];
			$safeHost .= ':' . $parsedHost['port'];

			$result[] = $safeHost;
		}

		return [
			'elastic_host'  => $result,
			'elastic_index' => $this->configService->getElasticIndex()
		];
	}


	/**
	 * @param IRunner $runner
	 */
	public function setRunner(IRunner $runner) {
		$this->runner = $runner;
	}


	/**
	 * Called when loading the platform.
	 *
	 * Loading some container and connect to ElasticSearch.
	 *
	 * @throws ConfigurationException
	 * @throws Exception
	 */
	public function loadPlatform() {
		try {
			$this->connectToElastic($this->configService->getElasticHost());
		} catch (ConfigurationException $e) {
			throw $e;
		}
	}


	/**
	 * not used yet.
	 *
	 * @return bool
	 */
	public function testPlatform(): bool {
		return $this->client->ping();
	}


	/**
	 * called before any index
	 *
	 * We create a general index.
	 *
	 * @throws ConfigurationException
	 * @throws BadRequest400Exception
	 */
	public function initializeIndex() {
		$this->indexService->initializeIndex($this->client);
	}


	/**
	 * resetIndex();
	 *
	 * Called when admin wants to remove an index specific to a $provider.
	 * $provider can be null, meaning a reset of the whole index.
	 *
	 * @param string $providerId
	 *
	 * @throws ConfigurationException
	 */
	public function resetIndex(string $providerId) {
		if ($providerId === 'all') {
			$this->indexService->resetIndexAll($this->client);
		} else {
			$this->indexService->resetIndex($this->client, $providerId);
		}
	}


	/**
	 * @param IndexDocument $document
	 *
	 * @return IIndex
	 */
	public function indexDocument(IndexDocument $document): IIndex {

		$document->initHash();

		try {
			$result = $this->indexService->indexDocument($this->client, $document);

			$index = $this->indexService->parseIndexResult($document->getIndex(), $result);

			$this->updateNewIndexResult(
				$document->getIndex(), json_encode($result), 'ok',
				IRunner::RESULT_TYPE_SUCCESS
			);

			return $index;
		} catch (Exception $e) {
			$this->updateNewIndexResult(
				$document->getIndex(), '', 'issue while indexing, testing with empty content',
				IRunner::RESULT_TYPE_WARNING
			);

			$this->manageIndexErrorException($document, $e);
		}

		try {
			$result = $this->indexDocumentError($document, $e);
			$index = $this->indexService->parseIndexResult($document->getIndex(), $result);

			$this->updateNewIndexResult(
				$document->getIndex(), json_encode($result), 'ok',
				IRunner::RESULT_TYPE_WARNING
			);

			return $index;
		} catch (Exception $e) {
			$this->updateNewIndexResult(
				$document->getIndex(), '', 'fail',
				IRunner::RESULT_TYPE_FAIL
			);
			$this->manageIndexErrorException($document, $e);
		}

		return $document->getIndex();
	}


	/**
	 * @param IndexDocument $document
	 * @param Exception $e
	 *
	 * @return array
	 * @throws AccessIsEmptyException
	 * @throws ConfigurationException
	 * @throws \Exception
	 */
	private function indexDocumentError(IndexDocument $document, Exception $e): array {

		$this->updateRunnerAction('indexDocumentWithoutContent', true);

		$document->setContent('');
//		$index = $document->getIndex();
//		$index->unsetStatus(Index::INDEX_CONTENT);

		$result = $this->indexService->indexDocument($this->client, $document);

		return $result;
	}


	/**
	 * @param IndexDocument $document
	 * @param Exception $e
	 */
	private function manageIndexErrorException(IndexDocument $document, Exception $e) {

		$message = $this->parseIndexErrorException($e);
		$document->getIndex()
				 ->addError($message, get_class($e), IIndex::ERROR_SEV_3);
		$this->updateNewIndexError(
			$document->getIndex(), $message, get_class($e), IIndex::ERROR_SEV_3
		);
	}


	/**
	 * @param Exception $e
	 *
	 * @return string
	 */
	private function parseIndexErrorException(Exception $e): string {

		$arr = json_decode($e->getMessage(), true);
		if (!is_array($arr)) {
			return $e->getMessage();
		}

		if (array_key_exists('reason', $arr['error']['root_cause'][0])) {
			return $arr['error']['root_cause'][0]['reason'];
		}

		return $e->getMessage();
	}


	/**
	 * {@inheritdoc}
	 * @throws ConfigurationException
	 */
	public function deleteIndexes(array $indexes) {
		try {
			$this->indexService->deleteIndexes($this->client, $indexes);
		} catch (ConfigurationException $e) {
			throw $e;
		}
	}


	/**
	 * {@inheritdoc}
	 * @throws Exception
	 */
	public function searchRequest(ISearchResult $result, DocumentAccess $access) {
		$this->searchService->searchRequest($this->client, $result, $access);
	}


	/**
	 * @param string $providerId
	 * @param string $documentId
	 *
	 * @return IndexDocument
	 * @throws ConfigurationException
	 */
	public function getDocument(string $providerId, string $documentId): IndexDocument {
		return $this->searchService->getDocument($this->client, $providerId, $documentId);
	}


	private function cleanHost($host) {
		return $this->withoutEndSlash($host, false, false);
	}

	/**
	 * @param array $hosts
	 *
	 * @throws Exception
	 */
	private function connectToElastic(array $hosts) {

		try {
			$hosts = array_map([$this, 'cleanHost'], $hosts);
			$this->client = ClientBuilder::create()
										 ->setHosts($hosts)
										 ->setRetries(3)
										 ->build();

//		}
//		catch (CouldNotConnectToHost $e) {
//			$this 'CouldNotConnectToHost';
//			$previous = $e->getPrevious();
//			if ($previous instanceof MaxRetriesException) {
//				echo "Max retries!";
//			}
		} catch (Exception $e) {
			throw $e;
//			echo ' ElasticSearchPlatform::load() Exception --- ' . $e->getMessage() . "\n";
		}
	}


	/**
	 * @param string $action
	 * @param bool $force
	 *
	 * @throws Exception
	 */
	private function updateRunnerAction(string $action, bool $force = false) {
		if ($this->runner === null) {
			return;
		}

		$this->runner->updateAction($action, $force);
	}


	/**
	 * @param IIndex $index
	 * @param string $message
	 * @param string $exception
	 * @param int $sev
	 */
	private function updateNewIndexError(IIndex $index, string $message, string $exception, int $sev
	) {
		if ($this->runner === null) {
			return;
		}

		$this->runner->newIndexError($index, $message, $exception, $sev);
	}


	/**
	 * @param IIndex $index
	 * @param string $message
	 * @param string $status
	 * @param int $type
	 */
	private function updateNewIndexResult(IIndex $index, string $message, string $status, int $type
	) {
		if ($this->runner === null) {
			return;
		}

		$this->runner->newIndexResult($index, $message, $status, $type);
	}


}


1			<?php
2			declare(strict_types=1);
3
4
5			/**
6			* FullTextSearch_ElasticSearch - Use Elasticsearch to index the content of your nextcloud
7			*
8			* This file is licensed under the Affero General Public License version 3 or
9			* later. See the COPYING file.
10			*
11			* @author Maxence Lange <[email protected]>
12			* @copyright 2018
13			* @license GNU AGPL version 3 or any later version
14			*
15			* This program is free software: you can redistribute it and/or modify
16			* it under the terms of the GNU Affero General Public License as
17			* published by the Free Software Foundation, either version 3 of the
18			* License, or (at your option) any later version.
19			*
20			* This program is distributed in the hope that it will be useful,
21			* but WITHOUT ANY WARRANTY; without even the implied warranty of
22			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23			* GNU Affero General Public License for more details.
24			*
25			* You should have received a copy of the GNU Affero General Public License
26			* along with this program. If not, see <http://www.gnu.org/licenses/>.
27			*
28			*/
29
30
31			namespace OCA\FullTextSearch_ElasticSearch\Platform;
32
33
34			use daita\MySmallPhpTools\Traits\TPathTools;
35			use Elasticsearch\Client;
36			use Elasticsearch\ClientBuilder;
37			use Elasticsearch\Common\Exceptions\BadRequest400Exception;
38			use Exception;
39			use OCA\FullTextSearch_ElasticSearch\Exceptions\AccessIsEmptyException;
40			use OCA\FullTextSearch_ElasticSearch\Exceptions\ConfigurationException;
41			use OCA\FullTextSearch_ElasticSearch\Service\ConfigService;
42			use OCA\FullTextSearch_ElasticSearch\Service\IndexService;
43			use OCA\FullTextSearch_ElasticSearch\Service\MiscService;
44			use OCA\FullTextSearch_ElasticSearch\Service\SearchService;
45			use OCP\FullTextSearch\IFullTextSearchPlatform;
46			use OCP\FullTextSearch\Model\DocumentAccess;
47			use OCP\FullTextSearch\Model\IIndex;
48			use OCP\FullTextSearch\Model\IndexDocument;
49			use OCP\FullTextSearch\Model\IRunner;
50			use OCP\FullTextSearch\Model\ISearchResult;
51
52
53			/**
54			* Class ElasticSearchPlatform
55			*
56			* @package OCA\FullTextSearch_ElasticSearch\Platform
57			*/
58			class ElasticSearchPlatform implements IFullTextSearchPlatform {
59
60
61			use TPathTools;
62
63
64			/** @var ConfigService */
65			private $configService;
66
67			/** @var IndexService */
68			private $indexService;
69
70			/** @var SearchService */
71			private $searchService;
72
73			/** @var MiscService */
74			private $miscService;
75
76			/** @var Client */
77			private $client;
78
79			/** @var IRunner */
80			private $runner;
81
82
83			/**
84			* ElasticSearchPlatform constructor.
85			*
86			* @param ConfigService $configService
87			* @param IndexService $indexService
88			* @param SearchService $searchService
89			* @param MiscService $miscService
90			*/
91			public function __construct(
92			ConfigService $configService, IndexService $indexService, SearchService $searchService,
93			MiscService $miscService
94			) {
95			$this->configService = $configService;
96			$this->indexService = $indexService;
97			$this->searchService = $searchService;
98			$this->miscService = $miscService;
99			}
100
101
102			/**
103			* return a unique Id of the platform.
104			*/
105			public function getId(): string {
106			return 'elastic_search';
107			}
108
109
110			/**
111			* return a unique Id of the platform.
112			*/
113			public function getName(): string {
114			return 'Elasticsearch';
115			}
116
117
118			/**
119			* @return array
120			* @throws ConfigurationException
121			*/
122			public function getConfiguration(): array {
123
124			$result = [];
125			$hosts = $this->configService->getElasticHost();
126
127			foreach ($hosts as $host) {
128			$parsedHost = parse_url($host);
129			$safeHost = $parsedHost['scheme'] . '://';
130			if (array_key_exists('user', $parsedHost)) {
131			$safeHost .= $parsedHost['user'] . ':' . '********' . '@';
132			}
133			$safeHost .= $parsedHost['host'];
134			$safeHost .= ':' . $parsedHost['port'];
135
136			$result[] = $safeHost;
137			}
138
139			return [
140			'elastic_host' => $result,
141			'elastic_index' => $this->configService->getElasticIndex()
142			];
143			}
144
145
146			/**
147			* @param IRunner $runner
148			*/
149			public function setRunner(IRunner $runner) {
150			$this->runner = $runner;
151			}
152
153
154			/**
155			* Called when loading the platform.
156			*
157			* Loading some container and connect to ElasticSearch.
158			*
159			* @throws ConfigurationException
160			* @throws Exception
161			*/
162			public function loadPlatform() {
163			try {
164			$this->connectToElastic($this->configService->getElasticHost());
165			} catch (ConfigurationException $e) {
166			throw $e;
167			}
168			}
169
170
171			/**
172			* not used yet.
173			*
174			* @return bool
175			*/
176			public function testPlatform(): bool {
177			return $this->client->ping();
178			}
179
180
181			/**
182			* called before any index
183			*
184			* We create a general index.
185			*
186			* @throws ConfigurationException
187			* @throws BadRequest400Exception
188			*/
189			public function initializeIndex() {
190			$this->indexService->initializeIndex($this->client);
191			}
192
193
194			/**
195			* resetIndex();
196			*
197			* Called when admin wants to remove an index specific to a $provider.
198			* $provider can be null, meaning a reset of the whole index.
199			*
200			* @param string $providerId
201			*
202			* @throws ConfigurationException
203			*/
204			public function resetIndex(string $providerId) {
205			if ($providerId === 'all') {
206			$this->indexService->resetIndexAll($this->client);
207			} else {
208			$this->indexService->resetIndex($this->client, $providerId);
209			}
210			}
211
212
213			/**
214			* @param IndexDocument $document
215			*
216			* @return IIndex
217			*/
218			public function indexDocument(IndexDocument $document): IIndex {
219
220			$document->initHash();
221
222			try {
223			$result = $this->indexService->indexDocument($this->client, $document);
224
225			$index = $this->indexService->parseIndexResult($document->getIndex(), $result);
226
227			$this->updateNewIndexResult(
228			$document->getIndex(), json_encode($result), 'ok',
229			IRunner::RESULT_TYPE_SUCCESS
230			);
231
232			return $index;
233			} catch (Exception $e) {
234			$this->updateNewIndexResult(
235			$document->getIndex(), '', 'issue while indexing, testing with empty content',
236			IRunner::RESULT_TYPE_WARNING
237			);
238
239			$this->manageIndexErrorException($document, $e);
240			}
241
242			try {
243			$result = $this->indexDocumentError($document, $e);
244			$index = $this->indexService->parseIndexResult($document->getIndex(), $result);
245
246			$this->updateNewIndexResult(
247			$document->getIndex(), json_encode($result), 'ok',
248			IRunner::RESULT_TYPE_WARNING
249			);
250
251			return $index;
252			} catch (Exception $e) {
253			$this->updateNewIndexResult(
254			$document->getIndex(), '', 'fail',
255			IRunner::RESULT_TYPE_FAIL
256			);
257			$this->manageIndexErrorException($document, $e);
258			}
259
260			return $document->getIndex();
261			}
262
263
264			/**
265			* @param IndexDocument $document
266			* @param Exception $e
267			*
268			* @return array
269			* @throws AccessIsEmptyException
270			* @throws ConfigurationException
271			* @throws \Exception
272			*/
273			private function indexDocumentError(IndexDocument $document, Exception $e): array {
274
275			$this->updateRunnerAction('indexDocumentWithoutContent', true);
276
277			$document->setContent('');
278			// $index = $document->getIndex();
279			// $index->unsetStatus(Index::INDEX_CONTENT);
280
281			$result = $this->indexService->indexDocument($this->client, $document);
282
283			return $result;
284			}
285
286
287			/**
288			* @param IndexDocument $document
289			* @param Exception $e
290			*/
291			private function manageIndexErrorException(IndexDocument $document, Exception $e) {
292
293			$message = $this->parseIndexErrorException($e);
294			$document->getIndex()
295			->addError($message, get_class($e), IIndex::ERROR_SEV_3);
296			$this->updateNewIndexError(
297			$document->getIndex(), $message, get_class($e), IIndex::ERROR_SEV_3
298			);
299			}
300
301
302			/**
303			* @param Exception $e
304			*
305			* @return string
306			*/
307			private function parseIndexErrorException(Exception $e): string {
308
309			$arr = json_decode($e->getMessage(), true);
310			if (!is_array($arr)) {
311			return $e->getMessage();
312			}
313
314			if (array_key_exists('reason', $arr['error']['root_cause'][0])) {
315			return $arr['error']['root_cause'][0]['reason'];
316			}
317
318			return $e->getMessage();
319			}
320
321
322			/**
323			* {@inheritdoc}
324			* @throws ConfigurationException
325			*/
326			public function deleteIndexes(array $indexes) {
327			try {
328			$this->indexService->deleteIndexes($this->client, $indexes);
329			} catch (ConfigurationException $e) {
330			throw $e;
331			}
332			}
333
334
335			/**
336			* {@inheritdoc}
337			* @throws Exception
338			*/
339			public function searchRequest(ISearchResult $result, DocumentAccess $access) {
340			$this->searchService->searchRequest($this->client, $result, $access);
341			}
342
343
344			/**
345			* @param string $providerId
346			* @param string $documentId
347			*
348			* @return IndexDocument
349			* @throws ConfigurationException
350			*/
351			public function getDocument(string $providerId, string $documentId): IndexDocument {
352			return $this->searchService->getDocument($this->client, $providerId, $documentId);
353			}
354
355
356			private function cleanHost($host) {
357			return $this->withoutEndSlash($host, false, false);
358			}
359
360			/**
361			* @param array $hosts
362			*
363			* @throws Exception
364			*/
365			private function connectToElastic(array $hosts) {
366
367			try {
368			$hosts = array_map([$this, 'cleanHost'], $hosts);
369			$this->client = ClientBuilder::create()
370			->setHosts($hosts)
371			->setRetries(3)
372			->build();
373
374			// }
375			// catch (CouldNotConnectToHost $e) {
376			// $this 'CouldNotConnectToHost';
377			// $previous = $e->getPrevious();
378			// if ($previous instanceof MaxRetriesException) {
379			// echo "Max retries!";
380			// }
381			} catch (Exception $e) {
382			throw $e;
383			// echo ' ElasticSearchPlatform::load() Exception --- ' . $e->getMessage() . "\n";
384			}
385			}
386
387
388			/**
389			* @param string $action
390			* @param bool $force
391			*
392			* @throws Exception
393			*/
394			private function updateRunnerAction(string $action, bool $force = false) {
395			if ($this->runner === null) {
396			return;
397			}
398
399			$this->runner->updateAction($action, $force);
400			}
401
402
403			/**
404			* @param IIndex $index
405			* @param string $message
406			* @param string $exception
407			* @param int $sev
408			*/
409			private function updateNewIndexError(IIndex $index, string $message, string $exception, int $sev
410			) {
411			if ($this->runner === null) {
412			return;
413			}
414
415			$this->runner->newIndexError($index, $message, $exception, $sev);
416			}
417
418
419			/**
420			* @param IIndex $index
421			* @param string $message
422			* @param string $status
423			* @param int $type
424			*/
425			private function updateNewIndexResult(IIndex $index, string $message, string $status, int $type
426			) {
427			if ($this->runner === null) {
428			return;
429			}
430
431			$this->runner->newIndexResult($index, $message, $status, $type);
432			}
433
434
435			}
436

nextcloud / fulltextsearch_elasticsearch

Push — master ( 0cdbd8...fb0474 )

ElasticSearchPlatform::indexDocuments() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like