SolrIndexTask::taskSetup() - Code Metrics - Inspection of "Original test was correct" - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — hans/valid-classes ( 1110df...3598c3 )

by Simon

created 2019-09-10 06:38 UTC

SolrIndexTask::taskSetup() A

↳ Parent: SolrIndexTask

Complexity

Conditions	3
Paths	4

Size

Total Lines	10
Code Lines	7

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	8
CRAP Score	3

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
eloc	7
c	2
b	0
f	0
dl	0
loc	10
ccs	8
cts	8
cp	1
rs	10
cc	3
nc	4
nop	1
crap	3

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\States\SiteState;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use GuzzleHttp\Exception\GuzzleException;
use Psr\Log\LoggerInterface;
use ReflectionException;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\ORM\ArrayList;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Versioned\Versioned;

/**
 * Class SolrIndexTask
 *
 * @description Index items to Solr through a tasks
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends BuildTask
{
    use LoggerTrait;
    /**
     * URLSegment of this task
     *
     * @var string
     */
    private static $segment = 'SolrIndexTask';
    /**
     * Store the current states for all instances of SiteState
     *
     * @var array
     */
    public $currentStates;
    /**
     * My name
     *
     * @var string
     */
    protected $title = 'Solr Index update';
    /**
     * What do I do?
     *
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';
    /**
     * Debug mode enabled, default false
     *
     * @var bool
     */
    protected $debug = false;
    /**
     * Singleton of {@link SolrCoreService}
     *
     * @var SolrCoreService
     */
    protected $service;

    /**
     * SolrIndexTask constructor. Sets up the document factory
     *
     * @throws ReflectionException
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Director::isDev() || Director::is_cli());
        $this->currentStates = SiteState::currentStates();
    }

    /**
     * Set the {@link SolrCoreService}
     *
     * @param SolrCoreService $service
     * @return SolrIndexTask
     */
    public function setService(SolrCoreService $service): SolrIndexTask
    {
        $this->service = $service;

        return $this;
    }

    /**
     * Set the debug mode
     *
     * @param bool $debug
     * @return SolrIndexTask
     */
    public function setDebug(bool $debug): SolrIndexTask
    {
        $this->debug = $debug;

        return $this;
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     * @todo defer to background because it may run out of memory
     */
    public function run($request)
    {
        $startTime = time();
        list($vars, $group, $isGroup) = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);

            $indexClasses = $index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $this->clearIndex($vars, $index);

            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
        }
        $this->getLogger()->info(
            sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
        );
        // Grab the latest logs from indexing if needed
        $solrLogger = new SolrLogger();
        $solrLogger->saveSolrLog('Config');

        return $groups;
    }

    /**
     * Set up the requirements for this task
     *
     * @param HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : $group;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * get the classes to run for this task execution
     *
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * Clear the given index if a full re-index is needed
     *
     * @param $vars
     * @param BaseIndex $index
     * @throws Exception
     */
    public function clearIndex($vars, BaseIndex $index)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
        }
    }

    /**
     * Index the classes for a specific index
     *
     * @param $classes
     * @param $isGroup
     * @param BaseIndex $index
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $index, $group);
        }

        return $groups;
    }

    /**
     * Index a single class for a given index. {@link static::indexClassForIndex()}
     *
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
    {
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);

        $batchLength = DocumentFactory::config()->get('batchLength');
        $groups = (int)ceil($class::get()->count() / $batchLength);
        $groups = $isGroup ? $group : $groups;
        while ($group <= $groups) { // Run from oldest to newest
            try {
                $this->doReindex($group, $class, $batchLength, $index);
            } catch (Exception $error) {
                $this->logException($index->getIndexName(), $group, $error);
                $group++;
                continue;
            }
            $group++;
            $this->getLogger()->info(sprintf('Indexed group %s', $group));
        }

        return $groups;
    }

    /**
     * Reindex the given group, for each state
     *
     * @param int $group
     * @param string $class
     * @param int $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function doReindex($group, $class, $batchLength, BaseIndex $index): void
    {
        foreach (SiteState::getStates() as $state) {
            if ($state !== 'default') {
                SiteState::withState($state);
            }
            $this->stateReindex($group, $class, $batchLength, $index);
        }

        // Reset the variants back to it's original state for the next round
        foreach ($this->currentStates as $variant => $value) {
            singleton($variant)->activateState($value);
        }
    }

    /**
     * Index a group of a class for a specific state and index
     *
     * @param $group
     * @param $class
     * @param $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function stateReindex($group, $class, $batchLength, BaseIndex $index): void
    {
        // Generate filtered list of local records
        $baseClass = DataObject::getSchema()->baseDataClass($class);
        /** @var DataList|DataObject[] $items */
        $items = DataObject::get($baseClass)
            ->sort('ID ASC')
            ->limit($batchLength, ($group * $batchLength));
        if ($items->count()) {
            $this->updateIndex($index, $items);
        }
    }

    /**
     * Execute the update on the client
     *
     * @param BaseIndex $index
     * @param $items
     * @throws Exception
     */
    private function updateIndex(BaseIndex $index, $items): void
    {
        $client = $index->getClient();
        $update = $client->createUpdate();
        $this->service->setInDebugMode($this->debug);
        $this->service->updateIndex($index, $items, $update);
        $update->addCommit();
        $client->update($update);
    }

    /**
     * Log an exception if it happens. Most are catched, these logs are for the developers
     * to identify problems and fix them.
     *
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function logException($index, int $group, Exception $exception): void
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            'Error indexing core %s on group %s,' . PHP_EOL .
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }
}


1		<?php
2
3
4		namespace Firesphere\SolrSearch\Tasks;
5
6		use Exception;
7		use Firesphere\SolrSearch\Factories\DocumentFactory;
8		use Firesphere\SolrSearch\Helpers\SolrLogger;
9		use Firesphere\SolrSearch\Indexes\BaseIndex;
10		use Firesphere\SolrSearch\Services\SolrCoreService;
11		use Firesphere\SolrSearch\States\SiteState;
12		use Firesphere\SolrSearch\Traits\LoggerTrait;
13		use GuzzleHttp\Exception\GuzzleException;
14		use Psr\Log\LoggerInterface;
15		use ReflectionException;
16		use SilverStripe\Control\Director;
17		use SilverStripe\Control\HTTPRequest;
18		use SilverStripe\Core\Injector\Injector;
19		use SilverStripe\Dev\BuildTask;
20		use SilverStripe\ORM\ArrayList;
21		use SilverStripe\ORM\DataList;
22		use SilverStripe\ORM\DataObject;
23		use SilverStripe\ORM\ValidationException;
24		use SilverStripe\Versioned\Versioned;
25
26		/**
27		* Class SolrIndexTask
28		*
29		* @description Index items to Solr through a tasks
30		* @package Firesphere\SolrSearch\Tasks
31		*/
32		class SolrIndexTask extends BuildTask
33		{
34		use LoggerTrait;
35		/**
36		* URLSegment of this task
37		*
38		* @var string
39		*/
40		private static $segment = 'SolrIndexTask';
41		/**
42		* Store the current states for all instances of SiteState
43		*
44		* @var array
45		*/
46		public $currentStates;
47		/**
48		* My name
49		*
50		* @var string
51		*/
52		protected $title = 'Solr Index update';
53		/**
54		* What do I do?
55		*
56		* @var string
57		*/
58		protected $description = 'Add or update documents to an existing Solr core.';
59		/**
60		* Debug mode enabled, default false
61		*
62		* @var bool
63		*/
64		protected $debug = false;
65		/**
66		* Singleton of {@link SolrCoreService}
67		*
68		* @var SolrCoreService
69		*/
70		protected $service;
71
72		/**
73		* SolrIndexTask constructor. Sets up the document factory
74		*
75		* @throws ReflectionException
76		*/
77	13	public function __construct()
78		{
79	13	parent::__construct();
80		// Only index live items.
81		// The old FTS module also indexed Draft items. This is unnecessary
82	13	Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
83	13	$this->setService(Injector::inst()->get(SolrCoreService::class));
84	13	$this->setLogger(Injector::inst()->get(LoggerInterface::class));
85	13	$this->setDebug(Director::isDev() \|\| Director::is_cli());
86	13	$this->currentStates = SiteState::currentStates();
87	13	}
88
89		/**
90		* Set the {@link SolrCoreService}
91		*
92		* @param SolrCoreService $service
93		* @return SolrIndexTask
94		*/
95	13	public function setService(SolrCoreService $service): SolrIndexTask
96		{
97	13	$this->service = $service;
98
99	13	return $this;
100		}
101
102		/**
103		* Set the debug mode
104		*
105		* @param bool $debug
106		* @return SolrIndexTask
107		*/
108	13	public function setDebug(bool $debug): SolrIndexTask
109		{
110	13	$this->debug = $debug;
111
112	13	return $this;
113		}
114
115		/**
116		* Implement this method in the task subclass to
117		* execute via the TaskRunner
118		*
119		* @param HTTPRequest $request
120		* @return int\|bool
121		* @throws Exception
122		* @throws GuzzleException
123		* @todo defer to background because it may run out of memory
124		*/
125	12	public function run($request)
126		{
127	12	$startTime = time();
128	12	list($vars, $group, $isGroup) = $this->taskSetup($request);
129	12	$groups = 0;
130	12	$indexes = $this->service->getValidIndexes($request->getVar('index'));
131
132	12	foreach ($indexes as $indexName) {
133		/** @var BaseIndex $index */
134	12	$index = Injector::inst()->get($indexName, false);
135
136	12	$indexClasses = $index->getClasses();
137	12	$classes = $this->getClasses($vars, $indexClasses);
138	12	if (!count($classes)) {
139	9	continue;
140		}
141
142	12	$this->clearIndex($vars, $index);
143
144	12	$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
145		}
146	12	$this->getLogger()->info(
147	12	sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
148		);
149		// Grab the latest logs from indexing if needed
150	12	$solrLogger = new SolrLogger();
151	12	$solrLogger->saveSolrLog('Config');
152
153	12	return $groups;
154		}
155
156		/**
157		* Set up the requirements for this task
158		*
159		* @param HTTPRequest $request
160		* @return array
161		*/
162	12	protected function taskSetup($request): array
163		{
164	12	$vars = $request->getVars();
165	12	$this->debug = $this->debug \|\| isset($vars['debug']);
166	12	$group = $vars['group'] ?? 0;
167	12	$start = $vars['start'] ?? 0;
168	12	$group = ($start > $group) ? $start : $group;
169	12	$isGroup = isset($vars['group']);
170
171	12	return [$vars, $group, $isGroup];
172		}
173
174		/**
175		* get the classes to run for this task execution
176		*
177		* @param $vars
178		* @param array $classes
179		* @return bool\|array
180		*/
181	12	protected function getClasses($vars, array $classes): array
182		{
183	12	if (isset($vars['class'])) {
184	1	return array_intersect($classes, [$vars['class']]);
185		}
186
187	11	return $classes;
188		}
189
190		/**
191		* Clear the given index if a full re-index is needed
192		*
193		* @param $vars
194		* @param BaseIndex $index
195		* @throws Exception
196		*/
197	12	public function clearIndex($vars, BaseIndex $index)
198		{
199	12	if (!empty($vars['clear'])) {
200	1	$this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
201	1	$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
202		}
203	12	}
204
205		/**
206		* Index the classes for a specific index
207		*
208		* @param $classes
209		* @param $isGroup
210		* @param BaseIndex $index
211		* @param $group
212		* @return int
213		* @throws Exception
214		* @throws GuzzleException
215		*/
216	12	protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
217		{
218	12	$groups = 0;
219	12	foreach ($classes as $class) {
220	12	$groups = $this->indexClass($isGroup, $class, $index, $group);
221		}
222
223	12	return $groups;
224		}
225
226		/**
227		* Index a single class for a given index. {@link static::indexClassForIndex()}
228		*
229		* @param bool $isGroup
230		* @param string $class
231		* @param BaseIndex $index
232		* @param int $group
233		* @return int
234		* @throws GuzzleException
235		* @throws ValidationException
236		*/
237	12	private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
238		{
239	12	$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);
240
241	12	$batchLength = DocumentFactory::config()->get('batchLength');
242	12	$groups = (int)ceil($class::get()->count() / $batchLength);
243	12	$groups = $isGroup ? $group : $groups;
244	12	while ($group <= $groups) { // Run from oldest to newest
245		try {
246	12	$this->doReindex($group, $class, $batchLength, $index);
247		} catch (Exception $error) {
248		$this->logException($index->getIndexName(), $group, $error);
249		$group++;
250		continue;
251		}
252	12	$group++;
253	12	$this->getLogger()->info(sprintf('Indexed group %s', $group));
254		}
255
256	12	return $groups;
257		}
258
259		/**
260		* Reindex the given group, for each state
261		*
262		* @param int $group
263		* @param string $class
264		* @param int $batchLength
265		* @param BaseIndex $index
266		* @throws Exception
267		*/
268	12	private function doReindex($group, $class, $batchLength, BaseIndex $index): void
269		{
270	12	foreach (SiteState::getStates() as $state) {
271	12	if ($state !== 'default') {
272		SiteState::withState($state);
273		}
274	12	$this->stateReindex($group, $class, $batchLength, $index);
275		}
276
277		// Reset the variants back to it's original state for the next round
278	12	foreach ($this->currentStates as $variant => $value) {
279	11	singleton($variant)->activateState($value);
280		}
281	12	}
282
283		/**
284		* Index a group of a class for a specific state and index
285		*
286		* @param $group
287		* @param $class
288		* @param $batchLength
289		* @param BaseIndex $index
290		* @throws Exception
291		*/
292	12	private function stateReindex($group, $class, $batchLength, BaseIndex $index): void
293		{
294		// Generate filtered list of local records
295	12	$baseClass = DataObject::getSchema()->baseDataClass($class);
296		/** @var DataList\|DataObject[] $items */
297	12	$items = DataObject::get($baseClass)
298	12	->sort('ID ASC')
299	12	->limit($batchLength, ($group * $batchLength));
300	12	if ($items->count()) {
301	1	$this->updateIndex($index, $items);
302		}
303	12	}
304
305		/**
306		* Execute the update on the client
307		*
308		* @param BaseIndex $index
309		* @param $items
310		* @throws Exception
311		*/
312	1	private function updateIndex(BaseIndex $index, $items): void
313		{
314	1	$client = $index->getClient();
315	1	$update = $client->createUpdate();
316	1	$this->service->setInDebugMode($this->debug);
317	1	$this->service->updateIndex($index, $items, $update);
318	1	$update->addCommit();
319	1	$client->update($update);
320	1	}
321
322		/**
323		* Log an exception if it happens. Most are catched, these logs are for the developers
324		* to identify problems and fix them.
325		*
326		* @param string $index
327		* @param int $group
328		* @param Exception $exception
329		* @throws GuzzleException
330		* @throws ValidationException
331		*/
332		private function logException($index, int $group, Exception $exception): void
333		{
334		$this->getLogger()->error($exception->getMessage());
335		$msg = sprintf(
336		'Error indexing core %s on group %s,' . PHP_EOL .
337		'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
338		$index,
339		$group
340		);
341		SolrLogger::logMessage('ERROR', $msg, $index);
342		}
343		}
344

Firesphere / silverstripe-solr-search

Push — hans/valid-classes ( 1110df...3598c3 )

SolrIndexTask::taskSetup() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like