SolrIndexTask::doReindex() - Code Metrics - Inspection of "Use default buffer size of 100" - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — hans/bufferadd ( a6c9e6...1f3340 )

by Simon

created 2019-10-20 07:53 UTC

SolrIndexTask::doReindex() A

↳ Parent: SolrIndexTask

Complexity

Conditions	4
Paths	3

Size

Total Lines	10
Code Lines	5

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	5
CRAP Score	4.074

Importance

Changes	11
Bugs	1	Features	0

Metric	Value
cc	4
eloc	5
c	11
b	1
f	0
nc	3
nop	5
dl	0
loc	10
ccs	5
cts	6
cp	0.8333
crap	4.074
rs	10

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\States\SiteState;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use GuzzleHttp\Exception\GuzzleException;
use Psr\Log\LoggerInterface;
use ReflectionException;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\Dev\Debug;
use SilverStripe\ORM\ArrayList;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Versioned\Versioned;
use Solarium\Plugin\BufferedAdd\BufferedAdd;
use Solarium\Plugin\BufferedAdd\Event\Events;
use Solarium\Plugin\BufferedAdd\Event\PreFlush as PreFlushEvent;

/**
 * Class SolrIndexTask
 *
 * @description Index items to Solr through a tasks
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends BuildTask
{
    use LoggerTrait;
    /**
     * URLSegment of this task
     *
     * @var string
     */
    private static $segment = 'SolrIndexTask';
    /**
     * Store the current states for all instances of SiteState
     *
     * @var array
     */
    public $currentStates;
    /**
     * My name
     *
     * @var string
     */
    protected $title = 'Solr Index update';
    /**
     * What do I do?
     *
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';
    /**
     * Debug mode enabled, default false
     *
     * @var bool
     */
    protected $debug = false;
    /**
     * Singleton of {@link SolrCoreService}
     *
     * @var SolrCoreService
     */
    protected $service;

    /**
     * SolrIndexTask constructor. Sets up the document factory
     *
     * @throws ReflectionException
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Director::isDev() || Director::is_cli());
        $currentStates = SiteState::currentStates();
        SiteState::setDefaultStates($currentStates);
    }

    /**
     * Set the {@link SolrCoreService}
     *
     * @param SolrCoreService $service
     * @return SolrIndexTask
     */
    public function setService(SolrCoreService $service): SolrIndexTask
    {
        $this->service = $service;

        return $this;
    }

    /**
     * Set the debug mode
     *
     * @param bool $debug
     * @return SolrIndexTask
     */
    public function setDebug(bool $debug): SolrIndexTask
    {
        $this->debug = $debug;

        return $this;
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     * @todo defer to background because it may run out of memory
     */
    public function run($request)
    {
        $startTime = time();
        list($vars, $group, $isGroup) = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);

            $indexClasses = $index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $this->clearIndex($vars, $index);

            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
        }
        $this->getLogger()->info(
            sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
        );
        // Grab the latest logs from indexing if needed
        $solrLogger = new SolrLogger();
        $solrLogger->saveSolrLog('Config');

        return $groups;
    }

    /**
     * Set up the requirements for this task
     *
     * @param HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : false;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * get the classes to run for this task execution
     *
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * Clear the given index if a full re-index is needed
     *
     * @param $vars
     * @param BaseIndex $index
     * @throws Exception
     */
    public function clearIndex($vars, BaseIndex $index)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
        }
    }

    /**
     * Index the classes for a specific index
     *
     * @param $classes
     * @param $isGroup
     * @param BaseIndex $index
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $index, $group);
        }

        return $groups;
    }

    /**
     * Index a single class for a given index. {@link static::indexClassForIndex()}
     *
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
    {
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);

        $batchLength = DocumentFactory::config()->get('batchLength');
        try {
            $this->doReindex($group, $class, $batchLength, $index, $isGroup);
        } catch (Exception $error) {
            $this->logException($index->getIndexName(), $group, $error);
            $group++;
        }
        return $group;
    }

    /**
     * Reindex the given group, for each state
     *
     * @param int $group
     * @param string $class
     * @param int $batchLength
     * @param BaseIndex $index
     * @param bool $isGroup
     * @throws ReflectionException
     * @throws Exception
     */
    private function doReindex($group, $class, $batchLength, BaseIndex $index, $isGroup): void
    {
        foreach (SiteState::getStates() as $state) {
            if ($state !== 'default' && !empty($state)) {
                SiteState::withState($state);
            }
            $this->stateReindex($group, $class, $batchLength, $index, $isGroup);
        }

        SiteState::withState(SiteState::DEFAULT_STATE);
    }

    /**
     * Index a group of a class for a specific state and index
     *
     * @param $group
     * @param $class
     * @param int $batchLength
     * @param BaseIndex $index
     * @param bool $isGroup
     * @throws Exception
     */
    private function stateReindex($group, $class, $batchLength, BaseIndex $index, $isGroup): void
    {
        // Generate filtered list of local records
        $baseClass = DataObject::getSchema()->baseDataClass($class);
        /** @var DataList|DataObject[] $items */
        $items = DataObject::get($baseClass)
            ->sort('ID ASC');
        if ($isGroup) {
            $items = $items->limit($batchLength, ($group * $batchLength));
        }
        if ($items->count()) {
            $this->updateIndex($index, $items);
        }
    }

    /**
     * Execute the update on the client
     *
     * @param BaseIndex $index
     * @param $items
     * @throws Exception
     */
    private function updateIndex(BaseIndex $index, $items): void
    {
        $this->service->setInDebugMode($this->debug);
        $this->service->updateIndex($index, $items);
    }

    /**
     * Log an exception if it happens. Most are catched, these logs are for the developers
     * to identify problems and fix them.
     *
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function logException($index, int $group, Exception $exception): void
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            'Error indexing core %s on group %s,' . PHP_EOL .
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }
}


1		<?php
2
3
4		namespace Firesphere\SolrSearch\Tasks;
5
6		use Exception;
7		use Firesphere\SolrSearch\Factories\DocumentFactory;
8		use Firesphere\SolrSearch\Helpers\SolrLogger;
9		use Firesphere\SolrSearch\Indexes\BaseIndex;
10		use Firesphere\SolrSearch\Services\SolrCoreService;
11		use Firesphere\SolrSearch\States\SiteState;
12		use Firesphere\SolrSearch\Traits\LoggerTrait;
13		use GuzzleHttp\Exception\GuzzleException;
14		use Psr\Log\LoggerInterface;
15		use ReflectionException;
16		use SilverStripe\Control\Director;
17		use SilverStripe\Control\HTTPRequest;
18		use SilverStripe\Core\Injector\Injector;
19		use SilverStripe\Dev\BuildTask;
20		use SilverStripe\Dev\Debug;
21		use SilverStripe\ORM\ArrayList;
22		use SilverStripe\ORM\DataList;
23		use SilverStripe\ORM\DataObject;
24		use SilverStripe\ORM\ValidationException;
25		use SilverStripe\Versioned\Versioned;
26		use Solarium\Plugin\BufferedAdd\BufferedAdd;
27		use Solarium\Plugin\BufferedAdd\Event\Events;
28		use Solarium\Plugin\BufferedAdd\Event\PreFlush as PreFlushEvent;
29
30		/**
31		* Class SolrIndexTask
32		*
33		* @description Index items to Solr through a tasks
34		* @package Firesphere\SolrSearch\Tasks
35		*/
36		class SolrIndexTask extends BuildTask
37		{
38		use LoggerTrait;
39		/**
40		* URLSegment of this task
41		*
42		* @var string
43		*/
44		private static $segment = 'SolrIndexTask';
45		/**
46		* Store the current states for all instances of SiteState
47		*
48		* @var array
49		*/
50		public $currentStates;
51		/**
52		* My name
53		*
54		* @var string
55		*/
56		protected $title = 'Solr Index update';
57		/**
58		* What do I do?
59		*
60		* @var string
61		*/
62		protected $description = 'Add or update documents to an existing Solr core.';
63		/**
64		* Debug mode enabled, default false
65		*
66		* @var bool
67		*/
68		protected $debug = false;
69		/**
70		* Singleton of {@link SolrCoreService}
71		*
72		* @var SolrCoreService
73		*/
74		protected $service;
75
76		/**
77		* SolrIndexTask constructor. Sets up the document factory
78		*
79		* @throws ReflectionException
80		*/
81	14	public function __construct()
82		{
83	14	parent::__construct();
84		// Only index live items.
85		// The old FTS module also indexed Draft items. This is unnecessary
86	14	Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
87	14	$this->setService(Injector::inst()->get(SolrCoreService::class));
88	14	$this->setLogger(Injector::inst()->get(LoggerInterface::class));
89	14	$this->setDebug(Director::isDev() \|\| Director::is_cli());
90	14	$currentStates = SiteState::currentStates();
91	14	SiteState::setDefaultStates($currentStates);
92	14	}
93
94		/**
95		* Set the {@link SolrCoreService}
96		*
97		* @param SolrCoreService $service
98		* @return SolrIndexTask
99		*/
100	14	public function setService(SolrCoreService $service): SolrIndexTask
101		{
102	14	$this->service = $service;
103
104	14	return $this;
105		}
106
107		/**
108		* Set the debug mode
109		*
110		* @param bool $debug
111		* @return SolrIndexTask
112		*/
113	14	public function setDebug(bool $debug): SolrIndexTask
114		{
115	14	$this->debug = $debug;
116
117	14	return $this;
118		}
119
120		/**
121		* Implement this method in the task subclass to
122		* execute via the TaskRunner
123		*
124		* @param HTTPRequest $request
125		* @return int\|bool
126		* @throws Exception
127		* @throws GuzzleException
128		* @todo defer to background because it may run out of memory
129		*/
130	13	public function run($request)
131		{
132	13	$startTime = time();
133	13	list($vars, $group, $isGroup) = $this->taskSetup($request);
134	13	$groups = 0;
135	13	$indexes = $this->service->getValidIndexes($request->getVar('index'));
136
137	13	foreach ($indexes as $indexName) {
138		/** @var BaseIndex $index */
139	13	$index = Injector::inst()->get($indexName, false);
140
141	13	$indexClasses = $index->getClasses();
142	13	$classes = $this->getClasses($vars, $indexClasses);
143	13	if (!count($classes)) {
144	10	continue;
145		}
146
147	13	$this->clearIndex($vars, $index);
148
149	13	$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
150		}
151	13	$this->getLogger()->info(
152	13	sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
153		);
154		// Grab the latest logs from indexing if needed
155	13	$solrLogger = new SolrLogger();
156	13	$solrLogger->saveSolrLog('Config');
157
158	13	return $groups;
159		}
160
161		/**
162		* Set up the requirements for this task
163		*
164		* @param HTTPRequest $request
165		* @return array
166		*/
167	13	protected function taskSetup($request): array
168		{
169	13	$vars = $request->getVars();
170	13	$this->debug = $this->debug \|\| isset($vars['debug']);
171	13	$group = $vars['group'] ?? 0;
172	13	$start = $vars['start'] ?? 0;
173	13	$group = ($start > $group) ? $start : false;
174	13	$isGroup = isset($vars['group']);
175
176	13	return [$vars, $group, $isGroup];
177		}
178
179		/**
180		* get the classes to run for this task execution
181		*
182		* @param $vars
183		* @param array $classes
184		* @return bool\|array
185		*/
186	13	protected function getClasses($vars, array $classes): array
187		{
188	13	if (isset($vars['class'])) {
189	1	return array_intersect($classes, [$vars['class']]);
190		}
191
192	12	return $classes;
193		}
194
195		/**
196		* Clear the given index if a full re-index is needed
197		*
198		* @param $vars
199		* @param BaseIndex $index
200		* @throws Exception
201		*/
202	13	public function clearIndex($vars, BaseIndex $index)
203		{
204	13	if (!empty($vars['clear'])) {
205	1	$this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
206	1	$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
207		}
208	13	}
209
210		/**
211		* Index the classes for a specific index
212		*
213		* @param $classes
214		* @param $isGroup
215		* @param BaseIndex $index
216		* @param $group
217		* @return int
218		* @throws Exception
219		* @throws GuzzleException
220		*/
221	13	protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
222		{
223	13	$groups = 0;
224	13	foreach ($classes as $class) {
225	13	$groups = $this->indexClass($isGroup, $class, $index, $group);
226		}
227
228	13	return $groups;
229		}
230
231		/**
232		* Index a single class for a given index. {@link static::indexClassForIndex()}
233		*
234		* @param bool $isGroup
235		* @param string $class
236		* @param BaseIndex $index
237		* @param int $group
238		* @return int
239		* @throws GuzzleException
240		* @throws ValidationException
241		*/
242	13	private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
243		{
244	13	$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);
245
246	13	$batchLength = DocumentFactory::config()->get('batchLength');
247		try {
248	13	$this->doReindex($group, $class, $batchLength, $index, $isGroup);
249		} catch (Exception $error) {
250		$this->logException($index->getIndexName(), $group, $error);
251		$group++;
252		}
253	13	return $group;
254		}
255
256		/**
257		* Reindex the given group, for each state
258		*
259		* @param int $group
260		* @param string $class
261		* @param int $batchLength
262		* @param BaseIndex $index
263		* @param bool $isGroup
264		* @throws ReflectionException
265		* @throws Exception
266		*/
267	13	private function doReindex($group, $class, $batchLength, BaseIndex $index, $isGroup): void
268		{
269	13	foreach (SiteState::getStates() as $state) {
270	13	if ($state !== 'default' && !empty($state)) {
271		SiteState::withState($state);
272		}
273	13	$this->stateReindex($group, $class, $batchLength, $index, $isGroup);
274		}
275
276	13	SiteState::withState(SiteState::DEFAULT_STATE);
277	13	}
278
279		/**
280		* Index a group of a class for a specific state and index
281		*
282		* @param $group
283		* @param $class
284		* @param int $batchLength
285		* @param BaseIndex $index
286		* @param bool $isGroup
287		* @throws Exception
288		*/
289	13	private function stateReindex($group, $class, $batchLength, BaseIndex $index, $isGroup): void
290		{
291		// Generate filtered list of local records
292	13	$baseClass = DataObject::getSchema()->baseDataClass($class);
293		/** @var DataList\|DataObject[] $items */
294	13	$items = DataObject::get($baseClass)
295	13	->sort('ID ASC');
296	13	if ($isGroup) {
297	2	$items = $items->limit($batchLength, ($group * $batchLength));
298		}
299	13	if ($items->count()) {
300	1	$this->updateIndex($index, $items);
301		}
302	13	}
303
304		/**
305		* Execute the update on the client
306		*
307		* @param BaseIndex $index
308		* @param $items
309		* @throws Exception
310		*/
311	1	private function updateIndex(BaseIndex $index, $items): void
312		{
313	1	$this->service->setInDebugMode($this->debug);
314	1	$this->service->updateIndex($index, $items);
315	1	}
316
317		/**
318		* Log an exception if it happens. Most are catched, these logs are for the developers
319		* to identify problems and fix them.
320		*
321		* @param string $index
322		* @param int $group
323		* @param Exception $exception
324		* @throws GuzzleException
325		* @throws ValidationException
326		*/
327		private function logException($index, int $group, Exception $exception): void
328		{
329		$this->getLogger()->error($exception->getMessage());
330		$msg = sprintf(
331		'Error indexing core %s on group %s,' . PHP_EOL .
332		'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
333		$index,
334		$group
335		);
336		SolrLogger::logMessage('ERROR', $msg, $index);
337		}
338		}
339

Firesphere / silverstripe-solr-search

Push — hans/bufferadd ( a6c9e6...1f3340 )

SolrIndexTask::doReindex() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like