SolrIndexTask::indexClass() - Code Metrics - Inspection of "Use the maximum amount of cores available" - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — bufferfix ( 848e8a...2b29dc )

by Simon

created 2019-11-22 23:14 UTC

SolrIndexTask::indexClass() B

↳ Parent: SolrIndexTask

Complexity

Conditions	9
Paths	24

Size

Total Lines	43
Code Lines	28

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	10
CRAP Score	9.9957

Importance

Changes	9
Bugs	1	Features	0

Metric	Value
cc	9
eloc	28
c	9
b	1
f	0
nc	24
nop	4
dl	0
loc	43
ccs	10
cts	13
cp	0.7692
crap	9.9957
rs	8.0555

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\States\SiteState;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use GuzzleHttp\Exception\GuzzleException;
use Psr\Log\LoggerInterface;
use ReflectionException;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\ORM\ArrayList;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Versioned\Versioned;

/**
 * Class SolrIndexTask
 *
 * @description Index items to Solr through a tasks
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends BuildTask
{
    use LoggerTrait;
    /**
     * URLSegment of this task
     *
     * @var string
     */
    private static $segment = 'SolrIndexTask';
    /**
     * Store the current states for all instances of SiteState
     *
     * @var array
     */
    public $currentStates;
    /**
     * My name
     *
     * @var string
     */
    protected $title = 'Solr Index update';
    /**
     * What do I do?
     *
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';
    /**
     * Debug mode enabled, default false
     *
     * @var bool
     */
    protected $debug = false;
    /**
     * Singleton of {@link SolrCoreService}
     *
     * @var SolrCoreService
     */
    protected $service;

    /**
     * SolrIndexTask constructor. Sets up the document factory
     *
     * @throws ReflectionException
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Director::isDev() || Director::is_cli());
        $currentStates = SiteState::currentStates();
        SiteState::setDefaultStates($currentStates);
    }

    /**
     * Set the {@link SolrCoreService}
     *
     * @param SolrCoreService $service
     * @return SolrIndexTask
     */
    public function setService(SolrCoreService $service): SolrIndexTask
    {
        $this->service = $service;

        return $this;
    }

    /**
     * Set the debug mode
     *
     * @param bool $debug
     * @return SolrIndexTask
     */
    public function setDebug(bool $debug): SolrIndexTask
    {
        $this->debug = $debug;

        return $this;
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     * @todo defer to background because it may run out of memory
     */
    public function run($request)
    {
        $startTime = time();
        list($vars, $group, $isGroup) = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);

            $indexClasses = $index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $this->clearIndex($vars, $index);

            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
        }
        $this->getLogger()->info(
            sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
        );
        // Grab the latest logs from indexing if needed
        $solrLogger = new SolrLogger();
        $solrLogger->saveSolrLog('Config');

        return $groups;
    }

    /**
     * Set up the requirements for this task
     *
     * @param HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : $group;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * get the classes to run for this task execution
     *
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * Clear the given index if a full re-index is needed
     *
     * @param $vars
     * @param BaseIndex $index
     * @throws Exception
     */
    public function clearIndex($vars, BaseIndex $index)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
        }
    }

    /**
     * Index the classes for a specific index
     *
     * @param $classes
     * @param $isGroup
     * @param BaseIndex $index
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $index, $group);
        }

        return $groups;
    }

    /**
     * Index a single class for a given index. {@link static::indexClassForIndex()}
     *
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
    {
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);

        $batchLength = DocumentFactory::config()->get('batchLength');
        $groups = (int)ceil($class::get()->count() / $batchLength);
        $groups = $isGroup ? $group : $groups;
        // How much cores do we have
        $cores = SolrCoreService::config()->get('cores');
        $pid = false;

        $pids = [];
        $this->getLogger()->info(sprintf('Total groups %s', $groups));
        while ($group <= $groups) { // Run from oldest to newest
            try {
                // for each core, start a grouped indexing
                for ($i = 0; $i < $cores; $i++) {
                    $pid = pcntl_fork();
                    // Couldn't fork, so continue to the next try
                    if ($pid === -1) {
                        continue;
                    }
                    $pids[$i] = $pid;
                    if (!$pid) {
                        $config = DB::getConfig();
                        DB::connect($config);
                        $this->doReindex($group, $class, $batchLength, $index);
                    }
                    $group++;
                }
                // Wait for each child to finish
                foreach ($pids as $pid) {
                    if ($pid) {
                        pcntl_waitpid($pid, $status);
                    }
                }
            } catch (Exception $error) {
                $this->logException($index->getIndexName(), $group, $error);
                $group++;
                continue;
            }
        }

        return $groups;
    }

    /**
     * Reindex the given group, for each state
     *
     * @param int $group
     * @param string $class
     * @param int $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function doReindex($group, $class, $batchLength, BaseIndex $index): void
    {
        foreach (SiteState::getStates() as $state) {
            if ($state !== 'default' && !empty($state)) {
                SiteState::withState($state);
            }
            $this->stateReindex($group, $class, $batchLength, $index);
        }

        SiteState::withState(SiteState::DEFAULT_STATE);
        $this->getLogger()->info(sprintf('Indexed group %s', $group));
        // Exit, we don't want this child process to linger around;
        exit;

        die(0);
function fx() {
    try {
        doSomething();
        return true;
    }
    catch (\Exception $e) {
        return false;
    }

    return false;
}
    }

    /**
     * Index a group of a class for a specific state and index
     *
     * @param $group
     * @param $class
     * @param $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function stateReindex($group, $class, $batchLength, BaseIndex $index): void
    {
        // Generate filtered list of local records
        $baseClass = DataObject::getSchema()->baseDataClass($class);
        /** @var DataList|DataObject[] $items */
        $items = DataObject::get($baseClass)
            ->sort('ID ASC')
            ->limit($batchLength, ($group * $batchLength));
        if ($items->count()) {
            $this->updateIndex($index, $items);
        }
    }

    /**
     * Execute the update on the client
     *
     * @param BaseIndex $index
     * @param $items
     * @throws Exception
     */
    private function updateIndex(BaseIndex $index, $items): void
    {
        $client = $index->getClient();
        $update = $client->createUpdate();
        $this->service->setInDebugMode($this->debug);
        $this->service->updateIndex($index, $items, $update);
        $update->addCommit();
        $client->update($update);
    }

    /**
     * Log an exception if it happens. Most are catched, these logs are for the developers
     * to identify problems and fix them.
     *
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function logException($index, int $group, Exception $exception): void
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            'Error indexing core %s on group %s,' . PHP_EOL .
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }
}


1		<?php
2
3
4		namespace Firesphere\SolrSearch\Tasks;
5
6		use Exception;
7		use Firesphere\SolrSearch\Factories\DocumentFactory;
8		use Firesphere\SolrSearch\Helpers\SolrLogger;
9		use Firesphere\SolrSearch\Indexes\BaseIndex;
10		use Firesphere\SolrSearch\Services\SolrCoreService;
11		use Firesphere\SolrSearch\States\SiteState;
12		use Firesphere\SolrSearch\Traits\LoggerTrait;
13		use GuzzleHttp\Exception\GuzzleException;
14		use Psr\Log\LoggerInterface;
15		use ReflectionException;
16		use SilverStripe\Control\Director;
17		use SilverStripe\Control\HTTPRequest;
18		use SilverStripe\Core\Injector\Injector;
19		use SilverStripe\Dev\BuildTask;
20		use SilverStripe\ORM\ArrayList;
21		use SilverStripe\ORM\DataList;
22		use SilverStripe\ORM\DataObject;
23		use SilverStripe\ORM\DB;
24		use SilverStripe\ORM\ValidationException;
25		use SilverStripe\Versioned\Versioned;
26
27		/**
28		* Class SolrIndexTask
29		*
30		* @description Index items to Solr through a tasks
31		* @package Firesphere\SolrSearch\Tasks
32		*/
33		class SolrIndexTask extends BuildTask
34		{
35		use LoggerTrait;
36		/**
37		* URLSegment of this task
38		*
39		* @var string
40		*/
41		private static $segment = 'SolrIndexTask';
42		/**
43		* Store the current states for all instances of SiteState
44		*
45		* @var array
46		*/
47		public $currentStates;
48		/**
49		* My name
50		*
51		* @var string
52		*/
53		protected $title = 'Solr Index update';
54		/**
55		* What do I do?
56		*
57		* @var string
58		*/
59		protected $description = 'Add or update documents to an existing Solr core.';
60		/**
61		* Debug mode enabled, default false
62		*
63		* @var bool
64		*/
65		protected $debug = false;
66		/**
67		* Singleton of {@link SolrCoreService}
68		*
69		* @var SolrCoreService
70		*/
71		protected $service;
72
73		/**
74		* SolrIndexTask constructor. Sets up the document factory
75		*
76		* @throws ReflectionException
77	14	*/
78		public function __construct()
79	14	{
80		parent::__construct();
81		// Only index live items.
82	14	// The old FTS module also indexed Draft items. This is unnecessary
83	14	Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
84	14	$this->setService(Injector::inst()->get(SolrCoreService::class));
85	14	$this->setLogger(Injector::inst()->get(LoggerInterface::class));
86	14	$this->setDebug(Director::isDev() \|\| Director::is_cli());
87	14	$currentStates = SiteState::currentStates();
88	14	SiteState::setDefaultStates($currentStates);
89		}
90
91		/**
92		* Set the {@link SolrCoreService}
93		*
94		* @param SolrCoreService $service
95		* @return SolrIndexTask
96	14	*/
97		public function setService(SolrCoreService $service): SolrIndexTask
98	14	{
99		$this->service = $service;
100	14
101		return $this;
102		}
103
104		/**
105		* Set the debug mode
106		*
107		* @param bool $debug
108		* @return SolrIndexTask
109	14	*/
110		public function setDebug(bool $debug): SolrIndexTask
111	14	{
112		$this->debug = $debug;
113	14
114		return $this;
115		}
116
117		/**
118		* Implement this method in the task subclass to
119		* execute via the TaskRunner
120		*
121		* @param HTTPRequest $request
122		* @return int\|bool
123		* @throws Exception
124		* @throws GuzzleException
125		* @todo defer to background because it may run out of memory
126	13	*/
127		public function run($request)
128	13	{
129	13	$startTime = time();
130	13	list($vars, $group, $isGroup) = $this->taskSetup($request);
131	13	$groups = 0;
132		$indexes = $this->service->getValidIndexes($request->getVar('index'));
133	13
134		foreach ($indexes as $indexName) {
135	13	/** @var BaseIndex $index */
136		$index = Injector::inst()->get($indexName, false);
137	13
138	13	$indexClasses = $index->getClasses();
139	13	$classes = $this->getClasses($vars, $indexClasses);
140	10	if (!count($classes)) {
141		continue;
142		}
143	13
144		$this->clearIndex($vars, $index);
145	13
146		$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
147	13	}
148	13	$this->getLogger()->info(
149		sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
150		);
151	13	// Grab the latest logs from indexing if needed
152	13	$solrLogger = new SolrLogger();
153		$solrLogger->saveSolrLog('Config');
154	13
155		return $groups;
156		}
157
158		/**
159		* Set up the requirements for this task
160		*
161		* @param HTTPRequest $request
162		* @return array
163	13	*/
164		protected function taskSetup($request): array
165	13	{
166	13	$vars = $request->getVars();
167	13	$this->debug = $this->debug \|\| isset($vars['debug']);
168	13	$group = $vars['group'] ?? 0;
169	13	$start = $vars['start'] ?? 0;
170		$group = ($start > $group) ? $start : $group;
171	13	$isGroup = isset($vars['group']);
172
173		return [$vars, $group, $isGroup];
174		}
175
176		/**
177		* get the classes to run for this task execution
178		*
179		* @param $vars
180		* @param array $classes
181	13	* @return bool\|array
182		*/
183	13	protected function getClasses($vars, array $classes): array
184	1	{
185		if (isset($vars['class'])) {
186		return array_intersect($classes, [$vars['class']]);
187	12	}
188
189		return $classes;
190		}
191
192		/**
193		* Clear the given index if a full re-index is needed
194		*
195		* @param $vars
196		* @param BaseIndex $index
197	13	* @throws Exception
198		*/
199	13	public function clearIndex($vars, BaseIndex $index)
200	1	{
201	1	if (!empty($vars['clear'])) {
202		$this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
203	13	$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
204		}
205		}
206
207		/**
208		* Index the classes for a specific index
209		*
210		* @param $classes
211		* @param $isGroup
212		* @param BaseIndex $index
213		* @param $group
214		* @return int
215	13	* @throws Exception
216		* @throws GuzzleException
217	13	*/
218	13	protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
219	13	{
220		$groups = 0;
221		foreach ($classes as $class) {
222	13	$groups = $this->indexClass($isGroup, $class, $index, $group);
223		}
224
225		return $groups;
226		}
227
228		/**
229		* Index a single class for a given index. {@link static::indexClassForIndex()}
230		*
231		* @param bool $isGroup
232		* @param string $class
233		* @param BaseIndex $index
234		* @param int $group
235	13	* @return int
236		* @throws GuzzleException
237	13	* @throws ValidationException
238		*/
239		private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
240	13	{
241		$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);
242
243		$batchLength = DocumentFactory::config()->get('batchLength');
244		$groups = (int)ceil($class::get()->count() / $batchLength);
245	13	$groups = $isGroup ? $group : $groups;
246		// How much cores do we have
247		$cores = SolrCoreService::config()->get('cores');
248		$pid = false;
		0 ignored issues – show Unused Code introduced 2019-11-22 23:21 UTC by Report Bug Copy Issue Report The assignment to `$pid` is dead and can be removed. Loading history...
249		$pids = [];
250		$this->getLogger()->info(sprintf('Total groups %s', $groups));
251		while ($group <= $groups) { // Run from oldest to newest
252		try {
253		// for each core, start a grouped indexing
254		for ($i = 0; $i < $cores; $i++) {
255		$pid = pcntl_fork();
256		// Couldn't fork, so continue to the next try
257	13	if ($pid === -1) {
258		continue;
259	13	}
260	13	$pids[$i] = $pid;
261		if (!$pid) {
262		$config = DB::getConfig();
263	13	DB::connect($config);
264		$this->doReindex($group, $class, $batchLength, $index);
265		}
266	13	$group++;
267	13	}
268		// Wait for each child to finish
269		foreach ($pids as $pid) {
270		if ($pid) {
271		pcntl_waitpid($pid, $status);
272		}
273		}
274		} catch (Exception $error) {
275		$this->logException($index->getIndexName(), $group, $error);
276		$group++;
277	13	continue;
278		}
279		}
280	13
281		return $groups;
282	13	}
283	13
284	13	/**
285	13	* Reindex the given group, for each state
286	13	*
287		* @param int $group
288	13	* @param string $class
289	1	* @param int $batchLength
290		* @param BaseIndex $index
291	13	* @throws Exception
292		*/
293		private function doReindex($group, $class, $batchLength, BaseIndex $index): void
294		{
295		foreach (SiteState::getStates() as $state) {
296		if ($state !== 'default' && !empty($state)) {
297		SiteState::withState($state);
298		}
299		$this->stateReindex($group, $class, $batchLength, $index);
300	1	}
301
302	1	SiteState::withState(SiteState::DEFAULT_STATE);
303	1	$this->getLogger()->info(sprintf('Indexed group %s', $group));
304	1	// Exit, we don't want this child process to linger around;
305		exit;
		0 ignored issues – show Best Practice introduced 2019-11-22 23:21 UTC by Report Bug Copy Issue Report Using `exit` here is not recommended. In general, usage of exit should be done with care and only when running in a scripting context like a CLI script. Loading history...
306		die(0);
		0 ignored issues – show Unused Code introduced 2019-11-22 23:21 UTC by Report Bug Copy Issue Report `ExitNode` is not reachable. This check looks for unreachable code. It uses sophisticated control flow analysis techniques to find statements which will never be executed. Unreachable code is most often the result of `return`, `die` or `exit` statements that have been added for debug purposes. function fx() { try { doSomething(); return true; } catch (\Exception $e) { return false; } return false; } In the above example, the last `return false` will never be executed, because a return statement has already been met in every possible execution path. Loading history... Best Practice introduced 2019-11-22 23:21 UTC by Report Bug Copy Issue Report Using `exit` here is not recommended. In general, usage of exit should be done with care and only when running in a scripting context like a CLI script. Loading history...
307		}
308
309		/**
310		* Index a group of a class for a specific state and index
311		*
312		* @param $group
313		* @param $class
314		* @param $batchLength
315		* @param BaseIndex $index
316		* @throws Exception
317		*/
318		private function stateReindex($group, $class, $batchLength, BaseIndex $index): void
319		{
320		// Generate filtered list of local records
321		$baseClass = DataObject::getSchema()->baseDataClass($class);
322		/** @var DataList\|DataObject[] $items */
323		$items = DataObject::get($baseClass)
324		->sort('ID ASC')
325		->limit($batchLength, ($group * $batchLength));
326		if ($items->count()) {
327		$this->updateIndex($index, $items);
328		}
329		}
330
331		/**
332		* Execute the update on the client
333		*
334		* @param BaseIndex $index
335		* @param $items
336		* @throws Exception
337		*/
338		private function updateIndex(BaseIndex $index, $items): void
339		{
340		$client = $index->getClient();
341		$update = $client->createUpdate();
342		$this->service->setInDebugMode($this->debug);
343		$this->service->updateIndex($index, $items, $update);
344		$update->addCommit();
345		$client->update($update);
346		}
347
348		/**
349		* Log an exception if it happens. Most are catched, these logs are for the developers
350		* to identify problems and fix them.
351		*
352		* @param string $index
353		* @param int $group
354		* @param Exception $exception
355		* @throws GuzzleException
356		* @throws ValidationException
357		*/
358		private function logException($index, int $group, Exception $exception): void
359		{
360		$this->getLogger()->error($exception->getMessage());
361		$msg = sprintf(
362		'Error indexing core %s on group %s,' . PHP_EOL .
363		'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
364		$index,
365		$group
366		);
367		SolrLogger::logMessage('ERROR', $msg, $index);
368		}
369		}
370

Firesphere / silverstripe-solr-search

Push — bufferfix ( 848e8a...2b29dc )

SolrIndexTask::indexClass() B

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like