SolrIndexTask::taskSetup() - Code Metrics - Inspection of "Hans/code cleanup" - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#164)

by Simon

created 2019-12-14 22:50 UTC

SolrIndexTask::taskSetup() A

↳ Parent: SolrIndexTask

Complexity

Conditions	3
Paths	4

Size

Total Lines	10
Code Lines	7

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	4
CRAP Score	3

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
eloc	7
c	2
b	0
f	0
dl	0
loc	10
rs	10
ccs	4
cts	4
cp	1
cc	3
nc	4
nop	1
crap	3

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\States\SiteState;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use Firesphere\SolrSearch\Traits\SolrIndexTrait;
use GuzzleHttp\Exception\GuzzleException;
use Psr\Log\LoggerInterface;
use ReflectionException;
use SilverStripe\Control\Controller;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\ORM\ArrayList;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Versioned\Versioned;

/**
 * Class SolrIndexTask
 *
 * @description Index items to Solr through a tasks
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends BuildTask
{
    use LoggerTrait;
    use SolrIndexTrait;
    /**
     * URLSegment of this task
     *
     * @var string
     */
    private static $segment = 'SolrIndexTask';
    /**
     * Store the current states for all instances of SiteState
     *
     * @var array
     */
    public $currentStates;
    /**
     * My name
     *
     * @var string
     */
    protected $title = 'Solr Index update';
    /**
     * What do I do?
     *
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';

    /**
     * SolrIndexTask constructor. Sets up the document factory
     *
     * @throws ReflectionException
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
        // If versioned is needed, a separate Versioned Search module is required
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Director::isDev() || Director::is_cli());
        $this->setBatchLength(DocumentFactory::config()->get('batchLength'));
        $cores = SolrCoreService::config()->get('cpucores') ?: 1;
        $this->setCores($cores);
        $currentStates = SiteState::currentStates();
        SiteState::setDefaultStates($currentStates);
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     */
    public function run($request)
    {
        $start = time();
        $this->getLogger()->info(date('Y-m-d H:i:s'));
        list($vars, $group, $isGroup) = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);
            $this->setIndex($index);

            $indexClasses = $this->index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $this->clearIndex($vars);

            $groups = $this->indexClassForIndex($classes, $isGroup, $group);
        }

        $this->getLogger()->info(date('Y-m-d H:i:s'));
        $this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));

        return $groups;
    }

    /**
     * Set up the requirements for this task
     *
     * @param HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : $group;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * get the classes to run for this task execution
     *
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * Clear the given index if a full re-index is needed
     *
     * @param $vars
     * @throws Exception
     */
    public function clearIndex($vars)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $this->index->getIndexName()));
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $this->index);
        }
    }

    /**
     * Index the classes for a specific index
     *
     * @param $classes
     * @param $isGroup
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $group);
        }

        return $groups;
    }

    /**
     * Index a single class for a given index. {@link static::indexClassForIndex()}
     *
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function indexClass($isGroup, $class, int $group): int
    {
        $index = $this->getIndex();
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
        $totalGroups = (int)ceil($class::get()->count() / $this->getBatchLength());
        $groups = $isGroup ? ($group + $this->cores - 1) : $totalGroups;
        $this->getLogger()->info(sprintf('Total groups %s', $totalGroups));
        do { // Run from oldest to newest
            try {
                // The unittest param is from phpunit.xml.dist, meant to bypass the exit(0) call
                if (function_exists('pcntl_fork') &&
                    !Controller::curr()->getRequest()->getVar('unittest')
                ) {
                    $group = $this->spawnChildren($class, $group, $groups);
                } else {
                    $this->doReindex($group, $class);
                }
            } catch (Exception $error) {
                $this->logException($index->getIndexName(), $group, $error);
                $group++;
                continue;
            }
            $group++;
        } while ($group <= $groups);

        return $totalGroups;
    }

    /**
     * For each core, spawn a child process that will handle a separate group.
     * This speeds up indexing through CLI massively.
     *
     * @param string $class Class to index
     * @param int $group Group to index
     * @param int $groups Total amount of groups
     * @return int Last group indexed
     * @throws Exception
     * @throws GuzzleException
     */
    private function spawnChildren($class, int $group, int $groups): int
    {
        $start = $group;
        $pids = [];
        $cores = $this->getCores();
        // for each core, start a grouped indexing
        for ($i = 0; $i < $cores; $i++) {
            $start = $group + $i;
            if ($start < $groups) {
                $this->runForkedChild($class, $pids, $i, $start);
            }
        }
        // Wait for each child to finish
        foreach ($pids as $key => $pid) {
            pcntl_waitpid($pid, $status);
            if ($status === 0) {
                unset($pids[$key]);
            }
        }
        $commit = $this->index->getClient()->createUpdate();
        $commit->addCommit();

        $this->index->getClient()->update($commit);

        return $start;
    }

    /**
     * Create a fork and run the child
     *
     * @param string $class Class to index
     * @param array $pids Array of all the child PID's
     * @param int $coreNumber Core number
     * @param int $start Start point for the objects
     * @return void
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function runForkedChild($class, array &$pids, int $coreNumber, int $start): void
    {
        $pid = pcntl_fork();
        // PID needs to be pushed before anything else, for some reason
        $pids[$coreNumber] = $pid;
        $config = DB::getConfig();
        DB::connect($config);
        $this->runChild($class, $pid, $start);
    }

    /**
     * Ren a single child index operation
     *
     * @param string $class Class to index
     * @param int $pid PID of the child
     * @param int $start Position to start
     * @throws GuzzleException
     * @throws ValidationException
     * @throws Exception
     */
    private function runChild($class, int $pid, int $start): void
    {
        if (!$pid) {
            try {
                $this->doReindex($start, $class, $pid);
            } catch (Exception $e) {
                SolrLogger::logMessage('ERROR', $e, $this->index->getIndexName());
                $msg = sprintf(
                    'Something went wrong while indexing %s on %s, see the logs for details',
                    $start,
                    $this->index->getIndexName()
                );
                throw new Exception($msg);
            }
        }
    }

    /**
     * Reindex the given group, for each state
     *
     * @param int $group
     * @param string $class
     * @param bool|int $pcntl
     * @throws Exception
     */
    private function doReindex($group, $class, $pcntl = false)
    {
        foreach (SiteState::getStates() as $state) {
            if ($state !== 'default' && !empty($state)) {
                SiteState::withState($state);
            }
            $this->stateReindex($group, $class);
        }

        SiteState::withState(SiteState::DEFAULT_STATE);
        $this->getLogger()->info(sprintf('Indexed group %s', $group));

        if ($pcntl !== false) {
            exit(0);

        }
    }

    /**
     * Index a group of a class for a specific state and index
     *
     * @param $group
     * @param $class
     * @throws Exception
     */
    private function stateReindex($group, $class): void
    {
        // Generate filtered list of local records
        $baseClass = DataObject::getSchema()->baseDataClass($class);
        /** @var DataList|DataObject[] $items */
        $items = DataObject::get($baseClass)
            ->sort('ID ASC')
            ->limit($this->getBatchLength(), ($group * $this->getBatchLength()));
        if ($items->count()) {
            $this->updateIndex($items);
        }
    }

    /**
     * Execute the update on the client
     *
     * @param $items
     * @throws Exception
     */
    private function updateIndex($items): void
    {
        $index = $this->getIndex();
        $client = $index->getClient();
        $update = $client->createUpdate();
        $this->service->setInDebugMode($this->debug);
        $this->service->updateIndex($index, $items, $update);
        $client->update($update);
    }

    /**
     * Log an exception if it happens. Most are catched, these logs are for the developers
     * to identify problems and fix them.
     *
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function logException($index, int $group, Exception $exception): void
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            'Error indexing core %s on group %s,' . PHP_EOL .
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }
}


1		<?php
2
3
4		namespace Firesphere\SolrSearch\Tasks;
5
6		use Exception;
7		use Firesphere\SolrSearch\Factories\DocumentFactory;
8		use Firesphere\SolrSearch\Helpers\SolrLogger;
9		use Firesphere\SolrSearch\Indexes\BaseIndex;
10		use Firesphere\SolrSearch\Services\SolrCoreService;
11		use Firesphere\SolrSearch\States\SiteState;
12		use Firesphere\SolrSearch\Traits\LoggerTrait;
13		use Firesphere\SolrSearch\Traits\SolrIndexTrait;
14		use GuzzleHttp\Exception\GuzzleException;
15		use Psr\Log\LoggerInterface;
16		use ReflectionException;
17		use SilverStripe\Control\Controller;
18		use SilverStripe\Control\Director;
19		use SilverStripe\Control\HTTPRequest;
20		use SilverStripe\Core\Injector\Injector;
21		use SilverStripe\Dev\BuildTask;
22		use SilverStripe\ORM\ArrayList;
23		use SilverStripe\ORM\DataList;
24		use SilverStripe\ORM\DataObject;
25		use SilverStripe\ORM\DB;
26		use SilverStripe\ORM\ValidationException;
27		use SilverStripe\Versioned\Versioned;
28
29		/**
30		* Class SolrIndexTask
31		*
32		* @description Index items to Solr through a tasks
33		* @package Firesphere\SolrSearch\Tasks
34		*/
35		class SolrIndexTask extends BuildTask
36		{
37		use LoggerTrait;
38		use SolrIndexTrait;
39		/**
40		* URLSegment of this task
41		*
42		* @var string
43		*/
44		private static $segment = 'SolrIndexTask';
45		/**
46		* Store the current states for all instances of SiteState
47		*
48		* @var array
49		*/
50		public $currentStates;
51		/**
52		* My name
53		*
54		* @var string
55		*/
56		protected $title = 'Solr Index update';
57		/**
58		* What do I do?
59		*
60		* @var string
61		*/
62		protected $description = 'Add or update documents to an existing Solr core.';
63
64		/**
65		* SolrIndexTask constructor. Sets up the document factory
66		*
67		* @throws ReflectionException
68		*/
69		public function __construct()
70		{
71		parent::__construct();
72		// Only index live items.
73		// The old FTS module also indexed Draft items. This is unnecessary
74		Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
75		// If versioned is needed, a separate Versioned Search module is required
76		$this->setService(Injector::inst()->get(SolrCoreService::class));
77		$this->setLogger(Injector::inst()->get(LoggerInterface::class));
78		$this->setDebug(Director::isDev() \|\| Director::is_cli());
79		$this->setBatchLength(DocumentFactory::config()->get('batchLength'));
80		$cores = SolrCoreService::config()->get('cpucores') ?: 1;
81		$this->setCores($cores);
82		$currentStates = SiteState::currentStates();
83		SiteState::setDefaultStates($currentStates);
84		}
85
86		/**
87	14	* Implement this method in the task subclass to
88		* execute via the TaskRunner
89	14	*
90		* @param HTTPRequest $request
91		* @return int\|bool
92	14	* @throws Exception
93		* @throws GuzzleException
94	14	*/
95	14	public function run($request)
96	14	{
97	14	$start = time();
98	14	$this->getLogger()->info(date('Y-m-d H:i:s'));
99	14	list($vars, $group, $isGroup) = $this->taskSetup($request);
100		$groups = 0;
101		$indexes = $this->service->getValidIndexes($request->getVar('index'));
102
103		foreach ($indexes as $indexName) {
104		/** @var BaseIndex $index */
105		$index = Injector::inst()->get($indexName, false);
106		$this->setIndex($index);
107	14
108		$indexClasses = $this->index->getClasses();
109	14	$classes = $this->getClasses($vars, $indexClasses);
110		if (!count($classes)) {
111	14	continue;
112		}
113
114		$this->clearIndex($vars);
115
116		$groups = $this->indexClassForIndex($classes, $isGroup, $group);
117		}
118
119		$this->getLogger()->info(date('Y-m-d H:i:s'));
120	14	$this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));
121
122	14	return $groups;
123		}
124	14
125		/**
126		* Set up the requirements for this task
127		*
128		* @param HTTPRequest $request
129		* @return array
130		*/
131		protected function taskSetup($request): array
132		{
133		$vars = $request->getVars();
134		$this->debug = $this->debug \|\| isset($vars['debug']);
135		$group = $vars['group'] ?? 0;
136	13	$start = $vars['start'] ?? 0;
137		$group = ($start > $group) ? $start : $group;
138	13	$isGroup = isset($vars['group']);
139	13
140	13	return [$vars, $group, $isGroup];
141	13	}
142	13
143		/**
144	13	* get the classes to run for this task execution
145		*
146	13	* @param $vars
147		* @param array $classes
148	13	* @return bool\|array
149	13	*/
150	13	protected function getClasses($vars, array $classes): array
151	10	{
152		if (isset($vars['class'])) {
153		return array_intersect($classes, [$vars['class']]);
154	13	}
155
156	13	return $classes;
157		}
158
159	13	/**
160	13	* Clear the given index if a full re-index is needed
161		*
162	13	* @param $vars
163		* @throws Exception
164		*/
165		public function clearIndex($vars)
166		{
167		if (!empty($vars['clear'])) {
168		$this->getLogger()->info(sprintf('Clearing index %s', $this->index->getIndexName()));
169		$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $this->index);
170		}
171	13	}
172
173	13	/**
174	13	* Index the classes for a specific index
175	13	*
176	13	* @param $classes
177	13	* @param $isGroup
178	13	* @param $group
179		* @return int
180	13	* @throws Exception
181		* @throws GuzzleException
182		*/
183		protected function indexClassForIndex($classes, $isGroup, $group): int
184		{
185		$groups = 0;
186		foreach ($classes as $class) {
187		$groups = $this->indexClass($isGroup, $class, $group);
188		}
189
190	13	return $groups;
191		}
192	13
193	1	/**
194		* Index a single class for a given index. {@link static::indexClassForIndex()}
195		*
196	12	* @param bool $isGroup
197		* @param string $class
198		* @param BaseIndex $index
199		* @param int $group
200		* @return int
201		* @throws GuzzleException
202		* @throws ValidationException
203		*/
204		private function indexClass($isGroup, $class, int $group): int
205		{
206	13	$index = $this->getIndex();
207		$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
208	13	$totalGroups = (int)ceil($class::get()->count() / $this->getBatchLength());
209	1	$groups = $isGroup ? ($group + $this->cores - 1) : $totalGroups;
210	1	$this->getLogger()->info(sprintf('Total groups %s', $totalGroups));
211		do { // Run from oldest to newest
212	13	try {
213		// The unittest param is from phpunit.xml.dist, meant to bypass the exit(0) call
214		if (function_exists('pcntl_fork') &&
215		!Controller::curr()->getRequest()->getVar('unittest')
216		) {
217		$group = $this->spawnChildren($class, $group, $groups);
218		} else {
219		$this->doReindex($group, $class);
220		}
221		} catch (Exception $error) {
222		$this->logException($index->getIndexName(), $group, $error);
223		$group++;
224		continue;
225	13	}
226		$group++;
227	13	} while ($group <= $groups);
228	13
229	13	return $totalGroups;
230		}
231
232	13	/**
233		* For each core, spawn a child process that will handle a separate group.
234		* This speeds up indexing through CLI massively.
235		*
236		* @param string $class Class to index
237		* @param int $group Group to index
238		* @param int $groups Total amount of groups
239		* @return int Last group indexed
240		* @throws Exception
241		* @throws GuzzleException
242		*/
243		private function spawnChildren($class, int $group, int $groups): int
244		{
245		$start = $group;
246	13	$pids = [];
247		$cores = $this->getCores();
248	13	// for each core, start a grouped indexing
249	13	for ($i = 0; $i < $cores; $i++) {
250	13	$start = $group + $i;
251	13	if ($start < $groups) {
252	13	$this->runForkedChild($class, $pids, $i, $start);
253	13	}
254		}
255		// Wait for each child to finish
256		foreach ($pids as $key => $pid) {
257	13	pcntl_waitpid($pid, $status);
258	13	if ($status === 0) {
259		unset($pids[$key]);
260		}
261		}
262	13	$commit = $this->index->getClient()->createUpdate();
263		$commit->addCommit();
264
265		$this->index->getClient()->update($commit);
266
267		return $start;
268		}
269	13
270	13	/**
271		* Create a fork and run the child
272	13	*
273		* @param string $class Class to index
274		* @param array $pids Array of all the child PID's
275		* @param int $coreNumber Core number
276		* @param int $start Start point for the objects
277		* @return void
278		* @throws GuzzleException
279		* @throws ValidationException
280		*/
281		private function runForkedChild($class, array &$pids, int $coreNumber, int $start): void
282		{
283		$pid = pcntl_fork();
284		// PID needs to be pushed before anything else, for some reason
285		$pids[$coreNumber] = $pid;
286		$config = DB::getConfig();
287		DB::connect($config);
288		$this->runChild($class, $pid, $start);
289		}
290
291		/**
292		* Ren a single child index operation
293		*
294		* @param string $class Class to index
295		* @param int $pid PID of the child
296		* @param int $start Position to start
297		* @throws GuzzleException
298		* @throws ValidationException
299		* @throws Exception
300		*/
301		private function runChild($class, int $pid, int $start): void
302		{
303		if (!$pid) {
304		try {
305		$this->doReindex($start, $class, $pid);
306		} catch (Exception $e) {
307		SolrLogger::logMessage('ERROR', $e, $this->index->getIndexName());
308		$msg = sprintf(
309		'Something went wrong while indexing %s on %s, see the logs for details',
310		$start,
311		$this->index->getIndexName()
312		);
313		throw new Exception($msg);
314		}
315		}
316		}
317
318		/**
319		* Reindex the given group, for each state
320		*
321		* @param int $group
322		* @param string $class
323		* @param bool\|int $pcntl
324		* @throws Exception
325		*/
326		private function doReindex($group, $class, $pcntl = false)
327		{
328		foreach (SiteState::getStates() as $state) {
329		if ($state !== 'default' && !empty($state)) {
330		SiteState::withState($state);
331		}
332		$this->stateReindex($group, $class);
333		}
334
335		SiteState::withState(SiteState::DEFAULT_STATE);
336		$this->getLogger()->info(sprintf('Indexed group %s', $group));
337
338		if ($pcntl !== false) {
339		exit(0);
		1 ignored issue – show Best Practice introduced 2019-11-24 21:19 UTC by Report Bug Copy Issue Report Using `exit` here is not recommended. In general, usage of exit should be done with care and only when running in a scripting context like a CLI script. Loading history...
340	13	}
341		}
342	13
343	13	/**
344		* Index a group of a class for a specific state and index
345		*
346	13	* @param $group
347		* @param $class
348		* @throws Exception
349	13	*/
350	13	private function stateReindex($group, $class): void
351		{
352	13	// Generate filtered list of local records
353		$baseClass = DataObject::getSchema()->baseDataClass($class);
354		/** @var DataList\|DataObject[] $items */
355	13	$items = DataObject::get($baseClass)
356		->sort('ID ASC')
357		->limit($this->getBatchLength(), ($group * $this->getBatchLength()));
358		if ($items->count()) {
359		$this->updateIndex($items);
360		}
361		}
362
363		/**
364		* Execute the update on the client
365	13	*
366		* @param $items
367		* @throws Exception
368	13	*/
369		private function updateIndex($items): void
370	13	{
371	13	$index = $this->getIndex();
372	13	$client = $index->getClient();
373	13	$update = $client->createUpdate();
374	1	$this->service->setInDebugMode($this->debug);
375		$this->service->updateIndex($index, $items, $update);
376	13	$client->update($update);
377		}
378
379		/**
380		* Log an exception if it happens. Most are catched, these logs are for the developers
381		* to identify problems and fix them.
382		*
383		* @param string $index
384		* @param int $group
385	1	* @param Exception $exception
386		* @throws GuzzleException
387	1	* @throws ValidationException
388	1	*/
389	1	private function logException($index, int $group, Exception $exception): void
390	1	{
391	1	$this->getLogger()->error($exception->getMessage());
392	1	$msg = sprintf(
393		'Error indexing core %s on group %s,' . PHP_EOL .
394		'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
395		$index,
396		$group
397		);
398		SolrLogger::logMessage('ERROR', $msg, $index);
399		}
400		}
401

Firesphere / silverstripe-solr-search

Pull Request — master (#164)

SolrIndexTask::taskSetup() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like