SolrIndexTask::spawnChildren() - Code Metrics - Inspection of "More readable forking" - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — bufferfix ( fef525...9ac6d4 )

by Simon

created 2019-11-24 21:13 UTC

SolrIndexTask::spawnChildren() B

↳ Parent: SolrIndexTask

Complexity

Conditions	7
Paths	12

Size

Total Lines	27
Code Lines	16

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	0
CRAP Score	56

Importance

Changes

Metric	Value
cc	7
eloc	16
c	0
b	0
f	0
nc	12
nop	5
dl	0
loc	27
ccs	0
cts	0
cp	0
crap	56
rs	8.8333

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\States\SiteState;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use GuzzleHttp\Exception\GuzzleException;
use Psr\Log\LoggerInterface;
use ReflectionException;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\Dev\Debug;
use SilverStripe\ORM\ArrayList;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Versioned\Versioned;

/**
 * Class SolrIndexTask
 *
 * @description Index items to Solr through a tasks
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends BuildTask
{
    use LoggerTrait;
    /**
     * URLSegment of this task
     *
     * @var string
     */
    private static $segment = 'SolrIndexTask';
    /**
     * Store the current states for all instances of SiteState
     *
     * @var array
     */
    public $currentStates;
    /**
     * My name
     *
     * @var string
     */
    protected $title = 'Solr Index update';
    /**
     * What do I do?
     *
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';
    /**
     * Debug mode enabled, default false
     *
     * @var bool
     */
    protected $debug = false;
    /**
     * Singleton of {@link SolrCoreService}
     *
     * @var SolrCoreService
     */
    protected $service;

    /**
     * SolrIndexTask constructor. Sets up the document factory
     *
     * @throws ReflectionException
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
        // If versioned is needed, a separate Versioned Search module is required
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Director::isDev() || Director::is_cli());
        $currentStates = SiteState::currentStates();
        SiteState::setDefaultStates($currentStates);
    }

    /**
     * Set the {@link SolrCoreService}
     *
     * @param SolrCoreService $service
     * @return SolrIndexTask
     */
    public function setService(SolrCoreService $service): SolrIndexTask
    {
        $this->service = $service;

        return $this;
    }

    /**
     * Set the debug mode
     *
     * @param bool $debug
     * @return SolrIndexTask
     */
    public function setDebug(bool $debug): SolrIndexTask
    {
        $this->debug = $debug;

        return $this;
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     * @todo defer to background because it may run out of memory
     */
    public function run($request)
    {
        $start = time();
        $this->getLogger()->info(date('Y-m-d H:i:s'));
        list($vars, $group, $isGroup) = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);

            $indexClasses = $index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $this->clearIndex($vars, $index);

            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
        }

        $this->getLogger()->info(date('Y-m-d H:i:s'));
        $this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));

        return $groups;
    }

    /**
     * Set up the requirements for this task
     *
     * @param HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : $group;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * get the classes to run for this task execution
     *
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * Clear the given index if a full re-index is needed
     *
     * @param $vars
     * @param BaseIndex $index
     * @throws Exception
     */
    public function clearIndex($vars, BaseIndex $index)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
        }
    }

    /**
     * Index the classes for a specific index
     *
     * @param $classes
     * @param $isGroup
     * @param BaseIndex $index
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $index, $group);
        }

        return $groups;
    }

    /**
     * Index a single class for a given index. {@link static::indexClassForIndex()}
     *
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
    {
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
        $batchLength = DocumentFactory::config()->get('batchLength');
        $groups = (int)ceil($class::get()->count() / $batchLength);
        $groups = $isGroup ? $group : $groups;
        $cores = SolrCoreService::config()->get('cores') ?: 1;
        $this->getLogger()->info(sprintf('Total groups %s', $groups));
        do { // Run from oldest to newest
            try {
                // Temporary workaround for CircleCI
                if (function_exists('pcntl_fork')) {
                    // for each core, start a grouped indexing
                    $group = $this->spawnChildren($class, $index, $group, $cores, $groups);
                } else {
                    $this->doReindex($group, $class, $batchLength, $index);
                }
            } catch (Exception $error) {
                $this->logException($index->getIndexName(), $group, $error);
                $group++;
                continue;
            }
            $group++;
        } while ($group <= $groups);

        return $groups;
    }

    /**
     * Reindex the given group, for each state
     *
     * @param int $group
     * @param string $class
     * @param int $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function doReindex($group, $class, BaseIndex $index, $pcntl = false)
    {
        foreach (SiteState::getStates() as $state) {
            if ($state !== 'default' && !empty($state)) {
                SiteState::withState($state);
            }
            $this->stateReindex($group, $class, $index);
        }

        SiteState::withState(SiteState::DEFAULT_STATE);
        $this->getLogger()->info(sprintf('Indexed group %s', $group));

        if ($pcntl) {
            exit(0);

        }
    }

    /**
     * Index a group of a class for a specific state and index
     *
     * @param $group
     * @param $class
     * @param $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function stateReindex($group, $class, BaseIndex $index): void
    {
        $batchLength = DocumentFactory::config()->get('batchLength');
        // Generate filtered list of local records
        $baseClass = DataObject::getSchema()->baseDataClass($class);
        /** @var DataList|DataObject[] $items */
        $items = DataObject::get($baseClass)
            ->sort('ID ASC')
            ->limit($batchLength, ($group * $batchLength));
        if ($items->count()) {
            $this->updateIndex($index, $items);
        }
    }

    /**
     * Execute the update on the client
     *
     * @param BaseIndex $index
     * @param $items
     * @throws Exception
     */
    private function updateIndex(BaseIndex $index, $items): void
    {
        $client = $index->getClient();
        $update = $client->createUpdate();
        $this->service->setInDebugMode($this->debug);
        $this->service->updateIndex($index, $items, $update);
        $update->addCommit();
        $client->update($update);
    }

    /**
     * Log an exception if it happens. Most are catched, these logs are for the developers
     * to identify problems and fix them.
     *
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function logException($index, int $group, Exception $exception): void
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            'Error indexing core %s on group %s,' . PHP_EOL .
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }

    /**
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @param int $cores
     * @param int $groups
     * @return array
     * @throws Exception
     */
    private function spawnChildren($class, BaseIndex $index, int $group, int $cores, int $groups): int
    {
        $pids = [];
        $start = $group;
        for ($i = 0; $i < $cores; $i++) {
            $start = $group + $i;
            $pid = pcntl_fork();
            // PID needs to be pushed before anything else, for some reason
            $pids[$i] = $pid;
            if (!$pid && $start <= $groups) {
                $config = DB::getConfig();
                DB::connect($config);
                $this->doReindex($start, $class, $index, true);
            }
        }

        // Wait for each child to finish
        foreach ($pids as $key => $pid) {
            if ($pid) {
                pcntl_waitpid($pid, $status);
                if ($status === 0) {
                    unset($pids[$key]);
                }
            }
        }

        return $start;

    }
}


1		<?php
2
3
4		namespace Firesphere\SolrSearch\Tasks;
5
6		use Exception;
7		use Firesphere\SolrSearch\Factories\DocumentFactory;
8		use Firesphere\SolrSearch\Helpers\SolrLogger;
9		use Firesphere\SolrSearch\Indexes\BaseIndex;
10		use Firesphere\SolrSearch\Services\SolrCoreService;
11		use Firesphere\SolrSearch\States\SiteState;
12		use Firesphere\SolrSearch\Traits\LoggerTrait;
13		use GuzzleHttp\Exception\GuzzleException;
14		use Psr\Log\LoggerInterface;
15		use ReflectionException;
16		use SilverStripe\Control\Director;
17		use SilverStripe\Control\HTTPRequest;
18		use SilverStripe\Core\Injector\Injector;
19		use SilverStripe\Dev\BuildTask;
20		use SilverStripe\Dev\Debug;
21		use SilverStripe\ORM\ArrayList;
22		use SilverStripe\ORM\DataList;
23		use SilverStripe\ORM\DataObject;
24		use SilverStripe\ORM\DB;
25		use SilverStripe\ORM\ValidationException;
26		use SilverStripe\Versioned\Versioned;
27
28		/**
29		* Class SolrIndexTask
30		*
31		* @description Index items to Solr through a tasks
32		* @package Firesphere\SolrSearch\Tasks
33		*/
34		class SolrIndexTask extends BuildTask
35		{
36		use LoggerTrait;
37		/**
38		* URLSegment of this task
39		*
40		* @var string
41		*/
42		private static $segment = 'SolrIndexTask';
43		/**
44		* Store the current states for all instances of SiteState
45		*
46		* @var array
47		*/
48		public $currentStates;
49		/**
50		* My name
51		*
52		* @var string
53		*/
54		protected $title = 'Solr Index update';
55		/**
56		* What do I do?
57		*
58		* @var string
59		*/
60		protected $description = 'Add or update documents to an existing Solr core.';
61		/**
62		* Debug mode enabled, default false
63		*
64		* @var bool
65		*/
66		protected $debug = false;
67		/**
68		* Singleton of {@link SolrCoreService}
69		*
70		* @var SolrCoreService
71		*/
72		protected $service;
73
74		/**
75		* SolrIndexTask constructor. Sets up the document factory
76		*
77	14	* @throws ReflectionException
78		*/
79	14	public function __construct()
80		{
81		parent::__construct();
82	14	// Only index live items.
83	14	// The old FTS module also indexed Draft items. This is unnecessary
84	14	Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
85	14	// If versioned is needed, a separate Versioned Search module is required
86	14	$this->setService(Injector::inst()->get(SolrCoreService::class));
87	14	$this->setLogger(Injector::inst()->get(LoggerInterface::class));
88	14	$this->setDebug(Director::isDev() \|\| Director::is_cli());
89		$currentStates = SiteState::currentStates();
90		SiteState::setDefaultStates($currentStates);
91		}
92
93		/**
94		* Set the {@link SolrCoreService}
95		*
96	14	* @param SolrCoreService $service
97		* @return SolrIndexTask
98	14	*/
99		public function setService(SolrCoreService $service): SolrIndexTask
100	14	{
101		$this->service = $service;
102
103		return $this;
104		}
105
106		/**
107		* Set the debug mode
108		*
109	14	* @param bool $debug
110		* @return SolrIndexTask
111	14	*/
112		public function setDebug(bool $debug): SolrIndexTask
113	14	{
114		$this->debug = $debug;
115
116		return $this;
117		}
118
119		/**
120		* Implement this method in the task subclass to
121		* execute via the TaskRunner
122		*
123		* @param HTTPRequest $request
124		* @return int\|bool
125		* @throws Exception
126	13	* @throws GuzzleException
127		* @todo defer to background because it may run out of memory
128	13	*/
129	13	public function run($request)
130	13	{
131	13	$start = time();
132		$this->getLogger()->info(date('Y-m-d H:i:s'));
133	13	list($vars, $group, $isGroup) = $this->taskSetup($request);
134		$groups = 0;
135	13	$indexes = $this->service->getValidIndexes($request->getVar('index'));
136
137	13	foreach ($indexes as $indexName) {
138	13	/** @var BaseIndex $index */
139	13	$index = Injector::inst()->get($indexName, false);
140	10
141		$indexClasses = $index->getClasses();
142		$classes = $this->getClasses($vars, $indexClasses);
143	13	if (!count($classes)) {
144		continue;
145	13	}
146
147	13	$this->clearIndex($vars, $index);
148	13
149		$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
150		}
151	13
152	13	$this->getLogger()->info(date('Y-m-d H:i:s'));
153		$this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));
154	13
155		return $groups;
156		}
157
158		/**
159		* Set up the requirements for this task
160		*
161		* @param HTTPRequest $request
162		* @return array
163	13	*/
164		protected function taskSetup($request): array
165	13	{
166	13	$vars = $request->getVars();
167	13	$this->debug = $this->debug \|\| isset($vars['debug']);
168	13	$group = $vars['group'] ?? 0;
169	13	$start = $vars['start'] ?? 0;
170		$group = ($start > $group) ? $start : $group;
171	13	$isGroup = isset($vars['group']);
172
173		return [$vars, $group, $isGroup];
174		}
175
176		/**
177		* get the classes to run for this task execution
178		*
179		* @param $vars
180		* @param array $classes
181	13	* @return bool\|array
182		*/
183	13	protected function getClasses($vars, array $classes): array
184	1	{
185		if (isset($vars['class'])) {
186		return array_intersect($classes, [$vars['class']]);
187	12	}
188
189		return $classes;
190		}
191
192		/**
193		* Clear the given index if a full re-index is needed
194		*
195		* @param $vars
196		* @param BaseIndex $index
197	13	* @throws Exception
198		*/
199	13	public function clearIndex($vars, BaseIndex $index)
200	1	{
201	1	if (!empty($vars['clear'])) {
202		$this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
203	13	$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
204		}
205		}
206
207		/**
208		* Index the classes for a specific index
209		*
210		* @param $classes
211		* @param $isGroup
212		* @param BaseIndex $index
213		* @param $group
214		* @return int
215	13	* @throws Exception
216		* @throws GuzzleException
217	13	*/
218	13	protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
219	13	{
220		$groups = 0;
221		foreach ($classes as $class) {
222	13	$groups = $this->indexClass($isGroup, $class, $index, $group);
223		}
224
225		return $groups;
226		}
227
228		/**
229		* Index a single class for a given index. {@link static::indexClassForIndex()}
230		*
231		* @param bool $isGroup
232		* @param string $class
233		* @param BaseIndex $index
234		* @param int $group
235	13	* @return int
236		* @throws GuzzleException
237	13	* @throws ValidationException
238		*/
239		private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
240	13	{
241		$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
242		$batchLength = DocumentFactory::config()->get('batchLength');
243		$groups = (int)ceil($class::get()->count() / $batchLength);
244		$groups = $isGroup ? $group : $groups;
245	13	$cores = SolrCoreService::config()->get('cores') ?: 1;
246		$this->getLogger()->info(sprintf('Total groups %s', $groups));
247		do { // Run from oldest to newest
248		try {
249		// Temporary workaround for CircleCI
250		if (function_exists('pcntl_fork')) {
251		// for each core, start a grouped indexing
252		$group = $this->spawnChildren($class, $index, $group, $cores, $groups);
253		} else {
254		$this->doReindex($group, $class, $batchLength, $index);
255		}
256		} catch (Exception $error) {
257	13	$this->logException($index->getIndexName(), $group, $error);
258		$group++;
259	13	continue;
260	13	}
261		$group++;
262		} while ($group <= $groups);
263	13
264		return $groups;
265		}
266	13
267	13	/**
268		* Reindex the given group, for each state
269		*
270		* @param int $group
271		* @param string $class
272		* @param int $batchLength
273		* @param BaseIndex $index
274		* @throws Exception
275		*/
276		private function doReindex($group, $class, BaseIndex $index, $pcntl = false)
277	13	{
278		foreach (SiteState::getStates() as $state) {
279		if ($state !== 'default' && !empty($state)) {
280	13	SiteState::withState($state);
281		}
282	13	$this->stateReindex($group, $class, $index);
283	13	}
284	13
285	13	SiteState::withState(SiteState::DEFAULT_STATE);
286	13	$this->getLogger()->info(sprintf('Indexed group %s', $group));
287
288	13	if ($pcntl) {
289	1	exit(0);
		1 ignored issue – show Best Practice introduced 2019-11-24 21:19 UTC by Report Bug Copy Issue Report Using `exit` here is not recommended. In general, usage of exit should be done with care and only when running in a scripting context like a CLI script. Loading history...
290		}
291	13	}
292
293		/**
294		* Index a group of a class for a specific state and index
295		*
296		* @param $group
297		* @param $class
298		* @param $batchLength
299		* @param BaseIndex $index
300	1	* @throws Exception
301		*/
302	1	private function stateReindex($group, $class, BaseIndex $index): void
303	1	{
304	1	$batchLength = DocumentFactory::config()->get('batchLength');
305		// Generate filtered list of local records
306		$baseClass = DataObject::getSchema()->baseDataClass($class);
307		/** @var DataList\|DataObject[] $items */
308		$items = DataObject::get($baseClass)
309		->sort('ID ASC')
310		->limit($batchLength, ($group * $batchLength));
311		if ($items->count()) {
312		$this->updateIndex($index, $items);
313		}
314		}
315
316		/**
317		* Execute the update on the client
318		*
319		* @param BaseIndex $index
320		* @param $items
321		* @throws Exception
322		*/
323		private function updateIndex(BaseIndex $index, $items): void
324		{
325		$client = $index->getClient();
326		$update = $client->createUpdate();
327		$this->service->setInDebugMode($this->debug);
328		$this->service->updateIndex($index, $items, $update);
329		$update->addCommit();
330		$client->update($update);
331		}
332
333		/**
334		* Log an exception if it happens. Most are catched, these logs are for the developers
335		* to identify problems and fix them.
336		*
337		* @param string $index
338		* @param int $group
339		* @param Exception $exception
340		* @throws GuzzleException
341		* @throws ValidationException
342		*/
343		private function logException($index, int $group, Exception $exception): void
344		{
345		$this->getLogger()->error($exception->getMessage());
346		$msg = sprintf(
347		'Error indexing core %s on group %s,' . PHP_EOL .
348		'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
349		$index,
350		$group
351		);
352		SolrLogger::logMessage('ERROR', $msg, $index);
353		}
354
355		/**
356		* @param string $class
357		* @param BaseIndex $index
358		* @param int $group
359		* @param int $cores
360		* @param int $groups
361		* @return array
362		* @throws Exception
363		*/
364		private function spawnChildren($class, BaseIndex $index, int $group, int $cores, int $groups): int
365		{
366		$pids = [];
367		$start = $group;
368		for ($i = 0; $i < $cores; $i++) {
369		$start = $group + $i;
370		$pid = pcntl_fork();
371		// PID needs to be pushed before anything else, for some reason
372		$pids[$i] = $pid;
373		if (!$pid && $start <= $groups) {
374		$config = DB::getConfig();
375		DB::connect($config);
376		$this->doReindex($start, $class, $index, true);
377		}
378		}
379
380		// Wait for each child to finish
381		foreach ($pids as $key => $pid) {
382		if ($pid) {
383		pcntl_waitpid($pid, $status);
384		if ($status === 0) {
385		unset($pids[$key]);
386		}
387		}
388		}
389
390		return $start;
		0 ignored issues – show Bug Best Practice introduced 2019-11-24 21:19 UTC by Report Bug Copy Issue Report The expression `return $start` returns the type `integer` which is incompatible with the documented return type `array`. Loading history...
391		}
392		}
393

Firesphere / silverstripe-solr-search

Push — bufferfix ( fef525...9ac6d4 )

SolrIndexTask::spawnChildren() B

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like