SolrIndexTask::run() - Code Metrics - Inspection of "Add markdown to HTML experiment for documentation..." - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 83fab6...77ac30 )

by Marco

created 2019-11-26 23:20 UTC

SolrIndexTask::run() A

↳ Parent: SolrIndexTask

Complexity

Conditions	3
Paths	3

Size

Total Lines	27
Code Lines	16

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	17
CRAP Score	3

Importance

Changes	11
Bugs	0	Features	0

Metric	Value
cc	3
eloc	16
c	11
b	0
f	0
nc	3
nop	1
dl	0
loc	27
ccs	17
cts	17
cp	1
crap	3
rs	9.7333

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\States\SiteState;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use GuzzleHttp\Exception\GuzzleException;
use Psr\Log\LoggerInterface;
use ReflectionException;
use SilverStripe\Control\Controller;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\Dev\Debug;
use SilverStripe\ORM\ArrayList;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Versioned\Versioned;

/**
 * Class SolrIndexTask
 *
 * @description Index items to Solr through a tasks
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends BuildTask
{
    use LoggerTrait;
    /**
     * URLSegment of this task
     *
     * @var string
     */
    private static $segment = 'SolrIndexTask';
    /**
     * Store the current states for all instances of SiteState
     *
     * @var array
     */
    public $currentStates;
    /**
     * My name
     *
     * @var string
     */
    protected $title = 'Solr Index update';
    /**
     * What do I do?
     *
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';
    /**
     * Debug mode enabled, default false
     *
     * @var bool
     */
    protected $debug = false;
    /**
     * Singleton of {@link SolrCoreService}
     *
     * @var SolrCoreService
     */
    protected $service;

    /**
     * @var int
     */
    protected $batchLength = 1;

    /**
     * SolrIndexTask constructor. Sets up the document factory
     *
     * @throws ReflectionException
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
        // If versioned is needed, a separate Versioned Search module is required
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Director::isDev() || Director::is_cli());
        $currentStates = SiteState::currentStates();
        SiteState::setDefaultStates($currentStates);
    }

    /**
     * Set the {@link SolrCoreService}
     *
     * @param SolrCoreService $service
     * @return SolrIndexTask
     */
    public function setService(SolrCoreService $service): SolrIndexTask
    {
        $this->service = $service;

        return $this;
    }

    /**
     * Set the debug mode
     *
     * @param bool $debug
     * @return SolrIndexTask
     */
    public function setDebug(bool $debug): SolrIndexTask
    {
        $this->debug = $debug;

        return $this;
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     */
    public function run($request)
    {
        $start = time();
        $this->getLogger()->info(date('Y-m-d H:i:s'));
        list($vars, $group, $isGroup) = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);

            $indexClasses = $index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $this->clearIndex($vars, $index);

            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
        }

        $this->getLogger()->info(date('Y-m-d H:i:s'));
        $this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));

        return $groups;
    }

    /**
     * Set up the requirements for this task
     *
     * @param HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : $group;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * get the classes to run for this task execution
     *
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * Clear the given index if a full re-index is needed
     *
     * @param $vars
     * @param BaseIndex $index
     * @throws Exception
     */
    public function clearIndex($vars, BaseIndex $index)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
        }
    }

    /**
     * Index the classes for a specific index
     *
     * @param $classes
     * @param $isGroup
     * @param BaseIndex $index
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $index, $group);
        }

        return $groups;
    }

    /**
     * Index a single class for a given index. {@link static::indexClassForIndex()}
     *
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
    {
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
        $this->batchLength = DocumentFactory::config()->get('batchLength');
        $groups = (int)ceil($class::get()->count() / $this->batchLength);
        $cores = SolrCoreService::config()->get('cores') ?: 1;
        $groups = $isGroup ? ($group + $cores - 1) : $groups;
        $this->getLogger()->info(sprintf('Total groups %s', $groups));
        do { // Run from oldest to newest
            try {
                // The unittest param is from phpunit.xml.dist, meant to bypass the exit(0) call
                if (function_exists('pcntl_fork') &&
                    !Controller::curr()->getRequest()->getVar('unittest')
                ) {
                    $group = $this->spawnChildren($class, $index, $group, $cores, $groups);
                } else {
                    $this->doReindex($group, $class, $index);
                }
            } catch (Exception $error) {
                $this->logException($index->getIndexName(), $group, $error);
                $group++;
                continue;
            }
            $group++;
        } while ($group <= $groups);

        return $groups;
    }

    /**
     * Reindex the given group, for each state
     *
     * @param int $group
     * @param string $class
     * @param BaseIndex $index
     * @param bool $pcntl
     * @throws Exception
     */
    private function doReindex($group, $class, BaseIndex $index, $pcntl = false)
    {
        foreach (SiteState::getStates() as $state) {
            if ($state !== 'default' && !empty($state)) {
                SiteState::withState($state);
            }
            $this->stateReindex($group, $class, $index);
        }

        SiteState::withState(SiteState::DEFAULT_STATE);
        $this->getLogger()->info(sprintf('Indexed group %s', $group));

        if ($pcntl) {
            exit(0);

        }
    }

    /**
     * Index a group of a class for a specific state and index
     *
     * @param $group
     * @param $class
     * @param BaseIndex $index
     * @throws Exception
     */
    private function stateReindex($group, $class, BaseIndex $index): void
    {
        // Generate filtered list of local records
        $baseClass = DataObject::getSchema()->baseDataClass($class);
        /** @var DataList|DataObject[] $items */
        $items = DataObject::get($baseClass)
            ->sort('ID ASC')
            ->limit($this->batchLength, ($group * $this->batchLength));
        if ($items->count()) {
            $this->updateIndex($index, $items);
        }
    }

    /**
     * Execute the update on the client
     *
     * @param BaseIndex $index
     * @param $items
     * @throws Exception
     */
    private function updateIndex(BaseIndex $index, $items): void
    {
        $client = $index->getClient();
        $update = $client->createUpdate();
        $this->service->setInDebugMode($this->debug);
        $this->service->updateIndex($index, $items, $update);
        $update->addCommit();
        $client->update($update);
    }

    /**
     * Log an exception if it happens. Most are catched, these logs are for the developers
     * to identify problems and fix them.
     *
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function logException($index, int $group, Exception $exception): void
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            'Error indexing core %s on group %s,' . PHP_EOL .
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }

    /**
     * For each core, spawn a child process that will handle a separate group.
     * This speeds up indexing through CLI massively.
     *
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @param int $cores
     * @param int $groups
     * @return int
     * @throws Exception
     */
    private function spawnChildren($class, BaseIndex $index, int $group, int $cores, int $groups): int
    {
        $start = $group;
        $pids = [];
        // for each core, start a grouped indexing
        for ($i = 0; $i < $cores; $i++) {
            $start =  $group + $i;
            if ($start < $groups) {
                $pid = pcntl_fork();
                // PID needs to be pushed before anything else, for some reason
                $pids[$i] = $pid;
                $config = DB::getConfig();
                DB::connect($config);
                if (!$pid) {
                    $this->doReindex($start, $class, $index, true);
                }
            }
        }
        // Wait for each child to finish
        foreach ($pids as $key => $pid) {
            pcntl_waitpid($pid, $status);
            if ($status === 0) {
                unset($pids[$key]);
            }
        }

        return $start;
    }
}


1		<?php
2
3
4		namespace Firesphere\SolrSearch\Tasks;
5
6		use Exception;
7		use Firesphere\SolrSearch\Factories\DocumentFactory;
8		use Firesphere\SolrSearch\Helpers\SolrLogger;
9		use Firesphere\SolrSearch\Indexes\BaseIndex;
10		use Firesphere\SolrSearch\Services\SolrCoreService;
11		use Firesphere\SolrSearch\States\SiteState;
12		use Firesphere\SolrSearch\Traits\LoggerTrait;
13		use GuzzleHttp\Exception\GuzzleException;
14		use Psr\Log\LoggerInterface;
15		use ReflectionException;
16		use SilverStripe\Control\Controller;
17		use SilverStripe\Control\Director;
18		use SilverStripe\Control\HTTPRequest;
19		use SilverStripe\Core\Injector\Injector;
20		use SilverStripe\Dev\BuildTask;
21		use SilverStripe\Dev\Debug;
22		use SilverStripe\ORM\ArrayList;
23		use SilverStripe\ORM\DataList;
24		use SilverStripe\ORM\DataObject;
25		use SilverStripe\ORM\DB;
26		use SilverStripe\ORM\ValidationException;
27		use SilverStripe\Versioned\Versioned;
28
29		/**
30		* Class SolrIndexTask
31		*
32		* @description Index items to Solr through a tasks
33		* @package Firesphere\SolrSearch\Tasks
34		*/
35		class SolrIndexTask extends BuildTask
36		{
37		use LoggerTrait;
38		/**
39		* URLSegment of this task
40		*
41		* @var string
42		*/
43		private static $segment = 'SolrIndexTask';
44		/**
45		* Store the current states for all instances of SiteState
46		*
47		* @var array
48		*/
49		public $currentStates;
50		/**
51		* My name
52		*
53		* @var string
54		*/
55		protected $title = 'Solr Index update';
56		/**
57		* What do I do?
58		*
59		* @var string
60		*/
61		protected $description = 'Add or update documents to an existing Solr core.';
62		/**
63		* Debug mode enabled, default false
64		*
65		* @var bool
66		*/
67		protected $debug = false;
68		/**
69		* Singleton of {@link SolrCoreService}
70		*
71		* @var SolrCoreService
72		*/
73		protected $service;
74
75		/**
76		* @var int
77		*/
78		protected $batchLength = 1;
79
80		/**
81		* SolrIndexTask constructor. Sets up the document factory
82		*
83		* @throws ReflectionException
84		*/
85	14	public function __construct()
86		{
87	14	parent::__construct();
88		// Only index live items.
89		// The old FTS module also indexed Draft items. This is unnecessary
90	14	Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
91		// If versioned is needed, a separate Versioned Search module is required
92	14	$this->setService(Injector::inst()->get(SolrCoreService::class));
93	14	$this->setLogger(Injector::inst()->get(LoggerInterface::class));
94	14	$this->setDebug(Director::isDev() \|\| Director::is_cli());
95	14	$currentStates = SiteState::currentStates();
96	14	SiteState::setDefaultStates($currentStates);
97	14	}
98
99		/**
100		* Set the {@link SolrCoreService}
101		*
102		* @param SolrCoreService $service
103		* @return SolrIndexTask
104		*/
105	14	public function setService(SolrCoreService $service): SolrIndexTask
106		{
107	14	$this->service = $service;
108
109	14	return $this;
110		}
111
112		/**
113		* Set the debug mode
114		*
115		* @param bool $debug
116		* @return SolrIndexTask
117		*/
118	14	public function setDebug(bool $debug): SolrIndexTask
119		{
120	14	$this->debug = $debug;
121
122	14	return $this;
123		}
124
125		/**
126		* Implement this method in the task subclass to
127		* execute via the TaskRunner
128		*
129		* @param HTTPRequest $request
130		* @return int\|bool
131		* @throws Exception
132		* @throws GuzzleException
133		*/
134	13	public function run($request)
135		{
136	13	$start = time();
137	13	$this->getLogger()->info(date('Y-m-d H:i:s'));
138	13	list($vars, $group, $isGroup) = $this->taskSetup($request);
139	13	$groups = 0;
140	13	$indexes = $this->service->getValidIndexes($request->getVar('index'));
141
142	13	foreach ($indexes as $indexName) {
143		/** @var BaseIndex $index */
144	13	$index = Injector::inst()->get($indexName, false);
145
146	13	$indexClasses = $index->getClasses();
147	13	$classes = $this->getClasses($vars, $indexClasses);
148	13	if (!count($classes)) {
149	10	continue;
150		}
151
152	13	$this->clearIndex($vars, $index);
153
154	13	$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
155		}
156
157	13	$this->getLogger()->info(date('Y-m-d H:i:s'));
158	13	$this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));
159
160	13	return $groups;
161		}
162
163		/**
164		* Set up the requirements for this task
165		*
166		* @param HTTPRequest $request
167		* @return array
168		*/
169	13	protected function taskSetup($request): array
170		{
171	13	$vars = $request->getVars();
172	13	$this->debug = $this->debug \|\| isset($vars['debug']);
173	13	$group = $vars['group'] ?? 0;
174	13	$start = $vars['start'] ?? 0;
175	13	$group = ($start > $group) ? $start : $group;
176	13	$isGroup = isset($vars['group']);
177
178	13	return [$vars, $group, $isGroup];
179		}
180
181		/**
182		* get the classes to run for this task execution
183		*
184		* @param $vars
185		* @param array $classes
186		* @return bool\|array
187		*/
188	13	protected function getClasses($vars, array $classes): array
189		{
190	13	if (isset($vars['class'])) {
191	1	return array_intersect($classes, [$vars['class']]);
192		}
193
194	12	return $classes;
195		}
196
197		/**
198		* Clear the given index if a full re-index is needed
199		*
200		* @param $vars
201		* @param BaseIndex $index
202		* @throws Exception
203		*/
204	13	public function clearIndex($vars, BaseIndex $index)
205		{
206	13	if (!empty($vars['clear'])) {
207	1	$this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
208	1	$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
209		}
210	13	}
211
212		/**
213		* Index the classes for a specific index
214		*
215		* @param $classes
216		* @param $isGroup
217		* @param BaseIndex $index
218		* @param $group
219		* @return int
220		* @throws Exception
221		* @throws GuzzleException
222		*/
223	13	protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
224		{
225	13	$groups = 0;
226	13	foreach ($classes as $class) {
227	13	$groups = $this->indexClass($isGroup, $class, $index, $group);
228		}
229
230	13	return $groups;
231		}
232
233		/**
234		* Index a single class for a given index. {@link static::indexClassForIndex()}
235		*
236		* @param bool $isGroup
237		* @param string $class
238		* @param BaseIndex $index
239		* @param int $group
240		* @return int
241		* @throws GuzzleException
242		* @throws ValidationException
243		*/
244	13	private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
245		{
246	13	$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
247	13	$this->batchLength = DocumentFactory::config()->get('batchLength');
248	13	$groups = (int)ceil($class::get()->count() / $this->batchLength);
249	13	$cores = SolrCoreService::config()->get('cores') ?: 1;
250	13	$groups = $isGroup ? ($group + $cores - 1) : $groups;
251	13	$this->getLogger()->info(sprintf('Total groups %s', $groups));
252		do { // Run from oldest to newest
253		try {
254		// The unittest param is from phpunit.xml.dist, meant to bypass the exit(0) call
255	13	if (function_exists('pcntl_fork') &&
256	13	!Controller::curr()->getRequest()->getVar('unittest')
257		) {
258		$group = $this->spawnChildren($class, $index, $group, $cores, $groups);
259		} else {
260	13	$this->doReindex($group, $class, $index);
261		}
262		} catch (Exception $error) {
263		$this->logException($index->getIndexName(), $group, $error);
264		$group++;
265		continue;
266		}
267	13	$group++;
268	13	} while ($group <= $groups);
269
270	13	return $groups;
271		}
272
273		/**
274		* Reindex the given group, for each state
275		*
276		* @param int $group
277		* @param string $class
278		* @param BaseIndex $index
279		* @param bool $pcntl
280		* @throws Exception
281		*/
282	13	private function doReindex($group, $class, BaseIndex $index, $pcntl = false)
283		{
284	13	foreach (SiteState::getStates() as $state) {
285	13	if ($state !== 'default' && !empty($state)) {
286		SiteState::withState($state);
287		}
288	13	$this->stateReindex($group, $class, $index);
289		}
290
291	13	SiteState::withState(SiteState::DEFAULT_STATE);
292	13	$this->getLogger()->info(sprintf('Indexed group %s', $group));
293
294	13	if ($pcntl) {
295		exit(0);
		1 ignored issue – show Best Practice introduced 2019-11-24 21:19 UTC by Report Bug Copy Issue Report Using `exit` here is not recommended. In general, usage of exit should be done with care and only when running in a scripting context like a CLI script. Loading history...
296		}
297	13	}
298
299		/**
300		* Index a group of a class for a specific state and index
301		*
302		* @param $group
303		* @param $class
304		* @param BaseIndex $index
305		* @throws Exception
306		*/
307	13	private function stateReindex($group, $class, BaseIndex $index): void
308		{
309		// Generate filtered list of local records
310	13	$baseClass = DataObject::getSchema()->baseDataClass($class);
311		/** @var DataList\|DataObject[] $items */
312	13	$items = DataObject::get($baseClass)
313	13	->sort('ID ASC')
314	13	->limit($this->batchLength, ($group * $this->batchLength));
315	13	if ($items->count()) {
316	1	$this->updateIndex($index, $items);
317		}
318	13	}
319
320		/**
321		* Execute the update on the client
322		*
323		* @param BaseIndex $index
324		* @param $items
325		* @throws Exception
326		*/
327	1	private function updateIndex(BaseIndex $index, $items): void
328		{
329	1	$client = $index->getClient();
330	1	$update = $client->createUpdate();
331	1	$this->service->setInDebugMode($this->debug);
332	1	$this->service->updateIndex($index, $items, $update);
333	1	$update->addCommit();
334	1	$client->update($update);
335	1	}
336
337		/**
338		* Log an exception if it happens. Most are catched, these logs are for the developers
339		* to identify problems and fix them.
340		*
341		* @param string $index
342		* @param int $group
343		* @param Exception $exception
344		* @throws GuzzleException
345		* @throws ValidationException
346		*/
347		private function logException($index, int $group, Exception $exception): void
348		{
349		$this->getLogger()->error($exception->getMessage());
350		$msg = sprintf(
351		'Error indexing core %s on group %s,' . PHP_EOL .
352		'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
353		$index,
354		$group
355		);
356		SolrLogger::logMessage('ERROR', $msg, $index);
357		}
358
359		/**
360		* For each core, spawn a child process that will handle a separate group.
361		* This speeds up indexing through CLI massively.
362		*
363		* @param string $class
364		* @param BaseIndex $index
365		* @param int $group
366		* @param int $cores
367		* @param int $groups
368		* @return int
369		* @throws Exception
370		*/
371		private function spawnChildren($class, BaseIndex $index, int $group, int $cores, int $groups): int
372		{
373		$start = $group;
374		$pids = [];
375		// for each core, start a grouped indexing
376		for ($i = 0; $i < $cores; $i++) {
377		$start = $group + $i;
378		if ($start < $groups) {
379		$pid = pcntl_fork();
380		// PID needs to be pushed before anything else, for some reason
381		$pids[$i] = $pid;
382		$config = DB::getConfig();
383		DB::connect($config);
384		if (!$pid) {
385		$this->doReindex($start, $class, $index, true);
386		}
387		}
388		}
389		// Wait for each child to finish
390		foreach ($pids as $key => $pid) {
391		pcntl_waitpid($pid, $status);
392		if ($status === 0) {
393		unset($pids[$key]);
394		}
395		}
396
397		return $start;
398		}
399		}
400

Firesphere / silverstripe-solr-search

Push — master ( 83fab6...77ac30 )

SolrIndexTask::run() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like