SolrIndexTask::spawnChildren() - Code Metrics - Inspection of "Synonym made easier" - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — hans/its-the-same ( a633db )

by Simon

created 2019-11-25 02:46 UTC

SolrIndexTask::spawnChildren() A

↳ Parent: SolrIndexTask

Complexity

Conditions	6
Paths	9

Size

Total Lines	22
Code Lines	12

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	6
eloc	12
c	0
b	0
f	0
nc	9
nop	5
dl	0
loc	22
rs	9.2222

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\States\SiteState;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use GuzzleHttp\Exception\GuzzleException;
use Psr\Log\LoggerInterface;
use ReflectionException;
use SilverStripe\Control\Controller;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\ORM\ArrayList;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\DB;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Versioned\Versioned;

/**
 * Class SolrIndexTask
 *
 * @description Index items to Solr through a tasks
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends BuildTask
{
    use LoggerTrait;
    /**
     * URLSegment of this task
     *
     * @var string
     */
    private static $segment = 'SolrIndexTask';
    /**
     * Store the current states for all instances of SiteState
     *
     * @var array
     */
    public $currentStates;
    /**
     * My name
     *
     * @var string
     */
    protected $title = 'Solr Index update';
    /**
     * What do I do?
     *
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';
    /**
     * Debug mode enabled, default false
     *
     * @var bool
     */
    protected $debug = false;
    /**
     * Singleton of {@link SolrCoreService}
     *
     * @var SolrCoreService
     */
    protected $service;

    /**
     * SolrIndexTask constructor. Sets up the document factory
     *
     * @throws ReflectionException
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
        // If versioned is needed, a separate Versioned Search module is required
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Director::isDev() || Director::is_cli());
        $currentStates = SiteState::currentStates();
        SiteState::setDefaultStates($currentStates);
    }

    /**
     * Set the {@link SolrCoreService}
     *
     * @param SolrCoreService $service
     * @return SolrIndexTask
     */
    public function setService(SolrCoreService $service): SolrIndexTask
    {
        $this->service = $service;

        return $this;
    }

    /**
     * Set the debug mode
     *
     * @param bool $debug
     * @return SolrIndexTask
     */
    public function setDebug(bool $debug): SolrIndexTask
    {
        $this->debug = $debug;

        return $this;
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     * @todo defer to background because it may run out of memory
     */
    public function run($request)
    {
        $start = time();
        $this->getLogger()->info(date('Y-m-d H:i:s'));
        list($vars, $group, $isGroup) = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);

            $indexClasses = $index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $this->clearIndex($vars, $index);

            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
        }

        $this->getLogger()->info(date('Y-m-d H:i:s'));
        $this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));

        return $groups;
    }

    /**
     * Set up the requirements for this task
     *
     * @param HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : $group;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * get the classes to run for this task execution
     *
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * Clear the given index if a full re-index is needed
     *
     * @param $vars
     * @param BaseIndex $index
     * @throws Exception
     */
    public function clearIndex($vars, BaseIndex $index)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
        }
    }

    /**
     * Index the classes for a specific index
     *
     * @param $classes
     * @param $isGroup
     * @param BaseIndex $index
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $index, $group);
        }

        return $groups;
    }

    /**
     * Index a single class for a given index. {@link static::indexClassForIndex()}
     *
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
    {
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
        $batchLength = DocumentFactory::config()->get('batchLength');
        $groups = (int)ceil($class::get()->count() / $batchLength);
        $groups = $isGroup ? $group : $groups;
        $cores = SolrCoreService::config()->get('cores') ?: 1;
        $this->getLogger()->info(sprintf('Total groups %s', $groups));
        do { // Run from oldest to newest
            try {
                // The unittest param is from phpunit.xml.dist, meant to bypass the exit(0) call
                if (function_exists('pcntl_fork') &&
                    !Controller::curr()->getRequest()->getVar('unittest')) {
                    // for each core, start a grouped indexing
                    $group = $this->spawnChildren($class, $index, $group, $cores, $groups);
                } else {
                    $this->doReindex($group, $class, $index);
                }
            } catch (Exception $error) {
                $this->logException($index->getIndexName(), $group, $error);
                $group++;
                continue;
            }
            $group++;
        } while ($group <= $groups);

        return $groups;
    }

    /**
     * Reindex the given group, for each state
     *
     * @param int $group
     * @param string $class
     * @param int $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function doReindex($group, $class, BaseIndex $index, $pcntl = false)
    {
        if ($pcntl) {
            $config = DB::getConfig();
            DB::connect($config);
        }
        foreach (SiteState::getStates() as $state) {
            if ($state !== 'default' && !empty($state)) {
                SiteState::withState($state);
            }
            $this->stateReindex($group, $class, $index);
        }

        SiteState::withState(SiteState::DEFAULT_STATE);
        $this->getLogger()->info(sprintf('Indexed group %s', $group));

        if ($pcntl) {
            exit(0);

        }
    }

    /**
     * Index a group of a class for a specific state and index
     *
     * @param $group
     * @param $class
     * @param $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function stateReindex($group, $class, BaseIndex $index): void
    {
        $batchLength = DocumentFactory::config()->get('batchLength');
        // Generate filtered list of local records
        $baseClass = DataObject::getSchema()->baseDataClass($class);
        /** @var DataList|DataObject[] $items */
        $items = DataObject::get($baseClass)
            ->sort('ID ASC')
            ->limit($batchLength, ($group * $batchLength));
        if ($items->count()) {
            $this->updateIndex($index, $items);
        }
    }

    /**
     * Execute the update on the client
     *
     * @param BaseIndex $index
     * @param $items
     * @throws Exception
     */
    private function updateIndex(BaseIndex $index, $items): void
    {
        $client = $index->getClient();
        $update = $client->createUpdate();
        $this->service->setInDebugMode($this->debug);
        $this->service->updateIndex($index, $items, $update);
        $update->addCommit();
        $client->update($update);
    }

    /**
     * Log an exception if it happens. Most are catched, these logs are for the developers
     * to identify problems and fix them.
     *
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function logException($index, int $group, Exception $exception): void
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            'Error indexing core %s on group %s,' . PHP_EOL .
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }

    /**
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @param int $cores
     * @param int $groups
     * @return int
     * @throws Exception
     */
    private function spawnChildren($class, BaseIndex $index, int $group, int $cores, int $groups): int
    {
        $pids = [];
        $start = $group;
        for ($i = 0; $i < $cores; $i++) {
            $pid = pcntl_fork();
            // PID needs to be pushed before anything else, for some reason
            $pids[$i] = $pid;
            $start = $group + $i;
            if (!$pid && $start <= $groups) {
                $this->doReindex($group + $i, $class, $index, true);
            }
        }

        // Wait for each child to finish
        foreach ($pids as $pid) {
            if ($pid) {
                pcntl_waitpid($pid, $status);
            }
        }

        return $start;
    }
}


1			<?php
2
3
4			namespace Firesphere\SolrSearch\Tasks;
5
6			use Exception;
7			use Firesphere\SolrSearch\Factories\DocumentFactory;
8			use Firesphere\SolrSearch\Helpers\SolrLogger;
9			use Firesphere\SolrSearch\Indexes\BaseIndex;
10			use Firesphere\SolrSearch\Services\SolrCoreService;
11			use Firesphere\SolrSearch\States\SiteState;
12			use Firesphere\SolrSearch\Traits\LoggerTrait;
13			use GuzzleHttp\Exception\GuzzleException;
14			use Psr\Log\LoggerInterface;
15			use ReflectionException;
16			use SilverStripe\Control\Controller;
17			use SilverStripe\Control\Director;
18			use SilverStripe\Control\HTTPRequest;
19			use SilverStripe\Core\Injector\Injector;
20			use SilverStripe\Dev\BuildTask;
21			use SilverStripe\ORM\ArrayList;
22			use SilverStripe\ORM\DataList;
23			use SilverStripe\ORM\DataObject;
24			use SilverStripe\ORM\DB;
25			use SilverStripe\ORM\ValidationException;
26			use SilverStripe\Versioned\Versioned;
27
28			/**
29			* Class SolrIndexTask
30			*
31			* @description Index items to Solr through a tasks
32			* @package Firesphere\SolrSearch\Tasks
33			*/
34			class SolrIndexTask extends BuildTask
35			{
36			use LoggerTrait;
37			/**
38			* URLSegment of this task
39			*
40			* @var string
41			*/
42			private static $segment = 'SolrIndexTask';
43			/**
44			* Store the current states for all instances of SiteState
45			*
46			* @var array
47			*/
48			public $currentStates;
49			/**
50			* My name
51			*
52			* @var string
53			*/
54			protected $title = 'Solr Index update';
55			/**
56			* What do I do?
57			*
58			* @var string
59			*/
60			protected $description = 'Add or update documents to an existing Solr core.';
61			/**
62			* Debug mode enabled, default false
63			*
64			* @var bool
65			*/
66			protected $debug = false;
67			/**
68			* Singleton of {@link SolrCoreService}
69			*
70			* @var SolrCoreService
71			*/
72			protected $service;
73
74			/**
75			* SolrIndexTask constructor. Sets up the document factory
76			*
77			* @throws ReflectionException
78			*/
79			public function __construct()
80			{
81			parent::__construct();
82			// Only index live items.
83			// The old FTS module also indexed Draft items. This is unnecessary
84			Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
85			// If versioned is needed, a separate Versioned Search module is required
86			$this->setService(Injector::inst()->get(SolrCoreService::class));
87			$this->setLogger(Injector::inst()->get(LoggerInterface::class));
88			$this->setDebug(Director::isDev() \|\| Director::is_cli());
89			$currentStates = SiteState::currentStates();
90			SiteState::setDefaultStates($currentStates);
91			}
92
93			/**
94			* Set the {@link SolrCoreService}
95			*
96			* @param SolrCoreService $service
97			* @return SolrIndexTask
98			*/
99			public function setService(SolrCoreService $service): SolrIndexTask
100			{
101			$this->service = $service;
102
103			return $this;
104			}
105
106			/**
107			* Set the debug mode
108			*
109			* @param bool $debug
110			* @return SolrIndexTask
111			*/
112			public function setDebug(bool $debug): SolrIndexTask
113			{
114			$this->debug = $debug;
115
116			return $this;
117			}
118
119			/**
120			* Implement this method in the task subclass to
121			* execute via the TaskRunner
122			*
123			* @param HTTPRequest $request
124			* @return int\|bool
125			* @throws Exception
126			* @throws GuzzleException
127			* @todo defer to background because it may run out of memory
128			*/
129			public function run($request)
130			{
131			$start = time();
132			$this->getLogger()->info(date('Y-m-d H:i:s'));
133			list($vars, $group, $isGroup) = $this->taskSetup($request);
134			$groups = 0;
135			$indexes = $this->service->getValidIndexes($request->getVar('index'));
136
137			foreach ($indexes as $indexName) {
138			/** @var BaseIndex $index */
139			$index = Injector::inst()->get($indexName, false);
140
141			$indexClasses = $index->getClasses();
142			$classes = $this->getClasses($vars, $indexClasses);
143			if (!count($classes)) {
144			continue;
145			}
146
147			$this->clearIndex($vars, $index);
148
149			$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
150			}
151
152			$this->getLogger()->info(date('Y-m-d H:i:s'));
153			$this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));
154
155			return $groups;
156			}
157
158			/**
159			* Set up the requirements for this task
160			*
161			* @param HTTPRequest $request
162			* @return array
163			*/
164			protected function taskSetup($request): array
165			{
166			$vars = $request->getVars();
167			$this->debug = $this->debug \|\| isset($vars['debug']);
168			$group = $vars['group'] ?? 0;
169			$start = $vars['start'] ?? 0;
170			$group = ($start > $group) ? $start : $group;
171			$isGroup = isset($vars['group']);
172
173			return [$vars, $group, $isGroup];
174			}
175
176			/**
177			* get the classes to run for this task execution
178			*
179			* @param $vars
180			* @param array $classes
181			* @return bool\|array
182			*/
183			protected function getClasses($vars, array $classes): array
184			{
185			if (isset($vars['class'])) {
186			return array_intersect($classes, [$vars['class']]);
187			}
188
189			return $classes;
190			}
191
192			/**
193			* Clear the given index if a full re-index is needed
194			*
195			* @param $vars
196			* @param BaseIndex $index
197			* @throws Exception
198			*/
199			public function clearIndex($vars, BaseIndex $index)
200			{
201			if (!empty($vars['clear'])) {
202			$this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
203			$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
204			}
205			}
206
207			/**
208			* Index the classes for a specific index
209			*
210			* @param $classes
211			* @param $isGroup
212			* @param BaseIndex $index
213			* @param $group
214			* @return int
215			* @throws Exception
216			* @throws GuzzleException
217			*/
218			protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
219			{
220			$groups = 0;
221			foreach ($classes as $class) {
222			$groups = $this->indexClass($isGroup, $class, $index, $group);
223			}
224
225			return $groups;
226			}
227
228			/**
229			* Index a single class for a given index. {@link static::indexClassForIndex()}
230			*
231			* @param bool $isGroup
232			* @param string $class
233			* @param BaseIndex $index
234			* @param int $group
235			* @return int
236			* @throws GuzzleException
237			* @throws ValidationException
238			*/
239			private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
240			{
241			$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
242			$batchLength = DocumentFactory::config()->get('batchLength');
243			$groups = (int)ceil($class::get()->count() / $batchLength);
244			$groups = $isGroup ? $group : $groups;
245			$cores = SolrCoreService::config()->get('cores') ?: 1;
246			$this->getLogger()->info(sprintf('Total groups %s', $groups));
247			do { // Run from oldest to newest
248			try {
249			// The unittest param is from phpunit.xml.dist, meant to bypass the exit(0) call
250			if (function_exists('pcntl_fork') &&
251			!Controller::curr()->getRequest()->getVar('unittest')) {
252			// for each core, start a grouped indexing
253			$group = $this->spawnChildren($class, $index, $group, $cores, $groups);
254			} else {
255			$this->doReindex($group, $class, $index);
256			}
257			} catch (Exception $error) {
258			$this->logException($index->getIndexName(), $group, $error);
259			$group++;
260			continue;
261			}
262			$group++;
263			} while ($group <= $groups);
264
265			return $groups;
266			}
267
268			/**
269			* Reindex the given group, for each state
270			*
271			* @param int $group
272			* @param string $class
273			* @param int $batchLength
274			* @param BaseIndex $index
275			* @throws Exception
276			*/
277			private function doReindex($group, $class, BaseIndex $index, $pcntl = false)
278			{
279			if ($pcntl) {
280			$config = DB::getConfig();
281			DB::connect($config);
282			}
283			foreach (SiteState::getStates() as $state) {
284			if ($state !== 'default' && !empty($state)) {
285			SiteState::withState($state);
286			}
287			$this->stateReindex($group, $class, $index);
288			}
289
290			SiteState::withState(SiteState::DEFAULT_STATE);
291			$this->getLogger()->info(sprintf('Indexed group %s', $group));
292
293			if ($pcntl) {
294			exit(0);
			1 ignored issue – show Best Practice introduced 2019-11-24 21:19 UTC by Report Bug Copy Issue Report Using `exit` here is not recommended. In general, usage of exit should be done with care and only when running in a scripting context like a CLI script. Loading history...
295			}
296			}
297
298			/**
299			* Index a group of a class for a specific state and index
300			*
301			* @param $group
302			* @param $class
303			* @param $batchLength
304			* @param BaseIndex $index
305			* @throws Exception
306			*/
307			private function stateReindex($group, $class, BaseIndex $index): void
308			{
309			$batchLength = DocumentFactory::config()->get('batchLength');
310			// Generate filtered list of local records
311			$baseClass = DataObject::getSchema()->baseDataClass($class);
312			/** @var DataList\|DataObject[] $items */
313			$items = DataObject::get($baseClass)
314			->sort('ID ASC')
315			->limit($batchLength, ($group * $batchLength));
316			if ($items->count()) {
317			$this->updateIndex($index, $items);
318			}
319			}
320
321			/**
322			* Execute the update on the client
323			*
324			* @param BaseIndex $index
325			* @param $items
326			* @throws Exception
327			*/
328			private function updateIndex(BaseIndex $index, $items): void
329			{
330			$client = $index->getClient();
331			$update = $client->createUpdate();
332			$this->service->setInDebugMode($this->debug);
333			$this->service->updateIndex($index, $items, $update);
334			$update->addCommit();
335			$client->update($update);
336			}
337
338			/**
339			* Log an exception if it happens. Most are catched, these logs are for the developers
340			* to identify problems and fix them.
341			*
342			* @param string $index
343			* @param int $group
344			* @param Exception $exception
345			* @throws GuzzleException
346			* @throws ValidationException
347			*/
348			private function logException($index, int $group, Exception $exception): void
349			{
350			$this->getLogger()->error($exception->getMessage());
351			$msg = sprintf(
352			'Error indexing core %s on group %s,' . PHP_EOL .
353			'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
354			$index,
355			$group
356			);
357			SolrLogger::logMessage('ERROR', $msg, $index);
358			}
359
360			/**
361			* @param string $class
362			* @param BaseIndex $index
363			* @param int $group
364			* @param int $cores
365			* @param int $groups
366			* @return int
367			* @throws Exception
368			*/
369			private function spawnChildren($class, BaseIndex $index, int $group, int $cores, int $groups): int
370			{
371			$pids = [];
372			$start = $group;
373			for ($i = 0; $i < $cores; $i++) {
374			$pid = pcntl_fork();
375			// PID needs to be pushed before anything else, for some reason
376			$pids[$i] = $pid;
377			$start = $group + $i;
378			if (!$pid && $start <= $groups) {
379			$this->doReindex($group + $i, $class, $index, true);
380			}
381			}
382
383			// Wait for each child to finish
384			foreach ($pids as $pid) {
385			if ($pid) {
386			pcntl_waitpid($pid, $status);
387			}
388			}
389
390			return $start;
391			}
392			}
393

Firesphere / silverstripe-solr-search

Push — hans/its-the-same ( a633db )

SolrIndexTask::spawnChildren() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like