SolrIndexTask::getClasses() - Code Metrics - Inspection of "Reduce memory footprint of SolrIndexTask" - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — hans/Index-all-fluent-options ( 42a725...2f1f10 )

by Simon

created 2019-09-06 07:51 UTC

SolrIndexTask::getClasses() A

↳ Parent: SolrIndexTask

Complexity

Conditions	2
Paths	2

Size

Total Lines	7
Code Lines	3

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	4
CRAP Score	2

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
eloc	3
c	1
b	0
f	0
dl	0
loc	7
ccs	4
cts	4
cp	1
rs	10
cc	2
nc	2
nop	2
crap	2

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\States\SiteState;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use GuzzleHttp\Exception\GuzzleException;
use PhpParser\Comment\Doc;
use Psr\Log\LoggerInterface;
use SilverStripe\Control\Director;
use SilverStripe\Control\HTTPRequest;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\BuildTask;
use SilverStripe\ORM\ArrayList;
use SilverStripe\ORM\DataList;
use SilverStripe\ORM\DataObject;
use SilverStripe\ORM\ValidationException;
use SilverStripe\Versioned\Versioned;

/**
 * Class SolrIndexTask
 * @description Index items to Solr through a tasks
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends BuildTask
{
    use LoggerTrait;
    /**
     * @var string
     */
    private static $segment = 'SolrIndexTask';

    /**
     * @var string
     */
    protected $title = 'Solr Index update';

    /**
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';

    /**
     * @var bool
     */
    protected $debug = false;

    /**
     * @var SolrCoreService
     */
    protected $service;

    /**
     * @var array Store the current states for all instances of SiteState
     */
    public static $currentStates;

    /**
     * SolrIndexTask constructor. Sets up the document factory
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Director::isDev() || Director::is_cli());
        self::$currentStates = SiteState::currentStates();
    }

    /**
     * @param SolrCoreService $service
     * @return SolrIndexTask
     */
    public function setService(SolrCoreService $service): SolrIndexTask
    {
        $this->service = $service;

        return $this;
    }

    /**
     * @param bool $debug
     * @return SolrIndexTask
     */
    public function setDebug(bool $debug): SolrIndexTask
    {
        $this->debug = $debug;

        return $this;
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     * @todo defer to background because it may run out of memory
     */
    public function run($request)
    {
        $startTime = time();
        [$vars, $group, $isGroup] = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);

            $indexClasses = $index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $this->clearIndex($vars, $index);

            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
        }
        $this->getLogger()->info(
            sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
        );
        // Grab the latest logs from indexing if needed
        $solrLogger = new SolrLogger();
        $solrLogger->saveSolrLog('Config');

        return $groups;
    }

    /**
     * @param HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : $group;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * Clear the given index if a full re-index is needed
     *
     * @param $vars
     * @param BaseIndex $index
     * @throws Exception
     */
    public function clearIndex($vars, BaseIndex $index)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
        }
    }

    /**
     * @param $classes
     * @param $isGroup
     * @param BaseIndex $index
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $index, $group);
        }

        return $groups;
    }

    /**
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
    {
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);

        $batchLength = DocumentFactory::config()->get('batchLength');
        $groups = (int)ceil($class::get()->count() / $batchLength);
        $groups = $isGroup ? $group : $groups;
        while ($group <= $groups) { // Run from oldest to newest
            try {
                $this->doReindex($group, $class, $batchLength, $index);
            } catch (Exception $error) {
                $this->logException($index->getIndexName(), $group, $error);
                $group++;
                continue;
            }
            $group++;
            $this->getLogger()->info(sprintf('Indexed group %s', $group));
        }

        return $groups;
    }

    /**
     * Reindex the given group, for each state
     * @param int $group
     * @param string $class
     * @param int $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function doReindex($group, $class, $batchLength, BaseIndex $index): void
    {
        foreach (SiteState::getStates() as $state) {
            if ($state !== 'default') {
                SiteState::withState($state);
            }

            // Generate filtered list of local records
            $baseClass = DataObject::getSchema()->baseDataClass($class);
            /** @var DataList|DataObject[] $items */
            $items = DataObject::get($baseClass)
                ->sort('ID ASC')
                ->limit($batchLength, ($group * $batchLength));
            if ($items->count()) {
                $client = $index->getClient();
                $update = $client->createUpdate();
                $this->updateClient($index, $items, $update, $client);
            }
        }

        // Reset the variants back to it's original state for the next round
        foreach (self::$currentStates as $variant => $value) {
            singleton($variant)->activateState($value);
        }
    }

    /**
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ValidationException
     */
    private function logException($index, int $group, Exception $exception)
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            'Error indexing core %s on group %s,' . PHP_EOL .
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }

    /**
     * @param BaseIndex $index
     * @param $items
     * @param $update
     * @param \Solarium\Core\Client\Client $client
     * @throws Exception
     */
    private function updateClient(BaseIndex $index, $items, $update, \Solarium\Core\Client\Client $client): void
    {
        $this->service->setInDebugMode($this->debug);
        $this->service->updateIndex($index, $items, $update);
        $update->addCommit();
        $client->update($update);
    }
}


1		<?php
2
3
4		namespace Firesphere\SolrSearch\Tasks;
5
6		use Exception;
7		use Firesphere\SolrSearch\Factories\DocumentFactory;
8		use Firesphere\SolrSearch\Helpers\SolrLogger;
9		use Firesphere\SolrSearch\Indexes\BaseIndex;
10		use Firesphere\SolrSearch\Services\SolrCoreService;
11		use Firesphere\SolrSearch\States\SiteState;
12		use Firesphere\SolrSearch\Traits\LoggerTrait;
13		use GuzzleHttp\Exception\GuzzleException;
14		use PhpParser\Comment\Doc;
15		use Psr\Log\LoggerInterface;
16		use SilverStripe\Control\Director;
17		use SilverStripe\Control\HTTPRequest;
18		use SilverStripe\Core\Injector\Injector;
19		use SilverStripe\Dev\BuildTask;
20		use SilverStripe\ORM\ArrayList;
21		use SilverStripe\ORM\DataList;
22		use SilverStripe\ORM\DataObject;
23		use SilverStripe\ORM\ValidationException;
24		use SilverStripe\Versioned\Versioned;
25
26		/**
27		* Class SolrIndexTask
28		* @description Index items to Solr through a tasks
29		* @package Firesphere\SolrSearch\Tasks
30		*/
31		class SolrIndexTask extends BuildTask
32		{
33		use LoggerTrait;
34		/**
35		* @var string
36		*/
37		private static $segment = 'SolrIndexTask';
38
39		/**
40		* @var string
41		*/
42		protected $title = 'Solr Index update';
43
44		/**
45		* @var string
46		*/
47		protected $description = 'Add or update documents to an existing Solr core.';
48
49		/**
50		* @var bool
51		*/
52		protected $debug = false;
53
54		/**
55		* @var SolrCoreService
56		*/
57		protected $service;
58
59		/**
60		* @var array Store the current states for all instances of SiteState
61		*/
62		public static $currentStates;
63
64		/**
65		* SolrIndexTask constructor. Sets up the document factory
66		*/
67	12	public function __construct()
68		{
69	12	parent::__construct();
70		// Only index live items.
71		// The old FTS module also indexed Draft items. This is unnecessary
72	12	Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
73	12	$this->setService(Injector::inst()->get(SolrCoreService::class));
74	12	$this->setLogger(Injector::inst()->get(LoggerInterface::class));
75	12	$this->setDebug(Director::isDev() \|\| Director::is_cli());
76	12	self::$currentStates = SiteState::currentStates();
77	12	}
78
79		/**
80		* @param SolrCoreService $service
81		* @return SolrIndexTask
82		*/
83	12	public function setService(SolrCoreService $service): SolrIndexTask
84		{
85	12	$this->service = $service;
86
87	12	return $this;
88		}
89
90		/**
91		* @param bool $debug
92		* @return SolrIndexTask
93		*/
94	12	public function setDebug(bool $debug): SolrIndexTask
95		{
96	12	$this->debug = $debug;
97
98	12	return $this;
99		}
100
101		/**
102		* Implement this method in the task subclass to
103		* execute via the TaskRunner
104		*
105		* @param HTTPRequest $request
106		* @return int\|bool
107		* @throws Exception
108		* @throws GuzzleException
109		* @todo defer to background because it may run out of memory
110		*/
111	11	public function run($request)
112		{
113	11	$startTime = time();
114	11	[$vars, $group, $isGroup] = $this->taskSetup($request);
115	11	$groups = 0;
116	11	$indexes = $this->service->getValidIndexes($request->getVar('index'));
117
118	11	foreach ($indexes as $indexName) {
119		/** @var BaseIndex $index */
120	11	$index = Injector::inst()->get($indexName, false);
121
122	11	$indexClasses = $index->getClasses();
123	11	$classes = $this->getClasses($vars, $indexClasses);
124	11	if (!count($classes)) {
125	8	continue;
126		}
127
128	11	$this->clearIndex($vars, $index);
129
130	11	$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
131		}
132	11	$this->getLogger()->info(
133	11	sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
134		);
135		// Grab the latest logs from indexing if needed
136	11	$solrLogger = new SolrLogger();
137	11	$solrLogger->saveSolrLog('Config');
138
139	11	return $groups;
140		}
141
142		/**
143		* @param HTTPRequest $request
144		* @return array
145		*/
146	11	protected function taskSetup($request): array
147		{
148	11	$vars = $request->getVars();
149	11	$this->debug = $this->debug \|\| isset($vars['debug']);
150	11	$group = $vars['group'] ?? 0;
151	11	$start = $vars['start'] ?? 0;
152	11	$group = ($start > $group) ? $start : $group;
153	11	$isGroup = isset($vars['group']);
154
155	11	return [$vars, $group, $isGroup];
156		}
157
158		/**
159		* @param $vars
160		* @param array $classes
161		* @return bool\|array
162		*/
163	11	protected function getClasses($vars, array $classes): array
164		{
165	11	if (isset($vars['class'])) {
166	1	return array_intersect($classes, [$vars['class']]);
167		}
168
169	10	return $classes;
170		}
171
172		/**
173		* Clear the given index if a full re-index is needed
174		*
175		* @param $vars
176		* @param BaseIndex $index
177		* @throws Exception
178		*/
179	11	public function clearIndex($vars, BaseIndex $index)
180		{
181	11	if (!empty($vars['clear'])) {
182	1	$this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
183	1	$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
184		}
185	11	}
186
187		/**
188		* @param $classes
189		* @param $isGroup
190		* @param BaseIndex $index
191		* @param $group
192		* @return int
193		* @throws Exception
194		* @throws GuzzleException
195		*/
196	11	protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
197		{
198	11	$groups = 0;
199	11	foreach ($classes as $class) {
200	11	$groups = $this->indexClass($isGroup, $class, $index, $group);
201		}
202
203	11	return $groups;
204		}
205
206		/**
207		* @param bool $isGroup
208		* @param string $class
209		* @param BaseIndex $index
210		* @param int $group
211		* @return int
212		* @throws GuzzleException
213		* @throws ValidationException
214		*/
215	11	private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
216		{
217	11	$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);
218
219	11	$batchLength = DocumentFactory::config()->get('batchLength');
220	11	$groups = (int)ceil($class::get()->count() / $batchLength);
221	11	$groups = $isGroup ? $group : $groups;
222	11	while ($group <= $groups) { // Run from oldest to newest
223		try {
224	11	$this->doReindex($group, $class, $batchLength, $index);
225		} catch (Exception $error) {
226		$this->logException($index->getIndexName(), $group, $error);
227		$group++;
228		continue;
229		}
230	11	$group++;
231	11	$this->getLogger()->info(sprintf('Indexed group %s', $group));
232		}
233
234	11	return $groups;
235		}
236
237		/**
238		* Reindex the given group, for each state
239		* @param int $group
240		* @param string $class
241		* @param int $batchLength
242		* @param BaseIndex $index
243		* @throws Exception
244		*/
245	11	private function doReindex($group, $class, $batchLength, BaseIndex $index): void
246		{
247	11	foreach (SiteState::getStates() as $state) {
248	11	if ($state !== 'default') {
249		SiteState::withState($state);
250		}
251
252		// Generate filtered list of local records
253	11	$baseClass = DataObject::getSchema()->baseDataClass($class);
254		/** @var DataList\|DataObject[] $items */
255	11	$items = DataObject::get($baseClass)
256	11	->sort('ID ASC')
257	11	->limit($batchLength, ($group * $batchLength));
258	11	if ($items->count()) {
259	1	$client = $index->getClient();
260	1	$update = $client->createUpdate();
261	11	$this->updateClient($index, $items, $update, $client);
262		}
263		}
264
265		// Reset the variants back to it's original state for the next round
266	11	foreach (self::$currentStates as $variant => $value) {
267		singleton($variant)->activateState($value);
268		}
269	11	}
270
271		/**
272		* @param string $index
273		* @param int $group
274		* @param Exception $exception
275		* @throws GuzzleException
276		* @throws ValidationException
277		*/
278		private function logException($index, int $group, Exception $exception)
279		{
280		$this->getLogger()->error($exception->getMessage());
281		$msg = sprintf(
282		'Error indexing core %s on group %s,' . PHP_EOL .
283		'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
284		$index,
285		$group
286		);
287		SolrLogger::logMessage('ERROR', $msg, $index);
288		}
289
290		/**
291		* @param BaseIndex $index
292		* @param $items
293		* @param $update
294		* @param \Solarium\Core\Client\Client $client
295		* @throws Exception
296		*/
297	1	private function updateClient(BaseIndex $index, $items, $update, \Solarium\Core\Client\Client $client): void
298		{
299	1	$this->service->setInDebugMode($this->debug);
300	1	$this->service->updateIndex($index, $items, $update);
301	1	$update->addCommit();
302	1	$client->update($update);
303	1	}
304		}
305

Firesphere / silverstripe-solr-search

Push — hans/Index-all-fluent-options ( 42a725...2f1f10 )

SolrIndexTask::getClasses() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like