SolrIndexTask::indexClass() - Code Metrics - Inspection of "It doesn't show the index name, of course" - Firesphere/silverstripe-solr-search - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — feature/Logging ( ed488c...abe5d9 )

by Simon

created 2019-08-24 04:58 UTC

SolrIndexTask::indexClass() A

↳ Parent: SolrIndexTask

Complexity

Conditions	4
Paths	6

Size

Total Lines	20
Code Lines	14

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	10
CRAP Score	4.3731

Importance

Changes	8
Bugs	1	Features	0

Metric	Value
eloc	14
c	8
b	1
f	0
dl	0
loc	20
ccs	10
cts	14
cp	0.7143
rs	9.7998
cc	4
nc	6
nop	4
crap	4.3731

<?php


namespace Firesphere\SolrSearch\Tasks;

use Exception;
use Firesphere\SolrSearch\Factories\DocumentFactory;
use Firesphere\SolrSearch\Helpers\SolrLogger;
use Firesphere\SolrSearch\Indexes\BaseIndex;
use Firesphere\SolrSearch\Services\SolrCoreService;
use Firesphere\SolrSearch\Traits\LoggerTrait;
use GuzzleHttp\Exception\GuzzleException;
use Psr\Log\LoggerInterface;
use SilverStripe\Control;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev;
use SilverStripe\ORM;
use SilverStripe\Versioned\Versioned;

/**
 * Class SolrIndexTask
 * @package Firesphere\SolrSearch\Tasks
 */
class SolrIndexTask extends Dev\BuildTask
{
    use LoggerTrait;
    /**
     * @var string
     */
    private static $segment = 'SolrIndexTask';

    /**
     * @var string
     */
    protected $title = 'Solr Index update';

    /**
     * @var string
     */
    protected $description = 'Add or update documents to an existing Solr core.';

    /**
     * @var bool
     */
    protected $debug = false;

    /**
     * @var SolrCoreService
     */
    protected $service;

    /**
     * SolrIndexTask constructor. Sets up the document factory
     */
    public function __construct()
    {
        parent::__construct();
        // Only index live items.
        // The old FTS module also indexed Draft items. This is unnecessary
        Versioned::set_reading_mode(Versioned::DRAFT . '.' . Versioned::LIVE);
        $this->setService(Injector::inst()->get(SolrCoreService::class));
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
        $this->setDebug(Control\Director::isDev() || Control\Director::is_cli());
    }

    /**
     * @param SolrCoreService $service
     * @return SolrIndexTask
     */
    public function setService(SolrCoreService $service): SolrIndexTask
    {
        $this->service = $service;

        return $this;
    }

    /**
     * @param bool $debug
     * @return SolrIndexTask
     */
    public function setDebug(bool $debug): SolrIndexTask
    {
        $this->debug = $debug;

        return $this;
    }

    /**
     * Implement this method in the task subclass to
     * execute via the TaskRunner
     *
     * @param Control\HTTPRequest $request
     * @return int|bool
     * @throws Exception
     * @throws GuzzleException
     * @todo defer to background because it may run out of memory
     */
    public function run($request)
    {
        $startTime = time();
        [$vars, $group, $isGroup] = $this->taskSetup($request);
        $groups = 0;
        $indexes = $this->service->getValidIndexes($request->getVar('index'));

        foreach ($indexes as $indexName) {
            /** @var BaseIndex $index */
            $index = Injector::inst()->get($indexName, false);

            $indexClasses = $index->getClasses();
            $classes = $this->getClasses($vars, $indexClasses);
            if (!count($classes)) {
                continue;
            }

            $vars = $this->clearIndex($vars, $indexName, $index);

            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
        }
        $this->getLogger()->info(
            sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
        );
        // Grab the latest logs from indexing if needed
        $solrLogger = new SolrLogger();
        $solrLogger->saveSolrLog('Config');

        return $groups;
    }

    /**
     * @param Control\HTTPRequest $request
     * @return array
     */
    protected function taskSetup($request): array
    {
        $vars = $request->getVars();
        $this->debug = $this->debug || isset($vars['debug']);
        $group = $vars['group'] ?? 0;
        $start = $vars['start'] ?? 0;
        $group = ($start > $group) ? $start : $group;
        $isGroup = isset($vars['group']);

        return [$vars, $group, $isGroup];
    }

    /**
     * @param $vars
     * @param array $classes
     * @return bool|array
     */
    protected function getClasses($vars, array $classes): array
    {
        if (isset($vars['class'])) {
            return array_intersect($classes, [$vars['class']]);
        }

        return $classes;
    }

    /**
     * @param $vars
     * @param $indexName
     * @param BaseIndex $index
     * @return mixed
     * @throws Exception
     */
    protected function clearIndex($vars, $indexName, BaseIndex $index)
    {
        if (!empty($vars['clear'])) {
            $this->getLogger()->info(sprintf('Clearing index %s', $indexName));
            $this->service->doManipulate(ORM\ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
        }

        return $vars;
    }

    /**
     * @param $classes
     * @param $isGroup
     * @param BaseIndex $index
     * @param $group
     * @return int
     * @throws Exception
     * @throws GuzzleException
     */
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
    {
        $groups = 0;
        foreach ($classes as $class) {
            $groups = $this->indexClass($isGroup, $class, $index, $group);
        }

        return $groups;
    }

    /**
     * @param bool $isGroup
     * @param string $class
     * @param BaseIndex $index
     * @param int $group
     * @return int
     * @throws GuzzleException
     * @throws ORM\ValidationException
     */
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
    {
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);

        $batchLength = DocumentFactory::config()->get('batchLength');
        $groups = (int)ceil($class::get()->count() / $batchLength);
        $groups = $isGroup ? $group : $groups;
        while ($group <= $groups) { // Run from oldest to newest
            try {
                $this->doReindex($group, $class, $batchLength, $index);
            } catch (Exception $error) {
                $this->logException($index->getIndexName(), $group, $error);
                $group++;
                continue;
            }
            $group++;
            $this->getLogger()->info(sprintf('Indexed group %s', $group));
        }

        return $groups;
    }

    /**
     * @param int $group
     * @param string $class
     * @param int $batchLength
     * @param BaseIndex $index
     * @throws Exception
     */
    private function doReindex($group, $class, $batchLength, BaseIndex $index): void
    {
        // Generate filtered list of local records
        $baseClass = ORM\DataObject::getSchema()->baseDataClass($class);
        $client = $index->getClient();
        /** @var ORM\DataList|ORM\DataObject[] $items */
        $items = ORM\DataObject::get($baseClass)
            ->sort('ID ASC')
            ->limit($batchLength, ($group * $batchLength));
        $update = $client->createUpdate();
        if ($items->count()) {
            $this->service->setInDebugMode($this->debug);
            $this->service->updateIndex($index, $items, $update);
            $update->addCommit();
            $client->update($update);
        }
    }

    /**
     * @param string $index
     * @param int $group
     * @param Exception $exception
     * @throws GuzzleException
     * @throws ORM\ValidationException
     */
    private function logException($index, int $group, Exception $exception)
    {
        $this->getLogger()->error($exception->getMessage());
        $msg = sprintf(
            "Error indexing core %s on group %s," . PHP_EOL .
            "Please log in to the CMS to find out more about Indexing errors" . PHP_EOL .
            'Last known error:',
            $index,
            $group
        );
        SolrLogger::logMessage('ERROR', $msg, $index);
    }
}


1		<?php
2
3
4		namespace Firesphere\SolrSearch\Tasks;
5
6		use Exception;
7		use Firesphere\SolrSearch\Factories\DocumentFactory;
8		use Firesphere\SolrSearch\Helpers\SolrLogger;
9		use Firesphere\SolrSearch\Indexes\BaseIndex;
10		use Firesphere\SolrSearch\Services\SolrCoreService;
11		use Firesphere\SolrSearch\Traits\LoggerTrait;
12		use GuzzleHttp\Exception\GuzzleException;
13		use Psr\Log\LoggerInterface;
14		use SilverStripe\Control;
15		use SilverStripe\Core\Injector\Injector;
16		use SilverStripe\Dev;
17		use SilverStripe\ORM;
18		use SilverStripe\Versioned\Versioned;
19
20		/**
21		* Class SolrIndexTask
22		* @package Firesphere\SolrSearch\Tasks
23		*/
24		class SolrIndexTask extends Dev\BuildTask
25		{
26		use LoggerTrait;
27		/**
28		* @var string
29		*/
30		private static $segment = 'SolrIndexTask';
31
32		/**
33		* @var string
34		*/
35		protected $title = 'Solr Index update';
36
37		/**
38		* @var string
39		*/
40		protected $description = 'Add or update documents to an existing Solr core.';
41
42		/**
43		* @var bool
44		*/
45		protected $debug = false;
46
47		/**
48		* @var SolrCoreService
49		*/
50		protected $service;
51
52		/**
53		* SolrIndexTask constructor. Sets up the document factory
54		*/
55	12	public function __construct()
56		{
57	12	parent::__construct();
58		// Only index live items.
59		// The old FTS module also indexed Draft items. This is unnecessary
60	12	Versioned::set_reading_mode(Versioned::DRAFT . '.' . Versioned::LIVE);
61	12	$this->setService(Injector::inst()->get(SolrCoreService::class));
62	12	$this->setLogger(Injector::inst()->get(LoggerInterface::class));
63	12	$this->setDebug(Control\Director::isDev() \|\| Control\Director::is_cli());
64	12	}
65
66		/**
67		* @param SolrCoreService $service
68		* @return SolrIndexTask
69		*/
70	12	public function setService(SolrCoreService $service): SolrIndexTask
71		{
72	12	$this->service = $service;
73
74	12	return $this;
75		}
76
77		/**
78		* @param bool $debug
79		* @return SolrIndexTask
80		*/
81	12	public function setDebug(bool $debug): SolrIndexTask
82		{
83	12	$this->debug = $debug;
84
85	12	return $this;
86		}
87
88		/**
89		* Implement this method in the task subclass to
90		* execute via the TaskRunner
91		*
92		* @param Control\HTTPRequest $request
93		* @return int\|bool
94		* @throws Exception
95		* @throws GuzzleException
96		* @todo defer to background because it may run out of memory
97		*/
98	11	public function run($request)
99		{
100	11	$startTime = time();
101	11	[$vars, $group, $isGroup] = $this->taskSetup($request);
102	11	$groups = 0;
103	11	$indexes = $this->service->getValidIndexes($request->getVar('index'));
104
105	11	foreach ($indexes as $indexName) {
106		/** @var BaseIndex $index */
107	11	$index = Injector::inst()->get($indexName, false);
108
109	11	$indexClasses = $index->getClasses();
110	11	$classes = $this->getClasses($vars, $indexClasses);
111	11	if (!count($classes)) {
112		continue;
113		}
114
115	11	$vars = $this->clearIndex($vars, $indexName, $index);
116
117	11	$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
118		}
119	11	$this->getLogger()->info(
120	11	sprintf('It took me %d seconds to do all the indexing%s', (time() - $startTime), PHP_EOL)
121		);
122		// Grab the latest logs from indexing if needed
123	11	$solrLogger = new SolrLogger();
124	11	$solrLogger->saveSolrLog('Config');
125
126	11	return $groups;
127		}
128
129		/**
130		* @param Control\HTTPRequest $request
131		* @return array
132		*/
133	11	protected function taskSetup($request): array
134		{
135	11	$vars = $request->getVars();
136	11	$this->debug = $this->debug \|\| isset($vars['debug']);
137	11	$group = $vars['group'] ?? 0;
138	11	$start = $vars['start'] ?? 0;
139	11	$group = ($start > $group) ? $start : $group;
140	11	$isGroup = isset($vars['group']);
141
142	11	return [$vars, $group, $isGroup];
143		}
144
145		/**
146		* @param $vars
147		* @param array $classes
148		* @return bool\|array
149		*/
150	11	protected function getClasses($vars, array $classes): array
151		{
152	11	if (isset($vars['class'])) {
153	1	return array_intersect($classes, [$vars['class']]);
154		}
155
156	10	return $classes;
157		}
158
159		/**
160		* @param $vars
161		* @param $indexName
162		* @param BaseIndex $index
163		* @return mixed
164		* @throws Exception
165		*/
166	11	protected function clearIndex($vars, $indexName, BaseIndex $index)
167		{
168	11	if (!empty($vars['clear'])) {
169	1	$this->getLogger()->info(sprintf('Clearing index %s', $indexName));
170	1	$this->service->doManipulate(ORM\ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
171		}
172
173	11	return $vars;
174		}
175
176		/**
177		* @param $classes
178		* @param $isGroup
179		* @param BaseIndex $index
180		* @param $group
181		* @return int
182		* @throws Exception
183		* @throws GuzzleException
184		*/
185	11	protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
186		{
187	11	$groups = 0;
188	11	foreach ($classes as $class) {
189	11	$groups = $this->indexClass($isGroup, $class, $index, $group);
190		}
191
192	11	return $groups;
193		}
194
195		/**
196		* @param bool $isGroup
197		* @param string $class
198		* @param BaseIndex $index
199		* @param int $group
200		* @return int
201		* @throws GuzzleException
202		* @throws ORM\ValidationException
203		*/
204	11	private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
205		{
206	11	$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()), []);
207
208	11	$batchLength = DocumentFactory::config()->get('batchLength');
209	11	$groups = (int)ceil($class::get()->count() / $batchLength);
210	11	$groups = $isGroup ? $group : $groups;
211	11	while ($group <= $groups) { // Run from oldest to newest
212		try {
213	11	$this->doReindex($group, $class, $batchLength, $index);
214		} catch (Exception $error) {
215		$this->logException($index->getIndexName(), $group, $error);
216		$group++;
217		continue;
218		}
219	11	$group++;
220	11	$this->getLogger()->info(sprintf('Indexed group %s', $group));
221		}
222
223	11	return $groups;
224		}
225
226		/**
227		* @param int $group
228		* @param string $class
229		* @param int $batchLength
230		* @param BaseIndex $index
231		* @throws Exception
232		*/
233	11	private function doReindex($group, $class, $batchLength, BaseIndex $index): void
234		{
235		// Generate filtered list of local records
236	11	$baseClass = ORM\DataObject::getSchema()->baseDataClass($class);
237	11	$client = $index->getClient();
238		/** @var ORM\DataList\|ORM\DataObject[] $items */
239	11	$items = ORM\DataObject::get($baseClass)
240	11	->sort('ID ASC')
241	11	->limit($batchLength, ($group * $batchLength));
242	11	$update = $client->createUpdate();
243	11	if ($items->count()) {
244	1	$this->service->setInDebugMode($this->debug);
245	1	$this->service->updateIndex($index, $items, $update);
246	1	$update->addCommit();
247	1	$client->update($update);
248		}
249	11	}
250
251		/**
252		* @param string $index
253		* @param int $group
254		* @param Exception $exception
255		* @throws GuzzleException
256		* @throws ORM\ValidationException
257		*/
258		private function logException($index, int $group, Exception $exception)
259		{
260		$this->getLogger()->error($exception->getMessage());
261		$msg = sprintf(
262		"Error indexing core %s on group %s," . PHP_EOL .
263		"Please log in to the CMS to find out more about Indexing errors" . PHP_EOL .
264		'Last known error:',
265		$index,
266		$group
267		);
268		SolrLogger::logMessage('ERROR', $msg, $index);
269		}
270		}
271

Firesphere / silverstripe-solr-search

Push — feature/Logging ( ed488c...abe5d9 )

SolrIndexTask::indexClass() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like