Passed
Push — hans/code-cleanup ( 7f6349 )
by Simon
05:32
created

SolrIndexTask::runChild()   A

Complexity

Conditions 3
Paths 3

Size

Total Lines 11
Code Lines 9

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 0
CRAP Score 12

Importance

Changes 1
Bugs 0 Features 0
Metric Value
cc 3
eloc 9
nc 3
nop 4
dl 0
loc 11
ccs 0
cts 0
cp 0
crap 12
rs 9.9666
c 1
b 0
f 0
1
<?php
2
3
4
namespace Firesphere\SolrSearch\Tasks;
5
6
use Exception;
7
use Firesphere\SolrSearch\Factories\DocumentFactory;
8
use Firesphere\SolrSearch\Helpers\SolrLogger;
9
use Firesphere\SolrSearch\Indexes\BaseIndex;
10
use Firesphere\SolrSearch\Models\SolrLog;
11
use Firesphere\SolrSearch\Services\SolrCoreService;
12
use Firesphere\SolrSearch\States\SiteState;
13
use Firesphere\SolrSearch\Traits\LoggerTrait;
14
use GuzzleHttp\Exception\GuzzleException;
15
use Psr\Log\LoggerInterface;
16
use ReflectionException;
17
use SilverStripe\Control\Controller;
18
use SilverStripe\Control\Director;
19
use SilverStripe\Control\HTTPRequest;
20
use SilverStripe\Core\Injector\Injector;
21
use SilverStripe\Dev\BuildTask;
22
use SilverStripe\ORM\ArrayList;
23
use SilverStripe\ORM\DataList;
24
use SilverStripe\ORM\DataObject;
25
use SilverStripe\ORM\DB;
26
use SilverStripe\ORM\ValidationException;
27
use SilverStripe\Versioned\Versioned;
28
29
/**
30
 * Class SolrIndexTask
31
 *
32
 * @description Index items to Solr through a tasks
33
 * @package Firesphere\SolrSearch\Tasks
34
 */
35
class SolrIndexTask extends BuildTask
36
{
37
    use LoggerTrait;
38
    /**
39
     * URLSegment of this task
40
     *
41
     * @var string
42
     */
43
    private static $segment = 'SolrIndexTask';
44
    /**
45
     * Store the current states for all instances of SiteState
46
     *
47
     * @var array
48
     */
49
    public $currentStates;
50
    /**
51
     * My name
52
     *
53
     * @var string
54
     */
55
    protected $title = 'Solr Index update';
56
    /**
57
     * What do I do?
58
     *
59
     * @var string
60
     */
61
    protected $description = 'Add or update documents to an existing Solr core.';
62
    /**
63
     * Debug mode enabled, default false
64
     *
65
     * @var bool
66
     */
67
    protected $debug = false;
68
    /**
69
     * Singleton of {@link SolrCoreService}
70
     *
71
     * @var SolrCoreService
72
     */
73
    protected $service;
74
75
    /**
76
     * Default batch length
77 14
     *
78
     * @var int
79 14
     */
80
    protected $batchLength = 1;
81
82 14
    /**
83 14
     * SolrIndexTask constructor. Sets up the document factory
84 14
     *
85 14
     * @throws ReflectionException
86 14
     */
87 14
    public function __construct()
88 14
    {
89
        parent::__construct();
90
        // Only index live items.
91
        // The old FTS module also indexed Draft items. This is unnecessary
92
        Versioned::set_reading_mode(Versioned::DEFAULT_MODE);
93
        // If versioned is needed, a separate Versioned Search module is required
94
        $this->setService(Injector::inst()->get(SolrCoreService::class));
95
        $this->setLogger(Injector::inst()->get(LoggerInterface::class));
96 14
        $this->setDebug(Director::isDev() || Director::is_cli());
97
        $currentStates = SiteState::currentStates();
98 14
        SiteState::setDefaultStates($currentStates);
99
    }
100 14
101
    /**
102
     * Set the {@link SolrCoreService}
103
     *
104
     * @param SolrCoreService $service
105
     * @return SolrIndexTask
106
     */
107
    public function setService(SolrCoreService $service): SolrIndexTask
108
    {
109 14
        $this->service = $service;
110
111 14
        return $this;
112
    }
113 14
114
    /**
115
     * Set the debug mode
116
     *
117
     * @param bool $debug
118
     * @return SolrIndexTask
119
     */
120
    public function setDebug(bool $debug): SolrIndexTask
121
    {
122
        $this->debug = $debug;
123
124
        return $this;
125
    }
126 13
127
    /**
128 13
     * Implement this method in the task subclass to
129 13
     * execute via the TaskRunner
130 13
     *
131 13
     * @param HTTPRequest $request
132
     * @return int|bool
133 13
     * @throws Exception
134
     * @throws GuzzleException
135 13
     */
136
    public function run($request)
137 13
    {
138 13
        $start = time();
139 13
        $this->getLogger()->info(date('Y-m-d H:i:s'));
140 10
        list($vars, $group, $isGroup) = $this->taskSetup($request);
141
        $groups = 0;
142
        $indexes = $this->service->getValidIndexes($request->getVar('index'));
143 13
144
        foreach ($indexes as $indexName) {
145 13
            /** @var BaseIndex $index */
146
            $index = Injector::inst()->get($indexName, false);
147 13
148 13
            $indexClasses = $index->getClasses();
149
            $classes = $this->getClasses($vars, $indexClasses);
150
            if (!count($classes)) {
151 13
                continue;
152 13
            }
153
154 13
            $this->clearIndex($vars, $index);
155
156
            $groups = $this->indexClassForIndex($classes, $isGroup, $index, $group);
157
        }
158
159
        $this->getLogger()->info(date('Y-m-d H:i:s'));
160
        $this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60));
161
162
        return $groups;
163 13
    }
164
165 13
    /**
166 13
     * Set up the requirements for this task
167 13
     *
168 13
     * @param HTTPRequest $request
169 13
     * @return array
170 13
     */
171
    protected function taskSetup($request): array
172 13
    {
173
        $vars = $request->getVars();
174
        $this->debug = $this->debug || isset($vars['debug']);
175
        $group = $vars['group'] ?? 0;
176
        $start = $vars['start'] ?? 0;
177
        $group = ($start > $group) ? $start : $group;
178
        $isGroup = isset($vars['group']);
179
180
        return [$vars, $group, $isGroup];
181
    }
182 13
183
    /**
184 13
     * get the classes to run for this task execution
185 1
     *
186
     * @param $vars
187
     * @param array $classes
188 12
     * @return bool|array
189
     */
190
    protected function getClasses($vars, array $classes): array
191
    {
192
        if (isset($vars['class'])) {
193
            return array_intersect($classes, [$vars['class']]);
194
        }
195
196
        return $classes;
197
    }
198 13
199
    /**
200 13
     * Clear the given index if a full re-index is needed
201 1
     *
202 1
     * @param $vars
203
     * @param BaseIndex $index
204 13
     * @throws Exception
205
     */
206
    public function clearIndex($vars, BaseIndex $index)
207
    {
208
        if (!empty($vars['clear'])) {
209
            $this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName()));
210
            $this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index);
211
        }
212
    }
213
214
    /**
215
     * Index the classes for a specific index
216
     *
217 13
     * @param $classes
218
     * @param $isGroup
219 13
     * @param BaseIndex $index
220 13
     * @param $group
221 13
     * @return int
222
     * @throws Exception
223
     * @throws GuzzleException
224 13
     */
225
    protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int
226
    {
227
        $groups = 0;
228
        foreach ($classes as $class) {
229
            $groups = $this->indexClass($isGroup, $class, $index, $group);
230
        }
231
232
        return $groups;
233
    }
234
235
    /**
236
     * Index a single class for a given index. {@link static::indexClassForIndex()}
237
     *
238 13
     * @param bool $isGroup
239
     * @param string $class
240 13
     * @param BaseIndex $index
241
     * @param int $group
242 13
     * @return int
243 13
     * @throws GuzzleException
244 13
     * @throws ValidationException
245 13
     */
246
    private function indexClass($isGroup, $class, BaseIndex $index, int $group): int
247 13
    {
248
        $this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName()));
249
        $this->batchLength = DocumentFactory::config()->get('batchLength');
250
        $totalGroups = (int)ceil($class::get()->count() / $this->batchLength);
251
        $cores = SolrCoreService::config()->get('cpucores') ?: 1;
252
        $groups = $isGroup ? ($group + $cores - 1) : $totalGroups;
253 13
        $this->getLogger()->info(sprintf('Total groups %s', $totalGroups));
254 13
        do { // Run from oldest to newest
255
            try {
256
                // The unittest param is from phpunit.xml.dist, meant to bypass the exit(0) call
257 13
                if (function_exists('pcntl_fork') &&
258
                    !Controller::curr()->getRequest()->getVar('unittest')
259
                ) {
260
                    $group = $this->spawnChildren($class, $index, $group, $cores, $groups);
261
                } else {
262
                    $this->doReindex($group, $class, $index);
263
                }
264
            } catch (Exception $error) {
265
                $this->logException($index->getIndexName(), $group, $error);
266
                $group++;
267
                continue;
268
            }
269 13
            $group++;
270
        } while ($group <= $groups);
271 13
272 13
        return $totalGroups;
273
    }
274
275 13
    /**
276
     * For each core, spawn a child process that will handle a separate group.
277
     * This speeds up indexing through CLI massively.
278 13
     *
279 13
     * @param string $class
280
     * @param BaseIndex $index
281
     * @param int $group
282
     * @param int $cores
283
     * @param int $groups
284
     * @return int
285
     * @throws Exception
286
     * @throws GuzzleException
287
     */
288
    private function spawnChildren($class, BaseIndex $index, int $group, int $cores, int $groups): int
289
    {
290 13
        $start = $group;
291
        $pids = [];
292
        // for each core, start a grouped indexing
293 13
        for ($i = 0; $i < $cores; $i++) {
294
            $start = $group + $i;
295 13
            if ($start < $groups) {
296 13
                $this->runForkedChild($class, $index, $pids, $i, $start);
297 13
            }
298 13
        }
299 1
        // Wait for each child to finish
300
        foreach ($pids as $key => $pid) {
301 13
            pcntl_waitpid($pid, $status);
302
            if ($status === 0) {
303
                unset($pids[$key]);
304
            }
305
        }
306
        $commit = $index->getClient()->createUpdate();
307
        $commit->addCommit();
308
309
        $index->getClient()->update($commit);
310 1
311
        return $start;
312 1
    }
313 1
314 1
    /**
315 1
     * Reindex the given group, for each state
316 1
     *
317 1
     * @param int $group
318 1
     * @param string $class
319
     * @param BaseIndex $index
320
     * @param bool $pcntl
321
     * @throws Exception
322
     */
323
    private function doReindex($group, $class, BaseIndex $index, $pcntl = false)
324
    {
325
        foreach (SiteState::getStates() as $state) {
326
            if ($state !== 'default' && !empty($state)) {
327
                SiteState::withState($state);
328
            }
329
            $this->stateReindex($group, $class, $index);
330
        }
331
332
        SiteState::withState(SiteState::DEFAULT_STATE);
333
        $this->getLogger()->info(sprintf('Indexed group %s', $group));
334
335
        if ($pcntl) {
336
            exit(0);
1 ignored issue
show
Best Practice introduced by
Using exit here is not recommended.

In general, usage of exit should be done with care and only when running in a scripting context like a CLI script.

Loading history...
337
        }
338
    }
339
340
    /**
341
     * Index a group of a class for a specific state and index
342
     *
343
     * @param $group
344
     * @param $class
345
     * @param BaseIndex $index
346
     * @throws Exception
347
     */
348
    private function stateReindex($group, $class, BaseIndex $index): void
349
    {
350
        // Generate filtered list of local records
351
        $baseClass = DataObject::getSchema()->baseDataClass($class);
352
        /** @var DataList|DataObject[] $items */
353
        $items = DataObject::get($baseClass)
354
            ->sort('ID ASC')
355
            ->limit($this->batchLength, ($group * $this->batchLength));
356
        if ($items->count()) {
357
            $this->updateIndex($index, $items);
358
        }
359
    }
360
361
    /**
362
     * Execute the update on the client
363
     *
364
     * @param BaseIndex $index
365
     * @param $items
366
     * @throws Exception
367
     */
368
    private function updateIndex(BaseIndex $index, $items): void
369
    {
370
        $client = $index->getClient();
371
        $update = $client->createUpdate();
372
        $this->service->setInDebugMode($this->debug);
373
        $this->service->updateIndex($index, $items, $update);
374
        $client->update($update);
375
    }
376
377
    /**
378
     * Log an exception if it happens. Most are catched, these logs are for the developers
379
     * to identify problems and fix them.
380
     *
381
     * @param string $index
382
     * @param int $group
383
     * @param Exception $exception
384
     * @throws GuzzleException
385
     * @throws ValidationException
386
     */
387
    private function logException($index, int $group, Exception $exception): void
388
    {
389
        $this->getLogger()->error($exception->getMessage());
390
        $msg = sprintf(
391
            'Error indexing core %s on group %s,' . PHP_EOL .
392
            'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL,
393
            $index,
394
            $group
395
        );
396
        SolrLogger::logMessage('ERROR', $msg, $index);
397
    }
398
399
    /**
400
     * @param $class
401
     * @param BaseIndex $index
402
     * @param int $pid
403
     * @param int $start
404
     * @throws GuzzleException
405
     * @throws ValidationException
406
     * @throws Exception
407
     */
408
    private function runChild($class, BaseIndex $index, int $pid, int $start): void
409
    {
410
        if (!$pid) {
411
            try {
412
                $this->doReindex($start, $class, $index, true);
413
            } catch (Exception $e) {
414
                SolrLogger::logMessage('ERROR', $e, $index->getIndexName());
415
                throw new Exception(
416
                    sprintf(
417
                        'Something went wrong while indexing %s, see the logs for details',
418
                        $start
419
                    )
420
                );
421
            }
422
        }
423
    }
424
425
    /**
426
     * @param $class
427
     * @param BaseIndex $index
428
     * @param array $pids
429
     * @param int $i
430
     * @param int $start
431
     * @return void
432
     * @throws GuzzleException
433
     * @throws ValidationException
434
     */
435
    private function runForkedChild($class, BaseIndex $index, array &$pids, int $i, int $start): void
436
    {
437
        $pid = pcntl_fork();
438
        // PID needs to be pushed before anything else, for some reason
439
        $pids[$i] = $pid;
440
        $config = DB::getConfig();
441
        DB::connect($config);
442
        $this->runChild($class, $index, $pid, $start);
443
    }
444
}
445