1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
|
4
|
|
|
namespace Firesphere\SolrSearch\Tasks; |
5
|
|
|
|
6
|
|
|
use Exception; |
7
|
|
|
use Firesphere\SolrSearch\Factories\DocumentFactory; |
8
|
|
|
use Firesphere\SolrSearch\Helpers\SolrLogger; |
9
|
|
|
use Firesphere\SolrSearch\Indexes\BaseIndex; |
10
|
|
|
use Firesphere\SolrSearch\Models\SolrLog; |
11
|
|
|
use Firesphere\SolrSearch\Services\SolrCoreService; |
12
|
|
|
use Firesphere\SolrSearch\States\SiteState; |
13
|
|
|
use Firesphere\SolrSearch\Traits\LoggerTrait; |
14
|
|
|
use GuzzleHttp\Exception\GuzzleException; |
15
|
|
|
use Psr\Log\LoggerInterface; |
16
|
|
|
use ReflectionException; |
17
|
|
|
use SilverStripe\Control\Controller; |
18
|
|
|
use SilverStripe\Control\Director; |
19
|
|
|
use SilverStripe\Control\HTTPRequest; |
20
|
|
|
use SilverStripe\Core\Injector\Injector; |
21
|
|
|
use SilverStripe\Dev\BuildTask; |
22
|
|
|
use SilverStripe\ORM\ArrayList; |
23
|
|
|
use SilverStripe\ORM\DataList; |
24
|
|
|
use SilverStripe\ORM\DataObject; |
25
|
|
|
use SilverStripe\ORM\DB; |
26
|
|
|
use SilverStripe\ORM\ValidationException; |
27
|
|
|
use SilverStripe\Versioned\Versioned; |
28
|
|
|
|
29
|
|
|
/** |
30
|
|
|
* Class SolrIndexTask |
31
|
|
|
* |
32
|
|
|
* @description Index items to Solr through a tasks |
33
|
|
|
* @package Firesphere\SolrSearch\Tasks |
34
|
|
|
*/ |
35
|
|
|
class SolrIndexTask extends BuildTask |
36
|
|
|
{ |
37
|
|
|
use LoggerTrait; |
38
|
|
|
/** |
39
|
|
|
* URLSegment of this task |
40
|
|
|
* |
41
|
|
|
* @var string |
42
|
|
|
*/ |
43
|
|
|
private static $segment = 'SolrIndexTask'; |
44
|
|
|
/** |
45
|
|
|
* Store the current states for all instances of SiteState |
46
|
|
|
* |
47
|
|
|
* @var array |
48
|
|
|
*/ |
49
|
|
|
public $currentStates; |
50
|
|
|
/** |
51
|
|
|
* My name |
52
|
|
|
* |
53
|
|
|
* @var string |
54
|
|
|
*/ |
55
|
|
|
protected $title = 'Solr Index update'; |
56
|
|
|
/** |
57
|
|
|
* What do I do? |
58
|
|
|
* |
59
|
|
|
* @var string |
60
|
|
|
*/ |
61
|
|
|
protected $description = 'Add or update documents to an existing Solr core.'; |
62
|
|
|
/** |
63
|
|
|
* Debug mode enabled, default false |
64
|
|
|
* |
65
|
|
|
* @var bool |
66
|
|
|
*/ |
67
|
|
|
protected $debug = false; |
68
|
|
|
/** |
69
|
|
|
* Singleton of {@link SolrCoreService} |
70
|
|
|
* |
71
|
|
|
* @var SolrCoreService |
72
|
|
|
*/ |
73
|
|
|
protected $service; |
74
|
|
|
|
75
|
|
|
/** |
76
|
|
|
* Default batch length |
77
|
14 |
|
* |
78
|
|
|
* @var int |
79
|
14 |
|
*/ |
80
|
|
|
protected $batchLength = 1; |
81
|
|
|
|
82
|
14 |
|
/** |
83
|
14 |
|
* SolrIndexTask constructor. Sets up the document factory |
84
|
14 |
|
* |
85
|
14 |
|
* @throws ReflectionException |
86
|
14 |
|
*/ |
87
|
14 |
|
public function __construct() |
88
|
14 |
|
{ |
89
|
|
|
parent::__construct(); |
90
|
|
|
// Only index live items. |
91
|
|
|
// The old FTS module also indexed Draft items. This is unnecessary |
92
|
|
|
Versioned::set_reading_mode(Versioned::DEFAULT_MODE); |
93
|
|
|
// If versioned is needed, a separate Versioned Search module is required |
94
|
|
|
$this->setService(Injector::inst()->get(SolrCoreService::class)); |
95
|
|
|
$this->setLogger(Injector::inst()->get(LoggerInterface::class)); |
96
|
14 |
|
$this->setDebug(Director::isDev() || Director::is_cli()); |
97
|
|
|
$currentStates = SiteState::currentStates(); |
98
|
14 |
|
SiteState::setDefaultStates($currentStates); |
99
|
|
|
} |
100
|
14 |
|
|
101
|
|
|
/** |
102
|
|
|
* Set the {@link SolrCoreService} |
103
|
|
|
* |
104
|
|
|
* @param SolrCoreService $service |
105
|
|
|
* @return SolrIndexTask |
106
|
|
|
*/ |
107
|
|
|
public function setService(SolrCoreService $service): SolrIndexTask |
108
|
|
|
{ |
109
|
14 |
|
$this->service = $service; |
110
|
|
|
|
111
|
14 |
|
return $this; |
112
|
|
|
} |
113
|
14 |
|
|
114
|
|
|
/** |
115
|
|
|
* Set the debug mode |
116
|
|
|
* |
117
|
|
|
* @param bool $debug |
118
|
|
|
* @return SolrIndexTask |
119
|
|
|
*/ |
120
|
|
|
public function setDebug(bool $debug): SolrIndexTask |
121
|
|
|
{ |
122
|
|
|
$this->debug = $debug; |
123
|
|
|
|
124
|
|
|
return $this; |
125
|
|
|
} |
126
|
13 |
|
|
127
|
|
|
/** |
128
|
13 |
|
* Implement this method in the task subclass to |
129
|
13 |
|
* execute via the TaskRunner |
130
|
13 |
|
* |
131
|
13 |
|
* @param HTTPRequest $request |
132
|
|
|
* @return int|bool |
133
|
13 |
|
* @throws Exception |
134
|
|
|
* @throws GuzzleException |
135
|
13 |
|
*/ |
136
|
|
|
public function run($request) |
137
|
13 |
|
{ |
138
|
13 |
|
$start = time(); |
139
|
13 |
|
$this->getLogger()->info(date('Y-m-d H:i:s')); |
140
|
10 |
|
list($vars, $group, $isGroup) = $this->taskSetup($request); |
141
|
|
|
$groups = 0; |
142
|
|
|
$indexes = $this->service->getValidIndexes($request->getVar('index')); |
143
|
13 |
|
|
144
|
|
|
foreach ($indexes as $indexName) { |
145
|
13 |
|
/** @var BaseIndex $index */ |
146
|
|
|
$index = Injector::inst()->get($indexName, false); |
147
|
13 |
|
|
148
|
13 |
|
$indexClasses = $index->getClasses(); |
149
|
|
|
$classes = $this->getClasses($vars, $indexClasses); |
150
|
|
|
if (!count($classes)) { |
151
|
13 |
|
continue; |
152
|
13 |
|
} |
153
|
|
|
|
154
|
13 |
|
$this->clearIndex($vars, $index); |
155
|
|
|
|
156
|
|
|
$groups = $this->indexClassForIndex($classes, $isGroup, $index, $group); |
157
|
|
|
} |
158
|
|
|
|
159
|
|
|
$this->getLogger()->info(date('Y-m-d H:i:s')); |
160
|
|
|
$this->getLogger()->info(sprintf('Time taken: %s minutes', (time() - $start) / 60)); |
161
|
|
|
|
162
|
|
|
return $groups; |
163
|
13 |
|
} |
164
|
|
|
|
165
|
13 |
|
/** |
166
|
13 |
|
* Set up the requirements for this task |
167
|
13 |
|
* |
168
|
13 |
|
* @param HTTPRequest $request |
169
|
13 |
|
* @return array |
170
|
13 |
|
*/ |
171
|
|
|
protected function taskSetup($request): array |
172
|
13 |
|
{ |
173
|
|
|
$vars = $request->getVars(); |
174
|
|
|
$this->debug = $this->debug || isset($vars['debug']); |
175
|
|
|
$group = $vars['group'] ?? 0; |
176
|
|
|
$start = $vars['start'] ?? 0; |
177
|
|
|
$group = ($start > $group) ? $start : $group; |
178
|
|
|
$isGroup = isset($vars['group']); |
179
|
|
|
|
180
|
|
|
return [$vars, $group, $isGroup]; |
181
|
|
|
} |
182
|
13 |
|
|
183
|
|
|
/** |
184
|
13 |
|
* get the classes to run for this task execution |
185
|
1 |
|
* |
186
|
|
|
* @param $vars |
187
|
|
|
* @param array $classes |
188
|
12 |
|
* @return bool|array |
189
|
|
|
*/ |
190
|
|
|
protected function getClasses($vars, array $classes): array |
191
|
|
|
{ |
192
|
|
|
if (isset($vars['class'])) { |
193
|
|
|
return array_intersect($classes, [$vars['class']]); |
194
|
|
|
} |
195
|
|
|
|
196
|
|
|
return $classes; |
197
|
|
|
} |
198
|
13 |
|
|
199
|
|
|
/** |
200
|
13 |
|
* Clear the given index if a full re-index is needed |
201
|
1 |
|
* |
202
|
1 |
|
* @param $vars |
203
|
|
|
* @param BaseIndex $index |
204
|
13 |
|
* @throws Exception |
205
|
|
|
*/ |
206
|
|
|
public function clearIndex($vars, BaseIndex $index) |
207
|
|
|
{ |
208
|
|
|
if (!empty($vars['clear'])) { |
209
|
|
|
$this->getLogger()->info(sprintf('Clearing index %s', $index->getIndexName())); |
210
|
|
|
$this->service->doManipulate(ArrayList::create([]), SolrCoreService::DELETE_TYPE_ALL, $index); |
211
|
|
|
} |
212
|
|
|
} |
213
|
|
|
|
214
|
|
|
/** |
215
|
|
|
* Index the classes for a specific index |
216
|
|
|
* |
217
|
13 |
|
* @param $classes |
218
|
|
|
* @param $isGroup |
219
|
13 |
|
* @param BaseIndex $index |
220
|
13 |
|
* @param $group |
221
|
13 |
|
* @return int |
222
|
|
|
* @throws Exception |
223
|
|
|
* @throws GuzzleException |
224
|
13 |
|
*/ |
225
|
|
|
protected function indexClassForIndex($classes, $isGroup, BaseIndex $index, $group): int |
226
|
|
|
{ |
227
|
|
|
$groups = 0; |
228
|
|
|
foreach ($classes as $class) { |
229
|
|
|
$groups = $this->indexClass($isGroup, $class, $index, $group); |
230
|
|
|
} |
231
|
|
|
|
232
|
|
|
return $groups; |
233
|
|
|
} |
234
|
|
|
|
235
|
|
|
/** |
236
|
|
|
* Index a single class for a given index. {@link static::indexClassForIndex()} |
237
|
|
|
* |
238
|
13 |
|
* @param bool $isGroup |
239
|
|
|
* @param string $class |
240
|
13 |
|
* @param BaseIndex $index |
241
|
|
|
* @param int $group |
242
|
13 |
|
* @return int |
243
|
13 |
|
* @throws GuzzleException |
244
|
13 |
|
* @throws ValidationException |
245
|
13 |
|
*/ |
246
|
|
|
private function indexClass($isGroup, $class, BaseIndex $index, int $group): int |
247
|
13 |
|
{ |
248
|
|
|
$this->getLogger()->info(sprintf('Indexing %s for %s', $class, $index->getIndexName())); |
249
|
|
|
$this->batchLength = DocumentFactory::config()->get('batchLength'); |
250
|
|
|
$totalGroups = (int)ceil($class::get()->count() / $this->batchLength); |
251
|
|
|
$cores = SolrCoreService::config()->get('cpucores') ?: 1; |
252
|
|
|
$groups = $isGroup ? ($group + $cores - 1) : $totalGroups; |
253
|
13 |
|
$this->getLogger()->info(sprintf('Total groups %s', $totalGroups)); |
254
|
13 |
|
do { // Run from oldest to newest |
255
|
|
|
try { |
256
|
|
|
// The unittest param is from phpunit.xml.dist, meant to bypass the exit(0) call |
257
|
13 |
|
if (function_exists('pcntl_fork') && |
258
|
|
|
!Controller::curr()->getRequest()->getVar('unittest') |
259
|
|
|
) { |
260
|
|
|
$group = $this->spawnChildren($class, $index, $group, $cores, $groups); |
261
|
|
|
} else { |
262
|
|
|
$this->doReindex($group, $class, $index); |
263
|
|
|
} |
264
|
|
|
} catch (Exception $error) { |
265
|
|
|
$this->logException($index->getIndexName(), $group, $error); |
266
|
|
|
$group++; |
267
|
|
|
continue; |
268
|
|
|
} |
269
|
13 |
|
$group++; |
270
|
|
|
} while ($group <= $groups); |
271
|
13 |
|
|
272
|
13 |
|
return $totalGroups; |
273
|
|
|
} |
274
|
|
|
|
275
|
13 |
|
/** |
276
|
|
|
* For each core, spawn a child process that will handle a separate group. |
277
|
|
|
* This speeds up indexing through CLI massively. |
278
|
13 |
|
* |
279
|
13 |
|
* @param string $class |
280
|
|
|
* @param BaseIndex $index |
281
|
|
|
* @param int $group |
282
|
|
|
* @param int $cores |
283
|
|
|
* @param int $groups |
284
|
|
|
* @return int |
285
|
|
|
* @throws Exception |
286
|
|
|
* @throws GuzzleException |
287
|
|
|
*/ |
288
|
|
|
private function spawnChildren($class, BaseIndex $index, int $group, int $cores, int $groups): int |
289
|
|
|
{ |
290
|
13 |
|
$start = $group; |
291
|
|
|
$pids = []; |
292
|
|
|
// for each core, start a grouped indexing |
293
|
13 |
|
for ($i = 0; $i < $cores; $i++) { |
294
|
|
|
$start = $group + $i; |
295
|
13 |
|
if ($start < $groups) { |
296
|
13 |
|
$this->runForkedChild($class, $index, $pids, $i, $start); |
297
|
13 |
|
} |
298
|
13 |
|
} |
299
|
1 |
|
// Wait for each child to finish |
300
|
|
|
foreach ($pids as $key => $pid) { |
301
|
13 |
|
pcntl_waitpid($pid, $status); |
302
|
|
|
if ($status === 0) { |
303
|
|
|
unset($pids[$key]); |
304
|
|
|
} |
305
|
|
|
} |
306
|
|
|
$commit = $index->getClient()->createUpdate(); |
307
|
|
|
$commit->addCommit(); |
308
|
|
|
|
309
|
|
|
$index->getClient()->update($commit); |
310
|
1 |
|
|
311
|
|
|
return $start; |
312
|
1 |
|
} |
313
|
1 |
|
|
314
|
1 |
|
/** |
315
|
1 |
|
* Reindex the given group, for each state |
316
|
1 |
|
* |
317
|
1 |
|
* @param int $group |
318
|
1 |
|
* @param string $class |
319
|
|
|
* @param BaseIndex $index |
320
|
|
|
* @param bool $pcntl |
321
|
|
|
* @throws Exception |
322
|
|
|
*/ |
323
|
|
|
private function doReindex($group, $class, BaseIndex $index, $pcntl = false) |
324
|
|
|
{ |
325
|
|
|
foreach (SiteState::getStates() as $state) { |
326
|
|
|
if ($state !== 'default' && !empty($state)) { |
327
|
|
|
SiteState::withState($state); |
328
|
|
|
} |
329
|
|
|
$this->stateReindex($group, $class, $index); |
330
|
|
|
} |
331
|
|
|
|
332
|
|
|
SiteState::withState(SiteState::DEFAULT_STATE); |
333
|
|
|
$this->getLogger()->info(sprintf('Indexed group %s', $group)); |
334
|
|
|
|
335
|
|
|
if ($pcntl) { |
336
|
|
|
exit(0); |
|
|
|
|
337
|
|
|
} |
338
|
|
|
} |
339
|
|
|
|
340
|
|
|
/** |
341
|
|
|
* Index a group of a class for a specific state and index |
342
|
|
|
* |
343
|
|
|
* @param $group |
344
|
|
|
* @param $class |
345
|
|
|
* @param BaseIndex $index |
346
|
|
|
* @throws Exception |
347
|
|
|
*/ |
348
|
|
|
private function stateReindex($group, $class, BaseIndex $index): void |
349
|
|
|
{ |
350
|
|
|
// Generate filtered list of local records |
351
|
|
|
$baseClass = DataObject::getSchema()->baseDataClass($class); |
352
|
|
|
/** @var DataList|DataObject[] $items */ |
353
|
|
|
$items = DataObject::get($baseClass) |
354
|
|
|
->sort('ID ASC') |
355
|
|
|
->limit($this->batchLength, ($group * $this->batchLength)); |
356
|
|
|
if ($items->count()) { |
357
|
|
|
$this->updateIndex($index, $items); |
358
|
|
|
} |
359
|
|
|
} |
360
|
|
|
|
361
|
|
|
/** |
362
|
|
|
* Execute the update on the client |
363
|
|
|
* |
364
|
|
|
* @param BaseIndex $index |
365
|
|
|
* @param $items |
366
|
|
|
* @throws Exception |
367
|
|
|
*/ |
368
|
|
|
private function updateIndex(BaseIndex $index, $items): void |
369
|
|
|
{ |
370
|
|
|
$client = $index->getClient(); |
371
|
|
|
$update = $client->createUpdate(); |
372
|
|
|
$this->service->setInDebugMode($this->debug); |
373
|
|
|
$this->service->updateIndex($index, $items, $update); |
374
|
|
|
$client->update($update); |
375
|
|
|
} |
376
|
|
|
|
377
|
|
|
/** |
378
|
|
|
* Log an exception if it happens. Most are catched, these logs are for the developers |
379
|
|
|
* to identify problems and fix them. |
380
|
|
|
* |
381
|
|
|
* @param string $index |
382
|
|
|
* @param int $group |
383
|
|
|
* @param Exception $exception |
384
|
|
|
* @throws GuzzleException |
385
|
|
|
* @throws ValidationException |
386
|
|
|
*/ |
387
|
|
|
private function logException($index, int $group, Exception $exception): void |
388
|
|
|
{ |
389
|
|
|
$this->getLogger()->error($exception->getMessage()); |
390
|
|
|
$msg = sprintf( |
391
|
|
|
'Error indexing core %s on group %s,' . PHP_EOL . |
392
|
|
|
'Please log in to the CMS to find out more about Indexing errors' . PHP_EOL, |
393
|
|
|
$index, |
394
|
|
|
$group |
395
|
|
|
); |
396
|
|
|
SolrLogger::logMessage('ERROR', $msg, $index); |
397
|
|
|
} |
398
|
|
|
|
399
|
|
|
/** |
400
|
|
|
* @param $class |
401
|
|
|
* @param BaseIndex $index |
402
|
|
|
* @param int $pid |
403
|
|
|
* @param int $start |
404
|
|
|
* @throws GuzzleException |
405
|
|
|
* @throws ValidationException |
406
|
|
|
* @throws Exception |
407
|
|
|
*/ |
408
|
|
|
private function runChild($class, BaseIndex $index, int $pid, int $start): void |
409
|
|
|
{ |
410
|
|
|
if (!$pid) { |
411
|
|
|
try { |
412
|
|
|
$this->doReindex($start, $class, $index, true); |
413
|
|
|
} catch (Exception $e) { |
414
|
|
|
SolrLogger::logMessage('ERROR', $e, $index->getIndexName()); |
415
|
|
|
throw new Exception( |
416
|
|
|
sprintf( |
417
|
|
|
'Something went wrong while indexing %s, see the logs for details', |
418
|
|
|
$start |
419
|
|
|
) |
420
|
|
|
); |
421
|
|
|
} |
422
|
|
|
} |
423
|
|
|
} |
424
|
|
|
|
425
|
|
|
/** |
426
|
|
|
* @param $class |
427
|
|
|
* @param BaseIndex $index |
428
|
|
|
* @param array $pids |
429
|
|
|
* @param int $i |
430
|
|
|
* @param int $start |
431
|
|
|
* @return void |
432
|
|
|
* @throws GuzzleException |
433
|
|
|
* @throws ValidationException |
434
|
|
|
*/ |
435
|
|
|
private function runForkedChild($class, BaseIndex $index, array &$pids, int $i, int $start): void |
436
|
|
|
{ |
437
|
|
|
$pid = pcntl_fork(); |
438
|
|
|
// PID needs to be pushed before anything else, for some reason |
439
|
|
|
$pids[$i] = $pid; |
440
|
|
|
$config = DB::getConfig(); |
441
|
|
|
DB::connect($config); |
442
|
|
|
$this->runChild($class, $index, $pid, $start); |
443
|
|
|
} |
444
|
|
|
} |
445
|
|
|
|
In general, usage of exit should be done with care and only when running in a scripting context like a CLI script.