1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* class BaseIndex|Firesphere\SolrSearch\Indexes\BaseIndex is the base for indexing items |
4
|
|
|
* |
5
|
|
|
* @package Firesphere\SolrSearch\Indexes |
6
|
|
|
* @author Simon `Firesphere` Erkelens; Marco `Sheepy` Hermo |
7
|
|
|
* @copyright Copyright (c) 2018 - now() Firesphere & Sheepy |
8
|
|
|
*/ |
9
|
|
|
|
10
|
|
|
namespace Firesphere\SolrSearch\Indexes; |
11
|
|
|
|
12
|
|
|
use Exception; |
13
|
|
|
use Firesphere\SolrSearch\Factories\QueryComponentFactory; |
14
|
|
|
use Firesphere\SolrSearch\Factories\SchemaFactory; |
15
|
|
|
use Firesphere\SolrSearch\Helpers\SolrLogger; |
16
|
|
|
use Firesphere\SolrSearch\Helpers\Synonyms; |
17
|
|
|
use Firesphere\SolrSearch\Interfaces\ConfigStore; |
18
|
|
|
use Firesphere\SolrSearch\Models\SearchSynonym; |
19
|
|
|
use Firesphere\SolrSearch\Queries\BaseQuery; |
20
|
|
|
use Firesphere\SolrSearch\Results\SearchResult; |
21
|
|
|
use Firesphere\SolrSearch\Services\SolrCoreService; |
22
|
|
|
use Firesphere\SolrSearch\States\SiteState; |
23
|
|
|
use Firesphere\SolrSearch\Traits\BaseIndexTrait; |
24
|
|
|
use Firesphere\SolrSearch\Traits\GetterSetterTrait; |
25
|
|
|
use GuzzleHttp\Exception\GuzzleException; |
26
|
|
|
use LogicException; |
27
|
|
|
use ReflectionException; |
28
|
|
|
use SilverStripe\Control\Director; |
29
|
|
|
use SilverStripe\Core\Config\Config; |
30
|
|
|
use SilverStripe\Core\Config\Configurable; |
31
|
|
|
use SilverStripe\Core\Extensible; |
32
|
|
|
use SilverStripe\Core\Injector\Injectable; |
33
|
|
|
use SilverStripe\Core\Injector\Injector; |
34
|
|
|
use SilverStripe\Dev\Deprecation; |
35
|
|
|
use SilverStripe\ORM\DataList; |
36
|
|
|
use SilverStripe\ORM\ValidationException; |
37
|
|
|
use SilverStripe\View\ArrayData; |
38
|
|
|
use Solarium\Core\Client\Adapter\Guzzle; |
39
|
|
|
use Solarium\Core\Client\Client; |
40
|
|
|
use Solarium\QueryType\Select\Query\Query; |
41
|
|
|
use Solarium\QueryType\Select\Result\Result; |
42
|
|
|
|
43
|
|
|
/** |
44
|
|
|
* Base for creating a new Solr core. |
45
|
|
|
* |
46
|
|
|
* Base index settings and methods. Should be extended with at least a name for the index. |
47
|
|
|
* This is an abstract class that can not be instantiated on it's own |
48
|
|
|
* |
49
|
|
|
* @package Firesphere\SolrSearch\Indexes |
50
|
|
|
*/ |
51
|
|
|
abstract class BaseIndex |
52
|
|
|
{ |
53
|
|
|
use Extensible; |
54
|
|
|
use Configurable; |
55
|
|
|
use Injectable; |
56
|
|
|
use GetterSetterTrait; |
57
|
|
|
use BaseIndexTrait; |
58
|
|
|
|
59
|
|
|
/** |
60
|
|
|
* Field types that can be added |
61
|
|
|
* Used in init to call build methods from configuration yml |
62
|
|
|
* |
63
|
|
|
* @array |
64
|
|
|
*/ |
65
|
|
|
private static $fieldTypes = [ |
66
|
|
|
'FulltextFields', |
67
|
|
|
'SortFields', |
68
|
|
|
'FilterFields', |
69
|
|
|
'BoostedFields', |
70
|
|
|
'CopyFields', |
71
|
|
|
'DefaultField', |
72
|
|
|
'AndFacetFields', |
73
|
|
|
'OrFacetField', |
74
|
|
|
'StoredFields', |
75
|
|
|
]; |
76
|
|
|
/** |
77
|
|
|
* {@link SchemaFactory} |
78
|
|
|
* |
79
|
|
|
* @var SchemaFactory Schema factory for generating the schema |
80
|
|
|
*/ |
81
|
|
|
protected $schemaFactory; |
82
|
|
|
/** |
83
|
|
|
* {@link QueryComponentFactory} |
84
|
|
|
* |
85
|
|
|
* @var QueryComponentFactory Generator for all components |
86
|
|
|
*/ |
87
|
|
|
protected $queryFactory; |
88
|
|
|
/** |
89
|
|
|
* @var array The query terms as an array |
90
|
|
|
*/ |
91
|
|
|
protected $queryTerms = []; |
92
|
|
|
/** |
93
|
|
|
* @var Query Query that will hit the client |
94
|
|
|
*/ |
95
|
|
|
protected $clientQuery; |
96
|
|
|
/** |
97
|
|
|
* @var bool Signify if a retry should occur if nothing was found and there are suggestions to follow |
98
|
|
|
*/ |
99
|
|
|
private $retry = false; |
100
|
|
|
|
101
|
|
|
/** |
102
|
|
|
* BaseIndex constructor. |
103
|
|
|
*/ |
104
|
51 |
|
public function __construct() |
105
|
|
|
{ |
106
|
|
|
// Set up the client |
107
|
51 |
|
$config = Config::inst()->get(SolrCoreService::class, 'config'); |
108
|
51 |
|
$config['endpoint'] = $this->getConfig($config['endpoint']); |
109
|
51 |
|
$this->client = new Client($config); |
110
|
51 |
|
$this->client->setAdapter(new Guzzle()); |
111
|
|
|
|
112
|
|
|
// Set up the schema service, only used in the generation of the schema |
113
|
51 |
|
$schemaFactory = Injector::inst()->get(SchemaFactory::class, false); |
114
|
51 |
|
$schemaFactory->setIndex($this); |
115
|
51 |
|
$schemaFactory->setStore(Director::isDev()); |
116
|
51 |
|
$this->schemaFactory = $schemaFactory; |
117
|
51 |
|
$this->queryFactory = Injector::inst()->get(QueryComponentFactory::class, false); |
118
|
|
|
|
119
|
51 |
|
$this->extend('onBeforeInit'); |
120
|
51 |
|
$this->init(); |
121
|
51 |
|
$this->extend('onAfterInit'); |
122
|
51 |
|
} |
123
|
|
|
|
124
|
|
|
/** |
125
|
|
|
* Build a full config for all given endpoints |
126
|
|
|
* This is to add the current index to e.g. an index or select |
127
|
|
|
* |
128
|
|
|
* @param array $endpoints |
129
|
|
|
* @return array |
130
|
|
|
*/ |
131
|
51 |
|
public function getConfig($endpoints): array |
132
|
|
|
{ |
133
|
51 |
|
foreach ($endpoints as $host => $endpoint) { |
134
|
51 |
|
$endpoints[$host]['core'] = $this->getIndexName(); |
135
|
|
|
} |
136
|
|
|
|
137
|
51 |
|
return $endpoints; |
138
|
|
|
} |
139
|
|
|
|
140
|
|
|
/** |
141
|
|
|
* Name of this index. |
142
|
|
|
* |
143
|
|
|
* @return string |
144
|
|
|
*/ |
145
|
|
|
abstract public function getIndexName(); |
146
|
|
|
|
147
|
|
|
/** |
148
|
|
|
* Required to initialise the fields. |
149
|
|
|
* It's loaded in to the non-static properties for backward compatibility with FTS |
150
|
|
|
* Also, it's a tad easier to use this way, loading the other way around would be very |
151
|
|
|
* memory intensive, as updating the config for each item is not efficient |
152
|
|
|
*/ |
153
|
47 |
|
public function init() |
154
|
|
|
{ |
155
|
47 |
|
$config = self::config()->get($this->getIndexName()); |
156
|
47 |
|
if (!$config) { |
157
|
47 |
|
Deprecation::notice('5', 'Please set an index name and use a config yml'); |
158
|
|
|
} |
159
|
|
|
|
160
|
47 |
|
if (!empty($this->getClasses())) { |
161
|
47 |
|
if (!$this->usedAllFields) { |
162
|
47 |
|
Deprecation::notice('5', 'It is advised to use a config YML for most cases'); |
163
|
|
|
} |
164
|
|
|
|
165
|
47 |
|
return; |
166
|
|
|
} |
167
|
|
|
|
168
|
39 |
|
$this->initFromConfig($config); |
169
|
39 |
|
} |
170
|
|
|
|
171
|
|
|
/** |
172
|
|
|
* Generate the config from yml if possible |
173
|
|
|
* @param array|null $config |
174
|
|
|
*/ |
175
|
39 |
|
protected function initFromConfig($config): void |
176
|
|
|
{ |
177
|
39 |
|
if (!$config || !array_key_exists('Classes', $config)) { |
178
|
2 |
|
throw new LogicException('No classes or config to index found!'); |
179
|
|
|
} |
180
|
|
|
|
181
|
39 |
|
$this->setClasses($config['Classes']); |
182
|
|
|
// Fallback for the refactor to And/Or FacetFields |
183
|
39 |
|
$config['AndFacetFields'] = $config['AndFacetFields'] ?? $config['FacetFields']; |
184
|
|
|
|
185
|
|
|
// For backward compatibility, copy the config to the protected values |
186
|
|
|
// Saves doubling up further down the line |
187
|
39 |
|
foreach (self::$fieldTypes as $type) { |
188
|
39 |
|
if (array_key_exists($type, $config)) { |
189
|
39 |
|
$method = 'set' . $type; |
190
|
39 |
|
$this->$method($config[$type]); |
191
|
|
|
} |
192
|
|
|
} |
193
|
39 |
|
} |
194
|
|
|
|
195
|
|
|
/** |
196
|
|
|
* Default returns a SearchResult. It can return an ArrayData if FTS Compat is enabled |
197
|
|
|
* |
198
|
|
|
* @param BaseQuery $query |
199
|
|
|
* @return SearchResult|ArrayData|mixed |
200
|
|
|
* @throws GuzzleException |
201
|
|
|
* @throws ValidationException |
202
|
|
|
* @throws ReflectionException |
203
|
|
|
* @throws Exception |
204
|
|
|
*/ |
205
|
6 |
|
public function doSearch(BaseQuery $query) |
206
|
|
|
{ |
207
|
6 |
|
SiteState::alterQuery($query); |
208
|
|
|
// Build the actual query parameters |
209
|
6 |
|
$this->clientQuery = $this->buildSolrQuery($query); |
210
|
|
|
// Set the sorting |
211
|
6 |
|
$this->clientQuery->addSorts($query->getSort()); |
212
|
|
|
|
213
|
6 |
|
$this->extend('onBeforeSearch', $query, $this->clientQuery); |
214
|
|
|
|
215
|
|
|
try { |
216
|
6 |
|
$result = $this->client->select($this->clientQuery); |
217
|
|
|
} catch (Exception $error) { |
218
|
|
|
// @codeCoverageIgnoreStart |
219
|
|
|
$logger = new SolrLogger(); |
220
|
|
|
$logger->saveSolrLog('Query'); |
221
|
|
|
throw $error; |
222
|
|
|
// @codeCoverageIgnoreEnd |
223
|
|
|
} |
224
|
|
|
|
225
|
|
|
// Handle the after search first. This gets a raw search result |
226
|
6 |
|
$this->extend('onAfterSearch', $result); |
227
|
6 |
|
$searchResult = new SearchResult($result, $query, $this); |
228
|
6 |
|
if ($this->doRetry($query, $result, $searchResult)) { |
229
|
2 |
|
return $this->spellcheckRetry($query, $searchResult); |
230
|
|
|
} |
231
|
|
|
|
232
|
|
|
// And then handle the search results, which is a useable object for SilverStripe |
233
|
6 |
|
$this->extend('updateSearchResults', $searchResult); |
234
|
|
|
|
235
|
6 |
|
return $searchResult; |
236
|
|
|
} |
237
|
|
|
|
238
|
|
|
/** |
239
|
|
|
* From the given BaseQuery, generate a Solarium ClientQuery object |
240
|
|
|
* |
241
|
|
|
* @param BaseQuery $query |
242
|
|
|
* @return Query |
243
|
|
|
*/ |
244
|
6 |
|
public function buildSolrQuery(BaseQuery $query): Query |
245
|
|
|
{ |
246
|
6 |
|
$clientQuery = $this->client->createSelect(); |
247
|
6 |
|
$factory = $this->buildFactory($query, $clientQuery); |
248
|
|
|
|
249
|
6 |
|
$clientQuery = $factory->buildQuery(); |
250
|
6 |
|
$this->queryTerms = $factory->getQueryArray(); |
251
|
|
|
|
252
|
6 |
|
$queryData = implode(' ', $this->queryTerms); |
253
|
6 |
|
$clientQuery->setQuery($queryData); |
254
|
|
|
|
255
|
6 |
|
return $clientQuery; |
256
|
|
|
} |
257
|
|
|
|
258
|
|
|
/** |
259
|
|
|
* Build a factory to use in the SolrQuery building. {@link static::buildSolrQuery()} |
260
|
|
|
* |
261
|
|
|
* @param BaseQuery $query |
262
|
|
|
* @param Query $clientQuery |
263
|
|
|
* @return QueryComponentFactory|mixed |
264
|
|
|
*/ |
265
|
6 |
|
protected function buildFactory(BaseQuery $query, Query $clientQuery) |
266
|
|
|
{ |
267
|
6 |
|
$factory = $this->queryFactory; |
268
|
|
|
|
269
|
6 |
|
$helper = $clientQuery->getHelper(); |
270
|
|
|
|
271
|
6 |
|
$factory->setQuery($query); |
272
|
6 |
|
$factory->setClientQuery($clientQuery); |
273
|
6 |
|
$factory->setHelper($helper); |
274
|
6 |
|
$factory->setIndex($this); |
275
|
|
|
|
276
|
6 |
|
return $factory; |
277
|
|
|
} |
278
|
|
|
|
279
|
|
|
/** |
280
|
|
|
* Check if the query should be retried with spellchecking |
281
|
|
|
* Conditions are: |
282
|
|
|
* It is not already a retry with spellchecking |
283
|
|
|
* Spellchecking is enabled |
284
|
|
|
* If spellchecking is enabled and nothing is found OR it should follow spellchecking none the less |
285
|
|
|
* There is a spellcheck output |
286
|
|
|
* |
287
|
|
|
* @param BaseQuery $query |
288
|
|
|
* @param Result $result |
289
|
|
|
* @param SearchResult $searchResult |
290
|
|
|
* @return bool |
291
|
|
|
*/ |
292
|
6 |
|
protected function doRetry(BaseQuery $query, Result $result, SearchResult $searchResult): bool |
293
|
|
|
{ |
294
|
6 |
|
return !$this->retry && |
295
|
6 |
|
$query->hasSpellcheck() && |
296
|
6 |
|
($query->shouldFollowSpellcheck() || $result->getNumFound() === 0) && |
297
|
6 |
|
$searchResult->getCollatedSpellcheck(); |
298
|
|
|
} |
299
|
|
|
|
300
|
|
|
/** |
301
|
|
|
* Retry the query with the first collated spellcheck found. |
302
|
|
|
* |
303
|
|
|
* @param BaseQuery $query |
304
|
|
|
* @param SearchResult $searchResult |
305
|
|
|
* @return SearchResult|mixed|ArrayData |
306
|
|
|
* @throws GuzzleException |
307
|
|
|
* @throws ValidationException |
308
|
|
|
* @throws ReflectionException |
309
|
|
|
*/ |
310
|
2 |
|
protected function spellcheckRetry(BaseQuery $query, SearchResult $searchResult) |
311
|
|
|
{ |
312
|
2 |
|
$terms = $query->getTerms(); |
313
|
2 |
|
$spellChecked = $searchResult->getCollatedSpellcheck(); |
314
|
|
|
// Remove the fuzzyness from the collated check |
315
|
2 |
|
$term = preg_replace('/~\d+/', '', $spellChecked); |
316
|
2 |
|
$terms[0]['text'] = $term; |
317
|
2 |
|
$query->setTerms($terms); |
318
|
2 |
|
$this->retry = true; |
319
|
|
|
|
320
|
2 |
|
return $this->doSearch($query); |
321
|
|
|
} |
322
|
|
|
|
323
|
|
|
/** |
324
|
|
|
* Get all fields that are required for indexing in a unique way |
325
|
|
|
* |
326
|
|
|
* @return array |
327
|
|
|
*/ |
328
|
10 |
|
public function getFieldsForIndexing(): array |
329
|
|
|
{ |
330
|
10 |
|
$facets = []; |
331
|
10 |
|
foreach ($this->getFacetFields() as $field) { |
332
|
9 |
|
$facets[] = $field['Field']; |
333
|
|
|
} |
334
|
|
|
// Return values to make the key reset |
335
|
|
|
// Only return unique values |
336
|
|
|
// And make it all a single array |
337
|
10 |
|
$fields = array_values( |
338
|
10 |
|
array_unique( |
339
|
10 |
|
array_merge( |
340
|
10 |
|
$this->getFulltextFields(), |
341
|
10 |
|
$this->getSortFields(), |
342
|
10 |
|
$facets, |
343
|
10 |
|
$this->getFilterFields() |
344
|
|
|
) |
345
|
|
|
) |
346
|
|
|
); |
347
|
|
|
|
348
|
10 |
|
$this->extend('updateFieldsForIndexing', $fields); |
349
|
|
|
|
350
|
10 |
|
return $fields; |
351
|
|
|
} |
352
|
|
|
|
353
|
|
|
/** |
354
|
|
|
* Upload config for this index to the given store |
355
|
|
|
* |
356
|
|
|
* @param ConfigStore $store |
357
|
|
|
*/ |
358
|
35 |
|
public function uploadConfig(ConfigStore $store): void |
359
|
|
|
{ |
360
|
|
|
// @todo use types/schema/elevate rendering |
361
|
|
|
// Upload the config files for this index |
362
|
|
|
// Create a default schema which we can manage later |
363
|
35 |
|
$schema = (string)$this->schemaFactory->generateSchema(); |
364
|
35 |
|
$store->uploadString( |
365
|
35 |
|
$this->getIndexName(), |
366
|
35 |
|
'schema.xml', |
367
|
35 |
|
$schema |
368
|
|
|
); |
369
|
|
|
|
370
|
35 |
|
$this->getSynonyms($store); |
371
|
|
|
|
372
|
|
|
// Upload additional files |
373
|
35 |
|
foreach (glob($this->schemaFactory->getExtrasPath() . '/*') as $file) { |
374
|
35 |
|
if (is_file($file)) { |
375
|
35 |
|
$store->uploadFile($this->getIndexName(), $file); |
376
|
|
|
} |
377
|
|
|
} |
378
|
35 |
|
} |
379
|
|
|
|
380
|
|
|
/** |
381
|
|
|
* Add synonyms. Public to be extendable |
382
|
|
|
* |
383
|
|
|
* @param ConfigStore $store Store to use to write synonyms |
384
|
|
|
* @param bool $defaults Include UK to US synonyms |
385
|
|
|
* @return string |
386
|
|
|
*/ |
387
|
35 |
|
public function getSynonyms($store = null, $defaults = true) |
388
|
|
|
{ |
389
|
35 |
|
$synonyms = Synonyms::getSynonymsAsString($defaults); |
390
|
|
|
/** @var DataList|SearchSynonym[] $syn */ |
391
|
35 |
|
$syn = SearchSynonym::get(); |
392
|
35 |
|
foreach ($syn as $synonym) { |
393
|
1 |
|
$synonyms .= $synonym->getCombinedSynonym(); |
394
|
|
|
} |
395
|
|
|
|
396
|
|
|
// Upload synonyms |
397
|
35 |
|
if ($store) { |
398
|
35 |
|
$store->uploadString( |
399
|
35 |
|
$this->getIndexName(), |
400
|
35 |
|
'synonyms.txt', |
401
|
35 |
|
$synonyms |
402
|
|
|
); |
403
|
|
|
} |
404
|
|
|
|
405
|
35 |
|
return $synonyms; |
406
|
|
|
} |
407
|
|
|
|
408
|
|
|
/** |
409
|
|
|
* Get the final, generated terms |
410
|
|
|
* |
411
|
|
|
* @return array |
412
|
|
|
*/ |
413
|
2 |
|
public function getQueryTerms(): array |
414
|
|
|
{ |
415
|
2 |
|
return $this->queryTerms; |
416
|
|
|
} |
417
|
|
|
|
418
|
|
|
/** |
419
|
|
|
* Get the QueryComponentFactory. {@link QueryComponentFactory} |
420
|
|
|
* |
421
|
|
|
* @return QueryComponentFactory |
422
|
|
|
*/ |
423
|
1 |
|
public function getQueryFactory(): QueryComponentFactory |
424
|
|
|
{ |
425
|
1 |
|
return $this->queryFactory; |
426
|
|
|
} |
427
|
|
|
|
428
|
|
|
/** |
429
|
|
|
* Retrieve the Solarium client Query object for this index operation |
430
|
|
|
* |
431
|
|
|
* @return Query |
432
|
|
|
*/ |
433
|
1 |
|
public function getClientQuery(): Query |
434
|
|
|
{ |
435
|
1 |
|
return $this->clientQuery; |
436
|
|
|
} |
437
|
|
|
} |
438
|
|
|
|