1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* class BaseIndex|Firesphere\SolrSearch\Indexes\BaseIndex is the base for indexing items |
4
|
|
|
* |
5
|
|
|
* @package Firesphere\SolrSearch\Indexes |
6
|
|
|
* @author Simon `Firesphere` Erkelens; Marco `Sheepy` Hermo |
7
|
|
|
* @copyright Copyright (c) 2018 - now() Firesphere & Sheepy |
8
|
|
|
*/ |
9
|
|
|
|
10
|
|
|
namespace Firesphere\SolrSearch\Indexes; |
11
|
|
|
|
12
|
|
|
use Exception; |
13
|
|
|
use Firesphere\SolrSearch\Factories\QueryComponentFactory; |
14
|
|
|
use Firesphere\SolrSearch\Helpers\SolrLogger; |
15
|
|
|
use Firesphere\SolrSearch\Helpers\Synonyms; |
16
|
|
|
use Firesphere\SolrSearch\Interfaces\ConfigStore; |
17
|
|
|
use Firesphere\SolrSearch\Models\SearchSynonym; |
18
|
|
|
use Firesphere\SolrSearch\Queries\BaseQuery; |
19
|
|
|
use Firesphere\SolrSearch\Results\SearchResult; |
20
|
|
|
use Firesphere\SolrSearch\Services\SchemaService; |
21
|
|
|
use Firesphere\SolrSearch\Services\SolrCoreService; |
22
|
|
|
use Firesphere\SolrSearch\States\SiteState; |
23
|
|
|
use Firesphere\SolrSearch\Traits\BaseIndexTrait; |
24
|
|
|
use Firesphere\SolrSearch\Traits\GetterSetterTrait; |
25
|
|
|
use GuzzleHttp\Exception\GuzzleException; |
26
|
|
|
use LogicException; |
27
|
|
|
use ReflectionException; |
28
|
|
|
use SilverStripe\Control\Director; |
29
|
|
|
use SilverStripe\Core\Config\Config; |
30
|
|
|
use SilverStripe\Core\Config\Configurable; |
31
|
|
|
use SilverStripe\Core\Extensible; |
32
|
|
|
use SilverStripe\Core\Injector\Injectable; |
33
|
|
|
use SilverStripe\Core\Injector\Injector; |
34
|
|
|
use SilverStripe\Dev\Deprecation; |
35
|
|
|
use SilverStripe\ORM\DataList; |
36
|
|
|
use SilverStripe\ORM\ValidationException; |
37
|
|
|
use SilverStripe\View\ArrayData; |
38
|
|
|
use Solarium\Core\Client\Adapter\Guzzle; |
39
|
|
|
use Solarium\Core\Client\Client; |
40
|
|
|
use Solarium\QueryType\Select\Query\Query; |
41
|
|
|
use Solarium\QueryType\Select\Result\Result; |
42
|
|
|
|
43
|
|
|
/** |
44
|
|
|
* Base for creating a new Solr core. |
45
|
|
|
* |
46
|
|
|
* Base index settings and methods. Should be extended with at least a name for the index. |
47
|
|
|
* This is an abstract class that can not be instantiated on it's own |
48
|
|
|
* |
49
|
|
|
* @package Firesphere\SolrSearch\Indexes |
50
|
|
|
*/ |
51
|
|
|
abstract class BaseIndex |
52
|
|
|
{ |
53
|
|
|
use Extensible; |
54
|
|
|
use Configurable; |
55
|
|
|
use Injectable; |
56
|
|
|
use GetterSetterTrait; |
57
|
|
|
use BaseIndexTrait; |
58
|
|
|
|
59
|
|
|
/** |
60
|
|
|
* Field types that can be added |
61
|
|
|
* Used in init to call build methods from configuration yml |
62
|
|
|
* |
63
|
|
|
* @array |
64
|
|
|
*/ |
65
|
|
|
private static $fieldTypes = [ |
66
|
|
|
'FulltextFields', |
67
|
|
|
'SortFields', |
68
|
|
|
'FilterFields', |
69
|
|
|
'BoostedFields', |
70
|
|
|
'CopyFields', |
71
|
|
|
'DefaultField', |
72
|
|
|
'FacetFields', |
73
|
|
|
'StoredFields', |
74
|
|
|
]; |
75
|
|
|
/** |
76
|
|
|
* {@link SchemaService} |
77
|
|
|
* |
78
|
|
|
* @var SchemaService |
79
|
|
|
*/ |
80
|
|
|
protected $schemaService; |
81
|
|
|
/** |
82
|
|
|
* {@link QueryComponentFactory} |
83
|
|
|
* |
84
|
|
|
* @var QueryComponentFactory |
85
|
|
|
*/ |
86
|
|
|
protected $queryFactory; |
87
|
|
|
/** |
88
|
|
|
* The query terms as an array |
89
|
|
|
* |
90
|
|
|
* @var array |
91
|
|
|
*/ |
92
|
|
|
protected $queryTerms = []; |
93
|
|
|
/** |
94
|
|
|
* Should a retry occur if nothing was found and there are suggestions to follow |
95
|
|
|
* |
96
|
|
|
* @var bool |
97
|
|
|
*/ |
98
|
|
|
private $retry = false; |
99
|
|
|
|
100
|
|
|
/** |
101
|
|
|
* BaseIndex constructor. |
102
|
|
|
*/ |
103
|
78 |
|
public function __construct() |
104
|
|
|
{ |
105
|
|
|
// Set up the client |
106
|
78 |
|
$config = Config::inst()->get(SolrCoreService::class, 'config'); |
107
|
78 |
|
$config['endpoint'] = $this->getConfig($config['endpoint']); |
108
|
78 |
|
$this->client = new Client($config); |
109
|
78 |
|
$this->client->setAdapter(new Guzzle()); |
110
|
|
|
|
111
|
|
|
// Set up the schema service, only used in the generation of the schema |
112
|
78 |
|
$schemaService = Injector::inst()->get(SchemaService::class, false); |
113
|
78 |
|
$schemaService->setIndex($this); |
114
|
78 |
|
$schemaService->setStore(Director::isDev()); |
115
|
78 |
|
$this->schemaService = $schemaService; |
116
|
78 |
|
$this->queryFactory = Injector::inst()->get(QueryComponentFactory::class, false); |
117
|
|
|
|
118
|
78 |
|
$this->extend('onBeforeInit'); |
119
|
78 |
|
$this->init(); |
120
|
78 |
|
$this->extend('onAfterInit'); |
121
|
78 |
|
} |
122
|
|
|
|
123
|
|
|
/** |
124
|
|
|
* Build a full config for all given endpoints |
125
|
|
|
* This is to add the current index to e.g. an index or select |
126
|
|
|
* |
127
|
|
|
* @param array $endpoints |
128
|
|
|
* @return array |
129
|
|
|
*/ |
130
|
78 |
|
public function getConfig($endpoints): array |
131
|
|
|
{ |
132
|
78 |
|
foreach ($endpoints as $host => $endpoint) { |
133
|
78 |
|
$endpoints[$host]['core'] = $this->getIndexName(); |
134
|
|
|
} |
135
|
|
|
|
136
|
78 |
|
return $endpoints; |
137
|
|
|
} |
138
|
|
|
|
139
|
|
|
/** |
140
|
|
|
* Name of this index. |
141
|
|
|
* |
142
|
|
|
* @return string |
143
|
|
|
*/ |
144
|
|
|
abstract public function getIndexName(); |
145
|
|
|
|
146
|
|
|
/** |
147
|
|
|
* Required to initialise the fields. |
148
|
|
|
* It's loaded in to the non-static properties for backward compatibility with FTS |
149
|
|
|
* Also, it's a tad easier to use this way, loading the other way around would be very |
150
|
|
|
* memory intensive, as updating the config for each item is not efficient |
151
|
|
|
*/ |
152
|
77 |
|
public function init() |
153
|
|
|
{ |
154
|
77 |
|
$config = self::config()->get($this->getIndexName()); |
155
|
77 |
|
if (!$config) { |
156
|
77 |
|
Deprecation::notice('5', 'Please set an index name and use a config yml'); |
157
|
|
|
} |
158
|
|
|
|
159
|
77 |
|
if (!empty($this->getClasses())) { |
160
|
77 |
|
if (!$this->usedAllFields) { |
161
|
77 |
|
Deprecation::notice('5', 'It is advised to use a config YML for most cases'); |
162
|
|
|
} |
163
|
|
|
|
164
|
77 |
|
return; |
165
|
|
|
} |
166
|
|
|
|
167
|
76 |
|
$this->initFromConfig($config); |
168
|
76 |
|
} |
169
|
|
|
|
170
|
|
|
/** |
171
|
|
|
* Generate the config from yml if possible |
172
|
|
|
* @param array|null $config |
173
|
|
|
*/ |
174
|
76 |
|
protected function initFromConfig($config): void |
175
|
|
|
{ |
176
|
76 |
|
if (!$config || !array_key_exists('Classes', $config)) { |
177
|
2 |
|
throw new LogicException('No classes or config to index found!'); |
178
|
|
|
} |
179
|
|
|
|
180
|
76 |
|
$this->setClasses($config['Classes']); |
181
|
|
|
|
182
|
|
|
// For backward compatibility, copy the config to the protected values |
183
|
|
|
// Saves doubling up further down the line |
184
|
76 |
|
foreach (self::$fieldTypes as $type) { |
185
|
76 |
|
if (array_key_exists($type, $config)) { |
186
|
76 |
|
$method = 'set' . $type; |
187
|
76 |
|
$this->$method($config[$type]); |
188
|
|
|
} |
189
|
|
|
} |
190
|
76 |
|
} |
191
|
|
|
|
192
|
|
|
/** |
193
|
|
|
* Default returns a SearchResult. It can return an ArrayData if FTS Compat is enabled |
194
|
|
|
* |
195
|
|
|
* @param BaseQuery $query |
196
|
|
|
* @return SearchResult|ArrayData|mixed |
197
|
|
|
* @throws GuzzleException |
198
|
|
|
* @throws ValidationException |
199
|
|
|
* @throws ReflectionException |
200
|
|
|
* @throws Exception |
201
|
|
|
*/ |
202
|
6 |
|
public function doSearch(BaseQuery $query) |
203
|
|
|
{ |
204
|
6 |
|
SiteState::alterQuery($query); |
205
|
|
|
// Build the actual query parameters |
206
|
6 |
|
$clientQuery = $this->buildSolrQuery($query); |
207
|
|
|
|
208
|
6 |
|
$this->extend('onBeforeSearch', $query, $clientQuery); |
209
|
|
|
|
210
|
|
|
try { |
211
|
6 |
|
$result = $this->client->select($clientQuery); |
212
|
|
|
} catch (Exception $error) { |
213
|
|
|
// @codeCoverageIgnoreStart |
214
|
|
|
$logger = new SolrLogger(); |
215
|
|
|
$logger->saveSolrLog('Query'); |
216
|
|
|
throw $error; |
217
|
|
|
// @codeCoverageIgnoreEnd |
218
|
|
|
} |
219
|
|
|
|
220
|
|
|
// Handle the after search first. This gets a raw search result |
221
|
6 |
|
$this->extend('onAfterSearch', $result); |
222
|
6 |
|
$searchResult = new SearchResult($result, $query, $this); |
223
|
6 |
|
if ($this->doRetry($query, $result, $searchResult)) { |
224
|
2 |
|
return $this->spellcheckRetry($query, $searchResult); |
225
|
|
|
} |
226
|
|
|
|
227
|
|
|
// And then handle the search results, which is a useable object for SilverStripe |
228
|
6 |
|
$this->extend('updateSearchResults', $searchResult); |
229
|
|
|
|
230
|
6 |
|
return $searchResult; |
231
|
|
|
} |
232
|
|
|
|
233
|
|
|
/** |
234
|
|
|
* From the given BaseQuery, generate a Solarium ClientQuery object |
235
|
|
|
* |
236
|
|
|
* @param BaseQuery $query |
237
|
|
|
* @return Query |
238
|
|
|
*/ |
239
|
6 |
|
public function buildSolrQuery(BaseQuery $query): Query |
240
|
|
|
{ |
241
|
6 |
|
$clientQuery = $this->client->createSelect(); |
242
|
6 |
|
$factory = $this->buildFactory($query, $clientQuery); |
243
|
|
|
|
244
|
6 |
|
$clientQuery = $factory->buildQuery(); |
245
|
6 |
|
$this->queryTerms = $factory->getQueryArray(); |
246
|
|
|
|
247
|
6 |
|
$queryData = implode(' ', $this->queryTerms); |
248
|
6 |
|
$clientQuery->setQuery($queryData); |
249
|
|
|
|
250
|
6 |
|
return $clientQuery; |
251
|
|
|
} |
252
|
|
|
|
253
|
|
|
/** |
254
|
|
|
* Build a factory to use in the SolrQuery building. {@link static::buildSolrQuery()} |
255
|
|
|
* |
256
|
|
|
* @param BaseQuery $query |
257
|
|
|
* @param Query $clientQuery |
258
|
|
|
* @return QueryComponentFactory|mixed |
259
|
|
|
*/ |
260
|
6 |
|
protected function buildFactory(BaseQuery $query, Query $clientQuery) |
261
|
|
|
{ |
262
|
6 |
|
$factory = $this->queryFactory; |
263
|
|
|
|
264
|
6 |
|
$helper = $clientQuery->getHelper(); |
265
|
|
|
|
266
|
6 |
|
$factory->setQuery($query); |
267
|
6 |
|
$factory->setClientQuery($clientQuery); |
268
|
6 |
|
$factory->setHelper($helper); |
269
|
6 |
|
$factory->setIndex($this); |
270
|
|
|
|
271
|
6 |
|
return $factory; |
272
|
|
|
} |
273
|
|
|
|
274
|
|
|
/** |
275
|
|
|
* Check if the query should be retried with spellchecking |
276
|
|
|
* Conditions are: |
277
|
|
|
* It is not already a retry with spellchecking |
278
|
|
|
* Spellchecking is enabled |
279
|
|
|
* If spellchecking is enabled and nothing is found OR it should follow spellchecking none the less |
280
|
|
|
* There is a spellcheck output |
281
|
|
|
* |
282
|
|
|
* @param BaseQuery $query |
283
|
|
|
* @param Result $result |
284
|
|
|
* @param SearchResult $searchResult |
285
|
|
|
* @return bool |
286
|
|
|
*/ |
287
|
6 |
|
protected function doRetry(BaseQuery $query, Result $result, SearchResult $searchResult): bool |
288
|
|
|
{ |
289
|
6 |
|
return !$this->retry && |
290
|
6 |
|
$query->hasSpellcheck() && |
291
|
6 |
|
($query->shouldFollowSpellcheck() || $result->getNumFound() === 0) && |
292
|
6 |
|
$searchResult->getCollatedSpellcheck(); |
293
|
|
|
} |
294
|
|
|
|
295
|
|
|
/** |
296
|
|
|
* Retry the query with the first collated spellcheck found. |
297
|
|
|
* |
298
|
|
|
* @param BaseQuery $query |
299
|
|
|
* @param SearchResult $searchResult |
300
|
|
|
* @return SearchResult|mixed|ArrayData |
301
|
|
|
* @throws GuzzleException |
302
|
|
|
* @throws ValidationException |
303
|
|
|
* @throws ReflectionException |
304
|
|
|
*/ |
305
|
2 |
|
protected function spellcheckRetry(BaseQuery $query, SearchResult $searchResult) |
306
|
|
|
{ |
307
|
2 |
|
$terms = $query->getTerms(); |
308
|
2 |
|
$spellChecked = $searchResult->getCollatedSpellcheck(); |
309
|
|
|
// Remove the fuzzyness from the collated check |
310
|
2 |
|
$term = preg_replace('/~\d+/', '', $spellChecked); |
311
|
2 |
|
$terms[0]['text'] = $term; |
312
|
2 |
|
$query->setTerms($terms); |
313
|
2 |
|
$this->retry = true; |
314
|
|
|
|
315
|
2 |
|
return $this->doSearch($query); |
316
|
|
|
} |
317
|
|
|
|
318
|
|
|
/** |
319
|
|
|
* Get all fields that are required for indexing in a unique way |
320
|
|
|
* |
321
|
|
|
* @return array |
322
|
|
|
*/ |
323
|
9 |
|
public function getFieldsForIndexing(): array |
324
|
|
|
{ |
325
|
9 |
|
$facets = []; |
326
|
9 |
|
foreach ($this->getFacetFields() as $field) { |
327
|
8 |
|
$facets[] = $field['Field']; |
328
|
|
|
} |
329
|
|
|
// Return values to make the key reset |
330
|
|
|
// Only return unique values |
331
|
|
|
// And make it all a single array |
332
|
9 |
|
$fields = array_values( |
333
|
9 |
|
array_unique( |
334
|
9 |
|
array_merge( |
335
|
9 |
|
$this->getFulltextFields(), |
336
|
9 |
|
$this->getSortFields(), |
337
|
9 |
|
$facets, |
338
|
9 |
|
$this->getFilterFields() |
339
|
|
|
) |
340
|
|
|
) |
341
|
|
|
); |
342
|
|
|
|
343
|
9 |
|
$this->extend('updateFieldsForIndexing', $fields); |
344
|
|
|
|
345
|
9 |
|
return $fields; |
346
|
|
|
} |
347
|
|
|
|
348
|
|
|
/** |
349
|
|
|
* Upload config for this index to the given store |
350
|
|
|
* |
351
|
|
|
* @param ConfigStore $store |
352
|
|
|
*/ |
353
|
35 |
|
public function uploadConfig(ConfigStore $store): void |
354
|
|
|
{ |
355
|
|
|
// @todo use types/schema/elevate rendering |
356
|
|
|
// Upload the config files for this index |
357
|
|
|
// Create a default schema which we can manage later |
358
|
35 |
|
$schema = (string)$this->schemaService->generateSchema(); |
359
|
35 |
|
$store->uploadString( |
360
|
35 |
|
$this->getIndexName(), |
361
|
35 |
|
'schema.xml', |
362
|
35 |
|
$schema |
363
|
|
|
); |
364
|
|
|
|
365
|
35 |
|
$this->getSynonyms($store); |
366
|
|
|
|
367
|
|
|
// Upload additional files |
368
|
35 |
|
foreach (glob($this->schemaService->getExtrasPath() . '/*') as $file) { |
369
|
35 |
|
if (is_file($file)) { |
370
|
35 |
|
$store->uploadFile($this->getIndexName(), $file); |
371
|
|
|
} |
372
|
|
|
} |
373
|
35 |
|
} |
374
|
|
|
|
375
|
|
|
/** |
376
|
|
|
* Add synonyms. Public to be extendable |
377
|
|
|
* |
378
|
|
|
* @param ConfigStore $store Store to use to write synonyms |
379
|
|
|
* @param bool $defaults Include UK to US synonyms |
380
|
|
|
* @return string |
381
|
|
|
*/ |
382
|
35 |
|
public function getSynonyms($store = null, $defaults = true) |
383
|
|
|
{ |
384
|
35 |
|
$synonyms = Synonyms::getSynonymsAsString($defaults); |
385
|
|
|
/** @var DataList|SearchSynonym[] $syn */ |
386
|
35 |
|
$syn = SearchSynonym::get(); |
387
|
35 |
|
foreach ($syn as $synonym) { |
388
|
1 |
|
$synonyms .= $synonym->getCombinedSynonym(); |
389
|
|
|
} |
390
|
|
|
|
391
|
|
|
// Upload synonyms |
392
|
35 |
|
if ($store) { |
393
|
35 |
|
$store->uploadString( |
394
|
35 |
|
$this->getIndexName(), |
395
|
35 |
|
'synonyms.txt', |
396
|
35 |
|
$synonyms |
397
|
|
|
); |
398
|
|
|
} |
399
|
|
|
|
400
|
35 |
|
return $synonyms; |
401
|
|
|
} |
402
|
|
|
|
403
|
|
|
/** |
404
|
|
|
* Get the final, generated terms |
405
|
|
|
* |
406
|
|
|
* @return array |
407
|
|
|
*/ |
408
|
2 |
|
public function getQueryTerms(): array |
409
|
|
|
{ |
410
|
2 |
|
return $this->queryTerms; |
411
|
|
|
} |
412
|
|
|
|
413
|
|
|
/** |
414
|
|
|
* Get the QueryComponentFactory. {@link QueryComponentFactory} |
415
|
|
|
* |
416
|
|
|
* @return QueryComponentFactory |
417
|
|
|
*/ |
418
|
1 |
|
public function getQueryFactory(): QueryComponentFactory |
419
|
|
|
{ |
420
|
1 |
|
return $this->queryFactory; |
421
|
|
|
} |
422
|
|
|
} |
423
|
|
|
|