1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* class BaseIndex|Firesphere\SolrSearch\Indexes\BaseIndex is the base for indexing items |
4
|
|
|
* |
5
|
|
|
* @package Firesphere\SolrSearch\Indexes |
6
|
|
|
* @author Simon `Firesphere` Erkelens; Marco `Sheepy` Hermo |
7
|
|
|
* @copyright Copyright (c) 2018 - now() Firesphere & Sheepy |
8
|
|
|
*/ |
9
|
|
|
|
10
|
|
|
namespace Firesphere\SolrSearch\Indexes; |
11
|
|
|
|
12
|
|
|
use Exception; |
13
|
|
|
use Firesphere\SolrSearch\Factories\QueryComponentFactory; |
14
|
|
|
use Firesphere\SolrSearch\Factories\SchemaFactory; |
15
|
|
|
use Firesphere\SolrSearch\Helpers\SolrLogger; |
16
|
|
|
use Firesphere\SolrSearch\Helpers\Synonyms; |
17
|
|
|
use Firesphere\SolrSearch\Interfaces\ConfigStore; |
18
|
|
|
use Firesphere\SolrSearch\Models\SearchSynonym; |
19
|
|
|
use Firesphere\SolrSearch\Queries\BaseQuery; |
20
|
|
|
use Firesphere\SolrSearch\Results\SearchResult; |
21
|
|
|
use Firesphere\SolrSearch\Services\SolrCoreService; |
22
|
|
|
use Firesphere\SolrSearch\States\SiteState; |
23
|
|
|
use Firesphere\SolrSearch\Traits\BaseIndexTrait; |
24
|
|
|
use Firesphere\SolrSearch\Traits\GetterSetterTrait; |
25
|
|
|
use GuzzleHttp\Exception\GuzzleException; |
26
|
|
|
use LogicException; |
27
|
|
|
use ReflectionException; |
28
|
|
|
use SilverStripe\Control\Director; |
29
|
|
|
use SilverStripe\Core\Config\Config; |
30
|
|
|
use SilverStripe\Core\Config\Configurable; |
31
|
|
|
use SilverStripe\Core\Extensible; |
32
|
|
|
use SilverStripe\Core\Injector\Injectable; |
33
|
|
|
use SilverStripe\Core\Injector\Injector; |
34
|
|
|
use SilverStripe\Dev\Deprecation; |
35
|
|
|
use SilverStripe\ORM\DataList; |
36
|
|
|
use SilverStripe\ORM\ValidationException; |
37
|
|
|
use SilverStripe\View\ArrayData; |
38
|
|
|
use Solarium\Core\Client\Adapter\Guzzle; |
39
|
|
|
use Solarium\Core\Client\Client; |
40
|
|
|
use Solarium\QueryType\Select\Query\Query; |
41
|
|
|
use Solarium\QueryType\Select\Result\Result; |
42
|
|
|
|
43
|
|
|
/** |
44
|
|
|
* Base for creating a new Solr core. |
45
|
|
|
* |
46
|
|
|
* Base index settings and methods. Should be extended with at least a name for the index. |
47
|
|
|
* This is an abstract class that can not be instantiated on it's own |
48
|
|
|
* |
49
|
|
|
* @package Firesphere\SolrSearch\Indexes |
50
|
|
|
*/ |
51
|
|
|
abstract class BaseIndex |
52
|
|
|
{ |
53
|
|
|
use Extensible; |
54
|
|
|
use Configurable; |
55
|
|
|
use Injectable; |
56
|
|
|
use GetterSetterTrait; |
57
|
|
|
use BaseIndexTrait; |
58
|
|
|
|
59
|
|
|
/** |
60
|
|
|
* Field types that can be added |
61
|
|
|
* Used in init to call build methods from configuration yml |
62
|
|
|
* |
63
|
|
|
* @array |
64
|
|
|
*/ |
65
|
|
|
private static $fieldTypes = [ |
66
|
|
|
'FulltextFields', |
67
|
|
|
'SortFields', |
68
|
|
|
'FilterFields', |
69
|
|
|
'BoostedFields', |
70
|
|
|
'CopyFields', |
71
|
|
|
'DefaultField', |
72
|
|
|
'FacetFields', |
73
|
|
|
'StoredFields', |
74
|
|
|
]; |
75
|
|
|
/** |
76
|
|
|
* {@link SchemaFactory} |
77
|
|
|
* |
78
|
|
|
* @var SchemaFactory Schema factory for generating the schema |
79
|
|
|
*/ |
80
|
|
|
protected $schemaFactory; |
81
|
|
|
/** |
82
|
|
|
* {@link QueryComponentFactory} |
83
|
|
|
* |
84
|
|
|
* @var QueryComponentFactory Generator for all components |
85
|
|
|
*/ |
86
|
|
|
protected $queryFactory; |
87
|
|
|
/** |
88
|
|
|
* @var array The query terms as an array |
89
|
|
|
*/ |
90
|
|
|
protected $queryTerms = []; |
91
|
|
|
/** |
92
|
|
|
* @var bool Signify if a retry should occur if nothing was found and there are suggestions to follow |
93
|
|
|
*/ |
94
|
|
|
private $retry = false; |
95
|
|
|
|
96
|
|
|
/** |
97
|
|
|
* BaseIndex constructor. |
98
|
|
|
*/ |
99
|
51 |
|
public function __construct() |
100
|
|
|
{ |
101
|
|
|
// Set up the client |
102
|
51 |
|
$config = Config::inst()->get(SolrCoreService::class, 'config'); |
103
|
51 |
|
$config['endpoint'] = $this->getConfig($config['endpoint']); |
104
|
51 |
|
$this->client = new Client($config); |
105
|
51 |
|
$this->client->setAdapter(new Guzzle()); |
106
|
|
|
|
107
|
|
|
// Set up the schema service, only used in the generation of the schema |
108
|
51 |
|
$schemaFactory = Injector::inst()->get(SchemaFactory::class, false); |
109
|
51 |
|
$schemaFactory->setIndex($this); |
110
|
51 |
|
$schemaFactory->setStore(Director::isDev()); |
111
|
51 |
|
$this->schemaFactory = $schemaFactory; |
112
|
51 |
|
$this->queryFactory = Injector::inst()->get(QueryComponentFactory::class, false); |
113
|
|
|
|
114
|
51 |
|
$this->extend('onBeforeInit'); |
115
|
51 |
|
$this->init(); |
116
|
51 |
|
$this->extend('onAfterInit'); |
117
|
51 |
|
} |
118
|
|
|
|
119
|
|
|
/** |
120
|
|
|
* Build a full config for all given endpoints |
121
|
|
|
* This is to add the current index to e.g. an index or select |
122
|
|
|
* |
123
|
|
|
* @param array $endpoints |
124
|
|
|
* @return array |
125
|
|
|
*/ |
126
|
51 |
|
public function getConfig($endpoints): array |
127
|
|
|
{ |
128
|
51 |
|
foreach ($endpoints as $host => $endpoint) { |
129
|
51 |
|
$endpoints[$host]['core'] = $this->getIndexName(); |
130
|
|
|
} |
131
|
|
|
|
132
|
51 |
|
return $endpoints; |
133
|
|
|
} |
134
|
|
|
|
135
|
|
|
/** |
136
|
|
|
* Name of this index. |
137
|
|
|
* |
138
|
|
|
* @return string |
139
|
|
|
*/ |
140
|
|
|
abstract public function getIndexName(); |
141
|
|
|
|
142
|
|
|
/** |
143
|
|
|
* Required to initialise the fields. |
144
|
|
|
* It's loaded in to the non-static properties for backward compatibility with FTS |
145
|
|
|
* Also, it's a tad easier to use this way, loading the other way around would be very |
146
|
|
|
* memory intensive, as updating the config for each item is not efficient |
147
|
|
|
*/ |
148
|
47 |
|
public function init() |
149
|
|
|
{ |
150
|
47 |
|
$config = self::config()->get($this->getIndexName()); |
151
|
47 |
|
if (!$config) { |
152
|
47 |
|
Deprecation::notice('5', 'Please set an index name and use a config yml'); |
153
|
|
|
} |
154
|
|
|
|
155
|
47 |
|
if (!empty($this->getClasses())) { |
156
|
47 |
|
if (!$this->usedAllFields) { |
157
|
47 |
|
Deprecation::notice('5', 'It is advised to use a config YML for most cases'); |
158
|
|
|
} |
159
|
|
|
|
160
|
47 |
|
return; |
161
|
|
|
} |
162
|
|
|
|
163
|
39 |
|
$this->initFromConfig($config); |
164
|
39 |
|
} |
165
|
|
|
|
166
|
|
|
/** |
167
|
|
|
* Generate the config from yml if possible |
168
|
|
|
* @param array|null $config |
169
|
|
|
*/ |
170
|
39 |
|
protected function initFromConfig($config): void |
171
|
|
|
{ |
172
|
39 |
|
if (!$config || !array_key_exists('Classes', $config)) { |
173
|
2 |
|
throw new LogicException('No classes or config to index found!'); |
174
|
|
|
} |
175
|
|
|
|
176
|
39 |
|
$this->setClasses($config['Classes']); |
177
|
|
|
|
178
|
|
|
// For backward compatibility, copy the config to the protected values |
179
|
|
|
// Saves doubling up further down the line |
180
|
39 |
|
foreach (self::$fieldTypes as $type) { |
181
|
39 |
|
if (array_key_exists($type, $config)) { |
182
|
39 |
|
$method = 'set' . $type; |
183
|
39 |
|
$this->$method($config[$type]); |
184
|
|
|
} |
185
|
|
|
} |
186
|
39 |
|
} |
187
|
|
|
|
188
|
|
|
/** |
189
|
|
|
* Default returns a SearchResult. It can return an ArrayData if FTS Compat is enabled |
190
|
|
|
* |
191
|
|
|
* @param BaseQuery $query |
192
|
|
|
* @return SearchResult|ArrayData|mixed |
193
|
|
|
* @throws GuzzleException |
194
|
|
|
* @throws ValidationException |
195
|
|
|
* @throws ReflectionException |
196
|
|
|
* @throws Exception |
197
|
|
|
*/ |
198
|
6 |
|
public function doSearch(BaseQuery $query) |
199
|
|
|
{ |
200
|
6 |
|
SiteState::alterQuery($query); |
201
|
|
|
// Build the actual query parameters |
202
|
6 |
|
$clientQuery = $this->buildSolrQuery($query); |
203
|
|
|
// Set the sorting |
204
|
6 |
|
$clientQuery->addSorts($query->getSort()); |
205
|
|
|
|
206
|
6 |
|
$this->extend('onBeforeSearch', $query, $clientQuery); |
207
|
|
|
|
208
|
|
|
try { |
209
|
6 |
|
$result = $this->client->select($clientQuery); |
210
|
|
|
} catch (Exception $error) { |
211
|
|
|
// @codeCoverageIgnoreStart |
212
|
|
|
$logger = new SolrLogger(); |
213
|
|
|
$logger->saveSolrLog('Query'); |
214
|
|
|
throw $error; |
215
|
|
|
// @codeCoverageIgnoreEnd |
216
|
|
|
} |
217
|
|
|
|
218
|
|
|
// Handle the after search first. This gets a raw search result |
219
|
6 |
|
$this->extend('onAfterSearch', $result); |
220
|
6 |
|
$searchResult = new SearchResult($result, $query, $this); |
221
|
6 |
|
if ($this->doRetry($query, $result, $searchResult)) { |
222
|
2 |
|
return $this->spellcheckRetry($query, $searchResult); |
223
|
|
|
} |
224
|
|
|
|
225
|
|
|
// And then handle the search results, which is a useable object for SilverStripe |
226
|
6 |
|
$this->extend('updateSearchResults', $searchResult); |
227
|
|
|
|
228
|
6 |
|
return $searchResult; |
229
|
|
|
} |
230
|
|
|
|
231
|
|
|
/** |
232
|
|
|
* From the given BaseQuery, generate a Solarium ClientQuery object |
233
|
|
|
* |
234
|
|
|
* @param BaseQuery $query |
235
|
|
|
* @return Query |
236
|
|
|
*/ |
237
|
6 |
|
public function buildSolrQuery(BaseQuery $query): Query |
238
|
|
|
{ |
239
|
6 |
|
$clientQuery = $this->client->createSelect(); |
240
|
6 |
|
$factory = $this->buildFactory($query, $clientQuery); |
241
|
|
|
|
242
|
6 |
|
$clientQuery = $factory->buildQuery(); |
243
|
6 |
|
$this->queryTerms = $factory->getQueryArray(); |
244
|
|
|
|
245
|
6 |
|
$queryData = implode(' ', $this->queryTerms); |
246
|
6 |
|
$clientQuery->setQuery($queryData); |
247
|
|
|
|
248
|
6 |
|
return $clientQuery; |
249
|
|
|
} |
250
|
|
|
|
251
|
|
|
/** |
252
|
|
|
* Build a factory to use in the SolrQuery building. {@link static::buildSolrQuery()} |
253
|
|
|
* |
254
|
|
|
* @param BaseQuery $query |
255
|
|
|
* @param Query $clientQuery |
256
|
|
|
* @return QueryComponentFactory|mixed |
257
|
|
|
*/ |
258
|
6 |
|
protected function buildFactory(BaseQuery $query, Query $clientQuery) |
259
|
|
|
{ |
260
|
6 |
|
$factory = $this->queryFactory; |
261
|
|
|
|
262
|
6 |
|
$helper = $clientQuery->getHelper(); |
263
|
|
|
|
264
|
6 |
|
$factory->setQuery($query); |
265
|
6 |
|
$factory->setClientQuery($clientQuery); |
266
|
6 |
|
$factory->setHelper($helper); |
267
|
6 |
|
$factory->setIndex($this); |
268
|
|
|
|
269
|
6 |
|
return $factory; |
270
|
|
|
} |
271
|
|
|
|
272
|
|
|
/** |
273
|
|
|
* Check if the query should be retried with spellchecking |
274
|
|
|
* Conditions are: |
275
|
|
|
* It is not already a retry with spellchecking |
276
|
|
|
* Spellchecking is enabled |
277
|
|
|
* If spellchecking is enabled and nothing is found OR it should follow spellchecking none the less |
278
|
|
|
* There is a spellcheck output |
279
|
|
|
* |
280
|
|
|
* @param BaseQuery $query |
281
|
|
|
* @param Result $result |
282
|
|
|
* @param SearchResult $searchResult |
283
|
|
|
* @return bool |
284
|
|
|
*/ |
285
|
6 |
|
protected function doRetry(BaseQuery $query, Result $result, SearchResult $searchResult): bool |
286
|
|
|
{ |
287
|
6 |
|
return !$this->retry && |
288
|
6 |
|
$query->hasSpellcheck() && |
289
|
6 |
|
($query->shouldFollowSpellcheck() || $result->getNumFound() === 0) && |
290
|
6 |
|
$searchResult->getCollatedSpellcheck(); |
291
|
|
|
} |
292
|
|
|
|
293
|
|
|
/** |
294
|
|
|
* Retry the query with the first collated spellcheck found. |
295
|
|
|
* |
296
|
|
|
* @param BaseQuery $query |
297
|
|
|
* @param SearchResult $searchResult |
298
|
|
|
* @return SearchResult|mixed|ArrayData |
299
|
|
|
* @throws GuzzleException |
300
|
|
|
* @throws ValidationException |
301
|
|
|
* @throws ReflectionException |
302
|
|
|
*/ |
303
|
2 |
|
protected function spellcheckRetry(BaseQuery $query, SearchResult $searchResult) |
304
|
|
|
{ |
305
|
2 |
|
$terms = $query->getTerms(); |
306
|
2 |
|
$spellChecked = $searchResult->getCollatedSpellcheck(); |
307
|
|
|
// Remove the fuzzyness from the collated check |
308
|
2 |
|
$term = preg_replace('/~\d+/', '', $spellChecked); |
309
|
2 |
|
$terms[0]['text'] = $term; |
310
|
2 |
|
$query->setTerms($terms); |
311
|
2 |
|
$this->retry = true; |
312
|
|
|
|
313
|
2 |
|
return $this->doSearch($query); |
314
|
|
|
} |
315
|
|
|
|
316
|
|
|
/** |
317
|
|
|
* Get all fields that are required for indexing in a unique way |
318
|
|
|
* |
319
|
|
|
* @return array |
320
|
|
|
*/ |
321
|
10 |
|
public function getFieldsForIndexing(): array |
322
|
|
|
{ |
323
|
10 |
|
$facets = []; |
324
|
10 |
|
foreach ($this->getFacetFields() as $field) { |
325
|
9 |
|
$facets[] = $field['Field']; |
326
|
|
|
} |
327
|
|
|
// Return values to make the key reset |
328
|
|
|
// Only return unique values |
329
|
|
|
// And make it all a single array |
330
|
10 |
|
$fields = array_values( |
331
|
10 |
|
array_unique( |
332
|
10 |
|
array_merge( |
333
|
10 |
|
$this->getFulltextFields(), |
334
|
10 |
|
$this->getSortFields(), |
335
|
10 |
|
$facets, |
336
|
10 |
|
$this->getFilterFields() |
337
|
|
|
) |
338
|
|
|
) |
339
|
|
|
); |
340
|
|
|
|
341
|
10 |
|
$this->extend('updateFieldsForIndexing', $fields); |
342
|
|
|
|
343
|
10 |
|
return $fields; |
344
|
|
|
} |
345
|
|
|
|
346
|
|
|
/** |
347
|
|
|
* Upload config for this index to the given store |
348
|
|
|
* |
349
|
|
|
* @param ConfigStore $store |
350
|
|
|
*/ |
351
|
35 |
|
public function uploadConfig(ConfigStore $store): void |
352
|
|
|
{ |
353
|
|
|
// @todo use types/schema/elevate rendering |
354
|
|
|
// Upload the config files for this index |
355
|
|
|
// Create a default schema which we can manage later |
356
|
35 |
|
$schema = (string)$this->schemaFactory->generateSchema(); |
357
|
35 |
|
$store->uploadString( |
358
|
35 |
|
$this->getIndexName(), |
359
|
35 |
|
'schema.xml', |
360
|
35 |
|
$schema |
361
|
|
|
); |
362
|
|
|
|
363
|
35 |
|
$this->getSynonyms($store); |
364
|
|
|
|
365
|
|
|
// Upload additional files |
366
|
35 |
|
foreach (glob($this->schemaFactory->getExtrasPath() . '/*') as $file) { |
367
|
35 |
|
if (is_file($file)) { |
368
|
35 |
|
$store->uploadFile($this->getIndexName(), $file); |
369
|
|
|
} |
370
|
|
|
} |
371
|
35 |
|
} |
372
|
|
|
|
373
|
|
|
/** |
374
|
|
|
* Add synonyms. Public to be extendable |
375
|
|
|
* |
376
|
|
|
* @param ConfigStore $store Store to use to write synonyms |
377
|
|
|
* @param bool $defaults Include UK to US synonyms |
378
|
|
|
* @return string |
379
|
|
|
*/ |
380
|
35 |
|
public function getSynonyms($store = null, $defaults = true) |
381
|
|
|
{ |
382
|
35 |
|
$synonyms = Synonyms::getSynonymsAsString($defaults); |
383
|
|
|
/** @var DataList|SearchSynonym[] $syn */ |
384
|
35 |
|
$syn = SearchSynonym::get(); |
385
|
35 |
|
foreach ($syn as $synonym) { |
386
|
1 |
|
$synonyms .= $synonym->getCombinedSynonym(); |
387
|
|
|
} |
388
|
|
|
|
389
|
|
|
// Upload synonyms |
390
|
35 |
|
if ($store) { |
391
|
35 |
|
$store->uploadString( |
392
|
35 |
|
$this->getIndexName(), |
393
|
35 |
|
'synonyms.txt', |
394
|
35 |
|
$synonyms |
395
|
|
|
); |
396
|
|
|
} |
397
|
|
|
|
398
|
35 |
|
return $synonyms; |
399
|
|
|
} |
400
|
|
|
|
401
|
|
|
/** |
402
|
|
|
* Get the final, generated terms |
403
|
|
|
* |
404
|
|
|
* @return array |
405
|
|
|
*/ |
406
|
2 |
|
public function getQueryTerms(): array |
407
|
|
|
{ |
408
|
2 |
|
return $this->queryTerms; |
409
|
|
|
} |
410
|
|
|
|
411
|
|
|
/** |
412
|
|
|
* Get the QueryComponentFactory. {@link QueryComponentFactory} |
413
|
|
|
* |
414
|
|
|
* @return QueryComponentFactory |
415
|
|
|
*/ |
416
|
1 |
|
public function getQueryFactory(): QueryComponentFactory |
417
|
|
|
{ |
418
|
1 |
|
return $this->queryFactory; |
419
|
|
|
} |
420
|
|
|
} |
421
|
|
|
|