1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
|
4
|
|
|
namespace Firesphere\SolrSearch\Indexes; |
5
|
|
|
|
6
|
|
|
use Exception; |
7
|
|
|
use Firesphere\SolrSearch\Factories\QueryComponentFactory; |
8
|
|
|
use Firesphere\SolrSearch\Helpers\SolrLogger; |
9
|
|
|
use Firesphere\SolrSearch\Helpers\Synonyms; |
10
|
|
|
use Firesphere\SolrSearch\Interfaces\ConfigStore; |
11
|
|
|
use Firesphere\SolrSearch\Models\SearchSynonym; |
12
|
|
|
use Firesphere\SolrSearch\Queries\BaseQuery; |
13
|
|
|
use Firesphere\SolrSearch\Results\SearchResult; |
14
|
|
|
use Firesphere\SolrSearch\Services\SchemaService; |
15
|
|
|
use Firesphere\SolrSearch\Services\SolrCoreService; |
16
|
|
|
use Firesphere\SolrSearch\States\SiteState; |
17
|
|
|
use Firesphere\SolrSearch\Traits\BaseIndexTrait; |
18
|
|
|
use Firesphere\SolrSearch\Traits\GetterSetterTrait; |
19
|
|
|
use GuzzleHttp\Exception\GuzzleException; |
20
|
|
|
use LogicException; |
21
|
|
|
use ReflectionException; |
22
|
|
|
use SilverStripe\Control\Controller; |
23
|
|
|
use SilverStripe\Control\Director; |
24
|
|
|
use SilverStripe\Core\Config\Config; |
25
|
|
|
use SilverStripe\Core\Config\Configurable; |
26
|
|
|
use SilverStripe\Core\Extensible; |
27
|
|
|
use SilverStripe\Core\Injector\Injectable; |
28
|
|
|
use SilverStripe\Core\Injector\Injector; |
29
|
|
|
use SilverStripe\Dev\Debug; |
30
|
|
|
use SilverStripe\Dev\Deprecation; |
31
|
|
|
use SilverStripe\ORM\ValidationException; |
32
|
|
|
use SilverStripe\View\ArrayData; |
33
|
|
|
use Solarium\Core\Client\Adapter\Guzzle; |
34
|
|
|
use Solarium\Core\Client\Client; |
35
|
|
|
use Solarium\Core\Client\Response; |
36
|
|
|
use Solarium\Exception\RuntimeException; |
37
|
|
|
use Solarium\QueryType\Select\Query\Query; |
38
|
|
|
use Solarium\QueryType\Select\Result\Result; |
39
|
|
|
|
40
|
|
|
/** |
41
|
|
|
* Base for creating a new Solr core. |
42
|
|
|
* |
43
|
|
|
* Base index settings and methods. Should be extended with at least a name for the index. |
44
|
|
|
* This is an abstract class that can not be instantiated on it's own |
45
|
|
|
* |
46
|
|
|
* @package Firesphere\SolrSearch\Indexes |
47
|
|
|
*/ |
48
|
|
|
abstract class BaseIndex |
49
|
|
|
{ |
50
|
|
|
use Extensible; |
51
|
|
|
use Configurable; |
52
|
|
|
use Injectable; |
53
|
|
|
use GetterSetterTrait; |
54
|
|
|
use BaseIndexTrait; |
55
|
|
|
/** |
56
|
|
|
* Session key for the query history |
57
|
|
|
*/ |
58
|
|
|
const SEARCH_HISTORY_KEY = 'query_history'; |
59
|
|
|
|
60
|
|
|
/** |
61
|
|
|
* Field types that can be added |
62
|
|
|
* Used in init to call build methods from configuration yml |
63
|
|
|
* |
64
|
|
|
* @array |
65
|
|
|
*/ |
66
|
|
|
private static $fieldTypes = [ |
67
|
|
|
'FulltextFields', |
68
|
|
|
'SortFields', |
69
|
|
|
'FilterFields', |
70
|
|
|
'BoostedFields', |
71
|
|
|
'CopyFields', |
72
|
|
|
'DefaultField', |
73
|
|
|
'FacetFields', |
74
|
|
|
'StoredFields', |
75
|
|
|
]; |
76
|
|
|
/** |
77
|
|
|
* {@link SchemaService} |
78
|
|
|
* |
79
|
|
|
* @var SchemaService |
80
|
|
|
*/ |
81
|
|
|
protected $schemaService; |
82
|
|
|
/** |
83
|
|
|
* {@link QueryComponentFactory} |
84
|
|
|
* |
85
|
|
|
* @var QueryComponentFactory |
86
|
|
|
*/ |
87
|
|
|
protected $queryFactory; |
88
|
|
|
/** |
89
|
|
|
* The query terms as an array |
90
|
|
|
* |
91
|
|
|
* @var array |
92
|
|
|
*/ |
93
|
|
|
protected $queryTerms = []; |
94
|
|
|
/** |
95
|
|
|
* Should a retry occur if nothing was found and there are suggestions to follow |
96
|
|
|
* |
97
|
|
|
* @var bool |
98
|
|
|
*/ |
99
|
|
|
private $retry = false; |
100
|
|
|
|
101
|
|
|
/** |
102
|
|
|
* BaseIndex constructor. |
103
|
|
|
*/ |
104
|
75 |
|
public function __construct() |
105
|
|
|
{ |
106
|
|
|
// Set up the client |
107
|
75 |
|
$config = Config::inst()->get(SolrCoreService::class, 'config'); |
108
|
75 |
|
$config['endpoint'] = $this->getConfig($config['endpoint']); |
109
|
75 |
|
$this->client = new Client($config); |
110
|
75 |
|
$this->client->setAdapter(new Guzzle()); |
111
|
75 |
|
$session = Controller::curr()->getRequest()->getSession(); |
112
|
75 |
|
$this->history = $session->get(self::SEARCH_HISTORY_KEY) ?: []; |
113
|
|
|
|
114
|
|
|
// Set up the schema service, only used in the generation of the schema |
115
|
75 |
|
$schemaService = Injector::inst()->get(SchemaService::class, false); |
116
|
75 |
|
$schemaService->setIndex($this); |
117
|
75 |
|
$schemaService->setStore(Director::isDev()); |
118
|
75 |
|
$this->schemaService = $schemaService; |
119
|
75 |
|
$this->queryFactory = Injector::inst()->get(QueryComponentFactory::class, false); |
120
|
|
|
|
121
|
75 |
|
$this->extend('onBeforeInit'); |
122
|
75 |
|
$this->init(); |
123
|
75 |
|
$this->extend('onAfterInit'); |
124
|
75 |
|
} |
125
|
|
|
|
126
|
|
|
/** |
127
|
|
|
* Build a full config for all given endpoints |
128
|
|
|
* This is to add the current index to e.g. an index or select |
129
|
|
|
* |
130
|
|
|
* @param array $endpoints |
131
|
|
|
* @return array |
132
|
|
|
*/ |
133
|
75 |
|
public function getConfig($endpoints): array |
134
|
|
|
{ |
135
|
75 |
|
foreach ($endpoints as $host => $endpoint) { |
136
|
75 |
|
$endpoints[$host]['core'] = $this->getIndexName(); |
137
|
|
|
} |
138
|
|
|
|
139
|
75 |
|
return $endpoints; |
140
|
|
|
} |
141
|
|
|
|
142
|
|
|
/** |
143
|
|
|
* Name of this index. |
144
|
|
|
* |
145
|
|
|
* @return string |
146
|
|
|
*/ |
147
|
|
|
abstract public function getIndexName(); |
148
|
|
|
|
149
|
|
|
/** |
150
|
|
|
* Required to initialise the fields. |
151
|
|
|
* It's loaded in to the non-static properties for backward compatibility with FTS |
152
|
|
|
* Also, it's a tad easier to use this way, loading the other way around would be very |
153
|
|
|
* memory intensive, as updating the config for each item is not efficient |
154
|
|
|
*/ |
155
|
74 |
|
public function init() |
156
|
|
|
{ |
157
|
74 |
|
$config = self::config()->get($this->getIndexName()); |
158
|
74 |
|
if (!$config) { |
159
|
74 |
|
Deprecation::notice('5', 'Please set an index name and use a config yml'); |
160
|
|
|
} |
161
|
|
|
|
162
|
74 |
|
if (!empty($this->getClasses())) { |
163
|
74 |
|
if (!$this->usedAllFields) { |
164
|
74 |
|
Deprecation::notice('5', 'It is advised to use a config YML for most cases'); |
165
|
|
|
} |
166
|
|
|
|
167
|
74 |
|
return; |
168
|
|
|
} |
169
|
|
|
|
170
|
74 |
|
$this->initFromConfig($config); |
171
|
74 |
|
} |
172
|
|
|
|
173
|
|
|
/** |
174
|
|
|
* Generate the config from yml if possible |
175
|
|
|
* @param array|null $config |
176
|
|
|
*/ |
177
|
74 |
|
protected function initFromConfig($config): void |
178
|
|
|
{ |
179
|
74 |
|
if (!$config || !array_key_exists('Classes', $config)) { |
180
|
2 |
|
throw new LogicException('No classes or config to index found!'); |
181
|
|
|
} |
182
|
|
|
|
183
|
74 |
|
$this->setClasses($config['Classes']); |
184
|
|
|
|
185
|
|
|
// For backward compatibility, copy the config to the protected values |
186
|
|
|
// Saves doubling up further down the line |
187
|
74 |
|
foreach (self::$fieldTypes as $type) { |
188
|
74 |
|
if (array_key_exists($type, $config)) { |
189
|
74 |
|
$method = 'set' . $type; |
190
|
74 |
|
$this->$method($config[$type]); |
191
|
|
|
} |
192
|
|
|
} |
193
|
74 |
|
} |
194
|
|
|
|
195
|
|
|
/** |
196
|
|
|
* Default returns a SearchResult. It can return an ArrayData if FTS Compat is enabled |
197
|
|
|
* |
198
|
|
|
* @param BaseQuery $query |
199
|
|
|
* @return SearchResult|ArrayData|mixed |
200
|
|
|
* @throws GuzzleException |
201
|
|
|
* @throws ValidationException |
202
|
|
|
* @throws ReflectionException |
203
|
|
|
*/ |
204
|
5 |
|
public function doSearch(BaseQuery $query) |
205
|
|
|
{ |
206
|
5 |
|
SiteState::alterQuery($query); |
207
|
|
|
// Build the actual query parameters |
208
|
5 |
|
$clientQuery = $this->buildSolrQuery($query); |
209
|
|
|
|
210
|
5 |
|
$this->extend('onBeforeSearch', $query, $clientQuery); |
211
|
|
|
|
212
|
|
|
try { |
213
|
5 |
|
$result = $this->client->select($clientQuery); |
214
|
|
|
} catch (Exception $e) { |
215
|
|
|
$logger = new SolrLogger(); |
216
|
|
|
$logger->saveSolrLog('Query'); |
217
|
|
|
} |
218
|
|
|
|
219
|
|
|
// Handle the after search first. This gets a raw search result |
220
|
5 |
|
$this->extend('onAfterSearch', $result); |
221
|
5 |
|
$searchResult = new SearchResult($result, $query, $this); |
222
|
5 |
|
if ($this->doRetry($query, $result, $searchResult)) { |
223
|
2 |
|
return $this->spellcheckRetry($query, $searchResult); |
224
|
|
|
} |
225
|
|
|
|
226
|
|
|
// And then handle the search results, which is a useable object for SilverStripe |
227
|
5 |
|
$this->extend('updateSearchResults', $searchResult); |
228
|
|
|
|
229
|
5 |
|
Controller::curr() |
230
|
5 |
|
->getRequest() |
231
|
5 |
|
->getSession() |
232
|
5 |
|
->set(self::SEARCH_HISTORY_KEY, $this->getHistory()); |
233
|
|
|
|
234
|
5 |
|
return $searchResult; |
235
|
|
|
} |
236
|
|
|
|
237
|
|
|
/** |
238
|
|
|
* From the given BaseQuery, generate a Solarium ClientQuery object |
239
|
|
|
* |
240
|
|
|
* @param BaseQuery $query |
241
|
|
|
* @return Query |
242
|
|
|
*/ |
243
|
5 |
|
public function buildSolrQuery(BaseQuery $query): Query |
244
|
|
|
{ |
245
|
5 |
|
$clientQuery = $this->client->createSelect(); |
246
|
5 |
|
$factory = $this->buildFactory($query, $clientQuery); |
247
|
|
|
|
248
|
5 |
|
$clientQuery = $factory->buildQuery(); |
249
|
5 |
|
$this->queryTerms = $factory->getQueryArray(); |
250
|
|
|
|
251
|
5 |
|
$queryData = implode(' ', $this->queryTerms); |
252
|
5 |
|
$clientQuery->setQuery($queryData); |
253
|
|
|
|
254
|
5 |
|
return $clientQuery; |
255
|
|
|
} |
256
|
|
|
|
257
|
|
|
/** |
258
|
|
|
* Build a factory to use in the SolrQuery building. {@link static::buildSolrQuery()} |
259
|
|
|
* |
260
|
|
|
* @param BaseQuery $query |
261
|
|
|
* @param Query $clientQuery |
262
|
|
|
* @return QueryComponentFactory|mixed |
263
|
|
|
*/ |
264
|
5 |
|
protected function buildFactory(BaseQuery $query, Query $clientQuery) |
265
|
|
|
{ |
266
|
5 |
|
$factory = $this->queryFactory; |
267
|
|
|
|
268
|
5 |
|
$helper = $clientQuery->getHelper(); |
269
|
|
|
|
270
|
5 |
|
$factory->setQuery($query); |
271
|
5 |
|
$factory->setClientQuery($clientQuery); |
272
|
5 |
|
$factory->setHelper($helper); |
273
|
5 |
|
$factory->setIndex($this); |
274
|
|
|
|
275
|
5 |
|
return $factory; |
276
|
|
|
} |
277
|
|
|
|
278
|
|
|
/** |
279
|
|
|
* Check if the query should be retried with spellchecking |
280
|
|
|
* Conditions are: |
281
|
|
|
* It is not already a retry with spellchecking |
282
|
|
|
* Spellchecking is enabled |
283
|
|
|
* If spellchecking is enabled and nothing is found OR it should follow spellchecking none the less |
284
|
|
|
* There is a spellcheck output |
285
|
|
|
* |
286
|
|
|
* @param BaseQuery $query |
287
|
|
|
* @param Result $result |
288
|
|
|
* @param SearchResult $searchResult |
289
|
|
|
* @return bool |
290
|
|
|
*/ |
291
|
5 |
|
protected function doRetry(BaseQuery $query, Result $result, SearchResult $searchResult): bool |
292
|
|
|
{ |
293
|
5 |
|
return !$this->retry && |
294
|
5 |
|
$query->hasSpellcheck() && |
295
|
5 |
|
($query->shouldFollowSpellcheck() || $result->getNumFound() === 0) && |
296
|
5 |
|
$searchResult->getCollatedSpellcheck(); |
297
|
|
|
} |
298
|
|
|
|
299
|
|
|
/** |
300
|
|
|
* Retry the query with the first collated spellcheck found. |
301
|
|
|
* |
302
|
|
|
* @param BaseQuery $query |
303
|
|
|
* @param SearchResult $searchResult |
304
|
|
|
* @return SearchResult|mixed|ArrayData |
305
|
|
|
* @throws GuzzleException |
306
|
|
|
* @throws ValidationException |
307
|
|
|
* @throws ReflectionException |
308
|
|
|
*/ |
309
|
2 |
|
protected function spellcheckRetry(BaseQuery $query, SearchResult $searchResult) |
310
|
|
|
{ |
311
|
2 |
|
$terms = $query->getTerms(); |
312
|
2 |
|
$spellChecked = $searchResult->getCollatedSpellcheck(); |
313
|
|
|
// Remove the fuzzyness from the collated check |
314
|
2 |
|
$term = preg_replace('/~\d+/', '', $spellChecked); |
315
|
2 |
|
$terms[0]['text'] = $term; |
316
|
2 |
|
$query->setTerms($terms); |
317
|
2 |
|
$this->retry = true; |
318
|
|
|
|
319
|
2 |
|
return $this->doSearch($query); |
320
|
|
|
} |
321
|
|
|
|
322
|
|
|
/** |
323
|
|
|
* Get all fields that are required for indexing in a unique way |
324
|
|
|
* |
325
|
|
|
* @return array |
326
|
|
|
*/ |
327
|
8 |
|
public function getFieldsForIndexing(): array |
328
|
|
|
{ |
329
|
8 |
|
$facets = []; |
330
|
8 |
|
foreach ($this->getFacetFields() as $field) { |
331
|
7 |
|
$facets[] = $field['Field']; |
332
|
|
|
} |
333
|
|
|
// Return values to make the key reset |
334
|
|
|
// Only return unique values |
335
|
|
|
// And make it all a single array |
336
|
8 |
|
$fields = array_values( |
337
|
8 |
|
array_unique( |
338
|
8 |
|
array_merge( |
339
|
8 |
|
$this->getFulltextFields(), |
340
|
8 |
|
$this->getSortFields(), |
341
|
8 |
|
$facets, |
342
|
8 |
|
$this->getFilterFields() |
343
|
|
|
) |
344
|
|
|
) |
345
|
|
|
); |
346
|
|
|
|
347
|
8 |
|
$this->extend('updateFieldsForIndexing', $fields); |
348
|
|
|
|
349
|
8 |
|
return $fields; |
350
|
|
|
} |
351
|
|
|
|
352
|
|
|
/** |
353
|
|
|
* Upload config for this index to the given store |
354
|
|
|
* |
355
|
|
|
* @param ConfigStore $store |
356
|
|
|
*/ |
357
|
34 |
|
public function uploadConfig(ConfigStore $store): void |
358
|
|
|
{ |
359
|
|
|
// @todo use types/schema/elevate rendering |
360
|
|
|
// Upload the config files for this index |
361
|
|
|
// Create a default schema which we can manage later |
362
|
34 |
|
$schema = (string)$this->schemaService->generateSchema(); |
363
|
34 |
|
$store->uploadString( |
364
|
34 |
|
$this->getIndexName(), |
365
|
34 |
|
'schema.xml', |
366
|
34 |
|
$schema |
367
|
|
|
); |
368
|
|
|
|
369
|
34 |
|
$this->getSynonyms($store); |
370
|
|
|
|
371
|
|
|
// Upload additional files |
372
|
34 |
|
foreach (glob($this->schemaService->getExtrasPath() . '/*') as $file) { |
373
|
34 |
|
if (is_file($file)) { |
374
|
34 |
|
$store->uploadFile($this->getIndexName(), $file); |
375
|
|
|
} |
376
|
|
|
} |
377
|
34 |
|
} |
378
|
|
|
|
379
|
|
|
/** |
380
|
|
|
* Add synonyms. Public to be extendable |
381
|
|
|
* |
382
|
|
|
* @param ConfigStore $store Store to use to write synonyms |
383
|
|
|
* @param bool $defaults Include UK to US synonyms |
384
|
|
|
* @return string |
385
|
|
|
*/ |
386
|
34 |
|
public function getSynonyms($store = null, $defaults = true) |
387
|
|
|
{ |
388
|
34 |
|
$synonyms = Synonyms::getSynonymsAsString($defaults); |
389
|
34 |
|
$syn = SearchSynonym::get(); |
390
|
34 |
|
foreach ($syn as $synonym) { |
391
|
1 |
|
$synonyms .= $synonym->Keyword . ',' . $synonym->Synonym . PHP_EOL; |
392
|
|
|
} |
393
|
|
|
|
394
|
|
|
// Upload synonyms |
395
|
34 |
|
if ($store) { |
396
|
34 |
|
$store->uploadString( |
397
|
34 |
|
$this->getIndexName(), |
398
|
34 |
|
'synonyms.txt', |
399
|
34 |
|
$synonyms |
400
|
|
|
); |
401
|
|
|
} |
402
|
|
|
|
403
|
34 |
|
return $synonyms; |
404
|
|
|
} |
405
|
|
|
|
406
|
|
|
/** |
407
|
|
|
* Get the final, generated terms |
408
|
|
|
* |
409
|
|
|
* @return array |
410
|
|
|
*/ |
411
|
2 |
|
public function getQueryTerms(): array |
412
|
|
|
{ |
413
|
2 |
|
return $this->queryTerms; |
414
|
|
|
} |
415
|
|
|
|
416
|
|
|
/** |
417
|
|
|
* Get the QueryComponentFactory. {@link QueryComponentFactory} |
418
|
|
|
* |
419
|
|
|
* @return QueryComponentFactory |
420
|
|
|
*/ |
421
|
1 |
|
public function getQueryFactory(): QueryComponentFactory |
422
|
|
|
{ |
423
|
1 |
|
return $this->queryFactory; |
424
|
|
|
} |
425
|
|
|
} |
426
|
|
|
|