1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
define('PRIVATE_FID_ROOT', 0x1); |
4
|
|
|
|
5
|
|
|
define('PR_FOLDER_ID', 0x67480014); |
6
|
|
|
define('PR_MID', 0x674A0014); |
7
|
|
|
define('PR_CHANGE_NUMBER', 0x67A40014); |
8
|
|
|
|
9
|
|
|
class IndexSqlite extends SQLite3 { |
10
|
|
|
private $username; |
11
|
|
|
private $count; |
12
|
|
|
private $store; |
13
|
|
|
private $session; |
14
|
|
|
private $openResult; |
15
|
|
|
|
16
|
|
|
private const DEBUG_SAMPLE_LIMIT = 5; |
17
|
|
|
|
18
|
|
|
private function logDebug(string $message, array $context = []): void { |
19
|
|
|
if (!DEBUG_FULLTEXT_SEARCH) { |
20
|
|
|
return; |
21
|
|
|
} |
22
|
|
|
$prefix = '[fts-debug][index] '; |
23
|
|
|
if (!empty($context)) { |
24
|
|
|
$encoded = json_encode($context, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); |
25
|
|
|
if ($encoded === false) { |
26
|
|
|
$encoded = 'context_encoding_failed'; |
27
|
|
|
} |
28
|
|
|
error_log($prefix . $message . ' ' . $encoded); |
29
|
|
|
} |
30
|
|
|
else { |
31
|
|
|
error_log($prefix . $message); |
32
|
|
|
} |
33
|
|
|
} |
34
|
|
|
|
35
|
|
|
private static function formatEntryIdForLog($entryid) { |
36
|
|
|
if ($entryid === null) { |
37
|
|
|
return null; |
38
|
|
|
} |
39
|
|
|
if (is_array($entryid)) { |
40
|
|
|
return array_map([self::class, 'formatEntryIdForLog'], $entryid); |
41
|
|
|
} |
42
|
|
|
if (!is_string($entryid)) { |
43
|
|
|
return $entryid; |
44
|
|
|
} |
45
|
|
|
|
46
|
|
|
return bin2hex($entryid); |
47
|
|
|
} |
48
|
|
|
|
49
|
|
|
private static function formatBinaryFieldForLog($value) { |
50
|
|
|
if ($value === null) { |
51
|
|
|
return null; |
52
|
|
|
} |
53
|
|
|
if (!is_string($value)) { |
54
|
|
|
return $value; |
55
|
|
|
} |
56
|
|
|
if (preg_match('/[^\x20-\x7E]/', $value)) { |
57
|
|
|
return bin2hex($value); |
58
|
|
|
} |
59
|
|
|
|
60
|
|
|
return $value; |
61
|
|
|
} |
62
|
|
|
|
63
|
|
|
private static function get_gc_value($eid) { |
64
|
|
|
$r0 = ($eid >> 56) & 0xFF; |
65
|
|
|
$r1 = ($eid >> 48) & 0xFF; |
66
|
|
|
$r2 = ($eid >> 40) & 0xFF; |
67
|
|
|
$r3 = ($eid >> 32) & 0xFF; |
68
|
|
|
$r4 = ($eid >> 24) & 0xFF; |
69
|
|
|
$r5 = ($eid >> 16) & 0xFF; |
70
|
|
|
$value = $r0 | ($r1 << 8) | ($r2 << 16) | ($r3 << 24) | ($r4 << 32) | ($r5 << 40); |
71
|
|
|
|
72
|
|
|
return $value; |
73
|
|
|
} |
74
|
|
|
|
75
|
|
|
public function __construct($username = null, $session = null, $store = null) { |
76
|
|
|
$this->username = $username ?? $GLOBALS["mapisession"]->getSMTPAddress(); |
77
|
|
|
$this->session = $session ?? $GLOBALS["mapisession"]->getSession(); |
78
|
|
|
$this->store = $store ?? $GLOBALS["mapisession"]->getDefaultMessageStore(); |
79
|
|
|
$indexPath = SQLITE_INDEX_PATH . '/' . $this->username . '/index.sqlite3'; |
80
|
|
|
|
81
|
|
|
try { |
82
|
|
|
$this->open($indexPath, SQLITE3_OPEN_READONLY); |
83
|
|
|
$this->openResult = 0; |
84
|
|
|
$this->logDebug('Opened index database', ['path' => $indexPath]); |
85
|
|
|
} |
86
|
|
|
catch (Exception $e) { |
87
|
|
|
error_log(sprintf("Error opening the index database: %s", $e)); |
88
|
|
|
$this->openResult = 1; |
89
|
|
|
$this->logDebug('Failed to open index database', [ |
90
|
|
|
'path' => $indexPath, |
91
|
|
|
'error' => $e->getMessage(), |
92
|
|
|
]); |
93
|
|
|
} |
94
|
|
|
} |
95
|
|
|
|
96
|
|
|
private function try_insert_content( |
97
|
|
|
$search_entryid, |
98
|
|
|
$row, |
99
|
|
|
$message_classes, |
100
|
|
|
$date_start, |
101
|
|
|
$date_end, |
102
|
|
|
$unread, |
103
|
|
|
$has_attachments |
104
|
|
|
) { |
105
|
|
|
// if match condition contains '@', $row['entryid'] will disappear. it seems a bug for php-sqlite |
106
|
|
|
if (empty($row['entryid'])) { |
107
|
|
|
$results = $this->query("SELECT entryid FROM msg_content WHERE message_id=" . $row['message_id']); |
108
|
|
|
$row1 = $results->fetchArray(SQLITE3_NUM); |
109
|
|
|
if ($row1 && !empty($row1[0])) { |
|
|
|
|
110
|
|
|
$row['entryid'] = $row1[0]; |
111
|
|
|
$this->logDebug('Recovered missing entryid from msg_content', [ |
112
|
|
|
'message_id' => $row['message_id'], |
113
|
|
|
]); |
114
|
|
|
} |
115
|
|
|
// abort if the entryid is not available |
116
|
|
|
else { |
117
|
|
|
error_log(sprintf("No entryid available, not possible to link the message %d.", $row['message_id'])); |
118
|
|
|
$this->logDebug('Missing entryid prevents linking message', [ |
119
|
|
|
'message_id' => $row['message_id'], |
120
|
|
|
]); |
121
|
|
|
|
122
|
|
|
return; |
123
|
|
|
} |
124
|
|
|
} |
125
|
|
|
if (is_array($message_classes) && $message_classes !== []) { |
126
|
|
|
$found = false; |
127
|
|
|
foreach ($message_classes as $message_class) { |
128
|
|
|
if (strncasecmp((string) $row['message_class'], (string) $message_class, strlen((string) $message_class)) == 0) { |
129
|
|
|
$found = true; |
130
|
|
|
break; |
131
|
|
|
} |
132
|
|
|
} |
133
|
|
|
if (!$found) { |
134
|
|
|
$this->logDebug('Skipping message because message class is filtered out', [ |
135
|
|
|
'message_id' => $row['message_id'] ?? null, |
136
|
|
|
'message_class' => $row['message_class'] ?? null, |
137
|
|
|
]); |
138
|
|
|
return; |
139
|
|
|
} |
140
|
|
|
} |
141
|
|
|
if ($date_start !== null && $row['date'] < $date_start) { |
142
|
|
|
$this->logDebug('Skipping message before start date filter', [ |
143
|
|
|
'message_id' => $row['message_id'] ?? null, |
144
|
|
|
'message_date' => $row['date'] ?? null, |
145
|
|
|
'date_start' => $date_start, |
146
|
|
|
]); |
147
|
|
|
return; |
148
|
|
|
} |
149
|
|
|
if ($date_end !== null && $row['date'] > $date_end) { |
150
|
|
|
$this->logDebug('Skipping message after end date filter', [ |
151
|
|
|
'message_id' => $row['message_id'] ?? null, |
152
|
|
|
'message_date' => $row['date'] ?? null, |
153
|
|
|
'date_end' => $date_end, |
154
|
|
|
]); |
155
|
|
|
return; |
156
|
|
|
} |
157
|
|
|
if ($unread && $row['readflag']) { |
158
|
|
|
$this->logDebug('Skipping message because unread flag filter is active', [ |
159
|
|
|
'message_id' => $row['message_id'] ?? null, |
160
|
|
|
'readflag' => $row['readflag'] ?? null, |
161
|
|
|
]); |
162
|
|
|
return; |
163
|
|
|
} |
164
|
|
|
if ($has_attachments && !$row['attach_indexed']) { |
165
|
|
|
$this->logDebug('Skipping message because attachment filter is active', [ |
166
|
|
|
'message_id' => $row['message_id'] ?? null, |
167
|
|
|
'attach_indexed' => $row['attach_indexed'] ?? null, |
168
|
|
|
]); |
169
|
|
|
return; |
170
|
|
|
} |
171
|
|
|
|
172
|
|
|
try { |
173
|
|
|
mapi_linkmessage($this->session, $search_entryid, $row['entryid']); |
174
|
|
|
} |
175
|
|
|
catch (Exception $e) { |
176
|
|
|
$details = [ |
177
|
|
|
'message_id' => $row['message_id'], |
178
|
|
|
'entryid' => self::formatBinaryFieldForLog($row['entryid']), |
179
|
|
|
'error' => $e->getMessage(), |
180
|
|
|
]; |
181
|
|
|
if (function_exists('mapi_last_hresult')) { |
182
|
|
|
$details['hresult'] = mapi_last_hresult(); |
183
|
|
|
} |
184
|
|
|
$this->logDebug('MAPI linkmessage failed', $details); |
185
|
|
|
return; |
186
|
|
|
} |
187
|
|
|
++$this->count; |
188
|
|
|
} |
189
|
|
|
|
190
|
|
|
private function result_full() { |
191
|
|
|
return $this->count >= MAX_FTS_RESULT_ITEMS; |
192
|
|
|
} |
193
|
|
|
|
194
|
|
|
public function search($search_entryid, $descriptor, $folder_entryid, $recursive) { |
195
|
|
|
$startTime = microtime(true); |
196
|
|
|
$this->logDebug('Search invoked', [ |
197
|
|
|
'user' => $this->username, |
198
|
|
|
'search_entryid' => self::formatEntryIdForLog($search_entryid), |
199
|
|
|
'folder_entryid' => self::formatEntryIdForLog($folder_entryid), |
200
|
|
|
'recursive' => (bool) $recursive, |
201
|
|
|
'descriptor' => $descriptor, |
202
|
|
|
]); |
203
|
|
|
if ($this->openResult) { |
204
|
|
|
$this->logDebug('Search aborted: index database unavailable', ['open_result' => $this->openResult]); |
205
|
|
|
return false; |
206
|
|
|
} |
207
|
|
|
$whereFolderids = ''; |
208
|
|
|
if (isset($folder_entryid)) { |
209
|
|
|
try { |
210
|
|
|
$folder = mapi_msgstore_openentry($this->store, $folder_entryid); |
211
|
|
|
if (!$folder) { |
212
|
|
|
return false; |
213
|
|
|
} |
214
|
|
|
$tmp_props = mapi_getprops($folder, [PR_FOLDER_ID]); |
215
|
|
|
if (empty($tmp_props[PR_FOLDER_ID])) { |
216
|
|
|
return false; |
217
|
|
|
} |
218
|
|
|
$folder_id = IndexSqlite::get_gc_value((int) $tmp_props[PR_FOLDER_ID]); |
219
|
|
|
$whereFolderids .= "c.folder_id in (" . $folder_id . ", "; |
220
|
|
|
if ($recursive) { |
221
|
|
|
$this->getWhereFolderids($folder, $whereFolderids); |
222
|
|
|
} |
223
|
|
|
$whereFolderids = substr($whereFolderids, 0, -2) . ") AND "; |
224
|
|
|
$this->logDebug('Folder scope resolved', [ |
225
|
|
|
'root_folder_gc_id' => $folder_id, |
226
|
|
|
'folder_clause' => rtrim($whereFolderids), |
227
|
|
|
]); |
228
|
|
|
} |
229
|
|
|
catch (Exception $e) { |
230
|
|
|
error_log(sprintf("Index: error getting folder information %s - %s", $this->username, $e)); |
231
|
|
|
$this->logDebug('Failed to resolve folder scope', ['error' => $e->getMessage()]); |
232
|
|
|
|
233
|
|
|
return false; |
234
|
|
|
} |
235
|
|
|
} |
236
|
|
|
$sql_string = "SELECT c.message_id, c.entryid, c.folder_id, " . |
237
|
|
|
"c.message_class, c.date, c.readflag, c.attach_indexed " . |
238
|
|
|
"FROM msg_content c " . |
239
|
|
|
"JOIN messages m ON c.message_id = m.rowid " . |
240
|
|
|
"WHERE "; |
241
|
|
|
if (!empty($whereFolderids)) { |
242
|
|
|
$sql_string .= $whereFolderids; |
243
|
|
|
} |
244
|
|
|
$ftsAst = $descriptor['ast'] ?? null; |
245
|
|
|
$message_classes = $descriptor['message_classes'] ?? null; |
246
|
|
|
$date_start = $descriptor['date_start'] ?? null; |
247
|
|
|
$date_end = $descriptor['date_end'] ?? null; |
248
|
|
|
$unread = !empty($descriptor['unread']); |
249
|
|
|
$has_attachments = !empty($descriptor['has_attachments']); |
250
|
|
|
$this->logDebug('Search filters resolved', [ |
251
|
|
|
'unread' => $unread, |
252
|
|
|
'has_attachments' => $has_attachments, |
253
|
|
|
'date_start' => $date_start, |
254
|
|
|
'date_end' => $date_end, |
255
|
|
|
'message_classes_count' => is_array($message_classes) ? count($message_classes) : null, |
256
|
|
|
]); |
257
|
|
|
|
258
|
|
|
$ftsQuery = $this->compileFtsExpression($ftsAst); |
259
|
|
|
if ($ftsQuery === null || $ftsQuery === '') { |
260
|
|
|
$this->logDebug('FTS query compilation returned empty expression', [ |
261
|
|
|
'ast' => $ftsAst, |
262
|
|
|
]); |
263
|
|
|
return false; |
264
|
|
|
} |
265
|
|
|
|
266
|
|
|
$sql_string .= "messages MATCH '" . $ftsQuery . "'"; |
267
|
|
|
$this->count = 0; |
268
|
|
|
$sql_string .= " ORDER BY c.date DESC LIMIT " . MAX_FTS_RESULT_ITEMS; |
269
|
|
|
$this->logDebug('Executing SQLite FTS query', ['sql' => $sql_string]); |
270
|
|
|
$results = $this->query($sql_string); |
271
|
|
|
if ($results === false) { |
272
|
|
|
$this->logDebug('SQLite query execution failed', [ |
273
|
|
|
'error_code' => $this->lastErrorCode(), |
274
|
|
|
'error_message' => $this->lastErrorMsg(), |
275
|
|
|
]); |
276
|
|
|
return false; |
277
|
|
|
} |
278
|
|
|
$matchedRows = 0; |
279
|
|
|
$sampleRows = []; |
280
|
|
|
while (($row = $results->fetchArray(SQLITE3_ASSOC)) && !$this->result_full()) { |
281
|
|
|
++$matchedRows; |
282
|
|
|
$previousCount = $this->count; |
283
|
|
|
$this->try_insert_content( |
284
|
|
|
$search_entryid, |
285
|
|
|
$row, |
286
|
|
|
$message_classes, |
287
|
|
|
$date_start, |
288
|
|
|
$date_end, |
289
|
|
|
$unread, |
290
|
|
|
$has_attachments |
291
|
|
|
); |
292
|
|
|
if ($this->count > $previousCount && count($sampleRows) < self::DEBUG_SAMPLE_LIMIT) { |
293
|
|
|
$sampleRows[] = [ |
294
|
|
|
'message_id' => $row['message_id'] ?? null, |
295
|
|
|
'entryid' => self::formatBinaryFieldForLog($row['entryid'] ?? null), |
296
|
|
|
'message_class' => $row['message_class'] ?? null, |
297
|
|
|
'date' => $row['date'] ?? null, |
298
|
|
|
'readflag' => $row['readflag'] ?? null, |
299
|
|
|
'attach_indexed' => $row['attach_indexed'] ?? null, |
300
|
|
|
]; |
301
|
|
|
} |
302
|
|
|
} |
303
|
|
|
$durationMs = (int) round((microtime(true) - $startTime) * 1000); |
304
|
|
|
$this->logDebug('Search completed', [ |
305
|
|
|
'fts_query' => $ftsQuery, |
306
|
|
|
'matched_rows' => $matchedRows, |
307
|
|
|
'linked_messages' => $this->count, |
308
|
|
|
'limit_reached' => $this->result_full(), |
309
|
|
|
'folder_clause' => $whereFolderids !== '' ? rtrim($whereFolderids) : null, |
310
|
|
|
'sample_messages' => $sampleRows, |
311
|
|
|
'duration_ms' => $durationMs, |
312
|
|
|
]); |
313
|
|
|
|
314
|
|
|
return true; |
315
|
|
|
} |
316
|
|
|
|
317
|
|
|
private function compileFtsExpression($ast) { |
318
|
|
|
if ($ast === null) { |
319
|
|
|
return null; |
320
|
|
|
} |
321
|
|
|
|
322
|
|
|
if (isset($ast['type']) && $ast['type'] === 'term') { |
323
|
|
|
$fields = $ast['fields'] ?? []; |
324
|
|
|
if (empty($fields)) { |
325
|
|
|
return null; |
326
|
|
|
} |
327
|
|
|
$escaped = SQLite3::escapeString($this->quote_words($ast['value'] ?? '')); |
328
|
|
|
$segments = []; |
329
|
|
|
foreach ($fields as $field) { |
330
|
|
|
$segments[] = $field . ':' . $escaped; |
331
|
|
|
} |
332
|
|
|
if (count($segments) === 1) { |
333
|
|
|
return $segments[0]; |
334
|
|
|
} |
335
|
|
|
return '(' . implode(' OR ', $segments) . ')'; |
336
|
|
|
} |
337
|
|
|
|
338
|
|
|
$operator = $ast['op'] ?? null; |
339
|
|
|
$children = $ast['children'] ?? []; |
340
|
|
|
if ($operator === 'NOT') { |
341
|
|
|
$child = $this->compileFtsExpression($children[0] ?? null); |
342
|
|
|
if ($child === null) { |
343
|
|
|
return null; |
344
|
|
|
} |
345
|
|
|
return 'NOT (' . $child . ')'; |
346
|
|
|
} |
347
|
|
|
if ($operator === 'AND' || $operator === 'OR') { |
348
|
|
|
$parts = []; |
349
|
|
|
foreach ($children as $child) { |
350
|
|
|
$compiled = $this->compileFtsExpression($child); |
351
|
|
|
if ($compiled !== null) { |
352
|
|
|
$parts[] = $compiled; |
353
|
|
|
} |
354
|
|
|
} |
355
|
|
|
if (empty($parts)) { |
356
|
|
|
return null; |
357
|
|
|
} |
358
|
|
|
if (count($parts) === 1) { |
359
|
|
|
return $parts[0]; |
360
|
|
|
} |
361
|
|
|
$wrapped = array_map(function ($segment) { |
362
|
|
|
return '(' . $segment . ')'; |
363
|
|
|
}, $parts); |
364
|
|
|
return implode(' ' . $operator . ' ', $wrapped); |
365
|
|
|
} |
366
|
|
|
|
367
|
|
|
return null; |
368
|
|
|
} |
369
|
|
|
|
370
|
|
|
private function quote_words($search_string) { |
371
|
|
|
return '"' . preg_replace("/(\\s+)/", '*" "', trim((string) $search_string)) . '"*'; |
372
|
|
|
} |
373
|
|
|
|
374
|
|
|
/** |
375
|
|
|
* Returns the restriction to filter hidden folders. |
376
|
|
|
* |
377
|
|
|
* @return array |
378
|
|
|
*/ |
379
|
|
|
private function getHiddenRestriction() { |
380
|
|
|
return |
381
|
|
|
[RES_OR, [ |
382
|
|
|
[RES_PROPERTY, |
383
|
|
|
[ |
384
|
|
|
RELOP => RELOP_EQ, |
385
|
|
|
ULPROPTAG => PR_ATTR_HIDDEN, |
386
|
|
|
VALUE => [PR_ATTR_HIDDEN => false], |
387
|
|
|
], |
388
|
|
|
], |
389
|
|
|
[RES_NOT, |
390
|
|
|
[ |
391
|
|
|
[RES_EXIST, |
392
|
|
|
[ |
393
|
|
|
ULPROPTAG => PR_ATTR_HIDDEN, |
394
|
|
|
], |
395
|
|
|
], |
396
|
|
|
], |
397
|
|
|
], |
398
|
|
|
]]; |
399
|
|
|
} |
400
|
|
|
|
401
|
|
|
/** |
402
|
|
|
* Returns the comma joined folderids for the WHERE clause in the SQL |
403
|
|
|
* statement. |
404
|
|
|
* |
405
|
|
|
* @param mixed $folder |
406
|
|
|
* @param string $whereFolderids |
407
|
|
|
*/ |
408
|
|
|
private function getWhereFolderids($folder, &$whereFolderids) { |
409
|
|
|
/** |
410
|
|
|
* remove hidden folders, folders with PR_ATTR_HIDDEN property set |
411
|
|
|
* should not be shown to the client. |
412
|
|
|
*/ |
413
|
|
|
$restriction = $this->getHiddenRestriction(); |
414
|
|
|
$hierarchy = mapi_folder_gethierarchytable($folder, CONVENIENT_DEPTH | MAPI_DEFERRED_ERRORS); |
415
|
|
|
mapi_table_restrict($hierarchy, $restriction, TBL_BATCH); |
416
|
|
|
$rows = mapi_table_queryallrows($hierarchy, [PR_FOLDER_ID]); |
417
|
|
|
foreach ($rows as $row) { |
418
|
|
|
if (isset($row[PR_FOLDER_ID])) { |
419
|
|
|
$whereFolderids .= IndexSqlite::get_gc_value((int) $row[PR_FOLDER_ID]) . ", "; |
420
|
|
|
} |
421
|
|
|
} |
422
|
|
|
} |
423
|
|
|
} |
424
|
|
|
|
This check marks implicit conversions of arrays to boolean values in a comparison. While in PHP an empty array is considered to be equal (but not identical) to false, this is not always apparent.
Consider making the comparison explicit by using
empty(..)
or! empty(...)
instead.