Passed
Push — main ( 5995aa...2135e0 )
by Lorenzo
03:01
created

getRecordsFromDb()   A

Complexity

Conditions 2
Paths 2

Size

Total Lines 18
Code Lines 8

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
cc 2
eloc 8
c 0
b 0
f 0
nc 2
nop 1
dl 0
loc 18
rs 10
1
<?php
2
3
namespace Padosoft\SuperCacheInvalidate\Console;
4
5
use Illuminate\Console\Command;
6
use Illuminate\Support\Facades\Cache;
7
use Illuminate\Support\Facades\DB;
8
use Carbon\Carbon;
9
use Padosoft\SuperCacheInvalidate\Helpers\SuperCacheInvalidationHelper;
10
11
class ProcessCacheInvalidationEventsCommand extends Command
12
{
13
    /**
14
     * The name and signature of the console command.
15
     *
16
     * @var string
17
     */
18
    protected $signature = 'supercache:process-invalidation
19
                            {--shard= : The shard number to process}
20
                            {--priority= : The priority level}
21
                            {--limit= : The maximum number of events to fetch per batch}
22
                            {--tag-batch-size= : The number of identifiers to process per invalidation batch}';
23
24
    /**
25
     * The console command description.
26
     *
27
     * @var string
28
     */
29
    protected $description = 'Process cache invalidation events for a given shard and priority';
30
31
    /**
32
     * Cache invalidation helper instance.
33
     */
34
    protected SuperCacheInvalidationHelper $helper;
35
36
    /**
37
     * Create a new command instance.
38
     */
39
    public function __construct(SuperCacheInvalidationHelper $helper)
40
    {
41
        parent::__construct();
42
        $this->helper = $helper;
43
    }
44
45
    /**
46
     * Process cache invalidation events.
47
     *
48
     * @param int $shardId      The shard number to process
49
     * @param int $priority     The priority level
50
     * @param int $limit        Maximum number of events to fetch per batch
51
     * @param int $tagBatchSize Number of identifiers to process per batch
52
     *
53
     * @throws \Exception
54
     */
55
    protected function processEvents(int $shardId, int $priority, int $limit, int $tagBatchSize): void
56
    {
57
        $processingStartTime = now();
58
        $invalidationWindow = config('super_cache_invalidate.invalidation_window');
59
60
        // Fetch a batch of unprocessed events
61
        $events = DB::table('cache_invalidation_events')
62
            ->where('processed', 0)
63
            ->where('shard', $shardId)
64
            ->where('priority', $priority)
65
            ->where('event_time', '<', $processingStartTime)
66
            ->orderBy('event_time')
67
            ->limit($limit)
68
            ->get()
69
        ;
70
71
        if ($events->isEmpty()) {
72
            // No more events to process
73
            return;
74
        }
75
76
        // Group events by type and identifier
77
        $eventsByIdentifier = $events->groupBy(function ($event) {
78
            return $event->type . ':' . $event->identifier;
79
        });
80
81
        $batchIdentifiers = [];
82
        $eventsToUpdate = [];
83
        $counter = 0;
84
85
        // Fetch associated identifiers for the events
86
        $eventIds = $events->pluck('id')->all();
87
88
        //retrive associated identifiers related to fetched event id
89
        $associations = DB::table('cache_invalidation_event_associations')
90
            ->whereIn('event_id', $eventIds)
91
            ->get()
92
            ->groupBy('event_id')
93
        ;
94
95
        // Prepare list of all identifiers to fetch last invalidation times
96
        $allIdentifiers = [];
97
98
        foreach ($eventsByIdentifier as $key => $eventsGroup) {
99
            $allIdentifiers[] = $key;
100
            foreach ($eventsGroup as $event) {
101
                $eventAssociations = $associations->get($event->id, collect());
102
                foreach ($eventAssociations as $assoc) {
103
                    $assocKey = $assoc->associated_type . ':' . $assoc->associated_identifier;
104
                    $allIdentifiers[] = $assocKey;
105
                }
106
            }
107
        }
108
109
        // Fetch last invalidation times in bulk
110
        $lastInvalidationTimes = $this->getLastInvalidationTimes(array_unique($allIdentifiers));
111
112
        foreach ($eventsByIdentifier as $key => $eventsGroup) {
113
            // Extract type and identifier
114
            [$type, $identifier] = explode(':', $key, 2);
115
116
            // Get associated identifiers for the events
117
            $associatedIdentifiers = [];
118
            foreach ($eventsGroup as $event) {
119
                $eventAssociations = $associations->get($event->id, collect());
120
                foreach ($eventAssociations as $assoc) {
121
                    $assocKey = $assoc->associated_type . ':' . $assoc->associated_identifier;
122
                    $associatedIdentifiers[$assocKey] = [
123
                        'type' => $assoc->associated_type,
124
                        'identifier' => $assoc->associated_identifier,
125
                    ];
126
                }
127
            }
128
129
            // Build a list of all identifiers to check
130
            $identifiersToCheck = [$key];
131
            $identifiersToCheck = array_merge($identifiersToCheck, array_keys($associatedIdentifiers));
132
133
            $lastInvalidationTimesSubset = array_intersect_key($lastInvalidationTimes, array_flip($identifiersToCheck));
134
135
            $shouldInvalidate = $this->shouldInvalidateMultiple($identifiersToCheck, $lastInvalidationTimesSubset, $invalidationWindow);
136
137
            if ($shouldInvalidate) {
138
                // Proceed to invalidate
139
                $latestEvent = $eventsGroup->last();
140
141
                // Accumulate identifiers and events
142
                $batchIdentifiers[] = [
143
                    'type' => $type,
144
                    'identifier' => $identifier,
145
                    'event' => $latestEvent,
146
                    'associated' => array_values($associatedIdentifiers),
147
                ];
148
149
                // Update last invalidation times for all identifiers
150
                $this->updateLastInvalidationTimes($identifiersToCheck);
151
152
                // Mark all events in the group as processed
153
                foreach ($eventsGroup as $event) {
154
                    $eventsToUpdate[] = $event->id;
155
                }
156
            } else {
157
                // Within the invalidation window, skip invalidation
158
                // Mark all events except the last one as processed
159
                $eventsToProcess = $eventsGroup->slice(0, -1);
160
                foreach ($eventsToProcess as $event) {
161
                    $eventsToUpdate[] = $event->id;
162
                }
163
                // The last event remains unprocessed
164
            }
165
166
            $counter++;
167
168
            // When we reach the batch size, process the accumulated identifiers
169
            if ($counter % $tagBatchSize === 0) {
170
                $this->processBatch($batchIdentifiers, $eventsToUpdate);
171
172
                // Reset the accumulators
173
                $batchIdentifiers = [];
174
                $eventsToUpdate = [];
175
            }
176
        }
177
178
        if (empty($batchIdentifiers)) {
179
            return;
180
        }
181
182
        // Process any remaining identifiers in the batch
183
        $this->processBatch($batchIdentifiers, $eventsToUpdate);
184
    }
185
186
    /**
187
     * Fetch last invalidation times for identifiers in bulk.
188
     *
189
     * @param  array $identifiers Array of 'type:identifier' strings
190
     * @return array Associative array of last invalidation times
191
     */
192
    protected function getLastInvalidationTimes(array $identifiers): array
193
    {
194
        // Extract types and identifiers into tuples
195
        $tuples = array_map(static function ($key) {
196
            return explode(':', $key, 2);
197
        }, $identifiers);
198
199
        if (empty($tuples)) {
200
            return [];
201
        }
202
203
        $records = $this->getRecordsFromDb($tuples);
204
205
        // Build associative array
206
        $lastInvalidationTimes = [];
207
        foreach ($records as $record) {
208
            $key = $record->identifier_type . ':' . $record->identifier;
209
            $lastInvalidationTimes[$key] = Carbon::parse($record->last_invalidated);
210
        }
211
212
        return $lastInvalidationTimes;
213
    }
214
215
    /**
216
     * Execute Query to get records from DB
217
     */
218
    protected function getRecordsFromDb(array $tuples): array
219
    {
220
        // Prepare placeholders and parameters
221
        $placeholders = implode(',', array_fill(0, count($tuples), '(?, ?)'));
222
        $params = [];
223
        foreach ($tuples as [$type, $identifier]) {
224
            $params[] = $type;
225
            $params[] = $identifier;
226
        }
227
228
        $sql = "SELECT identifier_type,
229
                        identifier,
230
                        last_invalidated
231
                FROM cache_invalidation_timestamps
232
                WHERE (identifier_type, identifier) IN ($placeholders)
233
                ";
234
235
        return DB::select($sql, $params);
236
    }
237
238
    /**
239
     * Determine whether to invalidate based on last invalidation times for multiple identifiers.
240
     *
241
     * @param  array $identifiers           Array of 'type:identifier' strings
242
     * @param  array $lastInvalidationTimes Associative array of last invalidation times
243
     * @param  int   $invalidationWindow    Invalidation window in seconds
244
     * @return bool  True if should invalidate, false otherwise
245
     */
246
    protected function shouldInvalidateMultiple(array $identifiers, array $lastInvalidationTimes, int $invalidationWindow): bool
247
    {
248
        $now = now();
249
        foreach ($identifiers as $key) {
250
            $lastInvalidated = $lastInvalidationTimes[$key] ?? null;
251
            if (!$lastInvalidated) {
252
                continue;
253
            }
254
            $elapsed = $now->diffInSeconds($lastInvalidated);
255
            if ($elapsed < $invalidationWindow) {
256
                // At least one identifier is within the invalidation window
257
                return false;
258
            }
259
        }
260
261
        // All identifiers are outside the invalidation window or have no record
262
        return true;
263
    }
264
265
    /**
266
     * Update the last invalidation times for multiple identifiers.
267
     *
268
     * @param array $identifiers Array of 'type:identifier' strings
269
     */
270
    protected function updateLastInvalidationTimes(array $identifiers): void
271
    {
272
        $now = now();
273
274
        foreach ($identifiers as $key) {
275
            [$type, $identifier] = explode(':', $key, 2);
276
            DB::table('cache_invalidation_timestamps')
277
                ->updateOrInsert(
278
                    ['identifier_type' => $type, 'identifier' => $identifier],
279
                    ['last_invalidated' => $now]
280
                )
281
            ;
282
        }
283
    }
284
285
    /**
286
     * Process a batch of identifiers and update events.
287
     *
288
     * @param array $batchIdentifiers Array of identifiers to invalidate
289
     * @param array $eventsToUpdate   Array of event IDs to mark as processed
290
     *
291
     * @throws \Exception
292
     */
293
    protected function processBatch(array $batchIdentifiers, array $eventsToUpdate): void
294
    {
295
        // Begin transaction for the batch
296
        DB::beginTransaction();
297
298
        try {
299
            // Separate keys and tags
300
            $keys = [];
301
            $tags = [];
302
            foreach ($batchIdentifiers as $item) {
303
                if ($item['type'] === 'key') {
304
                    $keys[] = $item['identifier'];
305
                } else {
306
                    $tags[] = $item['identifier'];
307
                }
308
309
                if (empty($item['associated'])) {
310
                    continue;
311
                }
312
313
                // Include associated identifiers
314
                foreach ($item['associated'] as $assoc) {
315
                    if ($assoc['type'] === 'key') {
316
                        $keys[] = $assoc['identifier'];
317
                        continue;
318
                    }
319
320
                    $tags[] = $assoc['identifier'];
321
                }
322
            }
323
324
            // Remove duplicates
325
            $keys = array_unique($keys);
326
            $tags = array_unique($tags);
327
328
            // Invalidate cache for keys
329
            if (!empty($keys)) {
330
                $this->invalidateKeys($keys);
331
            }
332
333
            // Invalidate cache for tags
334
            if (!empty($tags)) {
335
                $this->invalidateTags($tags);
336
            }
337
338
            // Mark events as processed
339
            DB::table('cache_invalidation_events')
340
                ->whereIn('id', $eventsToUpdate)
341
                ->update(['processed' => 1])
342
            ;
343
344
            // Commit transaction
345
            DB::commit();
346
        } catch (\Exception $e) {
347
            // Rollback transaction on error
348
            DB::rollBack();
349
            throw $e;
350
        }
351
    }
352
353
    /**
354
     * Invalidate cache keys.
355
     *
356
     * @param array $keys Array of cache keys to invalidate
357
     */
358
    protected function invalidateKeys(array $keys): void
359
    {
360
        $callback = config('super_cache_invalidate.key_invalidation_callback');
361
362
        if (is_callable($callback)) {
363
            call_user_func($callback, $keys);
364
365
            return;
366
        }
367
368
        //default invalidation method
369
        foreach ($keys as $key) {
370
            Cache::forget($key);
371
        }
372
    }
373
374
    /**
375
     * Invalidate cache tags.
376
     *
377
     * @param array $tags Array of cache tags to invalidate
378
     */
379
    protected function invalidateTags(array $tags): void
380
    {
381
        $callback = config('super_cache_invalidate.tag_invalidation_callback');
382
383
        if (is_callable($callback)) {
384
            call_user_func($callback, $tags);
385
386
            return;
387
        }
388
389
        //default invalidation method
390
        Cache::tags($tags)->flush();
391
    }
392
393
    /**
394
     * Execute the console command.
395
     */
396
    public function handle(): void
397
    {
398
        $shardId = (int) $this->option('shard');
399
        $priority = (int) $this->option('priority');
400
        $limit = $this->option('limit') ?? config('super_cache_invalidate.processing_limit');
401
        $limit = (int)$limit;
402
        $tagBatchSize = $this->option('tag-batch-size') ?? config('super_cache_invalidate.tag_batch_size');
403
        $tagBatchSize = (int)$tagBatchSize;
404
        $lockTimeout = (int) config('super_cache_invalidate.lock_timeout');
405
406
        if ($shardId === 0 && $priority === 0) {
407
            $this->error('Shard and priority are required and must be non-zero integers.');
408
409
            return;
410
        }
411
412
        $lockValue = $this->helper->acquireShardLock($shardId, $lockTimeout);
413
414
        if (!$lockValue) {
415
            $this->info("Another process is handling shard $shardId.");
416
417
            return;
418
        }
419
420
        try {
421
            $this->processEvents($shardId, $priority, $limit, $tagBatchSize);
422
        } catch (\Exception $e) {
423
            $this->error('An error occourred in ' . __METHOD__ . ': ' . $e->getTraceAsString());
424
        } finally {
425
            $this->helper->releaseShardLock($shardId, $lockValue);
426
        }
427
    }
428
}
429