BinariesService::daytopost()   A
last analyzed

Complexity

Conditions 4
Paths 4

Size

Total Lines 21
Code Lines 11

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
eloc 11
c 0
b 0
f 0
dl 0
loc 21
rs 9.9
cc 4
nc 4
nop 2
1
<?php
2
3
declare(strict_types=1);
4
5
namespace App\Services\Binaries;
6
7
use App\Models\Settings;
8
use App\Models\UsenetGroup;
9
use App\Services\NNTP\NNTPService;
10
use Illuminate\Support\Carbon;
11
use Illuminate\Support\Facades\DB;
12
use Illuminate\Support\Facades\Log;
13
use Illuminate\Support\Str;
14
15
/**
16
 * Service for downloading and processing of Usenet binary headers.
17
 *
18
 * This service orchestrates the header processing workflow using:
19
 * - HeaderParser for parsing and filtering headers
20
 * - HeaderStorageService for storing headers to database
21
 * - MissedPartHandler for part repair tracking
22
 * - BinariesConfig for configuration
23
 */
24
class BinariesService
25
{
26
    private BinariesConfig $config;
0 ignored issues
show
Bug introduced by
The type App\Services\Binaries\BinariesConfig was not found. Maybe you did not declare it correctly or list all dependencies?

The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. excluded_paths: ["lib/*"], you can move it to the dependency path list as follows:

filter:
    dependency_paths: ["lib/*"]

For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths

Loading history...
27
28
    private HeaderParser $headerParser;
29
30
    private HeaderStorageService $headerStorage;
31
32
    private MissedPartHandler $missedPartHandler;
33
34
    private ?NNTPService $nntp = null;
35
36
    // Timing metrics
37
    private float $timeHeaders = 0;
38
39
    private float $timeCleaning = 0;
40
41
    private float $timeInsert = 0;
42
43
    private \DateTime $startLoop;
44
45
    private \DateTime $startCleaning;
46
47
    private \DateTime $startPR;
48
49
    private \DateTime $startUpdate;
50
51
    // Scan state
52
    private array $groupMySQL = [];
53
54
    private int $first = 0;
55
56
    private int $last = 0;
57
58
    private int $notYEnc = 0;
59
60
    private int $headersBlackListed = 0;
61
62
    private array $headersReceived = [];
63
64
    public function __construct(
65
        ?BinariesConfig $config = null,
66
        ?HeaderParser $headerParser = null,
67
        ?HeaderStorageService $headerStorage = null,
68
        ?MissedPartHandler $missedPartHandler = null,
69
        ?NNTPService $nntp = null
70
    ) {
71
        $this->config = $config ?? BinariesConfig::fromSettings();
72
        $this->headerParser = $headerParser ?? new HeaderParser;
73
        $this->headerStorage = $headerStorage ?? new HeaderStorageService(config: $this->config);
74
        $this->missedPartHandler = $missedPartHandler ?? new MissedPartHandler(
75
            $this->config->partRepairLimit,
76
            $this->config->partRepairMaxTries
77
        );
78
        $this->nntp = $nntp;
79
        $this->startUpdate = now();
80
    }
81
82
    /**
83
     * Set NNTP connection (for external injection).
84
     */
85
    public function setNntp(NNTPService $nntp): void
86
    {
87
        $this->nntp = $nntp;
88
    }
89
90
    /**
91
     * Get the NNTP connection, creating one if needed.
92
     */
93
    public function getNntp(): NNTPService
94
    {
95
        if ($this->nntp === null) {
96
            $this->nntp = new NNTPService;
97
        }
98
99
        return $this->nntp;
0 ignored issues
show
Bug Best Practice introduced by
The expression return $this->nntp could return the type null which is incompatible with the type-hinted return App\Services\NNTP\NNTPService. Consider adding an additional type-check to rule them out.
Loading history...
100
    }
101
102
    /**
103
     * Get the configuration object.
104
     */
105
    public function getConfig(): BinariesConfig
106
    {
107
        return $this->config;
108
    }
109
110
    /**
111
     * Get the message buffer size.
112
     */
113
    public function getMessageBuffer(): int
114
    {
115
        return $this->config->messageBuffer;
116
    }
117
118
    /**
119
     * Download new headers for all active groups.
120
     *
121
     * @param  int  $maxHeaders  (Optional) How many headers to download max.
122
     *
123
     * @throws \Exception
124
     * @throws \Throwable
125
     */
126
    public function updateAllGroups(int $maxHeaders = 100000): void
127
    {
128
        $groups = UsenetGroup::getActive()->toArray();
129
        $groupCount = \count($groups);
130
131
        if ($groupCount === 0) {
132
            $this->log(
133
                'No groups specified. Ensure groups are added to NNTmux\'s database for updating.',
134
                __FUNCTION__,
135
                'warning'
136
            );
137
138
            return;
139
        }
140
141
        $counter = 1;
142
        $allTime = now();
143
144
        $this->log(
145
            'Updating: '.$groupCount.' group(s) - Using compression? '.($this->config->compressedHeaders ? 'Yes' : 'No'),
146
            __FUNCTION__,
147
            'header'
148
        );
149
150
        foreach ($groups as $group) {
151
            $this->log(
152
                'Starting group '.$counter.' of '.$groupCount,
153
                __FUNCTION__,
154
                'header'
155
            );
156
157
            try {
158
                $this->updateGroup($group, $maxHeaders);
159
            } catch (\Throwable $e) {
160
                $this->logError('Error updating group '.$group['name'].': '.$e->getMessage());
161
            }
162
163
            $counter++;
164
        }
165
166
        $endTime = now()->diffInSeconds($allTime, true);
167
        $this->log(
168
            'Updating completed in '.$endTime.Str::plural(' second', $endTime),
169
            __FUNCTION__,
170
            'primary'
171
        );
172
    }
173
174
    /**
175
     * Log the indexer start time.
176
     */
177
    public function logIndexerStart(): void
178
    {
179
        Settings::query()->where('name', '=', 'last_run_time')->update(['value' => now()]);
180
    }
181
182
    /**
183
     * Download new headers for a single group.
184
     *
185
     * @param  array  $groupMySQL  Array of MySQL results for a single group.
186
     * @param  int  $maxHeaders  (Optional) How many headers to download max.
187
     *
188
     * @throws \Exception
189
     * @throws \Throwable
190
     */
191
    public function updateGroup(array $groupMySQL, int $maxHeaders = 0): void
192
    {
193
        $startGroup = now();
194
        $this->logIndexerStart();
195
196
        $nntp = $this->getNntp();
197
198
        // Select the group on the NNTP server
199
        $groupNNTP = $this->selectNntpGroup($groupMySQL, $nntp);
200
        if ($groupNNTP === null) {
201
            return;
202
        }
203
204
        if ($this->config->echoCli) {
205
            cli()->primary('Processing '.$groupMySQL['name']);
206
        }
207
208
        // Attempt to repair any missing parts before grabbing new ones
209
        if ((int) $groupMySQL['last_record'] !== 0 && $this->config->partRepair) {
210
            if ($this->config->echoCli) {
211
                cli()->primary('Part repair enabled. Checking for missing parts.');
212
            }
213
            $this->partRepair($groupMySQL);
214
        } elseif ($this->config->echoCli && (int) $groupMySQL['last_record'] !== 0) {
215
            cli()->primary('Part repair disabled by user.');
216
        }
217
218
        // Generate postdate for first record, for those that upgraded
219
        if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] !== 0) {
220
            $groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP);
221
            UsenetGroup::query()->where('id', $groupMySQL['id'])->update([
222
                'first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'], date_default_timezone_get()),
223
            ]);
224
        }
225
226
        // Calculate article range
227
        $range = $this->calculateArticleRange($groupMySQL, $groupNNTP, $maxHeaders);
228
229
        if ($range['total'] <= 0) {
230
            $this->outputNoNewArticles($groupMySQL, $groupNNTP, $range);
231
232
            return;
233
        }
234
235
        $this->outputNewArticlesInfo($groupMySQL, $groupNNTP, $range);
236
        $this->processArticleRange($groupMySQL, $groupNNTP, $range);
237
238
        if ($this->config->echoCli) {
239
            $endGroup = now()->diffInSeconds($startGroup, true);
240
            cli()->primary(
241
                PHP_EOL.'Group '.$groupMySQL['name'].' processed in '.$endGroup.Str::plural(' second', $endGroup)
242
            );
243
        }
244
    }
245
246
    /**
247
     * Loop over range of wanted headers, insert headers into DB.
248
     *
249
     * @param  array  $groupMySQL  The group info from mysql.
250
     * @param  int  $first  The oldest wanted header.
251
     * @param  int  $last  The newest wanted header.
252
     * @param  string  $type  Is this part repair or update or backfill?
253
     * @param  array|null  $missingParts  If we are running in part repair, the list of missing article numbers.
254
     * @return array Empty on failure.
255
     *
256
     * @throws \Exception
257
     * @throws \Throwable
258
     */
259
    public function scan(array $groupMySQL, int $first, int $last, string $type = 'update', ?array $missingParts = null): array
260
    {
261
        $this->startLoop = now();
262
        $this->groupMySQL = $groupMySQL;
263
        $this->last = $last;
264
        $this->first = $first;
265
        $this->notYEnc = $this->headersBlackListed = 0;
266
        $this->headersReceived = [];
267
268
        $returnArray = [];
269
        $partRepair = ($type === 'partrepair');
270
        $addToPartRepair = ($type === 'update' && $this->config->partRepair);
271
272
        // Download headers from NNTP
273
        $headers = $this->downloadHeaders($partRepair);
274
        if ($headers === null) {
275
            if ($partRepair) {
276
                $this->missedPartHandler->incrementRangeAttempts($groupMySQL['id'], $first, $last);
277
            }
278
279
            return $returnArray;
280
        }
281
282
        $this->startCleaning = now();
283
        $this->timeHeaders = $this->startCleaning->diffInSeconds($this->startLoop, true);
0 ignored issues
show
Bug introduced by
The method diffInSeconds() does not exist on DateTime. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-call  annotation

283
        /** @scrutinizer ignore-call */ 
284
        $this->timeHeaders = $this->startCleaning->diffInSeconds($this->startLoop, true);

This check looks for calls to methods that do not seem to exist on a given type. It looks for the method on the type itself as well as in inherited classes or implemented interfaces.

This is most likely a typographical error or the method has been renamed.

Loading history...
284
285
        $msgCount = \count($headers);
286
        if ($msgCount < 1) {
287
            return $returnArray;
288
        }
289
290
        // Extract article range info
291
        $returnArray = $this->headerParser->getArticleRange($headers);
292
293
        // Parse and filter headers
294
        $this->headerParser->reset();
295
        $parseResult = $this->headerParser->parse($headers, $groupMySQL['name'], $partRepair, $missingParts);
296
297
        $this->headersReceived = array_column($headers, 'Number');
298
        $this->headersReceived = array_filter($this->headersReceived);
299
        $this->notYEnc = $parseResult['notYEnc'];
300
        $this->headersBlackListed = $parseResult['blacklisted'];
301
302
        // Update blacklist last_activity
303
        $this->headerParser->flushBlacklistUpdates();
304
305
        unset($headers);
306
307
        if ($this->config->echoCli && ! $partRepair) {
308
            $this->outputHeaderInitial();
309
        }
310
311
        // Store headers
312
        $this->startUpdate = now();  // Reset before storage begins
313
        $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning, true);
314
315
        $headersNotInserted = [];
316
        if (! empty($parseResult['headers'])) {
317
            try {
318
                $headersNotInserted = $this->headerStorage->store($parseResult['headers'], $groupMySQL, $addToPartRepair);
319
            } catch (\Throwable $e) {
320
                $this->logError('storeHeaders failed: '.$e->getMessage());
321
            }
322
        }
323
324
        $this->startPR = now();
325
        $this->timeInsert = $this->startPR->diffInSeconds($this->startUpdate, true);
326
327
        // Handle repaired parts
328
        if ($partRepair && ! empty($parseResult['repaired'])) {
329
            $this->missedPartHandler->removeRepairedParts($parseResult['repaired'], $groupMySQL['id']);
330
        }
331
332
        // Handle part repair tracking
333
        if ($addToPartRepair) {
334
            $this->handlePartRepairTracking($headersNotInserted, $parseResult['headers']);
335
        }
336
337
        $this->outputHeaderDuration();
338
339
        return $returnArray;
340
    }
341
342
    /**
343
     * Attempt to get missing article headers.
344
     *
345
     * @param  array  $groupArr  The info for this group from mysql.
346
     *
347
     * @throws \Exception
348
     * @throws \Throwable
349
     */
350
    public function partRepair(array $groupArr): void
351
    {
352
        $missingParts = $this->missedPartHandler->getMissingParts($groupArr['id']);
353
        $missingCount = \count($missingParts);
354
355
        if ($missingCount === 0) {
356
            $this->missedPartHandler->cleanupExhaustedParts($groupArr['id']);
357
358
            return;
359
        }
360
361
        if ($this->config->echoCli) {
362
            cli()->primary('Attempting to repair '.number_format($missingCount).' parts.');
363
        }
364
365
        // Group into continuous ranges
366
        $ranges = $this->groupMissingPartsIntoRanges($missingParts);
367
368
        // Download missing parts in ranges
369
        foreach ($ranges as $range) {
370
            if ($this->config->echoCli) {
371
                echo \chr(random_int(45, 46)).PHP_EOL;
372
            }
373
374
            $this->scan($groupArr, $range['partfrom'], $range['partto'], 'partrepair', $range['partlist']);
375
        }
376
377
        // Calculate parts repaired
378
        $lastPartNumber = $missingParts[$missingCount - 1]->numberid;
379
        $remainingCount = $this->missedPartHandler->getCount($groupArr['id'], $lastPartNumber);
380
        $partsRepaired = $missingCount - $remainingCount;
381
382
        // Update attempts on remaining parts
383
        if (isset($missingParts[$missingCount - 1]->id)) {
384
            $this->missedPartHandler->incrementAttempts($groupArr['id'], $lastPartNumber);
385
        }
386
387
        if ($this->config->echoCli) {
388
            cli()->primary(PHP_EOL.number_format($partsRepaired).' parts repaired.');
389
        }
390
391
        // Remove articles that exceeded max tries
392
        $this->missedPartHandler->cleanupExhaustedParts($groupArr['id']);
393
    }
394
395
    /**
396
     * Returns unix time for an article number.
397
     *
398
     * @param  int|string  $post  The article number to get the time from.
399
     * @param  array  $groupData  Usenet group info from NNTP selectGroup method.
400
     * @return int Timestamp.
401
     *
402
     * @throws \Exception
403
     */
404
    public function postdate(int|string $post, array $groupData): int
405
    {
406
        $nntp = $this->getNntp();
407
        $currentPost = (int) $post;
408
        $attempts = 0;
409
        $date = 0;
410
411
        do {
412
            // Try to get the article date locally first
413
            $local = DB::select(
414
                sprintf(
415
                    'SELECT c.date AS date FROM collections c
416
                    INNER JOIN binaries b ON(c.id=b.collections_id)
417
                    INNER JOIN parts p ON(b.id=p.binaries_id)
418
                    WHERE p.number = %s',
419
                    $currentPost
420
                )
421
            );
422
423
            if (! empty($local)) {
424
                $date = $local[0]->date;
425
                break;
426
            }
427
428
            // Try usenet
429
            $header = $nntp->getXOVER((string) $currentPost);
430
            if (! NNTPService::isError($header) && isset($header[0]['Date']) && $header[0]['Date'] !== '') {
431
                $date = $header[0]['Date'];
432
                break;
433
            }
434
435
            // Try a different article number
436
            $currentPost = $this->getNextArticleToTry($currentPost, $groupData);
437
            if ($currentPost === null) {
438
                break;
439
            }
440
        } while ($attempts++ <= 20);
441
442
        if (! $date) {
443
            return time();
444
        }
445
446
        return strtotime($date);
447
    }
448
449
    /**
450
     * Returns article number based on # of days.
451
     *
452
     * @param  int  $days  How many days back we want to go.
453
     * @param  array  $data  Group data from usenet.
454
     *
455
     * @throws \Exception
456
     */
457
    public function daytopost(int $days, array $data): string
458
    {
459
        $goalTime = now()->subDays($days)->timestamp;
460
461
        $firstDate = $this->postdate($data['first'], $data);
462
        if ($goalTime < $firstDate) {
463
            return $data['first'];
464
        }
465
466
        $lastDate = $this->postdate($data['last'], $data);
467
        if ($goalTime > $lastDate) {
468
            return $data['last'];
469
        }
470
471
        if ($this->config->echoCli) {
472
            cli()->primary(
473
                'Searching for an approximate article number for group '.$data['group'].' '.$days.' days back.'
474
            );
475
        }
476
477
        return $this->binarySearchArticleByDate($goalTime, $data);
478
    }
479
480
    // ==================== Private Helper Methods ====================
481
482
    private function selectNntpGroup(array &$groupMySQL, NNTPService $nntp): ?array
483
    {
484
        $groupNNTP = $nntp->selectGroup($groupMySQL['name']);
485
486
        if (NNTPService::isError($groupNNTP)) {
487
            $groupNNTP = $nntp->dataError($nntp, $groupMySQL['name']);
488
489
            if (isset($groupNNTP['code']) && (int) $groupNNTP['code'] === 411) {
490
                UsenetGroup::disableIfNotExist($groupMySQL['id']);
491
            }
492
493
            if (NNTPService::isError($groupNNTP)) {
494
                return null;
495
            }
496
        }
497
498
        return $groupNNTP;
499
    }
500
501
    private function calculateArticleRange(array $groupMySQL, array $groupNNTP, int $maxHeaders): array
502
    {
503
        if ((int) $groupMySQL['last_record'] === 0) {
504
            return $this->calculateNewGroupRange($groupNNTP);
505
        }
506
507
        return $this->calculateExistingGroupRange($groupMySQL, $groupNNTP, $maxHeaders);
508
    }
509
510
    private function calculateNewGroupRange(array $groupNNTP): array
511
    {
512
        if ($this->config->newGroupScanByDays) {
513
            $first = (int) $this->daytopost($this->config->newGroupDaysToScan, $groupNNTP);
514
        } elseif ($groupNNTP['first'] >= ($groupNNTP['last'] - ($this->config->newGroupMessagesToScan + $this->config->messageBuffer))) {
515
            $first = (int) $groupNNTP['first'];
516
        } else {
517
            $first = (int) ($groupNNTP['last'] - ($this->config->newGroupMessagesToScan + $this->config->messageBuffer));
518
        }
519
520
        $leaveOver = $this->config->messageBuffer;
521
        $last = $groupLast = (int) ($groupNNTP['last'] - $leaveOver);
522
523
        if ($last < $first) {
524
            $last = $groupLast = $first;
525
        }
526
527
        $total = (int) ($groupLast - $first);
528
        $realTotal = (int) ($groupNNTP['last'] - $first);
529
530
        return [
531
            'first' => $first,
532
            'last' => $last,
533
            'groupLast' => $groupLast,
534
            'total' => $total,
535
            'realTotal' => $realTotal,
536
            'leaveOver' => $leaveOver,
537
            'isNew' => true,
538
        ];
539
    }
540
541
    private function calculateExistingGroupRange(array $groupMySQL, array $groupNNTP, int $maxHeaders): array
542
    {
543
        $first = (int) $groupMySQL['last_record'];
544
        $totalCount = (int) ($groupNNTP['last'] - $first);
545
546
        if ($totalCount > ($this->config->messageBuffer * 2)) {
547
            $leaveOver = (int) round($totalCount % $this->config->messageBuffer, 0, PHP_ROUND_HALF_DOWN) + $this->config->messageBuffer;
548
        } else {
549
            $leaveOver = (int) round($totalCount / 2, 0, PHP_ROUND_HALF_DOWN);
550
        }
551
552
        $last = $groupLast = (int) ($groupNNTP['last'] - $leaveOver);
553
554
        if ($last < $first) {
555
            $last = $groupLast = $first;
556
        }
557
558
        $total = (int) ($groupLast - $first);
559
        $realTotal = (int) ($groupNNTP['last'] - $first);
560
561
        // Apply max headers limit
562
        if ($maxHeaders > 0 && $maxHeaders < ($groupLast - $first)) {
563
            $groupLast = $last = (int) ($first + $maxHeaders);
564
            $total = (int) ($groupLast - $first);
565
        }
566
567
        return [
568
            'first' => $first,
569
            'last' => $last,
570
            'groupLast' => $groupLast,
571
            'total' => $total,
572
            'realTotal' => $realTotal,
573
            'leaveOver' => $leaveOver,
574
            'isNew' => false,
575
        ];
576
    }
577
578
    private function processArticleRange(array &$groupMySQL, array $groupNNTP, array $range): void
579
    {
580
        $first = (int) $range['first'];
581
        $last = (int) $range['last'];
582
        $groupLast = (int) $range['groupLast'];
583
        $done = false;
584
585
        while (! $done) {
586
            // Calculate chunk bounds
587
            if ($range['total'] > $this->config->messageBuffer) {
588
                $last = (int) min($first + $this->config->messageBuffer, $groupLast);
589
            }
590
591
            $first++;
592
593
            if ($this->config->echoCli) {
594
                cli()->header(
595
                    PHP_EOL.'Getting '.number_format($last - $first + 1).' articles ('.number_format($first).
596
                    ' to '.number_format($last).') from '.$groupMySQL['name'].' - ('.
597
                    number_format($groupLast - $last).' articles in queue).'
598
                );
599
            }
600
601
            // Scan this chunk
602
            $scanSummary = $this->scan($groupMySQL, $first, $last);
603
604
            // Update group record
605
            $this->updateGroupAfterScan($groupMySQL, $groupNNTP, $scanSummary, $last);
606
607
            if ($last === $groupLast) {
608
                $done = true;
609
            } else {
610
                $first = $last;
611
            }
612
        }
613
    }
614
615
    private function updateGroupAfterScan(array &$groupMySQL, array $groupNNTP, array $scanSummary, int $last): void
616
    {
617
        if (! empty($scanSummary)) {
618
            // New group - update first record
619
            if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] === 0) {
620
                $groupMySQL['first_record'] = $scanSummary['firstArticleNumber'];
621
                $groupMySQL['first_record_postdate'] = isset($scanSummary['firstArticleDate'])
622
                    ? strtotime($scanSummary['firstArticleDate'])
623
                    : $this->postdate($groupMySQL['first_record'], $groupNNTP);
624
625
                UsenetGroup::query()->where('id', $groupMySQL['id'])->update([
626
                    'first_record' => $scanSummary['firstArticleNumber'],
627
                    'first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'], date_default_timezone_get()),
628
                ]);
629
            }
630
631
            $lastArticleDate = isset($scanSummary['lastArticleDate'])
632
                ? strtotime($scanSummary['lastArticleDate'])
633
                : $this->postdate($scanSummary['lastArticleNumber'], $groupNNTP);
634
635
            UsenetGroup::query()->where('id', $groupMySQL['id'])->update([
636
                'last_record' => $scanSummary['lastArticleNumber'],
637
                'last_record_postdate' => Carbon::createFromTimestamp($lastArticleDate, date_default_timezone_get()),
638
                'last_updated' => now(),
639
            ]);
640
        } else {
641
            UsenetGroup::query()->where('id', $groupMySQL['id'])->update([
642
                'last_record' => $last,
643
                'last_updated' => now(),
644
            ]);
645
        }
646
    }
647
648
    private function downloadHeaders(bool $partRepair): ?array
649
    {
650
        $nntp = $this->getNntp();
651
652
        if ($partRepair) {
653
            $headers = $nntp->getOverview($this->first.'-'.$this->last, true, false);
654
        } else {
655
            $headers = $nntp->getXOVER($this->first.'-'.$this->last);
656
        }
657
658
        if (NNTPService::isError($headers) || ! \is_array($headers)) {
659
            if ($partRepair) {
660
                return null;
661
            }
662
663
            // Retry without compression
664
            $nntp->doQuit();
665
            if ($nntp->doConnect(false) !== true) {
666
                return null;
667
            }
668
669
            $nntp->selectGroup($this->groupMySQL['name']);
670
            $headers = $nntp->getXOVER($this->first.'-'.$this->last);
671
            $nntp->enableCompression();
672
673
            if (NNTPService::isError($headers) || ! \is_array($headers)) {
674
                if (\is_object($headers) && isset($headers->code, $headers->message)) {
675
                    $message = ((int) $headers->code === 0 ? 'Unknown error' : $headers->message);
676
                    $this->log("Code {$headers->code}: $message\nSkipping group: {$this->groupMySQL['name']}", __FUNCTION__, 'error');
677
                } else {
678
                    $this->log("Unknown error\nSkipping group: {$this->groupMySQL['name']}", __FUNCTION__, 'error');
679
                }
680
681
                return null;
682
            }
683
        }
684
685
        return $headers;
686
    }
687
688
    private function handlePartRepairTracking(array $headersNotInserted, array $parsedHeaders): void
689
    {
690
        $notInsertedCount = \count($headersNotInserted);
691
        if ($notInsertedCount > 0) {
692
            $this->missedPartHandler->addMissingParts($headersNotInserted, $this->groupMySQL['id']);
693
            $this->log($notInsertedCount.' articles failed to insert!', __FUNCTION__, 'warning');
694
        }
695
696
        // Check for missing headers in range
697
        $expectedCount = $this->last - $this->first - $this->notYEnc - $this->headersBlackListed + 1;
698
        if ($expectedCount > \count($this->headersReceived)) {
699
            $rangeNotReceived = array_diff(range($this->first, $this->last), $this->headersReceived);
700
            $notReceivedCount = \count($rangeNotReceived);
701
702
            if ($notReceivedCount > 0) {
703
                $this->missedPartHandler->addMissingParts($rangeNotReceived, $this->groupMySQL['id']);
704
705
                if ($this->config->echoCli) {
706
                    cli()->alternate(
707
                        'Server did not return '.$notReceivedCount.' articles from '.$this->groupMySQL['name'].'.'
708
                    );
709
                }
710
            }
711
        }
712
    }
713
714
    private function groupMissingPartsIntoRanges(array $missingParts): array
715
    {
716
        $ranges = [];
717
        $partList = [];
718
        $firstPart = $lastNum = $missingParts[0]->numberid;
719
720
        foreach ($missingParts as $part) {
721
            if (($part->numberid - $firstPart) > ($this->config->messageBuffer / 4)) {
722
                $ranges[] = [
723
                    'partfrom' => $firstPart,
724
                    'partto' => $lastNum,
725
                    'partlist' => $partList,
726
                ];
727
                $firstPart = $part->numberid;
728
                $partList = [];
729
            }
730
            $partList[] = $part->numberid;
731
            $lastNum = $part->numberid;
732
        }
733
734
        $ranges[] = [
735
            'partfrom' => $firstPart,
736
            'partto' => $lastNum,
737
            'partlist' => $partList,
738
        ];
739
740
        return $ranges;
741
    }
742
743
    private function getNextArticleToTry(int $currentPost, array $groupData): ?int
744
    {
745
        if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) {
746
            $tempPost = (int) round($currentPost / (random_int(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP);
747
            if ($tempPost < $groupData['first']) {
748
                $tempPost = $groupData['first'];
749
            }
750
        } else {
751
            $tempPost = (int) round((random_int(1005, 1012) / 1000) * $currentPost, 0, PHP_ROUND_HALF_UP);
752
            if ($tempPost > $groupData['last']) {
753
                $tempPost = $groupData['last'];
754
            }
755
        }
756
757
        // If we got the same article number, give up
758
        if ($tempPost === $currentPost) {
759
            return null;
760
        }
761
762
        return $tempPost;
763
    }
764
765
    private function binarySearchArticleByDate(int $goalTime, array $data): string
766
    {
767
        $wantedArticle = (int) round(($data['last'] + $data['first']) / 2);
768
        $aMax = $data['last'];
769
        $aMin = $data['first'];
770
        $oldArticle = $articleTime = null;
771
772
        while (true) {
773
            if ($wantedArticle <= $data['first'] || $wantedArticle >= $data['last']) {
774
                break;
775
            }
776
777
            $reallyOldArticle = $oldArticle;
778
            $oldArticle = $wantedArticle;
779
780
            $articleTime = $this->postdate($wantedArticle, $data);
781
782
            if (! $articleTime) {
783
                $wantedArticle = random_int($aMin, $aMax);
784
                $articleTime = $this->postdate($wantedArticle, $data);
785
            }
786
787
            if ($articleTime < $goalTime) {
788
                $aMin = $oldArticle;
789
                $wantedArticle = (int) round(($aMax + $oldArticle) / 2);
790
                if ($this->config->echoCli) {
791
                    echo '-';
792
                }
793
            } elseif ($articleTime > $goalTime) {
794
                $aMax = $oldArticle;
795
                $wantedArticle = (int) round(($aMin + $oldArticle) / 2);
796
                if ($this->config->echoCli) {
797
                    echo '+';
798
                }
799
            } else {
800
                break;
801
            }
802
803
            if ($reallyOldArticle === $wantedArticle && ($goalTime - $articleTime) <= 0) {
804
                break;
805
            }
806
        }
807
808
        if ($this->config->echoCli) {
809
            $goalCarbon = Carbon::createFromTimestamp($goalTime, date_default_timezone_get());
810
            $articleCarbon = Carbon::createFromTimestamp($articleTime, date_default_timezone_get());
811
            $diffDays = $goalCarbon->diffInDays($articleCarbon, true);
812
            cli()->primary(
813
                PHP_EOL.'Found article #'.$wantedArticle.' which has a date of '.date('r', $articleTime).
814
                ', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.$diffDays.' days.'
815
            );
816
        }
817
818
        return (string) $wantedArticle;
819
    }
820
821
    // ==================== Output Methods ====================
822
823
    private function outputNoNewArticles(array $groupMySQL, array $groupNNTP, array $range): void
824
    {
825
        if ($this->config->echoCli) {
826
            cli()->primary(
827
                'No new articles for '.$groupMySQL['name'].' (first '.number_format((int) $range['first']).
828
                ', last '.number_format((int) $range['last']).', grouplast '.number_format((int) $groupMySQL['last_record']).
829
                ', total '.number_format((int) $range['total']).")\n".'Server oldest: '.number_format((int) $groupNNTP['first']).
830
                ' Server newest: '.number_format((int) $groupNNTP['last']).' Local newest: '.number_format((int) $groupMySQL['last_record'])
831
            );
832
        }
833
    }
834
835
    private function outputNewArticlesInfo(array $groupMySQL, array $groupNNTP, array $range): void
836
    {
837
        if (! $this->config->echoCli) {
838
            return;
839
        }
840
841
        $message = $range['isNew']
842
            ? 'New group '.$groupNNTP['group'].' starting with '.
843
              ($this->config->newGroupScanByDays
844
                  ? $this->config->newGroupDaysToScan.' days'
845
                  : number_format($this->config->newGroupMessagesToScan).' messages').' worth.'
846
            : 'Group '.$groupNNTP['group'].' has '.number_format((int) $range['realTotal']).' new articles.';
847
848
        cli()->primary(
849
            $message.
850
            ' Leaving '.number_format((int) $range['leaveOver']).
851
            " for next pass.\nServer oldest: ".number_format((int) $groupNNTP['first']).
852
            ' Server newest: '.number_format((int) $groupNNTP['last']).
853
            ' Local newest: '.number_format((int) $groupMySQL['last_record'])
854
        );
855
    }
856
857
    private function outputHeaderInitial(): void
858
    {
859
        cli()->primary(
860
            'Received '.\count($this->headersReceived).
861
            ' articles of '.number_format($this->last - $this->first + 1).' requested, '.
862
            $this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.'
863
        );
864
    }
865
866
    private function outputHeaderDuration(): void
867
    {
868
        if (! $this->config->echoCli) {
869
            return;
870
        }
871
872
        $currentMicroTime = now();
873
        cli()->alternateOver(number_format($this->timeHeaders, 2).'s').
874
        cli()->primaryOver(' to download articles, ').
875
        cli()->alternateOver(number_format($this->timeCleaning, 2).'s').
876
        cli()->primaryOver(' to process collections, ').
877
        cli()->alternateOver(number_format($this->timeInsert, 2).'s').
878
        cli()->primaryOver(' to insert binaries/parts, ').
879
        cli()->alternateOver(number_format($currentMicroTime->diffInSeconds($this->startPR, true), 2).'s').
880
        cli()->primaryOver(' for part repair, ').
881
        cli()->alternateOver(number_format($currentMicroTime->diffInSeconds($this->startLoop, true), 2).'s').
882
        cli()->primary(' total.');
883
    }
884
885
    // ==================== Logging Methods ====================
886
887
    private function log(string $message, string $method, string $color): void
888
    {
889
        if ($this->config->echoCli) {
890
            cli()->$color($message.' ['.__CLASS__."::$method]");
891
        }
892
    }
893
894
    private function logError(string $message): void
895
    {
896
        if ($this->config->echoCli) {
897
            cli()->error($message);
898
        }
899
        if (config('app.debug')) {
900
            Log::error($message);
901
        }
902
    }
903
}
904