Completed
Push — dev ( adcbf7...b0d05b )
by Darko
09:57
created

ProcessReleases::postProcessReleases()   A

Complexity

Conditions 3
Paths 3

Size

Total Lines 8
Code Lines 6

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 0
CRAP Score 12

Importance

Changes 0
Metric Value
eloc 6
dl 0
loc 8
ccs 0
cts 0
cp 0
rs 10
c 0
b 0
f 0
cc 3
nc 3
nop 2
crap 12
1
<?php
2
3
namespace Blacklight\processing;
4
5
use App\Models\Collection;
6
use Blacklight\NZB;
7
use Blacklight\NNTP;
8
use App\Models\Predb;
9
use Blacklight\Genres;
10
use App\Models\Release;
11
use App\Models\Category;
12
use App\Models\Settings;
13
use Blacklight\ColorCLI;
14
use Blacklight\Releases;
15
use Blacklight\Categorize;
16
use App\Models\UsenetGroup;
17
use Illuminate\Support\Str;
18
use App\Models\ReleaseRegex;
19
use Blacklight\ConsoleTools;
20
use Blacklight\ReleaseImage;
21
use App\Models\ReleasesGroups;
22
use Blacklight\ReleaseCleaning;
23
use Illuminate\Support\Facades\DB;
24
use Illuminate\Support\Facades\Log;
25
26
class ProcessReleases
27
{
28
    public const COLLFC_DEFAULT = 0; // Collection has default filecheck status
29
    public const COLLFC_COMPCOLL = 1; // Collection is a complete collection
30
    public const COLLFC_COMPPART = 2; // Collection is a complete collection and has all parts available
31
    public const COLLFC_SIZED = 3; // Collection has been calculated for total size
32
    public const COLLFC_INSERTED = 4; // Collection has been inserted into releases
33
    public const COLLFC_DELETE = 5; // Collection is ready for deletion
34
    public const COLLFC_TEMPCOMP = 15; // Collection is complete and being checked for complete parts
35
    public const COLLFC_ZEROPART = 16; // Collection has a 00/0XX designator (temporary)
36
37
    public const FILE_INCOMPLETE = 0; // We don't have all the parts yet for the file (binaries table partcheck column).
38
    public const FILE_COMPLETE = 1; // We have all the parts for the file (binaries table partcheck column).
39
40
    /**
41
     * @var int
42
     */
43
    public $collectionDelayTime;
44
45
    /**
46
     * @var int
47
     */
48
    public $crossPostTime;
49
50
    /**
51
     * @var int
52
     */
53
    public $releaseCreationLimit;
54
55
    /**
56
     * @var int
57
     */
58
    public $completion;
59
60
    /**
61
     * @var bool
62
     */
63
    public $echoCLI;
64
65
    /**
66
     * @var \PDO
67
     */
68
    public $pdo;
69
70
    /**
71
     * @var \Blacklight\ConsoleTools
72
     */
73
    public $consoleTools;
74
75
    /**
76
     * @var \Blacklight\NZB
77
     */
78
    public $nzb;
79
80
    /**
81
     * @var \Blacklight\ReleaseCleaning
82
     */
83
    public $releaseCleaning;
84
85
    /**
86
     * @var \Blacklight\Releases
87
     */
88
    public $releases;
89
90
    /**
91
     * @var \Blacklight\ReleaseImage
92
     */
93
    public $releaseImage;
94
95
    /**
96
     * Time (hours) to wait before delete a stuck/broken collection.
97
     *
98
     *
99
     * @var int
100
     */
101
    private $collectionTimeout;
102
103
    /**
104
     * @var \Blacklight\ColorCLI
105
     */
106
    protected $colorCli;
107
108
    /**
109
     * @param array $options Class instances / Echo to cli ?
110
     *
111
     * @throws \Exception
112
     */
113
    public function __construct(array $options = [])
114
    {
115
        $defaults = [
116
            'Echo'            => true,
117
            'ConsoleTools'    => null,
118
            'Groups'          => null,
119
            'NZB'             => null,
120
            'ReleaseCleaning' => null,
121
            'ReleaseImage'    => null,
122
            'Releases'        => null,
123
            'Settings'        => null,
124
        ];
125
        $options += $defaults;
126
127
        $this->echoCLI = ($options['Echo'] && config('nntmux.echocli'));
128
129
        $this->consoleTools = ($options['ConsoleTools'] instanceof ConsoleTools ? $options['ConsoleTools'] : new ConsoleTools());
130
        $this->nzb = ($options['NZB'] instanceof NZB ? $options['NZB'] : new NZB());
131
        $this->releaseCleaning = ($options['ReleaseCleaning'] instanceof ReleaseCleaning ? $options['ReleaseCleaning'] : new ReleaseCleaning());
132
        $this->releases = ($options['Releases'] instanceof Releases ? $options['Releases'] : new Releases());
133
        $this->releaseImage = ($options['ReleaseImage'] instanceof ReleaseImage ? $options['ReleaseImage'] : new ReleaseImage());
134
        $this->colorCli = new ColorCLI();
135
136
        $dummy = Settings::settingValue('..delaytime');
137
        $this->collectionDelayTime = ($dummy !== '' ? (int) $dummy : 2);
0 ignored issues
show
introduced by
The condition $dummy !== '' is always true.
Loading history...
138
        $dummy = Settings::settingValue('..crossposttime');
139
        $this->crossPostTime = ($dummy !== '' ? (int) $dummy : 2);
0 ignored issues
show
introduced by
The condition $dummy !== '' is always true.
Loading history...
140
        $dummy = Settings::settingValue('..maxnzbsprocessed');
141
        $this->releaseCreationLimit = ($dummy !== '' ? (int) $dummy : 1000);
0 ignored issues
show
introduced by
The condition $dummy !== '' is always true.
Loading history...
142
        $dummy = Settings::settingValue('..completionpercent');
143
        $this->completion = ($dummy !== '' ? (int) $dummy : 0);
0 ignored issues
show
introduced by
The condition $dummy !== '' is always true.
Loading history...
144
        if ($this->completion > 100) {
145
            $this->completion = 100;
146
            $this->colorCli->error(PHP_EOL.'You have an invalid setting for completion. It cannot be higher than 100.');
147
        }
148
        $this->collectionTimeout = (int) Settings::settingValue('indexer.processing.collection_timeout');
149
    }
150
151
    /**
152
     * Main method for creating releases/NZB files from collections.
153
     *
154
     * @param int              $categorize
155
     * @param int              $postProcess
156
     * @param string           $groupName (optional)
157
     * @param \Blacklight\NNTP $nntp
158
     * @param bool             $echooutput
159
     *
160
     * @return int
161
     * @throws \Throwable
162
     */
163
    public function processReleases($categorize, $postProcess, $groupName, &$nntp, $echooutput): int
164
    {
165
        $this->echoCLI = ($echooutput && config('nntmux.echocli'));
166
        $groupID = '';
167
168
        if (! empty($groupName) && $groupName !== 'mgr') {
169
            $groupInfo = UsenetGroup::getByName($groupName);
170
            if ($groupInfo !== null) {
171
                $groupID = $groupInfo['id'];
172
            }
173
        }
174
175
        if ($this->echoCLI) {
176
            $this->colorCli->header('Starting release update process ('.now()->format('Y-m-d H:i:s').')');
177
        }
178
179
        if (! file_exists(Settings::settingValue('..nzbpath'))) {
180
            if ($this->echoCLI) {
181
                $this->colorCli->error('Bad or missing nzb directory - '.Settings::settingValue('..nzbpath'));
182
            }
183
184
            return 0;
185
        }
186
187
        $this->processIncompleteCollections($groupID);
188
        $this->processCollectionSizes($groupID);
189
        $this->deleteUnwantedCollections($groupID);
190
191
        $totalReleasesAdded = 0;
192
        do {
193
            $releasesCount = $this->createReleases($groupID);
194
            $totalReleasesAdded += $releasesCount['added'];
195
196
            $nzbFilesAdded = $this->createNZBs($groupID);
197
198
            $this->categorizeReleases($categorize, $groupID);
199
            $this->postProcessReleases($postProcess, $nntp);
200
            $this->deleteCollections($groupID);
201
202
            // This loops as long as the number of releases or nzbs added was >= the limit (meaning there are more waiting to be created)
203
        } while (
204
            ($releasesCount['added'] + $releasesCount['dupes']) >= $this->releaseCreationLimit
205
            || $nzbFilesAdded >= $this->releaseCreationLimit
206
        );
207
208
        // Only run if non-mgr as mgr is not specific to group
209
        if ($groupName !== 'mgr') {
210
            $this->deletedReleasesByGroup($groupID);
211
            $this->deleteReleases();
212
        }
213
214
        return $totalReleasesAdded;
215
    }
216
217
    /**
218
     * Return all releases to other->misc category.
219
     *
220
     * @param string $where Optional "where" query parameter.
221
     *
222
     * @void
223
     */
224
    public function resetCategorize($where = ''): void
225
    {
226
        DB::update(
227
            sprintf('UPDATE releases SET categories_id = %d, iscategorized = 0 %s', Category::OTHER_MISC, $where)
228
        );
229
    }
230
231
    /**
232
     * Categorizes releases.
233
     *
234
     * @param string $type  name or searchname | Categorize using the search name or subject.
235
     * @param string $where Optional "where" query parameter.
236
     *
237
     * @return int Quantity of categorized releases.
238
     * @throws \Exception
239
     */
240
    public function categorizeRelease($type, $where = ''): int
241
    {
242
        $cat = new Categorize();
243
        $categorized = $total = 0;
0 ignored issues
show
Unused Code introduced by
The assignment to $total is dead and can be removed.
Loading history...
244
        $releases = Release::fromQuery(
245
            sprintf(
246
                '
247
				SELECT id, fromname, %s, groups_id
248
				FROM releases %s',
249
                $type,
250
                $where
251
            )
252
        );
253
        if (\count($releases) > 0) {
254
            $total = \count($releases);
255
            foreach ($releases as $release) {
256
                $catId = $cat->determineCategory($release->groups_id, $release->{$type}, $release->fromname);
257
                Release::query()->where('id', $release->id)->update(['categories_id' => $catId['categories_id'], 'iscategorized' => 1]);
258
                try {
259
                    $taggedRelease = Release::find($release->id);
260
                    $taggedRelease->retag($catId['tags']);
261
                } catch (\Throwable $e) {
262
                    //Just pass this part, tag is not created for some reason, exception is thrown and blocks release creation
263
                }
264
                $categorized++;
265
                if ($this->echoCLI) {
266
                    $this->consoleTools->overWritePrimary(
267
                        'Categorizing: '.$this->consoleTools->percentString($categorized, $total)
268
                    );
269
                }
270
            }
271
        }
272
        if ($this->echoCLI && $categorized > 0) {
273
            echo PHP_EOL;
274
        }
275
276
        return $categorized;
277
    }
278
279
    /**
280
     * @param $groupID
281
     *
282
     * @throws \Exception
283
     * @throws \Throwable
284
     */
285
    public function processIncompleteCollections($groupID): void
286
    {
287
        $startTime = now();
288
289
        if ($this->echoCLI) {
290
            $this->colorCli->header('Process Releases -> Attempting to find complete collections.');
291
        }
292
293
        $where = (! empty($groupID) ? ' AND c.groups_id = '.$groupID.' ' : ' ');
294
295
        $this->processStuckCollections($where);
296
        $this->collectionFileCheckStage1($where);
297
        $this->collectionFileCheckStage2($where);
298
        $this->collectionFileCheckStage3($where);
299
        $this->collectionFileCheckStage4($where);
300
        $this->collectionFileCheckStage5($where);
301
        $this->collectionFileCheckStage6($where);
302
303
        if ($this->echoCLI) {
304
            $count = DB::selectOne(
305
                sprintf(
306
                    '
307
					SELECT COUNT(c.id) AS complete
308
					FROM collections c
309
					WHERE c.filecheck = %d %s',
310
                    self::COLLFC_COMPPART,
311
                    $where
312
                )
313
            );
314
315
            $totalTime = now()->diffInSeconds($startTime);
316
317
            $this->colorCli->primary(
318
                ($count === null ? 0 : $count->complete).' collections were found to be complete. Time: '.
319
                    $totalTime.Str::plural(' second', $totalTime),
320
                true
321
                );
322
        }
323
    }
324
325
    /**
326
     * @param $groupID
327
     *
328
     * @throws \Exception
329
     */
330
    public function processCollectionSizes($groupID): void
331
    {
332
        $startTime = now();
333
334
        if ($this->echoCLI) {
335
            $this->colorCli->header('Process Releases -> Calculating collection sizes (in bytes).');
336
        }
337
        // Get the total size in bytes of the collection for collections where filecheck = 2.
338
        DB::transaction(function () use ($groupID, $startTime) {
339
            $checked = DB::update(
340
                sprintf(
341
                    '
342
				UPDATE collections c
343
				SET c.filesize =
344
				(
345
					SELECT COALESCE(SUM(b.partsize), 0)
346
					FROM binaries b
347
					WHERE b.collections_id = c.id
348
				),
349
				c.filecheck = %d
350
				WHERE c.filecheck = %d
351
				AND c.filesize = 0 %s',
352
                    self::COLLFC_SIZED,
353
                    self::COLLFC_COMPPART,
354
                    (! empty($groupID) ? ' AND c.groups_id = '.$groupID : ' ')
355
                )
356
            );
357
            if ($checked > 0 && $this->echoCLI) {
358
                $this->colorCli->primary(
359
                    $checked.' collections set to filecheck = 3(size calculated)',
360
                    true
361
                );
362
                $totalTime = now()->diffInSeconds($startTime);
363
                $this->colorCli->primary($totalTime.Str::plural(' second', $totalTime), true);
364
            }
365
        }, 3);
366
    }
367
368
    /**
369
     * @param $groupID
370
     *
371
     * @throws \Exception
372
     * @throws \Throwable
373
     */
374
    public function deleteUnwantedCollections($groupID): void
375
    {
376
        $startTime = now();
377
378
        if ($this->echoCLI) {
379
            $this->colorCli->header('Process Releases -> Delete collections smaller/larger than minimum size/file count from group/site setting.');
380
        }
381
382
        $groupID === '' ? $groupIDs = UsenetGroup::getActiveIDs() : $groupIDs = [['id' => $groupID]];
383
384
        $minSizeDeleted = $maxSizeDeleted = $minFilesDeleted = 0;
385
386
        $maxSizeSetting = Settings::settingValue('.release.maxsizetoformrelease');
387
        $minSizeSetting = Settings::settingValue('.release.minsizetoformrelease');
388
        $minFilesSetting = Settings::settingValue('.release.minfilestoformrelease');
389
390
        foreach ($groupIDs as $grpID) {
391
            $groupMinSizeSetting = $groupMinFilesSetting = 0;
392
393
            $groupMinimums = UsenetGroup::getGroupByID($grpID['id']);
394
            if ($groupMinimums !== null) {
395
                if (! empty($groupMinimums['minsizetoformrelease']) && $groupMinimums['minsizetoformrelease'] > 0) {
396
                    $groupMinSizeSetting = (int) $groupMinimums['minsizetoformrelease'];
397
                }
398
                if (! empty($groupMinimums['minfilestoformrelease']) && $groupMinimums['minfilestoformrelease'] > 0) {
399
                    $groupMinFilesSetting = (int) $groupMinimums['minfilestoformrelease'];
400
                }
401
            }
402
403
            if (DB::selectOne(sprintf('
404
						SELECT SQL_NO_CACHE id
405
						FROM collections c
406
						WHERE c.filecheck = %d
407
						AND c.filesize > 0', self::COLLFC_SIZED)) !== null) {
408
                DB::transaction(function () use (
409
                    $groupMinSizeSetting,
410
                    $minSizeSetting,
411
                    $minSizeDeleted,
412
                    $maxSizeSetting,
413
                    $maxSizeDeleted,
414
                    $minFilesSetting,
415
                    $groupMinFilesSetting,
416
                    $minFilesDeleted,
417
                    $startTime
418
                ) {
419
                    $deleteQuery = DB::delete(sprintf('
420
						DELETE c FROM collections c
421
						WHERE c.filecheck = %d
422
						AND c.filesize > 0
423
						AND GREATEST(%d, %d) > 0
424
						AND c.filesize < GREATEST(%d, %d)', self::COLLFC_SIZED, $groupMinSizeSetting, $minSizeSetting, $groupMinSizeSetting, $minSizeSetting));
425
426
                    if ($deleteQuery > 0) {
427
                        $minSizeDeleted += $deleteQuery;
428
                    }
429
430
                    if ($maxSizeSetting > 0) {
431
                        $deleteQuery = DB::delete(sprintf('
432
							DELETE c FROM collections c
433
							WHERE c.filecheck = %d
434
							AND c.filesize > %d', self::COLLFC_SIZED, $maxSizeSetting));
435
436
                        if ($deleteQuery > 0) {
437
                            $maxSizeDeleted += $deleteQuery;
438
                        }
439
                    }
440
441
                    if ($minFilesSetting > 0 || $groupMinFilesSetting > 0) {
442
                        $deleteQuery = DB::delete(sprintf('
443
						DELETE c FROM collections c
444
						WHERE c.filecheck = %d
445
						AND GREATEST(%d, %d) > 0
446
						AND c.totalfiles < GREATEST(%d, %d)', self::COLLFC_SIZED, $groupMinFilesSetting, $minFilesSetting, $groupMinFilesSetting, $minFilesSetting));
447
448
                        if ($deleteQuery > 0) {
449
                            $minFilesDeleted += $deleteQuery;
450
                        }
451
                    }
452
453
                    $totalTime = now()->diffInSeconds($startTime);
454
455
                    if ($this->echoCLI) {
456
                        $this->colorCli->primary('Deleted '.($minSizeDeleted + $maxSizeDeleted + $minFilesDeleted).' collections: '.PHP_EOL.$minSizeDeleted.' smaller than, '.$maxSizeDeleted.' bigger than, '.$minFilesDeleted.' with less files than site/group settings in: '.$totalTime.Str::plural(' second', $totalTime), true);
457
                    }
458
                }, 3);
459
            }
460
        }
461
    }
462
463
    /**
464
     * @param int|string $groupID (optional)
465
     *
466
     * @return array
467
     * @throws \Throwable
468
     */
469
    public function createReleases($groupID): array
470
    {
471
        $startTime = now();
472
473
        $categorize = new Categorize();
474
        $returnCount = $duplicate = 0;
475
476
        if ($this->echoCLI) {
477
            $this->colorCli->header('Process Releases -> Create releases from complete collections.');
478
        }
479
480
        $collections = DB::select(
481
            sprintf(
482
                '
483
				SELECT SQL_NO_CACHE c.*, g.name AS gname
484
				FROM collections c
485
				INNER JOIN usenet_groups g ON c.groups_id = g.id
486
				WHERE %s c.filecheck = %d
487
				AND c.filesize > 0
488
				LIMIT %d',
489
                (! empty($groupID) ? ' c.groups_id = '.$groupID.' AND ' : ' '),
490
                self::COLLFC_SIZED,
491
                $this->releaseCreationLimit
492
            )
493
        );
494
495
        if ($this->echoCLI && \count($collections) > 0) {
496
            $this->colorCli->primary(\count($collections).' Collections ready to be converted to releases.', true);
497
        }
498
499
        foreach ($collections as $collection) {
500
            $cleanRelName = utf8_encode(str_replace(['#', '@', '$', '%', '^', '§', '¨', '©', 'Ö'], '', $collection->subject));
501
            $fromName = utf8_encode(
502
                trim($collection->fromname, "'")
503
                );
504
505
            // Look for duplicates, duplicates match on releases.name, releases.fromname and releases.size
506
            // A 1% variance in size is considered the same size when the subject and poster are the same
507
            $dupeCheck = Release::query()
508
                    ->where(['name' => $cleanRelName, 'fromname' => $fromName])
509
                    ->whereBetween('size', [$collection->filesize * .99, $collection->filesize * 1.01])
510
                    ->first(['id']);
511
512
            if ($dupeCheck === null) {
513
                $cleanedName = $this->releaseCleaning->releaseCleaner(
514
                    $collection->subject,
515
                    $collection->fromname,
516
                    $collection->gname
517
                    );
518
519
                if (\is_array($cleanedName)) {
520
                    $properName = $cleanedName['properlynamed'];
521
                    $preID = $cleanedName['predb'] ?? false;
522
                    $cleanedName = $cleanedName['cleansubject'];
523
                } else {
524
                    $properName = true;
525
                    $preID = false;
526
                }
527
528
                if ($preID === false && $cleanedName !== '') {
529
                    // try to match the cleaned searchname to predb title or filename here
530
                    $preMatch = Predb::matchPre($cleanedName);
531
                    if ($preMatch !== false) {
532
                        $cleanedName = $preMatch['title'];
533
                        $preID = $preMatch['predb_id'];
534
                        $properName = true;
535
                    }
536
                }
537
538
                $determinedCategory = $categorize->determineCategory($collection->groups_id, $cleanedName);
539
540
                $releaseID = Release::insertRelease(
541
                    [
542
                            'name' => $cleanRelName,
543
                            'searchname' => ! empty($cleanedName) ? utf8_encode($cleanedName) : $cleanRelName,
544
                            'totalpart' => $collection->totalfiles,
545
                            'groups_id' => $collection->groups_id,
546
                            'guid' => createGUID(),
547
                            'postdate' => $collection->date,
548
                            'fromname' => $fromName,
549
                            'size' => $collection->filesize,
550
                            'categories_id' => $determinedCategory['categories_id'],
551
                            'isrenamed' => $properName === true ? 1 : 0,
552
                            'predb_id' => $preID === false ? 0 : $preID,
553
                            'nzbstatus' => NZB::NZB_NONE,
554
                        ]
555
                    );
556
                try {
557
                    $release = Release::find($releaseID);
558
                    $release->retag($determinedCategory['tags']);
559
                } catch (\Throwable $e) {
560
                    Log::debug($e->getTraceAsString());
561
                    //Just pass this part, tag is not created for some reason, exception is thrown and blocks release creation
562
                }
563
564
                if ($releaseID !== null) {
565
                    // Update collections table to say we inserted the release.
566
                    DB::transaction(function () use ($collection, $releaseID) {
567
                        DB::update(
568
                            sprintf(
569
                                '
570
								UPDATE collections
571
								SET filecheck = %d, releases_id = %d
572
								WHERE id = %d',
573
                                self::COLLFC_INSERTED,
574
                                $releaseID,
575
                                $collection->id
576
                            )
577
                        );
578
                    }, 3);
579
580
                    // Add the id of regex that matched the collection and release name to release_regexes table
581
                    ReleaseRegex::insertIgnore([
582
                            'releases_id'            => $releaseID,
583
                            'collection_regex_id'    => $collection->collection_regexes_id,
584
                            'naming_regex_id'        => $cleanedName['id'] ?? 0,
585
                        ]);
586
587
                    if (preg_match_all('#(\S+):\S+#', $collection->xref, $matches)) {
588
                        foreach ($matches[1] as $grp) {
589
                            //check if the group name is in a valid format
590
                            $grpTmp = UsenetGroup::isValidGroup($grp);
591
                            if ($grpTmp !== false) {
592
                                //check if the group already exists in database
593
                                $xrefGrpID = UsenetGroup::getIDByName($grpTmp);
0 ignored issues
show
Bug introduced by
It seems like $grpTmp can also be of type true; however, parameter $name of App\Models\UsenetGroup::getIDByName() does only seem to accept string, maybe add an additional type check? ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

593
                                $xrefGrpID = UsenetGroup::getIDByName(/** @scrutinizer ignore-type */ $grpTmp);
Loading history...
594
                                if ($xrefGrpID === '') {
595
                                    $xrefGrpID = UsenetGroup::addGroup(
596
                                        [
597
                                                'name'                  => $grpTmp,
598
                                                'description'           => 'Added by Release processing',
599
                                                'backfill_target'       => 1,
600
                                                'first_record'          => 0,
601
                                                'last_record'           => 0,
602
                                                'active'                => 0,
603
                                                'backfill'              => 0,
604
                                                'minfilestoformrelease' => '',
605
                                                'minsizetoformrelease'  => '',
606
                                            ]
607
                                        );
608
                                }
609
610
                                $relGroupsChk = ReleasesGroups::query()->where(
611
                                    [
612
                                            ['releases_id', '=', $releaseID],
613
                                            ['groups_id', '=', $xrefGrpID],
614
                                        ]
615
                                    )->first();
616
617
                                if ($relGroupsChk === null) {
618
                                    ReleasesGroups::query()->insert(
619
                                        [
620
                                                'releases_id' => $releaseID,
621
                                                'groups_id'   => $xrefGrpID,
622
                                            ]
623
                                        );
624
                                }
625
                            }
626
                        }
627
                    }
628
629
                    $returnCount++;
630
631
                    if ($this->echoCLI) {
632
                        echo "Added $returnCount releases.\r";
633
                    }
634
                }
635
            } else {
636
                // The release was already in the DB, so delete the collection.
637
                DB::transaction(function () use ($collection) {
638
                    DB::delete(
639
                        sprintf(
640
                            '
641
							DELETE c
642
							FROM collections c
643
							WHERE c.collectionhash = %s',
644
                            escapeString($collection->collectionhash)
645
                        )
646
                    );
647
                }, 3);
648
649
                $duplicate++;
650
            }
651
        }
652
653
        $totalTime = now()->diffInSeconds($startTime);
654
655
        if ($this->echoCLI) {
656
            $this->colorCli->primary(
657
                PHP_EOL.
658
                    number_format($returnCount).
659
                    ' Releases added and '.
660
                    number_format($duplicate).
661
                    ' duplicate collections deleted in '.
662
                    $totalTime.Str::plural(' second', $totalTime),
663
                true
664
                );
665
        }
666
667
        return ['added' => $returnCount, 'dupes' => $duplicate];
668
    }
669
670
    /**
671
     * Create NZB files from complete releases.
672
     *
673
     * @param int|string $groupID (optional)
674
     *
675
     * @return int
676
     * @throws \Throwable
677
     */
678
    public function createNZBs($groupID): int
679
    {
680
        $startTime = now();
681
682
        if ($this->echoCLI) {
683
            $this->colorCli->header('Process Releases -> Create the NZB, delete collections/binaries/parts.');
684
        }
685
686
        $releases = Release::fromQuery(
687
            sprintf(
688
                "
689
				SELECT SQL_NO_CACHE
690
					CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title,
691
					r.name, r.id, r.guid
692
				FROM releases r
693
				INNER JOIN categories c ON r.categories_id = c.id
694
				INNER JOIN categories cp ON cp.id = c.parentid
695
				WHERE %s nzbstatus = 0",
696
                ! empty($groupID) ? ' r.groups_id = '.$groupID.' AND ' : ' '
697
            )
698
        );
699
700
        $nzbCount = 0;
701
702
        if (\count($releases) > 0) {
703
            $total = \count($releases);
704
            // Init vars for writing the NZB's.
705
            $this->nzb->initiateForWrite();
706
            foreach ($releases as $release) {
707
                if ($this->nzb->writeNzbForReleaseId($release->id, $release->guid, $release->name, $release->title)) {
0 ignored issues
show
Bug introduced by
The property title does not seem to exist on App\Models\Release. Are you sure there is no database migration missing?

Checks if undeclared accessed properties appear in database migrations and if the creating migration is correct.

Loading history...
708
                    $nzbCount++;
709
                    if ($this->echoCLI) {
710
                        echo "Creating NZBs and deleting Collections: $nzbCount/$total.\r";
711
                    }
712
                }
713
            }
714
        }
715
716
        $totalTime = now()->diffInSeconds($startTime);
717
718
        if ($this->echoCLI) {
719
            $this->colorCli->primary(
720
                number_format($nzbCount).' NZBs created/Collections deleted in '.
721
                    $totalTime.Str::plural(' second', $totalTime).PHP_EOL.
722
                    'Total time: '.$totalTime.Str::plural(' second', $totalTime),
723
                true
724
            );
725
        }
726
727
        return $nzbCount;
728
    }
729
730
    /**
731
     * Categorize releases.
732
     *
733
     * @param int        $categorize
734
     * @param int|string $groupID (optional)
735
     *
736
     * @void
737
     * @throws \Exception
738
     */
739
    public function categorizeReleases($categorize, $groupID = ''): void
740
    {
741
        $startTime = now();
742
        if ($this->echoCLI) {
743
            $this->colorCli->header('Process Releases -> Categorize releases.');
744
        }
745
        switch ((int) $categorize) {
746
            case 2:
0 ignored issues
show
Coding Style introduced by
case statements should be defined using a colon.

As per the PSR-2 coding standard, case statements should not be wrapped in curly braces. There is no need for braces, since each case is terminated by the next break.

There is also the option to use a semicolon instead of a colon, this is discouraged because many programmers do not even know it works and the colon is universal between programming languages.

switch ($expr) {
    case "A": { //wrong
        doSomething();
        break;
    }
    case "B"; //wrong
        doSomething();
        break;
    case "C": //right
        doSomething();
        break;
}

To learn more about the PSR-2 coding standard, please refer to the PHP-Fig.

Loading history...
747
                $type = 'searchname';
748
                break;
749
            case 1:
0 ignored issues
show
Coding Style introduced by
case statements should be defined using a colon.

As per the PSR-2 coding standard, case statements should not be wrapped in curly braces. There is no need for braces, since each case is terminated by the next break.

There is also the option to use a semicolon instead of a colon, this is discouraged because many programmers do not even know it works and the colon is universal between programming languages.

switch ($expr) {
    case "A": { //wrong
        doSomething();
        break;
    }
    case "B"; //wrong
        doSomething();
        break;
    case "C": //right
        doSomething();
        break;
}

To learn more about the PSR-2 coding standard, please refer to the PHP-Fig.

Loading history...
750
            default:
0 ignored issues
show
Coding Style introduced by
The default body in a switch statement must start on the line following the statement.

According to the PSR-2, the body of a default statement must start on the line immediately following the statement.

switch ($expr) {
    default:
        doSomething(); //right
        break;
}


switch ($expr) {
    default:

        doSomething(); //wrong
        break;
}

To learn more about the PSR-2 coding standard, please refer to the PHP-Fig.

Loading history...
751
752
                $type = 'name';
753
                break;
754
        }
755
        $this->categorizeRelease(
756
            $type,
757
            (! empty($groupID)
758
                ? 'WHERE categories_id = '.Category::OTHER_MISC.' AND iscategorized = 0 AND groups_id = '.$groupID
759
                : 'WHERE categories_id = '.Category::OTHER_MISC.' AND iscategorized = 0')
760
        );
761
762
        $totalTime = now()->diffInSeconds($startTime);
763
764
        if ($this->echoCLI) {
765
            $this->colorCli->primary($totalTime.Str::plural(' second', $totalTime));
766
        }
767
    }
768
769
    /**
770
     * Post-process releases.
771
     *
772
     * @param int  $postProcess
773
     * @param NNTP $nntp
774
     *
775
     * @void
776
     * @throws \Exception
777
     */
778
    public function postProcessReleases($postProcess, &$nntp): void
779
    {
780
        if ((int) $postProcess === 1) {
781
            (new PostProcess(['Echo' => $this->echoCLI]))->processAll($nntp);
782
        } elseif ($this->echoCLI) {
783
            $this->colorCli->info(
784
                'Post-processing is not running inside the Process Releases class.'.PHP_EOL.
785
                    'If you are using tmux or screen they might have their own scripts running Post-processing.'
786
                );
787
        }
788
    }
789
790
    /**
791
     * @param $groupID
792
     *
793
     * @throws \Exception
794
     * @throws \Throwable
795
     */
796
    public function deleteCollections($groupID): void
0 ignored issues
show
Unused Code introduced by
The parameter $groupID is not used and could be removed. ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-unused  annotation

796
    public function deleteCollections(/** @scrutinizer ignore-unused */ $groupID): void

This check looks for parameters that have been defined for a function or method, but which are not used in the method body.

Loading history...
797
    {
798
        $startTime = now();
799
800
        $deletedCount = 0;
801
802
        // CBP older than retention.
803
        if ($this->echoCLI) {
804
            echo
805
                $this->colorCli->header('Process Releases -> Delete finished collections.'.PHP_EOL).
0 ignored issues
show
Bug introduced by
Are you sure the usage of $this->colorCli->header(...ght\processing\PHP_EOL) targeting Blacklight\ColorCLI::header() seems to always return null.

This check looks for function or method calls that always return null and whose return value is used.

class A
{
    function getObject()
    {
        return null;
    }

}

$a = new A();
if ($a->getObject()) {

The method getObject() can return nothing but null, so it makes no sense to use the return value.

The reason is most likely that a function or method is imcomplete or has been reduced for debug purposes.

Loading history...
Bug introduced by
Are you sure $this->colorCli->header(...ght\processing\PHP_EOL) of type void can be used in concatenation? ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

805
                /** @scrutinizer ignore-type */ $this->colorCli->header('Process Releases -> Delete finished collections.'.PHP_EOL).
Loading history...
806
                $this->colorCli->primary(sprintf(
0 ignored issues
show
Bug introduced by
Are you sure the usage of $this->colorCli->primary...etentionhours')), true) targeting Blacklight\ColorCLI::primary() seems to always return null.

This check looks for function or method calls that always return null and whose return value is used.

class A
{
    function getObject()
    {
        return null;
    }

}

$a = new A();
if ($a->getObject()) {

The method getObject() can return nothing but null, so it makes no sense to use the return value.

The reason is most likely that a function or method is imcomplete or has been reduced for debug purposes.

Loading history...
807
                    'Deleting collections/binaries/parts older than %d hours.',
808
                    Settings::settingValue('..partretentionhours')
809
                ), true);
810
        }
811
812
        DB::transaction(function () use ($deletedCount, $startTime) {
813
            $deleted = 0;
814
            $deleteQuery = Collection::query()
815
                ->where('dateadded', '<', now()->subHours(Settings::settingValue('..partretentionhours')))
0 ignored issues
show
Bug introduced by
App\Models\Settings::set...'..partretentionhours') of type App\Models\Settings is incompatible with the type integer expected by parameter $value of Carbon\Carbon::subHours(). ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

815
                ->where('dateadded', '<', now()->subHours(/** @scrutinizer ignore-type */ Settings::settingValue('..partretentionhours')))
Loading history...
816
                ->delete();
817
            if ($deleteQuery > 0) {
818
                $deleted = $deleteQuery;
819
                $deletedCount += $deleted;
820
            }
821
            $firstQuery = $fourthQuery = now();
822
823
            $totalTime = $firstQuery->diffInSeconds($startTime);
824
825
            if ($this->echoCLI) {
826
                $this->colorCli->primary(
827
                    'Finished deleting '.$deleted.' old collections/binaries/parts in '.
828
                    $totalTime.Str::plural(' second', $totalTime),
829
                    true
830
                );
831
            }
832
833
            // Cleanup orphaned collections, binaries and parts
834
            // this really shouldn't happen, but just incase - so we only run 1/200 of the time
835
            if (random_int(0, 200) <= 1) {
836
                // CBP collection orphaned with no binaries or parts.
837
                if ($this->echoCLI) {
838
                    echo $this->colorCli->header('Process Releases -> Remove CBP orphans.'.PHP_EOL).$this->colorCli->primary('Deleting orphaned collections.');
0 ignored issues
show
Bug introduced by
Are you sure $this->colorCli->header(...ght\processing\PHP_EOL) of type void can be used in concatenation? ( Ignorable by Annotation )

If this is a false-positive, you can also ignore this issue in your code via the ignore-type  annotation

838
                    echo /** @scrutinizer ignore-type */ $this->colorCli->header('Process Releases -> Remove CBP orphans.'.PHP_EOL).$this->colorCli->primary('Deleting orphaned collections.');
Loading history...
Bug introduced by
Are you sure the usage of $this->colorCli->primary...orphaned collections.') targeting Blacklight\ColorCLI::primary() seems to always return null.

This check looks for function or method calls that always return null and whose return value is used.

class A
{
    function getObject()
    {
        return null;
    }

}

$a = new A();
if ($a->getObject()) {

The method getObject() can return nothing but null, so it makes no sense to use the return value.

The reason is most likely that a function or method is imcomplete or has been reduced for debug purposes.

Loading history...
Bug introduced by
Are you sure the usage of $this->colorCli->header(...ght\processing\PHP_EOL) targeting Blacklight\ColorCLI::header() seems to always return null.

This check looks for function or method calls that always return null and whose return value is used.

class A
{
    function getObject()
    {
        return null;
    }

}

$a = new A();
if ($a->getObject()) {

The method getObject() can return nothing but null, so it makes no sense to use the return value.

The reason is most likely that a function or method is imcomplete or has been reduced for debug purposes.

Loading history...
839
                }
840
841
                $deleted = 0;
842
                $deleteQuery = Collection::query()->whereNull('binaries.id')->orWhereNull('parts.binaries_id')->leftJoin('binaries', 'collections.id', '=', 'binaries.collections_id')->leftJoin('parts', 'binaries.id', '=', 'parts.binaries_id')->delete();
843
844
                if ($deleteQuery > 0) {
845
                    $deleted = $deleteQuery;
846
                    $deletedCount += $deleted;
847
                }
848
849
                $totalTime = now()->diffInSeconds($firstQuery);
850
851
                if ($this->echoCLI) {
852
                    $this->colorCli->primary('Finished deleting '.$deleted.' orphaned collections in '.$totalTime.Str::plural(' second', $totalTime), true);
853
                }
854
            }
855
856
            if ($this->echoCLI) {
857
                $this->colorCli->primary('Deleting collections that were missed after NZB creation.', true);
858
            }
859
860
            $deleted = 0;
861
            // Collections that were missing on NZB creation.
862
            $collections = Collection::query()->where('releases.nzbstatus', '=', 1)->leftJoin('releases', 'releases.id', '=', 'collections.releases_id')->select('collections.id')->get();
863
864
            foreach ($collections as $collection) {
865
                $deleted++;
866
                Collection::query()->where('id', $collection->id)->delete();
867
            }
868
            $deletedCount += $deleted;
869
870
            $colDelTime = now()->diffInSeconds($fourthQuery);
871
            $totalTime = $fourthQuery->diffInSeconds($startTime);
872
873
            if ($this->echoCLI) {
874
                $this->colorCli->primary('Finished deleting '.$deleted.' collections missed after NZB creation in '.$colDelTime.Str::plural(' second', $colDelTime).PHP_EOL.'Removed '.number_format($deletedCount).' parts/binaries/collection rows in '.$totalTime.Str::plural(' second', $totalTime), true);
875
            }
876
        }, 3);
877
    }
878
879
    /**
880
     * Delete unwanted releases based on admin settings.
881
     * This deletes releases based on group.
882
     *
883
     * @param int|string $groupID (optional)
884
     *
885
     * @void
886
     * @throws \Exception
887
     */
888
    public function deletedReleasesByGroup($groupID = ''): void
889
    {
890
        $startTime = now();
891
        $minSizeDeleted = $maxSizeDeleted = $minFilesDeleted = 0;
892
893
        if ($this->echoCLI) {
894
            $this->colorCli->header('Process Releases -> Delete releases smaller/larger than minimum size/file count from group/site setting.');
895
        }
896
897
        $groupID === '' ? $groupIDs = UsenetGroup::getActiveIDs() : $groupIDs = [['id' => $groupID]];
898
899
        $maxSizeSetting = Settings::settingValue('.release.maxsizetoformrelease');
900
        $minSizeSetting = Settings::settingValue('.release.minsizetoformrelease');
901
        $minFilesSetting = Settings::settingValue('.release.minfilestoformrelease');
902
903
        foreach ($groupIDs as $grpID) {
904
            $releases = Release::fromQuery(
905
                sprintf(
906
                    '
907
					SELECT SQL_NO_CACHE r.guid, r.id
908
					FROM releases r
909
					INNER JOIN usenet_groups g ON g.id = r.groups_id
910
					WHERE r.groups_id = %d
911
					AND greatest(IFNULL(g.minsizetoformrelease, 0), %d) > 0
912
					AND r.size < greatest(IFNULL(g.minsizetoformrelease, 0), %d)',
913
                    $grpID['id'],
914
                    $minSizeSetting,
915
                    $minSizeSetting
916
                )
917
            );
918
            foreach ($releases as $release) {
919
                $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
920
                $minSizeDeleted++;
921
            }
922
923
            if ($maxSizeSetting > 0) {
924
                $releases = Release::fromQuery(
925
                    sprintf(
926
                        '
927
						SELECT SQL_NO_CACHE id, guid
928
						FROM releases
929
						WHERE groups_id = %d
930
						AND size > %d',
931
                        $grpID['id'],
932
                        $maxSizeSetting
933
                    )
934
                );
935
                foreach ($releases as $release) {
936
                    $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
937
                    $maxSizeDeleted++;
938
                }
939
            }
940
            if ($minFilesSetting > 0) {
941
                $releases = Release::fromQuery(
942
                    sprintf(
943
                        '
944
				SELECT SQL_NO_CACHE r.id, r.guid
945
				FROM releases r
946
				INNER JOIN usenet_groups g ON g.id = r.groups_id
947
				WHERE r.groups_id = %d
948
				AND greatest(IFNULL(g.minfilestoformrelease, 0), %d) > 0
949
				AND r.totalpart < greatest(IFNULL(g.minfilestoformrelease, 0), %d)',
950
                        $grpID['id'],
951
                        $minFilesSetting,
952
                        $minFilesSetting
953
                     )
954
                 );
955
                foreach ($releases as $release) {
956
                    $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
957
                    $minFilesDeleted++;
958
                }
959
            }
960
        }
961
962
        $totalTime = now()->diffInSeconds($startTime);
963
964
        if ($this->echoCLI) {
965
            $this->colorCli->primary(
966
                'Deleted '.($minSizeDeleted + $maxSizeDeleted + $minFilesDeleted).
967
                    ' releases: '.PHP_EOL.
968
                    $minSizeDeleted.' smaller than, '.$maxSizeDeleted.' bigger than, '.$minFilesDeleted.
969
                    ' with less files than site/groups setting in: '.
970
                    $totalTime.Str::plural(' second', $totalTime),
971
                true
972
                );
973
        }
974
    }
975
976
    /**
977
     * Delete releases using admin settings.
978
     * This deletes releases, regardless of group.
979
     *
980
     * @void
981
     * @throws \Exception
982
     */
983
    public function deleteReleases(): void
984
    {
985
        $startTime = now();
986
        $genres = new Genres();
987
        $passwordDeleted = $duplicateDeleted = $retentionDeleted = $completionDeleted = $disabledCategoryDeleted = 0;
988
        $disabledGenreDeleted = $miscRetentionDeleted = $miscHashedDeleted = $categoryMinSizeDeleted = 0;
989
990
        // Delete old releases and finished collections.
991
        if ($this->echoCLI) {
992
            $this->colorCli->header('Process Releases -> Delete old releases and passworded releases.');
993
        }
994
995
        // Releases past retention.
996
        if ((int) Settings::settingValue('..releaseretentiondays') !== 0) {
997
            $releases = Release::fromQuery(
998
                sprintf(
999
                    'SELECT id, guid FROM releases WHERE postdate < (NOW() - INTERVAL %d DAY)',
1000
                    (int) Settings::settingValue('..releaseretentiondays')
1001
                )
1002
            );
1003
            foreach ($releases as $release) {
1004
                $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1005
                $retentionDeleted++;
1006
            }
1007
        }
1008
1009
        // Passworded releases.
1010
        if ((int) Settings::settingValue('..deletepasswordedrelease') === 1) {
1011
            $releases = Release::fromQuery(
1012
                sprintf(
1013
                    'SELECT id, guid FROM releases WHERE passwordstatus = %d',
1014
                    Releases::PASSWD_RAR
1015
                )
1016
            );
1017
            foreach ($releases as $release) {
1018
                $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1019
                $passwordDeleted++;
1020
            }
1021
        }
1022
1023
        // Possibly passworded releases.
1024
        if ((int) Settings::settingValue('..deletepossiblerelease') === 1) {
1025
            $releases = Release::fromQuery(
1026
                sprintf(
1027
                    'SELECT id, guid FROM releases WHERE passwordstatus = %d',
1028
                    Releases::PASSWD_POTENTIAL
1029
                )
1030
            );
1031
            foreach ($releases as $release) {
1032
                $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1033
                $passwordDeleted++;
1034
            }
1035
        }
1036
1037
        if ((int) $this->crossPostTime !== 0) {
1038
            // Crossposted releases.
1039
            $releases = Release::fromQuery(
1040
                sprintf(
1041
                    'SELECT id, guid FROM releases WHERE adddate > (NOW() - INTERVAL %d HOUR) GROUP BY name HAVING COUNT(name) > 1',
1042
                    $this->crossPostTime
1043
                )
1044
            );
1045
            foreach ($releases as $release) {
1046
                $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1047
                $duplicateDeleted++;
1048
            }
1049
        }
1050
1051
        if ($this->completion > 0) {
1052
            $releases = Release::fromQuery(
1053
                sprintf('SELECT id, guid FROM releases WHERE completion < %d AND completion > 0', $this->completion)
1054
            );
1055
            foreach ($releases as $release) {
1056
                $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1057
                $completionDeleted++;
1058
            }
1059
        }
1060
1061
        // Disabled categories.
1062
        $disabledCategories = Category::getDisabledIDs();
1063
        if (\count($disabledCategories) > 0) {
1064
            foreach ($disabledCategories as $disabledCategory) {
1065
                $releases = Release::fromQuery(
1066
                    sprintf('SELECT id, guid FROM releases WHERE categories_id = %d', (int) $disabledCategory['id'])
1067
                );
1068
                foreach ($releases as $release) {
1069
                    $disabledCategoryDeleted++;
1070
                    $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1071
                }
1072
            }
1073
        }
1074
1075
        // Delete smaller than category minimum sizes.
1076
        $categories = Category::fromQuery(
1077
            '
1078
			SELECT SQL_NO_CACHE c.id AS id,
1079
			CASE WHEN c.minsizetoformrelease = 0 THEN cp.minsizetoformrelease ELSE c.minsizetoformrelease END AS minsize
1080
			FROM categories c
1081
			INNER JOIN categories cp ON cp.id = c.parentid
1082
			WHERE c.parentid IS NOT NULL'
1083
        );
1084
1085
        foreach ($categories as $category) {
1086
            if ((int) $category->minsize > 0) {
0 ignored issues
show
Bug introduced by
The property minsize does not exist on App\Models\Category. Did you mean minsizetoformrelease?
Loading history...
1087
                $releases = Release::fromQuery(
1088
                    sprintf(
1089
                        '
1090
							SELECT id, guid
1091
							FROM releases
1092
							WHERE categories_id = %d
1093
							AND size < %d
1094
							LIMIT 1000',
1095
                        (int) $category->id,
1096
                        (int) $category->minsize
1097
                        )
1098
                    );
1099
                foreach ($releases as $release) {
1100
                    $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1101
                    $categoryMinSizeDeleted++;
1102
                }
1103
            }
1104
        }
1105
1106
        // Disabled music genres.
1107
        $genrelist = $genres->getDisabledIDs();
1108
        if (\count($genrelist) > 0) {
1109
            foreach ($genrelist as $genre) {
1110
                $releases = Release::fromQuery(
1111
                    sprintf(
1112
                        '
1113
						SELECT id, guid
1114
						FROM releases r
1115
						INNER JOIN
1116
						(
1117
							SELECT id AS mid
1118
							FROM musicinfo
1119
							WHERE musicinfo.genre_id = %d
1120
						) mi ON musicinfo_id = mi.mid',
1121
                        (int) $genre['id']
1122
                    )
1123
                );
1124
                foreach ($releases as $release) {
1125
                    $disabledGenreDeleted++;
1126
                    $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1127
                }
1128
            }
1129
        }
1130
1131
        // Misc other.
1132
        if (Settings::settingValue('..miscotherretentionhours') > 0) {
1133
            $releases = Release::fromQuery(
1134
                sprintf(
1135
                    '
1136
					SELECT SQL_NO_CACHE id, guid
1137
					FROM releases
1138
					WHERE categories_id = %d
1139
					AND adddate <= NOW() - INTERVAL %d HOUR',
1140
                    Category::OTHER_MISC,
1141
                    (int) Settings::settingValue('..miscotherretentionhours')
1142
                )
1143
            );
1144
            foreach ($releases as $release) {
1145
                $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1146
                $miscRetentionDeleted++;
1147
            }
1148
        }
1149
1150
        // Misc hashed.
1151
        if ((int) Settings::settingValue('..mischashedretentionhours') > 0) {
1152
            $releases = Release::fromQuery(
1153
                sprintf(
1154
                    '
1155
					SELECT SQL_NO_CACHE id, guid
1156
					FROM releases
1157
					WHERE categories_id = %d
1158
					AND adddate <= NOW() - INTERVAL %d HOUR',
1159
                    Category::OTHER_HASHED,
1160
                    (int) Settings::settingValue('..mischashedretentionhours')
1161
                )
1162
            );
1163
            foreach ($releases as $release) {
1164
                $this->releases->deleteSingle(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage);
1165
                $miscHashedDeleted++;
1166
            }
1167
        }
1168
1169
        if ($this->echoCLI) {
1170
            $this->colorCli->primary(
1171
                'Removed releases: '.
1172
                    number_format($retentionDeleted).
1173
                    ' past retention, '.
1174
                    number_format($passwordDeleted).
1175
                    ' passworded, '.
1176
                    number_format($duplicateDeleted).
1177
                    ' crossposted, '.
1178
                    number_format($disabledCategoryDeleted).
1179
                    ' from disabled categories, '.
1180
                    number_format($categoryMinSizeDeleted).
1181
                    ' smaller than category settings, '.
1182
                    number_format($disabledGenreDeleted).
1183
                    ' from disabled music genres, '.
1184
                    number_format($miscRetentionDeleted).
1185
                    ' from misc->other '.
1186
                    number_format($miscHashedDeleted).
1187
                    ' from misc->hashed'.
1188
                    (
1189
                        $this->completion > 0
1190
                        ? ', '.number_format($completionDeleted).' under '.$this->completion.'% completion.'
1191
                        : '.'
1192
                    ),
1193
                true
1194
                );
1195
1196
            $totalDeleted = (
1197
                $retentionDeleted + $passwordDeleted + $duplicateDeleted + $disabledCategoryDeleted +
1198
                $disabledGenreDeleted + $miscRetentionDeleted + $miscHashedDeleted + $completionDeleted +
1199
                $categoryMinSizeDeleted
1200
            );
1201
            if ($totalDeleted > 0) {
1202
                $totalTime = now()->diffInSeconds($startTime);
1203
                $this->colorCli->primary(
1204
                    'Removed '.number_format($totalDeleted).' releases in '.
1205
                        $totalTime.Str::plural(' second', $totalTime),
1206
                    true
1207
                    );
1208
            }
1209
        }
1210
    }
1211
1212
    /**
1213
     * Formulate part of a query to prevent deletion of currently inserting parts / binaries / collections.
1214
     *
1215
     * @param string $groupName
1216
     * @param int    $difference
1217
     *
1218
     * @return string
1219
     */
1220
    private function maxQueryFormulator($groupName, $difference): string
0 ignored issues
show
Unused Code introduced by
The method maxQueryFormulator() is not used, and could be removed.

This check looks for private methods that have been defined, but are not used inside the class.

Loading history...
1221
    {
1222
        $maxID = DB::selectOne(
1223
            sprintf(
1224
                '
1225
				SELECT IFNULL(MAX(id),0) AS max
1226
				FROM %s',
1227
                $groupName
1228
            )
1229
        );
1230
1231
        return empty($maxID->max) || $maxID->max < $difference ? 0 : $maxID->max - $difference;
1232
    }
1233
1234
    /**
1235
     * Look if we have all the files in a collection (which have the file count in the subject).
1236
     * Set file check to complete.
1237
     * This means the the binary table has the same count as the file count in the subject, but
1238
     * the collection might not be complete yet since we might not have all the articles in the parts table.
1239
     *
1240
     * @param string $where
1241
     *
1242
     * @void
1243
     * @throws \Throwable
1244
     */
1245
    private function collectionFileCheckStage1(&$where): void
1246
    {
1247
        DB::transaction(function () use ($where) {
1248
            DB::update(
1249
                sprintf(
1250
                    '
1251
				UPDATE collections c
1252
				INNER JOIN
1253
				(
1254
					SELECT c.id
1255
					FROM collections c
1256
					INNER JOIN binaries b ON b.collections_id = c.id
1257
					WHERE c.totalfiles > 0
1258
					AND c.filecheck = %d %s
1259
					GROUP BY b.collections_id, c.totalfiles, c.id
1260
					HAVING COUNT(b.id) IN (c.totalfiles, c.totalfiles + 1)
1261
				) r ON c.id = r.id
1262
				SET filecheck = %d',
1263
                    self::COLLFC_DEFAULT,
1264
                    $where,
1265
                    self::COLLFC_COMPCOLL
1266
                )
1267
            );
1268
        }, 3);
1269
    }
1270
1271
    /**
1272
     * The first query sets filecheck to COLLFC_ZEROPART if there's a file that starts with 0 (ex. [00/100]).
1273
     * The second query sets filecheck to COLLFC_TEMPCOMP on everything left over, so anything that starts with 1 (ex. [01/100]).
1274
     *
1275
     * This is done because some collections start at 0 and some at 1, so if you were to assume the collection is complete
1276
     * at 0 then you would never get a complete collection if it starts with 1 and if it starts, you can end up creating
1277
     * a incomplete collection, since you assumed it was complete.
1278
     *
1279
     * @param string $where
1280
     *
1281
     * @void
1282
     * @throws \Throwable
1283
     */
1284
    private function collectionFileCheckStage2(&$where): void
1285
    {
1286
        DB::transaction(function () use ($where) {
1287
            DB::update(
1288
                sprintf(
1289
                    '
1290
				UPDATE collections c
1291
				INNER JOIN
1292
				(
1293
					SELECT c.id
1294
					FROM collections c
1295
					INNER JOIN binaries b ON b.collections_id = c.id
1296
					WHERE b.filenumber = 0
1297
					AND c.totalfiles > 0
1298
					AND c.filecheck = %d %s
1299
					GROUP BY c.id
1300
				) r ON c.id = r.id
1301
				SET c.filecheck = %d',
1302
                    self::COLLFC_COMPCOLL,
1303
                    $where,
1304
                    self::COLLFC_ZEROPART
1305
                )
1306
            );
1307
        }, 3);
1308
1309
        DB::transaction(function () use ($where) {
1310
            DB::update(
1311
                sprintf(
1312
                    '
1313
				UPDATE collections c
1314
				SET filecheck = %d
1315
				WHERE filecheck = %d %s',
1316
                    self::COLLFC_TEMPCOMP,
1317
                    self::COLLFC_COMPCOLL,
1318
                    $where
1319
                )
1320
            );
1321
        }, 3);
1322
    }
1323
1324
    /**
1325
     * Check if the files (binaries table) in a complete collection has all the parts.
1326
     * If we have all the parts, set binaries table partcheck to FILE_COMPLETE.
1327
     *
1328
     * @param string $where
1329
     *
1330
     * @void
1331
     * @throws \Throwable
1332
     */
1333
    private function collectionFileCheckStage3($where): void
1334
    {
1335
        DB::transaction(function () use ($where) {
1336
            DB::update(
1337
                sprintf(
1338
                    '
1339
				UPDATE binaries b
1340
				INNER JOIN
1341
				(
1342
					SELECT b.id
1343
					FROM binaries b
1344
					INNER JOIN collections c ON c.id = b.collections_id
1345
					WHERE c.filecheck = %d
1346
					AND b.partcheck = %d %s
1347
					AND b.currentparts = b.totalparts
1348
					GROUP BY b.id, b.totalparts
1349
				) r ON b.id = r.id
1350
				SET b.partcheck = %d',
1351
                    self::COLLFC_TEMPCOMP,
1352
                    self::FILE_INCOMPLETE,
1353
                    $where,
1354
                    self::FILE_COMPLETE
1355
            )
1356
        );
1357
        }, 3);
1358
1359
        DB::transaction(function () use ($where) {
1360
            DB::update(
1361
                sprintf(
1362
                    '
1363
				UPDATE binaries b
1364
				INNER JOIN
1365
				(
1366
					SELECT b.id
1367
					FROM binaries b
1368
					INNER JOIN collections c ON c.id = b.collections_id
1369
					WHERE c.filecheck = %d
1370
					AND b.partcheck = %d %s
1371
					AND b.currentparts >= (b.totalparts + 1)
1372
					GROUP BY b.id, b.totalparts
1373
				) r ON b.id = r.id
1374
				SET b.partcheck = %d',
1375
                    self::COLLFC_ZEROPART,
1376
                    self::FILE_INCOMPLETE,
1377
                    $where,
1378
                    self::FILE_COMPLETE
1379
                )
1380
            );
1381
        }, 3);
1382
    }
1383
1384
    /**
1385
     * Check if all files (binaries table) for a collection are complete (if they all have the "parts").
1386
     * Set collections filecheck column to COLLFC_COMPPART.
1387
     * This means the collection is complete.
1388
     *
1389
     * @param string $where
1390
     *
1391
     * @void
1392
     * @throws \Throwable
1393
     */
1394
    private function collectionFileCheckStage4(&$where): void
1395
    {
1396
        DB::transaction(function () use ($where) {
1397
            DB::update(
1398
                sprintf(
1399
                    '
1400
				UPDATE collections c INNER JOIN
1401
					(SELECT c.id FROM collections c
1402
					INNER JOIN binaries b ON c.id = b.collections_id
1403
					WHERE b.partcheck = 1 AND c.filecheck IN (%d, %d) %s
1404
					GROUP BY b.collections_id, c.totalfiles, c.id HAVING COUNT(b.id) >= c.totalfiles)
1405
				r ON c.id = r.id SET filecheck = %d',
1406
                    self::COLLFC_TEMPCOMP,
1407
                    self::COLLFC_ZEROPART,
1408
                    $where,
1409
                    self::COLLFC_COMPPART
1410
                )
1411
            );
1412
        }, 3);
1413
    }
1414
1415
    /**
1416
     * If not all files (binaries table) had their parts on the previous stage,
1417
     * reset the collection filecheck column to COLLFC_COMPCOLL so we reprocess them next time.
1418
     *
1419
     * @param string $where
1420
     *
1421
     * @void
1422
     * @throws \Throwable
1423
     */
1424
    private function collectionFileCheckStage5(&$where): void
1425
    {
1426
        DB::transaction(function () use ($where) {
1427
            DB::update(
1428
                sprintf(
1429
                    '
1430
				UPDATE collections c
1431
				SET filecheck = %d
1432
				WHERE filecheck IN (%d, %d) %s',
1433
                    self::COLLFC_COMPCOLL,
1434
                    self::COLLFC_TEMPCOMP,
1435
                    self::COLLFC_ZEROPART,
1436
                    $where
1437
                )
1438
            );
1439
        }, 3);
1440
    }
1441
1442
    /**
1443
     * If a collection did not have the file count (ie: [00/12]) or the collection is incomplete after
1444
     * $this->collectionDelayTime hours, set the collection to complete to create it into a release/nzb.
1445
     *
1446
     * @param string $where
1447
     *
1448
     * @void
1449
     * @throws \Throwable
1450
     */
1451
    private function collectionFileCheckStage6(&$where): void
1452
    {
1453
        DB::transaction(function () use ($where) {
1454
            DB::update(
1455
                sprintf(
1456
                    "
1457
				UPDATE collections c SET filecheck = %d, totalfiles = (SELECT COUNT(b.id) FROM binaries b WHERE b.collections_id = c.id)
1458
				WHERE c.dateadded < NOW() - INTERVAL '%d' HOUR
1459
				AND c.filecheck IN (%d, %d, 3) %s",
1460
                    self::COLLFC_COMPPART,
1461
                    $this->collectionDelayTime,
1462
                    self::COLLFC_DEFAULT,
1463
                    self::COLLFC_COMPCOLL,
1464
                    $where
1465
                )
1466
            );
1467
        }, 3);
1468
    }
1469
1470
    /**
1471
     * If a collection has been stuck for $this->collectionTimeout hours, delete it, it's bad.
1472
     *
1473
     * @param string $where
1474
     *
1475
     * @void
1476
     * @throws \Exception
1477
     * @throws \Throwable
1478
     */
1479
    private function processStuckCollections($where): void
1480
    {
1481
        $lastRun = Settings::settingValue('indexer.processing.last_run_time');
1482
1483
        DB::transaction(function () use ($where, $lastRun) {
1484
            $obj = DB::delete(
1485
                sprintf(
1486
                    '
1487
                DELETE c FROM collections c
1488
                WHERE
1489
                    c.added <
1490
                    DATE_SUB(%s, INTERVAL %d HOUR)
1491
                %s',
1492
                    escapeString($lastRun),
1493
                    $this->collectionTimeout,
1494
                    $where
1495
                )
1496
            );
1497
            if ($this->echoCLI && $obj > 0) {
1498
                $this->colorCli->primary('Deleted '.$obj.' broken/stuck collections.', true);
1499
            }
1500
        }, 3);
1501
    }
1502
}
1503