|
1
|
|
|
<?php |
|
2
|
|
|
|
|
3
|
|
|
namespace App\Console\Commands; |
|
4
|
|
|
|
|
5
|
|
|
use App\Models\Category; |
|
6
|
|
|
use App\Models\Predb; |
|
7
|
|
|
use App\Models\Release; |
|
8
|
|
|
use Blacklight\NameFixer; |
|
9
|
|
|
use Blacklight\Nfo; |
|
10
|
|
|
use Blacklight\NNTP; |
|
11
|
|
|
use Blacklight\NZBContents; |
|
12
|
|
|
use Blacklight\processing\PostProcess; |
|
13
|
|
|
use Illuminate\Console\Command; |
|
14
|
|
|
use Illuminate\Support\Facades\DB; |
|
15
|
|
|
|
|
16
|
|
|
class ReleasesFixNamesGroup extends Command |
|
17
|
|
|
{ |
|
18
|
|
|
/** |
|
19
|
|
|
* The name and signature of the console command. |
|
20
|
|
|
* |
|
21
|
|
|
* @var string |
|
22
|
|
|
*/ |
|
23
|
|
|
protected $signature = 'releases:fix-names-group |
|
24
|
|
|
{type : Type of fix (standard|predbft)} |
|
25
|
|
|
{--guid-char= : GUID character to process (for standard type)} |
|
26
|
|
|
{--limit=1000 : Maximum releases to process} |
|
27
|
|
|
{--thread=1 : Thread number (for predbft type)}'; |
|
28
|
|
|
|
|
29
|
|
|
/** |
|
30
|
|
|
* The console command description. |
|
31
|
|
|
* |
|
32
|
|
|
* @var string |
|
33
|
|
|
*/ |
|
34
|
|
|
protected $description = 'Fix release names using various methods (group-based processing)'; |
|
35
|
|
|
|
|
36
|
|
|
private NameFixer $nameFixer; |
|
37
|
|
|
|
|
38
|
|
|
private int $checked = 0; |
|
39
|
|
|
|
|
40
|
|
|
/** |
|
41
|
|
|
* Execute the console command. |
|
42
|
|
|
*/ |
|
43
|
|
|
public function handle(): int |
|
44
|
|
|
{ |
|
45
|
|
|
$type = $this->argument('type'); |
|
46
|
|
|
$maxPerRun = (int) $this->option('limit'); |
|
47
|
|
|
|
|
48
|
|
|
$this->nameFixer = new NameFixer; |
|
49
|
|
|
|
|
50
|
|
|
switch ($type) { |
|
51
|
|
|
case 'standard': |
|
52
|
|
|
return $this->processStandard($maxPerRun); |
|
53
|
|
|
|
|
54
|
|
|
case 'predbft': |
|
55
|
|
|
return $this->processPredbFulltext($maxPerRun); |
|
56
|
|
|
|
|
57
|
|
|
default: |
|
58
|
|
|
$this->error("Invalid type: {$type}. Use 'standard' or 'predbft'"); |
|
59
|
|
|
|
|
60
|
|
|
return Command::FAILURE; |
|
61
|
|
|
} |
|
62
|
|
|
} |
|
63
|
|
|
|
|
64
|
|
|
/** |
|
65
|
|
|
* Process standard name fixing |
|
66
|
|
|
*/ |
|
67
|
|
|
protected function processStandard(int $maxPerRun): int |
|
68
|
|
|
{ |
|
69
|
|
|
$guidChar = $this->option('guid-char'); |
|
70
|
|
|
|
|
71
|
|
|
if ($guidChar === null) { |
|
|
|
|
|
|
72
|
|
|
$this->error('--guid-char is required for standard type'); |
|
73
|
|
|
|
|
74
|
|
|
return Command::FAILURE; |
|
75
|
|
|
} |
|
76
|
|
|
|
|
77
|
|
|
$this->info("Processing releases with GUID starting with: {$guidChar}"); |
|
78
|
|
|
$this->info("Maximum per run: {$maxPerRun}"); |
|
79
|
|
|
|
|
80
|
|
|
// Allow for larger filename return sets |
|
81
|
|
|
DB::statement('SET SESSION group_concat_max_len = 65536'); |
|
82
|
|
|
|
|
83
|
|
|
// Find releases to process |
|
84
|
|
|
$releases = $this->fetchReleases($guidChar, $maxPerRun); |
|
85
|
|
|
|
|
86
|
|
|
if ($releases->isEmpty()) { |
|
87
|
|
|
$this->info('No releases to process'); |
|
88
|
|
|
|
|
89
|
|
|
return Command::SUCCESS; |
|
90
|
|
|
} |
|
91
|
|
|
|
|
92
|
|
|
$this->info("Found {$releases->count()} releases to process"); |
|
93
|
|
|
$bar = $this->output->createProgressBar($releases->count()); |
|
94
|
|
|
$bar->start(); |
|
95
|
|
|
|
|
96
|
|
|
$nntp = null; |
|
|
|
|
|
|
97
|
|
|
$nzbcontents = null; |
|
98
|
|
|
|
|
99
|
|
|
foreach ($releases as $release) { |
|
100
|
|
|
$this->checked++; |
|
101
|
|
|
$this->nameFixer->reset(); |
|
102
|
|
|
|
|
103
|
|
|
// Process UID |
|
104
|
|
|
if ((int) $release->proc_uid === NameFixer::PROC_UID_NONE && |
|
105
|
|
|
(! empty($release->uid) || ! empty($release->mediainfo))) { |
|
106
|
|
|
|
|
107
|
|
|
if (! empty($release->uid)) { |
|
108
|
|
|
$this->nameFixer->checkName($release, true, 'UID, ', 1, true); |
|
109
|
|
|
} |
|
110
|
|
|
|
|
111
|
|
|
if (empty($this->nameFixer->matched) && ! empty($release->mediainfo)) { |
|
112
|
|
|
$this->nameFixer->checkName($release, true, 'Mediainfo, ', 1, true); |
|
113
|
|
|
} |
|
114
|
|
|
} |
|
115
|
|
|
|
|
116
|
|
|
$this->nameFixer->_updateSingleColumn('proc_uid', NameFixer::PROC_UID_DONE, $release->releases_id); |
|
117
|
|
|
|
|
118
|
|
|
if ($this->nameFixer->matched) { |
|
119
|
|
|
$bar->advance(); |
|
120
|
|
|
|
|
121
|
|
|
continue; |
|
122
|
|
|
} |
|
123
|
|
|
|
|
124
|
|
|
// Process CRC32 |
|
125
|
|
|
if ((int) $release->proc_crc32 === NameFixer::PROC_CRC_NONE && ! empty($release->crc)) { |
|
126
|
|
|
$this->nameFixer->reset(); |
|
127
|
|
|
$this->nameFixer->checkName($release, true, 'CRC32, ', 1, true); |
|
128
|
|
|
} |
|
129
|
|
|
|
|
130
|
|
|
$this->nameFixer->_updateSingleColumn('proc_crc32', NameFixer::PROC_CRC_DONE, $release->releases_id); |
|
131
|
|
|
|
|
132
|
|
|
if ($this->nameFixer->matched) { |
|
133
|
|
|
$bar->advance(); |
|
134
|
|
|
|
|
135
|
|
|
continue; |
|
136
|
|
|
} |
|
137
|
|
|
|
|
138
|
|
|
// Process SRR |
|
139
|
|
|
if ((int) $release->proc_srr === NameFixer::PROC_SRR_NONE) { |
|
140
|
|
|
$this->nameFixer->reset(); |
|
141
|
|
|
$this->nameFixer->checkName($release, true, 'SRR, ', 1, true); |
|
142
|
|
|
} |
|
143
|
|
|
|
|
144
|
|
|
$this->nameFixer->_updateSingleColumn('proc_srr', NameFixer::PROC_SRR_DONE, $release->releases_id); |
|
145
|
|
|
|
|
146
|
|
|
if ($this->nameFixer->matched) { |
|
147
|
|
|
$bar->advance(); |
|
148
|
|
|
|
|
149
|
|
|
continue; |
|
150
|
|
|
} |
|
151
|
|
|
|
|
152
|
|
|
// Process PAR2 hash |
|
153
|
|
|
if ((int) $release->proc_hash16k === NameFixer::PROC_HASH16K_NONE && ! empty($release->hash)) { |
|
154
|
|
|
$this->nameFixer->reset(); |
|
155
|
|
|
$this->nameFixer->checkName($release, true, 'PAR2 hash, ', 1, true); |
|
156
|
|
|
} |
|
157
|
|
|
|
|
158
|
|
|
$this->nameFixer->_updateSingleColumn('proc_hash16k', NameFixer::PROC_HASH16K_DONE, $release->releases_id); |
|
159
|
|
|
|
|
160
|
|
|
if ($this->nameFixer->matched) { |
|
161
|
|
|
$bar->advance(); |
|
162
|
|
|
|
|
163
|
|
|
continue; |
|
164
|
|
|
} |
|
165
|
|
|
|
|
166
|
|
|
// Process NFO |
|
167
|
|
|
if ((int) $release->nfostatus === Nfo::NFO_FOUND && |
|
168
|
|
|
(int) $release->proc_nfo === NameFixer::PROC_NFO_NONE && |
|
169
|
|
|
! empty($release->textstring) && |
|
170
|
|
|
! preg_match('/^=newz\[NZB\]=\w+/', $release->textstring)) { |
|
171
|
|
|
|
|
172
|
|
|
$this->nameFixer->reset(); |
|
173
|
|
|
$this->nameFixer->checkName($release, true, 'NFO, ', 1, true); |
|
174
|
|
|
} |
|
175
|
|
|
|
|
176
|
|
|
$this->nameFixer->_updateSingleColumn('proc_nfo', NameFixer::PROC_NFO_DONE, $release->releases_id); |
|
177
|
|
|
|
|
178
|
|
|
if ($this->nameFixer->matched) { |
|
179
|
|
|
$bar->advance(); |
|
180
|
|
|
|
|
181
|
|
|
continue; |
|
182
|
|
|
} |
|
183
|
|
|
|
|
184
|
|
|
// Process filenames |
|
185
|
|
|
if ((int) $release->fileid > 0 && (int) $release->proc_files === NameFixer::PROC_FILES_NONE) { |
|
186
|
|
|
$this->nameFixer->reset(); |
|
187
|
|
|
$fileNames = explode('|', $release->filestring); |
|
188
|
|
|
|
|
189
|
|
|
if (is_array($fileNames)) { |
|
190
|
|
|
$releaseFile = $release; |
|
191
|
|
|
foreach ($fileNames as $fileName) { |
|
192
|
|
|
if ($this->nameFixer->matched === false) { |
|
193
|
|
|
$releaseFile->textstring = $fileName; |
|
194
|
|
|
$this->nameFixer->checkName($releaseFile, true, 'Filenames, ', 1, true); |
|
195
|
|
|
} |
|
196
|
|
|
} |
|
197
|
|
|
} |
|
198
|
|
|
} |
|
199
|
|
|
|
|
200
|
|
|
$this->nameFixer->_updateSingleColumn('proc_files', NameFixer::PROC_FILES_DONE, $release->releases_id); |
|
201
|
|
|
|
|
202
|
|
|
if ($this->nameFixer->matched) { |
|
203
|
|
|
$bar->advance(); |
|
204
|
|
|
|
|
205
|
|
|
continue; |
|
206
|
|
|
} |
|
207
|
|
|
|
|
208
|
|
|
// Process PAR2 |
|
209
|
|
|
if ((int) $release->proc_par2 === NameFixer::PROC_PAR2_NONE) { |
|
210
|
|
|
// Initialize NZB contents if needed |
|
211
|
|
|
if (! isset($nzbcontents)) { |
|
212
|
|
|
$nntp = new NNTP; |
|
213
|
|
|
$compressedHeaders = config('nntmux_nntp.compressed_headers'); |
|
214
|
|
|
|
|
215
|
|
|
if ((config('nntmux_nntp.use_alternate_nntp_server') === true |
|
216
|
|
|
? $nntp->doConnect($compressedHeaders, true) |
|
217
|
|
|
: $nntp->doConnect()) !== true) { |
|
218
|
|
|
$this->warn('Unable to connect to usenet for PAR2 processing'); |
|
219
|
|
|
} else { |
|
220
|
|
|
$Nfo = new Nfo; |
|
221
|
|
|
$nzbcontents = new NZBContents([ |
|
|
|
|
|
|
222
|
|
|
'Echo' => false, |
|
223
|
|
|
'NNTP' => $nntp, |
|
224
|
|
|
'Nfo' => $Nfo, |
|
225
|
|
|
'PostProcess' => new PostProcess(['Nfo' => $Nfo, 'NameFixer' => $this->nameFixer]), |
|
|
|
|
|
|
226
|
|
|
]); |
|
227
|
|
|
} |
|
228
|
|
|
} |
|
229
|
|
|
|
|
230
|
|
|
if (isset($nzbcontents)) { |
|
231
|
|
|
$nzbcontents->checkPAR2($release->guid, $release->releases_id, $release->groups_id, 1, true); |
|
232
|
|
|
} |
|
233
|
|
|
} |
|
234
|
|
|
|
|
235
|
|
|
$this->nameFixer->_updateSingleColumn('proc_par2', NameFixer::PROC_PAR2_DONE, $release->releases_id); |
|
236
|
|
|
|
|
237
|
|
|
$bar->advance(); |
|
238
|
|
|
} |
|
239
|
|
|
|
|
240
|
|
|
$bar->finish(); |
|
241
|
|
|
$this->newLine(2); |
|
242
|
|
|
|
|
243
|
|
|
$this->info("✅ Processed {$this->checked} releases"); |
|
244
|
|
|
$this->info("✅ Fixed {$this->nameFixer->fixed} release names"); |
|
245
|
|
|
|
|
246
|
|
|
return Command::SUCCESS; |
|
247
|
|
|
} |
|
248
|
|
|
|
|
249
|
|
|
/** |
|
250
|
|
|
* Process PreDB fulltext matching |
|
251
|
|
|
*/ |
|
252
|
|
|
protected function processPredbFulltext(int $maxPerRun): int |
|
253
|
|
|
{ |
|
254
|
|
|
$thread = (int) $this->option('thread'); |
|
255
|
|
|
$offset = $thread * $maxPerRun - $maxPerRun; |
|
256
|
|
|
|
|
257
|
|
|
$this->info('Processing PreDB fulltext matching'); |
|
258
|
|
|
$this->info("Thread: {$thread}, Limit: {$maxPerRun}, Offset: {$offset}"); |
|
259
|
|
|
|
|
260
|
|
|
$pres = Predb::fromQuery( |
|
261
|
|
|
sprintf( |
|
262
|
|
|
' |
|
263
|
|
|
SELECT p.id AS predb_id, p.title, p.source, p.searched |
|
264
|
|
|
FROM predb p |
|
265
|
|
|
WHERE LENGTH(p.title) >= 15 AND p.title NOT REGEXP "[\"\<\> ]" |
|
266
|
|
|
AND p.searched = 0 |
|
267
|
|
|
AND p.predate < (NOW() - INTERVAL 1 DAY) |
|
268
|
|
|
ORDER BY p.predate ASC |
|
269
|
|
|
LIMIT %s |
|
270
|
|
|
OFFSET %s', |
|
271
|
|
|
$maxPerRun, |
|
272
|
|
|
$offset |
|
273
|
|
|
) |
|
274
|
|
|
); |
|
275
|
|
|
|
|
276
|
|
|
if ($pres->isEmpty()) { |
|
277
|
|
|
$this->info('No PreDB entries to process'); |
|
278
|
|
|
|
|
279
|
|
|
return Command::SUCCESS; |
|
280
|
|
|
} |
|
281
|
|
|
|
|
282
|
|
|
$this->info("Found {$pres->count()} PreDB entries to process"); |
|
283
|
|
|
$bar = $this->output->createProgressBar($pres->count()); |
|
284
|
|
|
$bar->start(); |
|
285
|
|
|
|
|
286
|
|
|
foreach ($pres as $pre) { |
|
287
|
|
|
$this->nameFixer->done = $this->nameFixer->matched = false; |
|
288
|
|
|
$searched = 0; |
|
289
|
|
|
|
|
290
|
|
|
$ftmatched = $this->nameFixer->matchPredbFT($pre, true, 1, true); |
|
291
|
|
|
|
|
292
|
|
|
if ($ftmatched > 0) { |
|
293
|
|
|
$searched = 1; |
|
294
|
|
|
} elseif ($ftmatched < 0) { |
|
295
|
|
|
$searched = -6; |
|
296
|
|
|
} else { |
|
297
|
|
|
$searched = $pre['searched'] - 1; |
|
298
|
|
|
} |
|
299
|
|
|
|
|
300
|
|
|
Predb::query()->where('id', $pre['predb_id'])->update(['searched' => $searched]); |
|
301
|
|
|
$this->checked++; |
|
302
|
|
|
|
|
303
|
|
|
$bar->advance(); |
|
304
|
|
|
} |
|
305
|
|
|
|
|
306
|
|
|
$bar->finish(); |
|
307
|
|
|
$this->newLine(2); |
|
308
|
|
|
|
|
309
|
|
|
$this->info("✅ Processed {$this->checked} PreDB entries"); |
|
310
|
|
|
|
|
311
|
|
|
return Command::SUCCESS; |
|
312
|
|
|
} |
|
313
|
|
|
|
|
314
|
|
|
/** |
|
315
|
|
|
* Fetch releases for processing |
|
316
|
|
|
*/ |
|
317
|
|
|
protected function fetchReleases(string $guidChar, int $maxPerRun) |
|
318
|
|
|
{ |
|
319
|
|
|
return Release::fromQuery(sprintf(" |
|
320
|
|
|
SELECT |
|
321
|
|
|
r.id AS releases_id, r.fromname, r.guid, r.groups_id, r.categories_id, r.name, r.searchname, r.proc_nfo, |
|
322
|
|
|
r.proc_uid, r.proc_files, r.proc_par2, r.ishashed, r.dehashstatus, r.nfostatus, |
|
323
|
|
|
r.size AS relsize, r.predb_id, r.proc_hash16k, r.proc_srr, r.proc_crc32, |
|
324
|
|
|
IFNULL(rf.releases_id, 0) AS fileid, IF(rf.ishashed = 1, rf.name, 0) AS filehash, |
|
325
|
|
|
IFNULL(GROUP_CONCAT(rf.name ORDER BY rf.name ASC SEPARATOR '|'), '') AS filestring, |
|
326
|
|
|
IFNULL(UNCOMPRESS(rn.nfo), '') AS textstring, |
|
327
|
|
|
IFNULL(ru.uniqueid, '') AS uid, |
|
328
|
|
|
IFNULL(ph.hash, 0) AS hash, |
|
329
|
|
|
IFNULL(rf.crc32, '') AS crc |
|
330
|
|
|
FROM releases r |
|
331
|
|
|
LEFT JOIN release_nfos rn ON rn.releases_id = r.id |
|
332
|
|
|
LEFT JOIN release_files rf ON rf.releases_id = r.id |
|
333
|
|
|
LEFT JOIN release_unique ru ON ru.releases_id = r.id |
|
334
|
|
|
LEFT JOIN par_hashes ph ON ph.releases_id = r.id |
|
335
|
|
|
WHERE r.leftguid = %s |
|
336
|
|
|
AND r.isrenamed = %d |
|
337
|
|
|
AND r.predb_id = 0 |
|
338
|
|
|
AND r.passwordstatus >= 0 |
|
339
|
|
|
AND r.nfostatus > %d |
|
340
|
|
|
AND ( |
|
341
|
|
|
(r.nfostatus = %d AND r.proc_nfo = %d) |
|
342
|
|
|
OR r.proc_files = %d |
|
343
|
|
|
OR r.proc_uid = %d |
|
344
|
|
|
OR r.proc_par2 = %d |
|
345
|
|
|
OR r.proc_srr = %d |
|
346
|
|
|
OR r.proc_hash16k = %d |
|
347
|
|
|
OR r.proc_crc32 = %d |
|
348
|
|
|
OR (r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0) |
|
349
|
|
|
) |
|
350
|
|
|
AND r.categories_id IN (%s) |
|
351
|
|
|
GROUP BY r.id |
|
352
|
|
|
ORDER BY r.id DESC |
|
353
|
|
|
LIMIT %s", |
|
354
|
|
|
escapeString($guidChar), |
|
355
|
|
|
NameFixer::IS_RENAMED_NONE, |
|
356
|
|
|
Nfo::NFO_UNPROC, |
|
357
|
|
|
Nfo::NFO_FOUND, |
|
358
|
|
|
NameFixer::PROC_NFO_NONE, |
|
359
|
|
|
NameFixer::PROC_FILES_NONE, |
|
360
|
|
|
NameFixer::PROC_UID_NONE, |
|
361
|
|
|
NameFixer::PROC_PAR2_NONE, |
|
362
|
|
|
NameFixer::PROC_SRR_NONE, |
|
363
|
|
|
NameFixer::PROC_HASH16K_NONE, |
|
364
|
|
|
NameFixer::PROC_CRC_NONE, |
|
365
|
|
|
Category::getCategoryOthersGroup(), |
|
366
|
|
|
$maxPerRun |
|
367
|
|
|
)); |
|
368
|
|
|
} |
|
369
|
|
|
} |
|
370
|
|
|
|