1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
namespace Blacklight; |
4
|
|
|
|
5
|
|
|
use App\Models\Group; |
6
|
|
|
use Blacklight\db\DB; |
7
|
|
|
use App\Models\Release; |
8
|
|
|
use App\Models\Settings; |
9
|
|
|
use Blacklight\utility\Utility; |
10
|
|
|
|
11
|
|
|
/** |
12
|
|
|
* Import NZB files into the database. |
13
|
|
|
* Class NZBImport. |
14
|
|
|
*/ |
15
|
|
|
class NZBImport |
16
|
|
|
{ |
17
|
|
|
/** |
18
|
|
|
* @var \Blacklight\db\DB |
19
|
|
|
*/ |
20
|
|
|
protected $pdo; |
21
|
|
|
|
22
|
|
|
/** |
23
|
|
|
* @var \Blacklight\Binaries |
24
|
|
|
*/ |
25
|
|
|
protected $binaries; |
26
|
|
|
|
27
|
|
|
/** |
28
|
|
|
* @var \Blacklight\ReleaseCleaning |
29
|
|
|
*/ |
30
|
|
|
protected $releaseCleaner; |
31
|
|
|
|
32
|
|
|
/** |
33
|
|
|
* @var bool|\stdClass |
34
|
|
|
*/ |
35
|
|
|
protected $site; |
36
|
|
|
|
37
|
|
|
/** |
38
|
|
|
* @var int |
39
|
|
|
*/ |
40
|
|
|
protected $crossPostt; |
41
|
|
|
|
42
|
|
|
/** |
43
|
|
|
* @var \Blacklight\Categorize |
44
|
|
|
*/ |
45
|
|
|
protected $category; |
46
|
|
|
|
47
|
|
|
/** |
48
|
|
|
* List of all the group names/ids in the DB. |
49
|
|
|
* @var array |
50
|
|
|
*/ |
51
|
|
|
protected $allGroups; |
52
|
|
|
|
53
|
|
|
/** |
54
|
|
|
* Was this run from the browser? |
55
|
|
|
* @var bool |
56
|
|
|
*/ |
57
|
|
|
protected $browser; |
58
|
|
|
|
59
|
|
|
/** |
60
|
|
|
* Return value for browser. |
61
|
|
|
* @var string |
62
|
|
|
*/ |
63
|
|
|
protected $retVal; |
64
|
|
|
|
65
|
|
|
/** |
66
|
|
|
* Guid of the current releases. |
67
|
|
|
* @var string |
68
|
|
|
*/ |
69
|
|
|
protected $relGuid; |
70
|
|
|
|
71
|
|
|
/** |
72
|
|
|
* @var bool |
73
|
|
|
*/ |
74
|
|
|
public $echoCLI; |
75
|
|
|
|
76
|
|
|
/** |
77
|
|
|
* @var \Blacklight\NZB |
78
|
|
|
*/ |
79
|
|
|
public $nzb; |
80
|
|
|
|
81
|
|
|
/** |
82
|
|
|
* @var string the MD5 hash of the first segment Message-ID of the NZB |
83
|
|
|
*/ |
84
|
|
|
protected $nzbGuid; |
85
|
|
|
|
86
|
|
|
/** |
87
|
|
|
* @var \Blacklight\Releases |
88
|
|
|
*/ |
89
|
|
|
private $releases; |
90
|
|
|
|
91
|
|
|
/** |
92
|
|
|
* Construct. |
93
|
|
|
* |
94
|
|
|
* @param array $options Class instances / various options. |
95
|
|
|
* @throws \Exception |
96
|
|
|
*/ |
97
|
|
|
public function __construct(array $options = []) |
98
|
|
|
{ |
99
|
|
|
$defaults = [ |
100
|
|
|
'Browser' => false, // Was this started from the browser? |
101
|
|
|
'Echo' => true, // Echo to CLI? |
102
|
|
|
'Binaries' => null, |
103
|
|
|
'Categorize' => null, |
104
|
|
|
'NZB' => null, |
105
|
|
|
'ReleaseCleaning' => null, |
106
|
|
|
'Releases' => null, |
107
|
|
|
'Settings' => null, |
108
|
|
|
]; |
109
|
|
|
$options += $defaults; |
110
|
|
|
|
111
|
|
|
$this->echoCLI = (! $this->browser && config('nntmux.echocli') && $options['Echo']); |
112
|
|
|
$this->pdo = ($options['Settings'] instanceof DB ? $options['Settings'] : new DB()); |
113
|
|
|
$this->binaries = ($options['Binaries'] instanceof Binaries ? $options['Binaries'] : new Binaries(['Settings' => $this->pdo, 'Echo' => $this->echoCLI])); |
114
|
|
|
$this->category = ($options['Categorize'] instanceof Categorize ? $options['Categorize'] : new Categorize(['Settings' => $this->pdo])); |
115
|
|
|
$this->nzb = ($options['NZB'] instanceof NZB ? $options['NZB'] : new NZB()); |
116
|
|
|
$this->releaseCleaner = ($options['ReleaseCleaning'] instanceof ReleaseCleaning ? $options['ReleaseCleaning'] : new ReleaseCleaning($this->pdo)); |
117
|
|
|
$this->releases = ($options['Releases'] instanceof Releases ? $options['Releases'] : new Releases(['settings' => $this->pdo])); |
118
|
|
|
|
119
|
|
|
$this->crossPostt = Settings::settingValue('..crossposttime') !== '' ? Settings::settingValue('..crossposttime') : 2; |
|
|
|
|
120
|
|
|
$this->browser = $options['Browser']; |
121
|
|
|
$this->retVal = ''; |
122
|
|
|
} |
123
|
|
|
|
124
|
|
|
/** |
125
|
|
|
* @param array $filesToProcess List of NZB files to import. |
126
|
|
|
* @param bool|string $useNzbName Use the NZB file name as release name? |
127
|
|
|
* @param bool $delete Delete the NZB when done? |
128
|
|
|
* @param bool $deleteFailed Delete the NZB if failed importing? |
129
|
|
|
* |
130
|
|
|
* @return string|bool |
131
|
|
|
* @throws \Exception |
132
|
|
|
*/ |
133
|
|
|
public function beginImport($filesToProcess, $useNzbName = false, $delete = true, $deleteFailed = true) |
134
|
|
|
{ |
135
|
|
|
// Get all the groups in the DB. |
136
|
|
|
if (! $this->getAllGroups()) { |
137
|
|
|
if ($this->browser) { |
138
|
|
|
return $this->retVal; |
139
|
|
|
} |
140
|
|
|
|
141
|
|
|
return false; |
142
|
|
|
} |
143
|
|
|
|
144
|
|
|
$start = date('Y-m-d H:i:s'); |
145
|
|
|
$nzbsImported = $nzbsSkipped = 0; |
146
|
|
|
|
147
|
|
|
// Loop over the file names. |
148
|
|
|
foreach ($filesToProcess as $nzbFile) { |
149
|
|
|
$this->nzbGuid = ''; |
150
|
|
|
|
151
|
|
|
// Check if the file is really there. |
152
|
|
|
if (is_file($nzbFile)) { |
153
|
|
|
|
154
|
|
|
// Get the contents of the NZB file as a string. |
155
|
|
|
if (strtolower(substr($nzbFile, -7)) === '.nzb.gz') { |
156
|
|
|
$nzbString = Utility::unzipGzipFile($nzbFile); |
157
|
|
|
} else { |
158
|
|
|
$nzbString = file_get_contents($nzbFile); |
159
|
|
|
} |
160
|
|
|
|
161
|
|
|
if ($nzbString === false) { |
162
|
|
|
$this->echoOut('ERROR: Unable to read: '.$nzbFile); |
163
|
|
|
|
164
|
|
|
if ($deleteFailed) { |
165
|
|
|
@unlink($nzbFile); |
|
|
|
|
166
|
|
|
} |
167
|
|
|
$nzbsSkipped++; |
168
|
|
|
continue; |
169
|
|
|
} |
170
|
|
|
|
171
|
|
|
// Load it as a XML object. |
172
|
|
|
$nzbXML = @simplexml_load_string($nzbString); |
173
|
|
|
if ($nzbXML === false || strtolower($nzbXML->getName()) !== 'nzb') { |
174
|
|
|
$this->echoOut('ERROR: Unable to load NZB XML data: '.$nzbFile); |
175
|
|
|
|
176
|
|
|
if ($deleteFailed) { |
177
|
|
|
@unlink($nzbFile); |
178
|
|
|
} |
179
|
|
|
$nzbsSkipped++; |
180
|
|
|
continue; |
181
|
|
|
} |
182
|
|
|
|
183
|
|
|
// Try to insert the NZB details into the DB. |
184
|
|
|
$inserted = $this->scanNZBFile($nzbXML, ($useNzbName ? str_ireplace('.nzb', '', basename($nzbFile)) : false)); |
185
|
|
|
|
186
|
|
|
if ($inserted) { |
187
|
|
|
|
188
|
|
|
// Try to copy the NZB to the NZB folder. |
189
|
|
|
$path = $this->nzb->getNZBPath($this->relGuid, 0, true); |
190
|
|
|
|
191
|
|
|
// Try to compress the NZB file in the NZB folder. |
192
|
|
|
$fp = gzopen($path, 'w5'); |
193
|
|
|
gzwrite($fp, $nzbString); |
194
|
|
|
gzclose($fp); |
195
|
|
|
|
196
|
|
|
if (! is_file($path)) { |
197
|
|
|
$this->echoOut('ERROR: Problem compressing NZB file to: '.$path); |
198
|
|
|
|
199
|
|
|
// Remove the release. |
200
|
|
|
Release::query()->where('guid', $this->relGuid)->delete(); |
201
|
|
|
|
202
|
|
|
if ($deleteFailed) { |
203
|
|
|
@unlink($nzbFile); |
204
|
|
|
} |
205
|
|
|
$nzbsSkipped++; |
206
|
|
|
continue; |
207
|
|
|
} else { |
208
|
|
|
$this->updateNzbGuid(); |
209
|
|
|
|
210
|
|
|
if ($delete) { |
211
|
|
|
// Remove the nzb file. |
212
|
|
|
@unlink($nzbFile); |
213
|
|
|
} |
214
|
|
|
|
215
|
|
|
$nzbsImported++; |
216
|
|
|
continue; |
217
|
|
|
} |
218
|
|
|
} else { |
219
|
|
|
$this->echoOut('ERROR: Failed to insert NZB!'); |
220
|
|
|
if ($deleteFailed) { |
221
|
|
|
@unlink($nzbFile); |
222
|
|
|
} |
223
|
|
|
$nzbsSkipped++; |
224
|
|
|
continue; |
225
|
|
|
} |
226
|
|
|
} else { |
227
|
|
|
$this->echoOut('ERROR: Unable to fetch: '.$nzbFile); |
228
|
|
|
$nzbsSkipped++; |
229
|
|
|
continue; |
230
|
|
|
} |
231
|
|
|
} |
232
|
|
|
$this->echoOut( |
233
|
|
|
'Proccessed '. |
234
|
|
|
$nzbsImported. |
235
|
|
|
' NZBs in '. |
236
|
|
|
(strtotime(date('Y-m-d H:i:s')) - strtotime($start)). |
237
|
|
|
' seconds, '. |
238
|
|
|
$nzbsSkipped. |
239
|
|
|
' NZBs were skipped.' |
240
|
|
|
); |
241
|
|
|
|
242
|
|
|
if ($this->browser) { |
243
|
|
|
return $this->retVal; |
244
|
|
|
} |
245
|
|
|
|
246
|
|
|
return true; |
247
|
|
|
} |
248
|
|
|
|
249
|
|
|
/** |
250
|
|
|
* @param object $nzbXML Reference of simpleXmlObject with NZB contents. |
251
|
|
|
* @param bool|string $useNzbName Use the NZB file name as release name? |
252
|
|
|
* @return bool |
253
|
|
|
* @throws \Exception |
254
|
|
|
*/ |
255
|
|
|
protected function scanNZBFile(&$nzbXML, $useNzbName = false): bool |
256
|
|
|
{ |
257
|
|
|
$binary_names = []; |
258
|
|
|
$totalFiles = $totalSize = $groupID = 0; |
259
|
|
|
$isBlackListed = $groupName = $firstName = $posterName = $postDate = false; |
260
|
|
|
|
261
|
|
|
// Go through the NZB, get the details, look if it's blacklisted, look if we have the groups. |
262
|
|
|
foreach ($nzbXML->file as $file) { |
263
|
|
|
$binary_names[] = $file['subject']; |
264
|
|
|
$totalFiles++; |
265
|
|
|
$groupID = -1; |
266
|
|
|
|
267
|
|
|
// Get the nzb info. |
268
|
|
|
if ($firstName === false) { |
269
|
|
|
$firstName = (string) $file->attributes()->subject; |
270
|
|
|
} |
271
|
|
|
if ($posterName === false) { |
272
|
|
|
$posterName = (string) $file->attributes()->poster; |
273
|
|
|
} |
274
|
|
|
if ($postDate === false) { |
275
|
|
|
$postDate = date('Y-m-d H:i:s', (string) $file->attributes()->date); |
|
|
|
|
276
|
|
|
} |
277
|
|
|
|
278
|
|
|
// Make a fake message array to use to check the blacklist. |
279
|
|
|
$msg = ['Subject' => (string) $file->attributes()->subject, 'From' => (string) $file->attributes()->poster, 'Message-ID' => '']; |
280
|
|
|
|
281
|
|
|
// Get the group names, group_id, check if it's blacklisted. |
282
|
|
|
$groupArr = []; |
283
|
|
|
foreach ($file->groups->group as $group) { |
284
|
|
|
$group = (string) $group; |
285
|
|
|
|
286
|
|
|
// If group_id is -1 try to get a group_id. |
287
|
|
|
if ($groupID === -1) { |
288
|
|
|
if (array_key_exists($group, $this->allGroups)) { |
289
|
|
|
$groupID = $this->allGroups[$group]; |
290
|
|
|
if (! $groupName) { |
291
|
|
|
$groupName = $group; |
292
|
|
|
} |
293
|
|
|
} else { |
294
|
|
|
$group = Group::isValidGroup($group); |
295
|
|
|
if ($group !== false) { |
296
|
|
|
$groupID = Group::addGroup([ |
297
|
|
|
'name' => $group, |
298
|
|
|
'description' => 'Added by NZBimport script.', |
299
|
|
|
'backfill_target' => 1, |
300
|
|
|
'minfilestoformrelease' => '', |
301
|
|
|
'minsizetoformrelease' => '', |
302
|
|
|
'first_record' => 0, |
303
|
|
|
'last_record' => 0, |
304
|
|
|
'active' => 0, |
305
|
|
|
'backfill' => 0, |
306
|
|
|
]); |
307
|
|
|
$this->allGroups[$group] = $groupID; |
308
|
|
|
|
309
|
|
|
$this->echoOut("Adding missing group: ($group)"); |
310
|
|
|
} |
311
|
|
|
} |
312
|
|
|
} |
313
|
|
|
// Add all the found groups to an array. |
314
|
|
|
$groupArr[] = $group; |
315
|
|
|
|
316
|
|
|
// Check if this NZB is blacklisted. |
317
|
|
|
if ($this->binaries->isBlackListed($msg, $group)) { |
|
|
|
|
318
|
|
|
$isBlackListed = true; |
319
|
|
|
break; |
320
|
|
|
} |
321
|
|
|
} |
322
|
|
|
|
323
|
|
|
// If we found a group and it's not blacklisted. |
324
|
|
|
if ($groupID !== -1 && ! $isBlackListed) { |
325
|
|
|
|
326
|
|
|
// Get the size of the release. |
327
|
|
|
if (count($file->segments->segment) > 0) { |
328
|
|
|
foreach ($file->segments->segment as $segment) { |
329
|
|
|
$totalSize += (int) $segment->attributes()->bytes; |
330
|
|
|
} |
331
|
|
|
} |
332
|
|
|
} else { |
333
|
|
|
if ($isBlackListed) { |
334
|
|
|
$errorMessage = 'Subject is blacklisted: '.utf8_encode(trim($firstName)); |
|
|
|
|
335
|
|
|
} else { |
336
|
|
|
$errorMessage = 'No group found for '.$firstName.' (one of '.implode(', ', $groupArr).' are missing'; |
|
|
|
|
337
|
|
|
} |
338
|
|
|
$this->echoOut($errorMessage); |
339
|
|
|
|
340
|
|
|
return false; |
341
|
|
|
} |
342
|
|
|
} |
343
|
|
|
|
344
|
|
|
// Sort values alphabetically but keep the keys intact |
345
|
|
|
if (count($binary_names) > 0) { |
346
|
|
|
asort($binary_names); |
347
|
|
|
foreach ($nzbXML->file as $file) { |
348
|
|
|
if ($file['subject'] == $binary_names[0]) { |
349
|
|
|
$this->nzbGuid = md5($file->segments->segment); |
350
|
|
|
break; |
351
|
|
|
} |
352
|
|
|
} |
353
|
|
|
} |
354
|
|
|
|
355
|
|
|
// Try to insert the NZB details into the DB. |
356
|
|
|
return $this->insertNZB( |
357
|
|
|
[ |
358
|
|
|
'subject' => $firstName, |
359
|
|
|
'useFName' => $useNzbName, |
360
|
|
|
'postDate' => empty($postDate) ? date('Y-m-d H:i:s') : $postDate, |
361
|
|
|
'from' => empty($posterName) ? '' : $posterName, |
362
|
|
|
'groups_id' => $groupID, |
363
|
|
|
'groupName' => $groupName, |
364
|
|
|
'totalFiles' => $totalFiles, |
365
|
|
|
'totalSize' => $totalSize, |
366
|
|
|
] |
367
|
|
|
); |
368
|
|
|
} |
369
|
|
|
|
370
|
|
|
/** |
371
|
|
|
* Insert the NZB details into the database. |
372
|
|
|
* |
373
|
|
|
* @param $nzbDetails |
374
|
|
|
* |
375
|
|
|
* @return bool |
376
|
|
|
* @throws \Exception |
377
|
|
|
*/ |
378
|
|
|
protected function insertNZB($nzbDetails): bool |
379
|
|
|
{ |
380
|
|
|
// Make up a GUID for the release. |
381
|
|
|
$this->relGuid = createGUID(); |
382
|
|
|
|
383
|
|
|
// Remove part count from subject. |
384
|
|
|
$partLess = preg_replace('/(\(\d+\/\d+\))*$/', 'yEnc', $nzbDetails['subject']); |
385
|
|
|
// Remove added yEnc from above and anything after. |
386
|
|
|
$subject = utf8_encode(trim(preg_replace('/yEnc.*$/i', 'yEnc', $partLess))); |
387
|
|
|
|
388
|
|
|
$renamed = 0; |
389
|
|
|
if ($nzbDetails['useFName']) { |
390
|
|
|
// If the user wants to use the file name.. use it. |
391
|
|
|
$cleanName = $nzbDetails['useFName']; |
392
|
|
|
$renamed = 1; |
393
|
|
|
} else { |
394
|
|
|
// Pass the subject through release cleaner to get a nicer name. |
395
|
|
|
$cleanName = $this->releaseCleaner->releaseCleaner($subject, $nzbDetails['from'], $nzbDetails['totalSize'], $nzbDetails['groupName']); |
396
|
|
|
if (isset($cleanName['properlynamed'])) { |
397
|
|
|
$cleanName = $cleanName['cleansubject']; |
398
|
|
|
$renamed = (isset($cleanName['properlynamed']) && $cleanName['properlynamed'] === true ? 1 : 0); |
399
|
|
|
} |
400
|
|
|
} |
401
|
|
|
|
402
|
|
|
$escapedSubject = $subject; |
403
|
|
|
$escapedFromName = $nzbDetails['from']; |
404
|
|
|
|
405
|
|
|
// Look for a duplicate on name, poster and size. |
406
|
|
|
$dupeCheck = Release::query()->where(['name' => $escapedSubject, 'fromname' => $escapedFromName])->whereBetween('size', [$nzbDetails['totalSize'] * 0.99, $nzbDetails['totalSize'] * 1.01])->first(['id']); |
407
|
|
|
|
408
|
|
|
if ($dupeCheck === null) { |
409
|
|
|
$escapedSearchName = $cleanName; |
410
|
|
|
// Insert the release into the DB. |
411
|
|
|
$relID = Release::insertRelease( |
412
|
|
|
[ |
413
|
|
|
'name' => $escapedSubject, |
414
|
|
|
'searchname' => $escapedSearchName, |
415
|
|
|
'totalpart' => $nzbDetails['totalFiles'], |
416
|
|
|
'groups_id' => $nzbDetails['groups_id'], |
417
|
|
|
'guid' => $this->relGuid, |
418
|
|
|
'postdate' => $nzbDetails['postDate'], |
419
|
|
|
'fromname' => $escapedFromName, |
420
|
|
|
'size' => $nzbDetails['totalSize'], |
421
|
|
|
'categories_id' => $this->category->determineCategory($nzbDetails['groups_id'], $cleanName, $escapedFromName), |
422
|
|
|
'isrenamed' => $renamed, |
423
|
|
|
'reqidstatus' => 0, |
424
|
|
|
'predb_id' => 0, |
425
|
|
|
'nzbstatus' => NZB::NZB_ADDED, |
426
|
|
|
] |
427
|
|
|
); |
428
|
|
|
} else { |
429
|
|
|
//$this->echoOut('This release is already in our DB so skipping: ' . $subject); |
|
|
|
|
430
|
|
|
return false; |
431
|
|
|
} |
432
|
|
|
|
433
|
|
|
if ($relID === null) { |
|
|
|
|
434
|
|
|
$this->echoOut('ERROR: Problem inserting: '.$subject); |
435
|
|
|
|
436
|
|
|
return false; |
437
|
|
|
} |
438
|
|
|
|
439
|
|
|
return true; |
440
|
|
|
} |
441
|
|
|
|
442
|
|
|
/** |
443
|
|
|
* Get all groups in the DB. |
444
|
|
|
* |
445
|
|
|
* @return bool |
446
|
|
|
*/ |
447
|
|
|
protected function getAllGroups(): bool |
448
|
|
|
{ |
449
|
|
|
$this->allGroups = []; |
450
|
|
|
$groups = Group::query()->get(['id', 'name']); |
451
|
|
|
|
452
|
|
|
if ($groups instanceof \Traversable) { |
|
|
|
|
453
|
|
|
foreach ($groups as $group) { |
454
|
|
|
$this->allGroups[$group['name']] = $group['id']; |
455
|
|
|
} |
456
|
|
|
} |
457
|
|
|
|
458
|
|
|
if (count($this->allGroups) === 0) { |
459
|
|
|
$this->echoOut('You have no groups in your database!'); |
460
|
|
|
|
461
|
|
|
return false; |
462
|
|
|
} |
463
|
|
|
|
464
|
|
|
return true; |
465
|
|
|
} |
466
|
|
|
|
467
|
|
|
/** |
468
|
|
|
* Echo message to browser or CLI. |
469
|
|
|
* |
470
|
|
|
* @param string $message |
471
|
|
|
*/ |
472
|
|
|
protected function echoOut($message): void |
473
|
|
|
{ |
474
|
|
|
if ($this->browser) { |
475
|
|
|
$this->retVal .= $message.'<br />'; |
476
|
|
|
} elseif ($this->echoCLI) { |
477
|
|
|
echo $message.PHP_EOL; |
478
|
|
|
} |
479
|
|
|
} |
480
|
|
|
|
481
|
|
|
/** |
482
|
|
|
* The function updates the NZB guid after there is no chance of deletion. |
483
|
|
|
*/ |
484
|
|
|
protected function updateNzbGuid(): void |
485
|
|
|
{ |
486
|
|
|
Release::query()->where('guid', $this->relGuid)->update(['nzb_guid' => sodium_hex2bin($this->nzbGuid)]); |
487
|
|
|
} |
488
|
|
|
} |
489
|
|
|
|