1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
namespace Blacklight; |
4
|
|
|
|
5
|
|
|
use App\Models\BinaryBlacklist; |
6
|
|
|
use App\Models\Collection; |
7
|
|
|
use App\Models\MissedPart; |
8
|
|
|
use App\Models\Settings; |
9
|
|
|
use App\Models\UsenetGroup; |
10
|
|
|
use Illuminate\Database\QueryException; |
11
|
|
|
use Illuminate\Support\Carbon; |
12
|
|
|
use Illuminate\Support\Facades\DB; |
13
|
|
|
use Illuminate\Support\Facades\Log; |
14
|
|
|
use Illuminate\Support\Str; |
15
|
|
|
|
16
|
|
|
/** |
17
|
|
|
* Class Binaries. |
18
|
|
|
*/ |
19
|
|
|
class Binaries |
20
|
|
|
{ |
21
|
|
|
public const OPTYPE_BLACKLIST = 1; |
22
|
|
|
|
23
|
|
|
public const OPTYPE_WHITELIST = 2; |
24
|
|
|
|
25
|
|
|
public const BLACKLIST_DISABLED = 0; |
26
|
|
|
|
27
|
|
|
public const BLACKLIST_ENABLED = 1; |
28
|
|
|
|
29
|
|
|
public const BLACKLIST_FIELD_SUBJECT = 1; |
30
|
|
|
|
31
|
|
|
public const BLACKLIST_FIELD_FROM = 2; |
32
|
|
|
|
33
|
|
|
public const BLACKLIST_FIELD_MESSAGEID = 3; |
34
|
|
|
|
35
|
|
|
public array $blackList = []; |
36
|
|
|
|
37
|
|
|
public array $whiteList = []; |
38
|
|
|
|
39
|
|
|
public int $messageBuffer; |
40
|
|
|
|
41
|
|
|
/** |
42
|
|
|
* @var \Blacklight\ColorCLI |
43
|
|
|
*/ |
44
|
|
|
protected mixed $colorCli; |
45
|
|
|
|
46
|
|
|
/** |
47
|
|
|
* @var \Blacklight\CollectionsCleaning |
48
|
|
|
*/ |
49
|
|
|
protected mixed $_collectionsCleaning; |
50
|
|
|
|
51
|
|
|
/** |
52
|
|
|
* @var \Blacklight\NNTP |
53
|
|
|
*/ |
54
|
|
|
protected mixed $_nntp; |
55
|
|
|
|
56
|
|
|
/** |
57
|
|
|
* Should we use header compression? |
58
|
|
|
*/ |
59
|
|
|
protected bool $_compressedHeaders; |
60
|
|
|
|
61
|
|
|
/** |
62
|
|
|
* Should we use part repair? |
63
|
|
|
*/ |
64
|
|
|
protected bool $_partRepair; |
65
|
|
|
|
66
|
|
|
protected \Closure|\PDO $_pdo; |
67
|
|
|
|
68
|
|
|
/** |
69
|
|
|
* How many days to go back on a new group? |
70
|
|
|
*/ |
71
|
|
|
protected bool $_newGroupScanByDays; |
72
|
|
|
|
73
|
|
|
/** |
74
|
|
|
* How many headers to download on new groups? |
75
|
|
|
*/ |
76
|
|
|
protected int $_newGroupMessagesToScan; |
77
|
|
|
|
78
|
|
|
/** |
79
|
|
|
* How many days to go back on new groups? |
80
|
|
|
*/ |
81
|
|
|
protected int $_newGroupDaysToScan; |
82
|
|
|
|
83
|
|
|
/** |
84
|
|
|
* How many headers to download per run of part repair? |
85
|
|
|
*/ |
86
|
|
|
protected int $_partRepairLimit; |
87
|
|
|
|
88
|
|
|
/** |
89
|
|
|
* Echo to cli? |
90
|
|
|
*/ |
91
|
|
|
protected bool $_echoCLI; |
92
|
|
|
|
93
|
|
|
/** |
94
|
|
|
* Max tries to download headers. |
95
|
|
|
*/ |
96
|
|
|
protected int $_partRepairMaxTries; |
97
|
|
|
|
98
|
|
|
/** |
99
|
|
|
* An array of BinaryBlacklist IDs that should have their activity date updated. |
100
|
|
|
*/ |
101
|
|
|
protected array $_binaryBlacklistIdsToUpdate = []; |
102
|
|
|
|
103
|
|
|
protected \DateTime $startCleaning; |
104
|
|
|
|
105
|
|
|
protected \DateTime $startLoop; |
106
|
|
|
|
107
|
|
|
/** |
108
|
|
|
* @var int How long it took in seconds to download headers |
109
|
|
|
*/ |
110
|
|
|
protected int $timeHeaders; |
111
|
|
|
|
112
|
|
|
/** |
113
|
|
|
* @var int How long it took in seconds to clean/parse headers |
114
|
|
|
*/ |
115
|
|
|
protected int $timeCleaning; |
116
|
|
|
|
117
|
|
|
protected \DateTime $startPR; |
118
|
|
|
|
119
|
|
|
protected \DateTime $startUpdate; |
120
|
|
|
|
121
|
|
|
/** |
122
|
|
|
* @var int The time it took to insert the headers |
123
|
|
|
*/ |
124
|
|
|
protected int $timeInsert; |
125
|
|
|
|
126
|
|
|
/** |
127
|
|
|
* @var array the header currently being scanned |
128
|
|
|
*/ |
129
|
|
|
protected array $header; |
130
|
|
|
|
131
|
|
|
/** |
132
|
|
|
* @var bool Should we add parts to part repair queue? |
133
|
|
|
*/ |
134
|
|
|
protected bool $addToPartRepair; |
135
|
|
|
|
136
|
|
|
/** |
137
|
|
|
* @var array Numbers of Headers received from the USP |
138
|
|
|
*/ |
139
|
|
|
protected array $headersReceived; |
140
|
|
|
|
141
|
|
|
/** |
142
|
|
|
* @var array The current newsgroup information being updated |
143
|
|
|
*/ |
144
|
|
|
protected array $groupMySQL; |
145
|
|
|
|
146
|
|
|
/** |
147
|
|
|
* @var int the last article number in the range |
148
|
|
|
*/ |
149
|
|
|
protected int $last; |
150
|
|
|
|
151
|
|
|
/** |
152
|
|
|
* @var int the first article number in the range |
153
|
|
|
*/ |
154
|
|
|
protected int $first; |
155
|
|
|
|
156
|
|
|
/** |
157
|
|
|
* @var int How many received headers were not yEnc encoded |
158
|
|
|
*/ |
159
|
|
|
protected int $notYEnc; |
160
|
|
|
|
161
|
|
|
/** |
162
|
|
|
* @var int How many received headers were blacklist matched |
163
|
|
|
*/ |
164
|
|
|
protected int $headersBlackListed; |
165
|
|
|
|
166
|
|
|
/** |
167
|
|
|
* @var array Header numbers that were not inserted |
168
|
|
|
*/ |
169
|
|
|
protected array $headersNotInserted; |
170
|
|
|
|
171
|
|
|
public function __construct() |
172
|
|
|
{ |
173
|
|
|
$this->startUpdate = now(); |
174
|
|
|
$this->timeCleaning = 0; |
175
|
|
|
|
176
|
|
|
$this->_echoCLI = config('nntmux.echocli'); |
177
|
|
|
|
178
|
|
|
$this->_pdo = DB::connection()->getPdo(); |
179
|
|
|
$this->colorCli = new ColorCLI(); |
180
|
|
|
$this->_nntp = new NNTP(); |
181
|
|
|
$this->_collectionsCleaning = new CollectionsCleaning(); |
182
|
|
|
|
183
|
|
|
$this->messageBuffer = Settings::settingValue('..maxmssgs') !== '' ? |
|
|
|
|
184
|
|
|
(int) Settings::settingValue('..maxmssgs') : 20000; |
185
|
|
|
$this->_compressedHeaders = (int) Settings::settingValue('..compressedheaders') === 1; |
186
|
|
|
$this->_partRepair = (int) Settings::settingValue('..partrepair') === 1; |
187
|
|
|
$this->_newGroupScanByDays = (int) Settings::settingValue('..newgroupscanmethod') === 1; |
188
|
|
|
$this->_newGroupMessagesToScan = Settings::settingValue('..newgroupmsgstoscan') !== '' ? (int) Settings::settingValue('..newgroupmsgstoscan') : 50000; |
|
|
|
|
189
|
|
|
$this->_newGroupDaysToScan = Settings::settingValue('..newgroupdaystoscan') !== '' ? (int) Settings::settingValue('..newgroupdaystoscan') : 3; |
|
|
|
|
190
|
|
|
$this->_partRepairLimit = Settings::settingValue('..maxpartrepair') !== '' ? (int) Settings::settingValue('..maxpartrepair') : 15000; |
|
|
|
|
191
|
|
|
$this->_partRepairMaxTries = (Settings::settingValue('..partrepairmaxtries') !== '' ? (int) Settings::settingValue('..partrepairmaxtries') : 3); |
|
|
|
|
192
|
|
|
|
193
|
|
|
$this->blackList = $this->whiteList = []; |
194
|
|
|
} |
195
|
|
|
|
196
|
|
|
/** |
197
|
|
|
* Download new headers for all active groups. |
198
|
|
|
* |
199
|
|
|
* @param int $maxHeaders (Optional) How many headers to download max. |
200
|
|
|
* |
201
|
|
|
* @throws \Exception |
202
|
|
|
* @throws \Throwable |
203
|
|
|
*/ |
204
|
|
|
public function updateAllGroups(int $maxHeaders = 100000): void |
205
|
|
|
{ |
206
|
|
|
$groups = UsenetGroup::getActive()->toArray(); |
207
|
|
|
|
208
|
|
|
$groupCount = \count($groups); |
209
|
|
|
if ($groupCount > 0) { |
210
|
|
|
$counter = 1; |
211
|
|
|
$allTime = now(); |
212
|
|
|
|
213
|
|
|
$this->log( |
214
|
|
|
'Updating: '.$groupCount.' group(s) - Using compression? '.($this->_compressedHeaders ? 'Yes' : 'No'), |
215
|
|
|
__FUNCTION__, |
216
|
|
|
'header' |
217
|
|
|
); |
218
|
|
|
|
219
|
|
|
// Loop through groups. |
220
|
|
|
foreach ($groups as $group) { |
221
|
|
|
$this->log( |
222
|
|
|
'Starting group '.$counter.' of '.$groupCount, |
223
|
|
|
__FUNCTION__, |
224
|
|
|
'header' |
225
|
|
|
); |
226
|
|
|
$this->updateGroup($group, $maxHeaders); |
227
|
|
|
$counter++; |
228
|
|
|
} |
229
|
|
|
|
230
|
|
|
$endTime = now()->diffInSeconds($allTime); |
231
|
|
|
$this->log( |
232
|
|
|
'Updating completed in '.$endTime.Str::plural(' second', $endTime), |
233
|
|
|
__FUNCTION__, |
234
|
|
|
'primary' |
235
|
|
|
); |
236
|
|
|
} else { |
237
|
|
|
$this->log( |
238
|
|
|
'No groups specified. Ensure groups are added to NNTmux\'s database for updating.', |
239
|
|
|
__FUNCTION__, |
240
|
|
|
'warning' |
241
|
|
|
); |
242
|
|
|
} |
243
|
|
|
} |
244
|
|
|
|
245
|
|
|
/** |
246
|
|
|
* When the indexer is started, log the date/time. |
247
|
|
|
*/ |
248
|
|
|
public function logIndexerStart(): void |
249
|
|
|
{ |
250
|
|
|
Settings::query()->where('setting', '=', 'last_run_time')->update(['value' => now()]); |
251
|
|
|
} |
252
|
|
|
|
253
|
|
|
/** |
254
|
|
|
* Download new headers for a single group. |
255
|
|
|
* |
256
|
|
|
* @param array $groupMySQL Array of MySQL results for a single group. |
257
|
|
|
* @param int $maxHeaders (Optional) How many headers to download max. |
258
|
|
|
* |
259
|
|
|
* @throws \Exception |
260
|
|
|
* @throws \Throwable |
261
|
|
|
*/ |
262
|
|
|
public function updateGroup(array $groupMySQL, int $maxHeaders = 0): void |
263
|
|
|
{ |
264
|
|
|
$startGroup = now(); |
265
|
|
|
$this->logIndexerStart(); |
266
|
|
|
|
267
|
|
|
// Select the group on the NNTP server, gets the latest info on it. |
268
|
|
|
$groupNNTP = $this->_nntp->selectGroup($groupMySQL['name']); |
269
|
|
|
if ($this->_nntp::isError($groupNNTP)) { |
270
|
|
|
$groupNNTP = $this->_nntp->dataError($this->_nntp, $groupMySQL['name']); |
271
|
|
|
|
272
|
|
|
if (isset($groupNNTP['code']) && (int) $groupNNTP['code'] === 411) { |
273
|
|
|
UsenetGroup::disableIfNotExist($groupMySQL['id']); |
274
|
|
|
} |
275
|
|
|
if ($this->_nntp::isError($groupNNTP)) { |
276
|
|
|
return; |
277
|
|
|
} |
278
|
|
|
} |
279
|
|
|
|
280
|
|
|
if ($this->_echoCLI) { |
281
|
|
|
$this->colorCli->primary('Processing '.$groupMySQL['name']); |
282
|
|
|
} |
283
|
|
|
|
284
|
|
|
// Attempt to repair any missing parts before grabbing new ones. |
285
|
|
|
if ((int) $groupMySQL['last_record'] !== 0) { |
286
|
|
|
if ($this->_partRepair) { |
287
|
|
|
if ($this->_echoCLI) { |
288
|
|
|
$this->colorCli->primary('Part repair enabled. Checking for missing parts.'); |
289
|
|
|
} |
290
|
|
|
$this->partRepair($groupMySQL); |
291
|
|
|
} elseif ($this->_echoCLI) { |
292
|
|
|
$this->colorCli->primary('Part repair disabled by user.'); |
293
|
|
|
} |
294
|
|
|
} |
295
|
|
|
|
296
|
|
|
// Generate postdate for first record, for those that upgraded. |
297
|
|
|
if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] !== 0) { |
298
|
|
|
$groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); |
299
|
|
|
UsenetGroup::query()->where('id', $groupMySQL['id'])->update(['first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'])]); |
300
|
|
|
} |
301
|
|
|
|
302
|
|
|
// Get first article we want aka the oldest. |
303
|
|
|
if ((int) $groupMySQL['last_record'] === 0) { |
304
|
|
|
if ($this->_newGroupScanByDays) { |
305
|
|
|
// For new newsgroups - determine here how far we want to go back using date. |
306
|
|
|
$first = $this->daytopost($this->_newGroupDaysToScan, $groupNNTP); |
307
|
|
|
} elseif ($groupNNTP['first'] >= ($groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer))) { |
308
|
|
|
// If what we want is lower than the groups first article, set the wanted first to the first. |
309
|
|
|
$first = $groupNNTP['first']; |
310
|
|
|
} else { |
311
|
|
|
// Or else, use the newest article minus how much we should get for new groups. |
312
|
|
|
$first = (string) ($groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer)); |
313
|
|
|
} |
314
|
|
|
|
315
|
|
|
// We will use this to subtract so we leave articles for the next time (in case the server doesn't have them yet) |
316
|
|
|
$leaveOver = $this->messageBuffer; |
317
|
|
|
|
318
|
|
|
// If this is not a new group, go from our newest to the servers newest. |
319
|
|
|
} else { |
320
|
|
|
// Set our oldest wanted to our newest local article. |
321
|
|
|
$first = $groupMySQL['last_record']; |
322
|
|
|
|
323
|
|
|
// This is how many articles we will grab. (the servers newest minus our newest). |
324
|
|
|
$totalCount = (string) ($groupNNTP['last'] - $first); |
325
|
|
|
|
326
|
|
|
// Check if the server has more articles than our loop limit x 2. |
327
|
|
|
if ($totalCount > ($this->messageBuffer * 2)) { |
328
|
|
|
// Get the remainder of $totalCount / $this->message buffer |
329
|
|
|
$leaveOver = round($totalCount % $this->messageBuffer, 0, PHP_ROUND_HALF_DOWN) + $this->messageBuffer; |
330
|
|
|
} else { |
331
|
|
|
// Else get half of the available. |
332
|
|
|
$leaveOver = round($totalCount / 2, 0, PHP_ROUND_HALF_DOWN); |
333
|
|
|
} |
334
|
|
|
} |
335
|
|
|
|
336
|
|
|
// The last article we want, aka the newest. |
337
|
|
|
$last = $groupLast = (string) ($groupNNTP['last'] - $leaveOver); |
338
|
|
|
|
339
|
|
|
// If the newest we want is older than the oldest we want somehow.. set them equal. |
340
|
|
|
if ($last < $first) { |
341
|
|
|
$last = $groupLast = $first; |
342
|
|
|
} |
343
|
|
|
|
344
|
|
|
// This is how many articles we are going to get. |
345
|
|
|
$total = (string) ($groupLast - $first); |
346
|
|
|
// This is how many articles are available (without $leaveOver). |
347
|
|
|
$realTotal = (string) ($groupNNTP['last'] - $first); |
348
|
|
|
|
349
|
|
|
// Check if we should limit the amount of fetched new headers. |
350
|
|
|
if ($maxHeaders > 0) { |
351
|
|
|
if ($maxHeaders < ($groupLast - $first)) { |
352
|
|
|
$groupLast = $last = (string) ($first + $maxHeaders); |
353
|
|
|
} |
354
|
|
|
$total = (string) ($groupLast - $first); |
355
|
|
|
} |
356
|
|
|
|
357
|
|
|
// If total is bigger than 0 it means we have new parts in the newsgroup. |
358
|
|
|
if ($total > 0) { |
359
|
|
|
if ($this->_echoCLI) { |
360
|
|
|
$this->colorCli->primary( |
361
|
|
|
( |
362
|
|
|
(int) $groupMySQL['last_record'] === 0 |
363
|
|
|
? 'New group '.$groupNNTP['group'].' starting with '. |
364
|
|
|
( |
365
|
|
|
$this->_newGroupScanByDays |
366
|
|
|
? $this->_newGroupDaysToScan.' days' |
367
|
|
|
: number_format($this->_newGroupMessagesToScan).' messages' |
368
|
|
|
).' worth.' |
369
|
|
|
: 'Group '.$groupNNTP['group'].' has '.number_format($realTotal).' new articles.' |
|
|
|
|
370
|
|
|
). |
371
|
|
|
' Leaving '.number_format($leaveOver). |
372
|
|
|
" for next pass.\nServer oldest: ".number_format($groupNNTP['first']). |
373
|
|
|
' Server newest: '.number_format($groupNNTP['last']). |
374
|
|
|
' Local newest: '.number_format($groupMySQL['last_record']) |
375
|
|
|
); |
376
|
|
|
} |
377
|
|
|
|
378
|
|
|
$done = false; |
379
|
|
|
// Get all the parts (in portions of $this->messageBuffer to not use too much memory). |
380
|
|
|
while (! $done) { |
381
|
|
|
// Increment last until we reach $groupLast (group newest article). |
382
|
|
|
if ($total > $this->messageBuffer) { |
383
|
|
|
if ((string) ($first + $this->messageBuffer) > $groupLast) { |
384
|
|
|
$last = $groupLast; |
385
|
|
|
} else { |
386
|
|
|
$last = (string) ($first + $this->messageBuffer); |
387
|
|
|
} |
388
|
|
|
} |
389
|
|
|
// Increment first so we don't get an article we already had. |
390
|
|
|
$first++; |
391
|
|
|
|
392
|
|
|
if ($this->_echoCLI) { |
393
|
|
|
$this->colorCli->header( |
394
|
|
|
PHP_EOL.'Getting '.number_format($last - $first + 1).' articles ('.number_format($first). |
395
|
|
|
' to '.number_format($last).') from '.$groupMySQL['name'].' - ('. |
396
|
|
|
number_format($groupLast - $last).' articles in queue).' |
397
|
|
|
); |
398
|
|
|
} |
399
|
|
|
|
400
|
|
|
// Get article headers from newsgroup. |
401
|
|
|
$scanSummary = $this->scan($groupMySQL, $first, $last); |
402
|
|
|
|
403
|
|
|
// Check if we fetched headers. |
404
|
|
|
if (! empty($scanSummary)) { |
405
|
|
|
// If new group, update first record & postdate |
406
|
|
|
if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] === 0) { |
407
|
|
|
$groupMySQL['first_record'] = $scanSummary['firstArticleNumber']; |
408
|
|
|
|
409
|
|
|
if (isset($scanSummary['firstArticleDate'])) { |
410
|
|
|
$groupMySQL['first_record_postdate'] = strtotime($scanSummary['firstArticleDate']); |
411
|
|
|
} else { |
412
|
|
|
$groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); |
413
|
|
|
} |
414
|
|
|
|
415
|
|
|
UsenetGroup::query() |
416
|
|
|
->where('id', $groupMySQL['id']) |
417
|
|
|
->update( |
418
|
|
|
[ |
419
|
|
|
'first_record' => $scanSummary['firstArticleNumber'], |
420
|
|
|
'first_record_postdate' => Carbon::createFromTimestamp( |
421
|
|
|
$groupMySQL['first_record_postdate'] |
422
|
|
|
), |
423
|
|
|
] |
424
|
|
|
); |
425
|
|
|
} |
426
|
|
|
|
427
|
|
|
$scanSummary['lastArticleDate'] = (isset($scanSummary['lastArticleDate']) ? strtotime($scanSummary['lastArticleDate']) : false); |
428
|
|
|
if (! is_numeric($scanSummary['lastArticleDate'])) { |
429
|
|
|
$scanSummary['lastArticleDate'] = $this->postdate($scanSummary['lastArticleNumber'], $groupNNTP); |
430
|
|
|
} |
431
|
|
|
|
432
|
|
|
UsenetGroup::query() |
433
|
|
|
->where('id', $groupMySQL['id']) |
434
|
|
|
->update( |
435
|
|
|
[ |
436
|
|
|
'last_record' => $scanSummary['lastArticleNumber'], |
437
|
|
|
'last_record_postdate' => Carbon::createFromTimestamp($scanSummary['lastArticleDate']), |
438
|
|
|
'last_updated' => now(), |
439
|
|
|
] |
440
|
|
|
); |
441
|
|
|
} else { |
442
|
|
|
// If we didn't fetch headers, update the record still. |
443
|
|
|
UsenetGroup::query() |
444
|
|
|
->where('id', $groupMySQL['id']) |
445
|
|
|
->update( |
446
|
|
|
[ |
447
|
|
|
'last_record' => $last, |
448
|
|
|
'last_updated' => now(), |
449
|
|
|
] |
450
|
|
|
); |
451
|
|
|
} |
452
|
|
|
|
453
|
|
|
if ((int) $last === (int) $groupLast) { |
454
|
|
|
$done = true; |
455
|
|
|
} else { |
456
|
|
|
$first = $last; |
457
|
|
|
} |
458
|
|
|
} |
459
|
|
|
|
460
|
|
|
if ($this->_echoCLI) { |
461
|
|
|
$endGroup = now()->diffInSeconds($startGroup); |
462
|
|
|
$this->colorCli->primary( |
463
|
|
|
PHP_EOL.'Group '.$groupMySQL['name'].' processed in '. |
464
|
|
|
$endGroup.Str::plural(' second', $endGroup) |
465
|
|
|
); |
466
|
|
|
} |
467
|
|
|
} elseif ($this->_echoCLI) { |
468
|
|
|
$this->colorCli->primary( |
469
|
|
|
'No new articles for '.$groupMySQL['name'].' (first '.number_format($first). |
470
|
|
|
', last '.number_format($last).', grouplast '.number_format($groupMySQL['last_record']). |
471
|
|
|
', total '.number_format($total).")\n".'Server oldest: '.number_format($groupNNTP['first']). |
472
|
|
|
' Server newest: '.number_format($groupNNTP['last']).' Local newest: '.number_format($groupMySQL['last_record']) |
473
|
|
|
); |
474
|
|
|
} |
475
|
|
|
} |
476
|
|
|
|
477
|
|
|
/** |
478
|
|
|
* Loop over range of wanted headers, insert headers into DB. |
479
|
|
|
* |
480
|
|
|
* @param array $groupMySQL The group info from mysql. |
481
|
|
|
* @param int $first The oldest wanted header. |
482
|
|
|
* @param int $last The newest wanted header. |
483
|
|
|
* @param string $type Is this part repair or update or backfill? |
484
|
|
|
* @param array|null $missingParts If we are running in part repair, the list of missing article numbers. |
485
|
|
|
* @return array Empty on failure. |
486
|
|
|
* |
487
|
|
|
* @throws \Exception |
488
|
|
|
* @throws \Throwable |
489
|
|
|
*/ |
490
|
|
|
public function scan(array $groupMySQL, int $first, int $last, string $type = 'update', array $missingParts = null): array |
491
|
|
|
{ |
492
|
|
|
// Start time of scan method and of fetching headers. |
493
|
|
|
$this->startLoop = now(); |
494
|
|
|
$this->groupMySQL = $groupMySQL; |
495
|
|
|
$this->last = $last; |
496
|
|
|
$this->first = $first; |
497
|
|
|
|
498
|
|
|
$this->notYEnc = $this->headersBlackListed = 0; |
499
|
|
|
|
500
|
|
|
$returnArray = $stdHeaders = []; |
501
|
|
|
|
502
|
|
|
$partRepair = ($type === 'partrepair'); |
503
|
|
|
$this->addToPartRepair = ($type === 'update' && $this->_partRepair); |
504
|
|
|
|
505
|
|
|
// Download the headers. |
506
|
|
|
if ($partRepair) { |
507
|
|
|
// This is slower but possibly is better with missing headers. |
508
|
|
|
$headers = $this->_nntp->getOverview($this->first.'-'.$this->last, true, false); |
509
|
|
|
} else { |
510
|
|
|
$headers = $this->_nntp->getXOVER($this->first.'-'.$this->last); |
511
|
|
|
} |
512
|
|
|
|
513
|
|
|
// If there was an error, try to reconnect. |
514
|
|
|
if ($this->_nntp::isError($headers)) { |
515
|
|
|
// Increment if part repair and return false. |
516
|
|
|
if ($partRepair) { |
517
|
|
|
MissedPart::query()->where('groups_id', $this->groupMySQL['id'])->where('numberid', ((int) $this->first === (int) $this->last ? '= '.$this->first : 'IN ('.implode(',', range($this->first, $this->last)).')'))->increment('attempts', 1); |
518
|
|
|
|
519
|
|
|
return $returnArray; |
520
|
|
|
} |
521
|
|
|
|
522
|
|
|
// This is usually a compression error, so try disabling compression. |
523
|
|
|
$this->_nntp->doQuit(); |
524
|
|
|
if ($this->_nntp->doConnect(false) !== true) { |
525
|
|
|
return $returnArray; |
526
|
|
|
} |
527
|
|
|
|
528
|
|
|
// Re-select group, download headers again without compression and re-enable compression. |
529
|
|
|
$this->_nntp->selectGroup($this->groupMySQL['name']); |
530
|
|
|
$headers = $this->_nntp->getXOVER($this->first.'-'.$this->last); |
531
|
|
|
$this->_nntp->enableCompression(); |
532
|
|
|
|
533
|
|
|
// Check if the non-compression headers have an error. |
534
|
|
|
if ($this->_nntp::isError($headers)) { |
535
|
|
|
$message = ((int) $headers->code === 0 ? 'Unknown error' : $headers->message); |
536
|
|
|
$this->log( |
537
|
|
|
"Code {$headers->code}: $message\nSkipping group: {$this->groupMySQL['name']}", |
538
|
|
|
__FUNCTION__, |
539
|
|
|
'error' |
540
|
|
|
); |
541
|
|
|
|
542
|
|
|
return $returnArray; |
543
|
|
|
} |
544
|
|
|
} |
545
|
|
|
|
546
|
|
|
// Start of processing headers. |
547
|
|
|
$this->startCleaning = now(); |
548
|
|
|
|
549
|
|
|
// End of the getting data from usenet. |
550
|
|
|
$this->timeHeaders = $this->startCleaning->diffInSeconds($this->startLoop); |
|
|
|
|
551
|
|
|
|
552
|
|
|
// Check if we got headers. |
553
|
|
|
$msgCount = \count($headers); |
554
|
|
|
|
555
|
|
|
if ($msgCount < 1) { |
556
|
|
|
return $returnArray; |
557
|
|
|
} |
558
|
|
|
|
559
|
|
|
$this->getHighLowArticleInfo($returnArray, $headers, $msgCount); |
560
|
|
|
|
561
|
|
|
$headersRepaired = $rangeNotReceived = $this->headersReceived = $this->headersNotInserted = []; |
562
|
|
|
|
563
|
|
|
foreach ($headers as $header) { |
564
|
|
|
// Check if we got the article or not. |
565
|
|
|
if (isset($header['Number'])) { |
566
|
|
|
$this->headersReceived[] = $header['Number']; |
567
|
|
|
} else { |
568
|
|
|
if ($this->addToPartRepair) { |
569
|
|
|
$rangeNotReceived[] = $header['Number']; |
570
|
|
|
} |
571
|
|
|
|
572
|
|
|
continue; |
573
|
|
|
} |
574
|
|
|
|
575
|
|
|
// If set we are running in partRepair mode. |
576
|
|
|
if ($partRepair && $missingParts !== null) { |
577
|
|
|
if (! \in_array($header['Number'], $missingParts, false)) { |
578
|
|
|
// If article isn't one that is missing skip it. |
579
|
|
|
continue; |
580
|
|
|
} |
581
|
|
|
// We got the part this time. Remove article from part repair. |
582
|
|
|
$headersRepaired[] = $header['Number']; |
583
|
|
|
} |
584
|
|
|
|
585
|
|
|
/* |
586
|
|
|
* Find part / total parts. Ignore if no part count found. |
587
|
|
|
* |
588
|
|
|
* \s* Trims the leading space. |
589
|
|
|
* (?!"Usenet Index Post) ignores these types of articles, they are useless. |
590
|
|
|
* (.+) Fetches the subject. |
591
|
|
|
* \s+ Trims trailing space after the subject. |
592
|
|
|
* \((\d+)\/(\d+)\) Gets the part count. |
593
|
|
|
* No ending ($) as there are cases of subjects with extra data after the part count. |
594
|
|
|
*/ |
595
|
|
|
if (preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $header['matches'])) { |
596
|
|
|
// Add yEnc to subjects that do not have them, but have the part number at the end of the header. |
597
|
|
|
if (stripos($header['Subject'], 'yEnc') === false) { |
598
|
|
|
$header['matches'][1] .= ' yEnc'; |
599
|
|
|
} |
600
|
|
|
} else { |
601
|
|
|
$this->notYEnc++; |
602
|
|
|
|
603
|
|
|
continue; |
604
|
|
|
} |
605
|
|
|
|
606
|
|
|
// Filter subject based on black/white list. |
607
|
|
|
if ($this->isBlackListed($header, $this->groupMySQL['name'])) { |
608
|
|
|
$this->headersBlackListed++; |
609
|
|
|
|
610
|
|
|
continue; |
611
|
|
|
} |
612
|
|
|
|
613
|
|
|
if (empty($header['Bytes'])) { |
614
|
|
|
$header['Bytes'] = (isset($this->header[':bytes']) ? $header[':bytes'] : 0); |
615
|
|
|
} |
616
|
|
|
|
617
|
|
|
$stdHeaders[] = $header; |
618
|
|
|
} |
619
|
|
|
|
620
|
|
|
unset($headers); // Reclaim memory now that headers are split. |
621
|
|
|
|
622
|
|
|
if (! empty($this->_binaryBlacklistIdsToUpdate)) { |
623
|
|
|
$this->updateBlacklistUsage(); |
624
|
|
|
} |
625
|
|
|
|
626
|
|
|
if ($this->_echoCLI && ! $partRepair) { |
627
|
|
|
$this->outputHeaderInitial(); |
628
|
|
|
} |
629
|
|
|
|
630
|
|
|
if (! empty($stdHeaders)) { |
631
|
|
|
$this->storeHeaders($stdHeaders); |
632
|
|
|
} |
633
|
|
|
unset($stdHeaders); |
634
|
|
|
|
635
|
|
|
// Start of part repair. |
636
|
|
|
$this->startPR = now(); |
637
|
|
|
|
638
|
|
|
// End of inserting. |
639
|
|
|
$this->timeInsert = $this->startPR->diffInSeconds($this->startUpdate); |
640
|
|
|
|
641
|
|
|
if ($partRepair && \count($headersRepaired) > 0) { |
642
|
|
|
$this->removeRepairedParts($headersRepaired, $this->groupMySQL['id']); |
643
|
|
|
} |
644
|
|
|
unset($headersRepaired); |
645
|
|
|
|
646
|
|
|
if ($this->addToPartRepair) { |
647
|
|
|
$notInsertedCount = \count($this->headersNotInserted); |
648
|
|
|
if ($notInsertedCount > 0) { |
649
|
|
|
$this->addMissingParts($this->headersNotInserted, $this->groupMySQL['id']); |
650
|
|
|
|
651
|
|
|
$this->log( |
652
|
|
|
$notInsertedCount.' articles failed to insert!', |
653
|
|
|
__FUNCTION__, |
654
|
|
|
'warning' |
655
|
|
|
); |
656
|
|
|
|
657
|
|
|
if (config('app.debug') === true) { |
658
|
|
|
Log::warning($notInsertedCount.' articles failed to insert!'); |
659
|
|
|
} |
660
|
|
|
} |
661
|
|
|
unset($this->headersNotInserted); |
662
|
|
|
|
663
|
|
|
// Check if we have any missing headers. |
664
|
|
|
if (($this->last - $this->first - $this->notYEnc - $this->headersBlackListed + 1) > \count($this->headersReceived)) { |
665
|
|
|
$rangeNotReceived = array_merge($rangeNotReceived, array_diff(range($this->first, $this->last), $this->headersReceived)); |
666
|
|
|
} |
667
|
|
|
$notReceivedCount = \count($rangeNotReceived); |
668
|
|
|
if ($notReceivedCount > 0) { |
669
|
|
|
$this->addMissingParts($rangeNotReceived, $this->groupMySQL['id']); |
670
|
|
|
|
671
|
|
|
if ($this->_echoCLI) { |
672
|
|
|
$this->colorCli->alternate( |
673
|
|
|
'Server did not return '.$notReceivedCount. |
674
|
|
|
' articles from '.$this->groupMySQL['name'].'.' |
675
|
|
|
); |
676
|
|
|
} |
677
|
|
|
} |
678
|
|
|
unset($rangeNotReceived); |
679
|
|
|
} |
680
|
|
|
|
681
|
|
|
$this->outputHeaderDuration(); |
682
|
|
|
|
683
|
|
|
return $returnArray; |
684
|
|
|
} |
685
|
|
|
|
686
|
|
|
/** |
687
|
|
|
* Parse headers into collections/binaries and store header data as parts. |
688
|
|
|
* |
689
|
|
|
* |
690
|
|
|
* |
691
|
|
|
* @throws \Exception |
692
|
|
|
* @throws \Throwable |
693
|
|
|
*/ |
694
|
|
|
protected function storeHeaders(array $headers = []): void |
695
|
|
|
{ |
696
|
|
|
$binariesUpdate = $collectionIDs = $articles = []; |
697
|
|
|
|
698
|
|
|
DB::beginTransaction(); |
699
|
|
|
|
700
|
|
|
$partsQuery = $partsCheck = 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '; |
701
|
|
|
|
702
|
|
|
// Loop articles, figure out files/parts. |
703
|
|
|
foreach ($headers as $this->header) { |
704
|
|
|
// Set up the info for inserting into parts/binaries/collections tables. |
705
|
|
|
if (! isset($articles[$this->header['matches'][1]])) { |
706
|
|
|
// Attempt to find the file count. If it is not found, set it to 0. |
707
|
|
|
$fileCount = $this->getFileCount($this->header['matches'][1]); |
708
|
|
|
if ($fileCount[1] === 0 && $fileCount[3] === 0) { |
709
|
|
|
$fileCount = $this->getFileCount($this->header['matches'][0]); |
710
|
|
|
} |
711
|
|
|
|
712
|
|
|
$collMatch = $this->_collectionsCleaning->collectionsCleaner( |
713
|
|
|
$this->header['matches'][1], $this->groupMySQL['name'] |
714
|
|
|
); |
715
|
|
|
|
716
|
|
|
// Used to group articles together when forming the release. |
717
|
|
|
$this->header['CollectionKey'] = $collMatch['name'].$fileCount[3]; |
718
|
|
|
|
719
|
|
|
// If this header's collection key isn't in memory, attempt to insert the collection |
720
|
|
|
if (! isset($collectionIDs[$this->header['CollectionKey']])) { |
721
|
|
|
/* Date from header should be a string this format: |
722
|
|
|
* 31 Mar 2014 15:36:04 GMT or 6 Oct 1998 04:38:40 -0500 |
723
|
|
|
* Still make sure it's not unix time, convert it to unix time if it is. |
724
|
|
|
*/ |
725
|
|
|
$this->header['Date'] = (is_numeric($this->header['Date']) ? $this->header['Date'] : strtotime($this->header['Date'])); |
726
|
|
|
|
727
|
|
|
// Get the current unixtime from PHP. |
728
|
|
|
$now = now()->timestamp; |
729
|
|
|
|
730
|
|
|
$xrefsData = Collection::whereCollectionhash(sha1($this->header['CollectionKey']))->value('xref'); |
731
|
|
|
|
732
|
|
|
$tempHeaderXrefs = []; |
733
|
|
|
foreach (explode(' ', $this->header['Xref']) as $headerXref) { |
734
|
|
|
if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $headerXref, $hit) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $headerXref, $hit)) { |
735
|
|
|
$tempHeaderXrefs[] = $hit[0]; |
736
|
|
|
} |
737
|
|
|
} |
738
|
|
|
|
739
|
|
|
$tempXrefsData = []; |
740
|
|
|
|
741
|
|
|
if ($xrefsData !== null) { |
742
|
|
|
foreach (explode(' ', $xrefsData) as $xrefData) { |
743
|
|
|
if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $xrefData, $match1) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $xrefData, $match1)) { |
744
|
|
|
$tempXrefsData[] = $match1[0]; |
745
|
|
|
} |
746
|
|
|
} |
747
|
|
|
} |
748
|
|
|
|
749
|
|
|
$finalXrefArray = []; |
750
|
|
|
foreach ($tempHeaderXrefs as $tempHeaderXref) { |
751
|
|
|
if (! in_array($tempHeaderXref, $tempXrefsData, false)) { |
752
|
|
|
$finalXrefArray[] = $tempHeaderXref; |
753
|
|
|
} |
754
|
|
|
} |
755
|
|
|
|
756
|
|
|
$finaXref = implode(' ', $finalXrefArray); |
757
|
|
|
|
758
|
|
|
$xref = sprintf('xref = CONCAT(xref, "\\n"%s ),', escapeString($finaXref)); |
759
|
|
|
|
760
|
|
|
$date = $this->header['Date'] > $now ? $now : $this->header['Date']; |
761
|
|
|
$unixtime = is_numeric($this->header['Date']) ? $date : $now; |
762
|
|
|
|
763
|
|
|
$random = random_bytes(16); |
764
|
|
|
$number = random_int(3, 9999999); |
765
|
|
|
$special = Str::random(5); |
766
|
|
|
$string = htmlspecialchars(str_shuffle('trusin_ @'.$number.$special), ENT_QUOTES); |
|
|
|
|
767
|
|
|
|
768
|
|
|
$collectionID = false; |
769
|
|
|
|
770
|
|
|
try { |
771
|
|
|
DB::insert(sprintf(" |
772
|
|
|
INSERT INTO collections (subject, fromname, date, xref, groups_id, |
773
|
|
|
totalfiles, collectionhash, collection_regexes_id, dateadded) |
774
|
|
|
VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', %d, NOW()) |
775
|
|
|
ON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'", escapeString(substr(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings()), 0, 255)), escapeString(mb_convert_encoding($this->header['From'], 'UTF-8', mb_list_encodings())), $unixtime, escapeString(implode(' ', $tempHeaderXrefs)), $this->groupMySQL['id'], $fileCount[3], sha1($this->header['CollectionKey']), $collMatch['id'], $xref, sodium_bin2hex($random))); |
|
|
|
|
776
|
|
|
$collectionID = $this->_pdo->lastInsertId(); |
|
|
|
|
777
|
|
|
DB::commit(); |
778
|
|
|
} catch (\Throwable $e) { |
779
|
|
|
if (config('app.debug') === true) { |
780
|
|
|
Log::error($e->getMessage()); |
781
|
|
|
} |
782
|
|
|
DB::rollBack(); |
783
|
|
|
} |
784
|
|
|
|
785
|
|
|
if ($collectionID === false) { |
786
|
|
|
if ($this->addToPartRepair) { |
787
|
|
|
$this->headersNotInserted[] = $this->header['Number']; |
788
|
|
|
} |
789
|
|
|
DB::rollBack(); |
790
|
|
|
DB::beginTransaction(); |
791
|
|
|
|
792
|
|
|
continue; |
793
|
|
|
} |
794
|
|
|
$collectionIDs[$this->header['CollectionKey']] = $collectionID; |
795
|
|
|
} else { |
796
|
|
|
$collectionID = $collectionIDs[$this->header['CollectionKey']]; |
797
|
|
|
} |
798
|
|
|
|
799
|
|
|
// Binary Hash should be unique to the group |
800
|
|
|
$hash = md5($this->header['matches'][1].$this->header['From'].$this->groupMySQL['id']); |
801
|
|
|
|
802
|
|
|
$binaryID = false; |
803
|
|
|
|
804
|
|
|
try { |
805
|
|
|
DB::insert(sprintf(" |
806
|
|
|
INSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) |
807
|
|
|
VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d) |
808
|
|
|
ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $hash, escapeString(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings())), $collectionID, $this->header['matches'][3], $fileCount[1], $this->header['Bytes'], $this->header['Bytes'])); |
809
|
|
|
$binaryID = $this->_pdo->lastInsertId(); |
810
|
|
|
DB::commit(); |
811
|
|
|
} catch (\Throwable $e) { |
812
|
|
|
if (config('app.debug') === true) { |
813
|
|
|
Log::error($e->getMessage()); |
814
|
|
|
} |
815
|
|
|
DB::rollBack(); |
816
|
|
|
} |
817
|
|
|
|
818
|
|
|
if ($binaryID === false) { |
819
|
|
|
if ($this->addToPartRepair) { |
820
|
|
|
$this->headersNotInserted[] = $this->header['Number']; |
821
|
|
|
} |
822
|
|
|
DB::rollBack(); |
823
|
|
|
DB::beginTransaction(); |
824
|
|
|
|
825
|
|
|
continue; |
826
|
|
|
} |
827
|
|
|
|
828
|
|
|
$binariesUpdate[$binaryID]['Size'] = 0; |
829
|
|
|
$binariesUpdate[$binaryID]['Parts'] = 0; |
830
|
|
|
|
831
|
|
|
$articles[$this->header['matches'][1]]['CollectionID'] = $collectionID; |
832
|
|
|
$articles[$this->header['matches'][1]]['BinaryID'] = $binaryID; |
833
|
|
|
} else { |
834
|
|
|
$binaryID = $articles[$this->header['matches'][1]]['BinaryID']; |
835
|
|
|
$binariesUpdate[$binaryID]['Size'] += $this->header['Bytes']; |
836
|
|
|
$binariesUpdate[$binaryID]['Parts']++; |
837
|
|
|
} |
838
|
|
|
|
839
|
|
|
// In case there are quotes in the message id |
840
|
|
|
$this->header['Message-ID'] = addslashes($this->header['Message-ID']); |
841
|
|
|
|
842
|
|
|
// Strip the < and >, saves space in DB. |
843
|
|
|
$this->header['Message-ID'][0] = "'"; |
844
|
|
|
|
845
|
|
|
$partsQuery .= |
846
|
|
|
'('.$binaryID.','.$this->header['Number'].','.rtrim($this->header['Message-ID'], '>')."',". |
847
|
|
|
$this->header['matches'][2].','.$this->header['Bytes'].'),'; |
848
|
|
|
} |
849
|
|
|
|
850
|
|
|
unset($headers); // Reclaim memory. |
851
|
|
|
|
852
|
|
|
// Start of inserting into SQL. |
853
|
|
|
$this->startUpdate = now(); |
854
|
|
|
|
855
|
|
|
// End of processing headers. |
856
|
|
|
$this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning); |
857
|
|
|
$binariesQuery = $binariesCheck = 'INSERT INTO binaries (id, partsize, currentparts) VALUES '; |
858
|
|
|
foreach ($binariesUpdate as $binaryID => $binary) { |
859
|
|
|
$binariesQuery .= '('.$binaryID.','.$binary['Size'].','.$binary['Parts'].'),'; |
860
|
|
|
} |
861
|
|
|
$binariesEnd = ' ON DUPLICATE KEY UPDATE partsize = VALUES(partsize) + partsize, currentparts = VALUES(currentparts) + currentparts'; |
862
|
|
|
$binariesQuery = rtrim($binariesQuery, ',').$binariesEnd; |
863
|
|
|
|
864
|
|
|
// Check if we got any binaries. If we did, try to insert them. |
865
|
|
|
if (\strlen($binariesCheck.$binariesEnd) === \strlen($binariesQuery) || $this->runQuery($binariesQuery)) { |
866
|
|
|
if (\strlen($partsQuery) === \strlen($partsCheck) || $this->runQuery(rtrim($partsQuery, ','))) { |
867
|
|
|
DB::commit(); |
868
|
|
|
} else { |
869
|
|
|
if ($this->addToPartRepair) { |
870
|
|
|
$this->headersNotInserted += $this->headersReceived; |
871
|
|
|
} |
872
|
|
|
DB::rollBack(); |
873
|
|
|
} |
874
|
|
|
} else { |
875
|
|
|
if ($this->addToPartRepair) { |
876
|
|
|
$this->headersNotInserted += $this->headersReceived; |
877
|
|
|
} |
878
|
|
|
DB::rollBack(); |
879
|
|
|
} |
880
|
|
|
} |
881
|
|
|
|
882
|
|
|
/** |
883
|
|
|
* Gets the First and Last Article Number and Date for the received headers. |
884
|
|
|
*/ |
885
|
|
|
protected function getHighLowArticleInfo(array &$returnArray, array $headers, int $msgCount): void |
886
|
|
|
{ |
887
|
|
|
// Get highest and lowest article numbers/dates. |
888
|
|
|
$iterator1 = 0; |
889
|
|
|
$iterator2 = $msgCount - 1; |
890
|
|
|
while (true) { |
891
|
|
|
if (! isset($returnArray['firstArticleNumber']) && isset($headers[$iterator1]['Number'])) { |
892
|
|
|
$returnArray['firstArticleNumber'] = $headers[$iterator1]['Number']; |
893
|
|
|
$returnArray['firstArticleDate'] = $headers[$iterator1]['Date']; |
894
|
|
|
} |
895
|
|
|
|
896
|
|
|
if (! isset($returnArray['lastArticleNumber']) && isset($headers[$iterator2]['Number'])) { |
897
|
|
|
$returnArray['lastArticleNumber'] = $headers[$iterator2]['Number']; |
898
|
|
|
$returnArray['lastArticleDate'] = $headers[$iterator2]['Date']; |
899
|
|
|
} |
900
|
|
|
|
901
|
|
|
// Break if we found non empty articles. |
902
|
|
|
if (isset($returnArray['firstArticleNumber, lastArticleNumber'])) { |
903
|
|
|
break; |
904
|
|
|
} |
905
|
|
|
|
906
|
|
|
// Break out if we couldn't find anything. |
907
|
|
|
if ($iterator1++ >= $msgCount - 1 || $iterator2-- <= 0) { |
908
|
|
|
break; |
909
|
|
|
} |
910
|
|
|
} |
911
|
|
|
} |
912
|
|
|
|
913
|
|
|
/** |
914
|
|
|
* Updates Blacklist Regex Timers in DB to reflect last usage. |
915
|
|
|
*/ |
916
|
|
|
protected function updateBlacklistUsage(): void |
917
|
|
|
{ |
918
|
|
|
BinaryBlacklist::query()->whereIn('id', $this->_binaryBlacklistIdsToUpdate)->update(['last_activity' => now()]); |
919
|
|
|
$this->_binaryBlacklistIdsToUpdate = []; |
920
|
|
|
} |
921
|
|
|
|
922
|
|
|
/** |
923
|
|
|
* Outputs the initial header scan results after yEnc check and blacklist routines. |
924
|
|
|
*/ |
925
|
|
|
protected function outputHeaderInitial(): void |
926
|
|
|
{ |
927
|
|
|
$this->colorCli->primary( |
928
|
|
|
'Received '.\count($this->headersReceived). |
929
|
|
|
' articles of '.number_format($this->last - $this->first + 1).' requested, '. |
930
|
|
|
$this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.' |
931
|
|
|
); |
932
|
|
|
} |
933
|
|
|
|
934
|
|
|
/** |
935
|
|
|
* Outputs speed metrics of the scan function to CLI. |
936
|
|
|
*/ |
937
|
|
|
protected function outputHeaderDuration(): void |
938
|
|
|
{ |
939
|
|
|
$currentMicroTime = now(); |
940
|
|
|
if ($this->_echoCLI) { |
941
|
|
|
$this->colorCli->alternateOver($this->timeHeaders.'s'). |
942
|
|
|
$this->colorCli->primaryOver(' to download articles, '). |
943
|
|
|
$this->colorCli->alternateOver($this->timeCleaning.'s'). |
944
|
|
|
$this->colorCli->primaryOver(' to process collections, '). |
945
|
|
|
$this->colorCli->alternateOver($this->timeInsert.'s'). |
946
|
|
|
$this->colorCli->primaryOver(' to insert binaries/parts, '). |
947
|
|
|
$this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startPR).'s'). |
948
|
|
|
$this->colorCli->primaryOver(' for part repair, '). |
949
|
|
|
$this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startLoop).'s'). |
950
|
|
|
$this->colorCli->primary(' total.'); |
951
|
|
|
} |
952
|
|
|
} |
953
|
|
|
|
954
|
|
|
/** |
955
|
|
|
* Attempt to get missing article headers. |
956
|
|
|
* |
957
|
|
|
* @param array $groupArr The info for this group from mysql. |
958
|
|
|
* |
959
|
|
|
* @throws \Exception |
960
|
|
|
* @throws \Throwable |
961
|
|
|
*/ |
962
|
|
|
public function partRepair(array $groupArr): void |
963
|
|
|
{ |
964
|
|
|
// Get all parts in part repair table. |
965
|
|
|
$missingParts = []; |
|
|
|
|
966
|
|
|
try { |
967
|
|
|
$missingParts = DB::select(sprintf(' |
968
|
|
|
SELECT * FROM missed_parts |
969
|
|
|
WHERE groups_id = %d AND attempts < %d |
970
|
|
|
ORDER BY numberid ASC LIMIT %d', $groupArr['id'], $this->_partRepairMaxTries, $this->_partRepairLimit)); |
971
|
|
|
} catch (\PDOException $e) { |
972
|
|
|
if ($e->getMessage() === 'SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction') { |
973
|
|
|
$this->colorCli->notice('Deadlock occurred'); |
974
|
|
|
DB::rollBack(); |
975
|
|
|
} |
976
|
|
|
} |
977
|
|
|
|
978
|
|
|
$missingCount = \count($missingParts); |
979
|
|
|
if ($missingCount > 0) { |
980
|
|
|
if ($this->_echoCLI) { |
981
|
|
|
$this->colorCli->primary( |
982
|
|
|
'Attempting to repair '. |
983
|
|
|
number_format($missingCount). |
984
|
|
|
' parts.' |
985
|
|
|
); |
986
|
|
|
} |
987
|
|
|
|
988
|
|
|
// Loop through each part to group into continuous ranges with a maximum range of messagebuffer/4. |
989
|
|
|
$ranges = $partList = []; |
990
|
|
|
$firstPart = $lastNum = $missingParts[0]->numberid; |
991
|
|
|
|
992
|
|
|
foreach ($missingParts as $part) { |
993
|
|
|
if (($part->numberid - $firstPart) > ($this->messageBuffer / 4)) { |
994
|
|
|
$ranges[] = [ |
995
|
|
|
'partfrom' => $firstPart, |
996
|
|
|
'partto' => $lastNum, |
997
|
|
|
'partlist' => $partList, |
998
|
|
|
]; |
999
|
|
|
|
1000
|
|
|
$firstPart = $part->numberid; |
1001
|
|
|
$partList = []; |
1002
|
|
|
} |
1003
|
|
|
$partList[] = $part->numberid; |
1004
|
|
|
$lastNum = $part->numberid; |
1005
|
|
|
} |
1006
|
|
|
|
1007
|
|
|
$ranges[] = [ |
1008
|
|
|
'partfrom' => $firstPart, |
1009
|
|
|
'partto' => $lastNum, |
1010
|
|
|
'partlist' => $partList, |
1011
|
|
|
]; |
1012
|
|
|
|
1013
|
|
|
// Download missing parts in ranges. |
1014
|
|
|
foreach ($ranges as $range) { |
1015
|
|
|
$partFrom = $range['partfrom']; |
1016
|
|
|
$partTo = $range['partto']; |
1017
|
|
|
$partList = $range['partlist']; |
1018
|
|
|
|
1019
|
|
|
if ($this->_echoCLI) { |
1020
|
|
|
echo \chr(random_int(45, 46)).PHP_EOL; |
1021
|
|
|
} |
1022
|
|
|
|
1023
|
|
|
// Get article headers from newsgroup. |
1024
|
|
|
$this->scan($groupArr, $partFrom, $partTo, 'partrepair', $partList); |
1025
|
|
|
} |
1026
|
|
|
|
1027
|
|
|
// Calculate parts repaired |
1028
|
|
|
$result = DB::select( |
1029
|
|
|
sprintf( |
1030
|
|
|
' |
1031
|
|
|
SELECT COUNT(id) AS num |
1032
|
|
|
FROM missed_parts |
1033
|
|
|
WHERE groups_id = %d |
1034
|
|
|
AND numberid <= %d', |
1035
|
|
|
$groupArr['id'], |
1036
|
|
|
$missingParts[$missingCount - 1]->numberid |
1037
|
|
|
) |
1038
|
|
|
); |
1039
|
|
|
|
1040
|
|
|
$partsRepaired = 0; |
1041
|
|
|
if ($result > 0) { |
1042
|
|
|
$partsRepaired = ($missingCount - $result[0]->num); |
1043
|
|
|
} |
1044
|
|
|
|
1045
|
|
|
// Update attempts on remaining parts for active group |
1046
|
|
|
if (isset($missingParts[$missingCount - 1]->id)) { |
1047
|
|
|
DB::update( |
1048
|
|
|
sprintf( |
1049
|
|
|
' |
1050
|
|
|
UPDATE missed_parts |
1051
|
|
|
SET attempts = attempts + 1 |
1052
|
|
|
WHERE groups_id = %d |
1053
|
|
|
AND numberid <= %d', |
1054
|
|
|
$groupArr['id'], |
1055
|
|
|
$missingParts[$missingCount - 1]->numberid |
1056
|
|
|
) |
1057
|
|
|
); |
1058
|
|
|
} |
1059
|
|
|
|
1060
|
|
|
if ($this->_echoCLI) { |
1061
|
|
|
$this->colorCli->primary( |
1062
|
|
|
PHP_EOL. |
1063
|
|
|
number_format($partsRepaired). |
1064
|
|
|
' parts repaired.' |
1065
|
|
|
); |
1066
|
|
|
} |
1067
|
|
|
} |
1068
|
|
|
|
1069
|
|
|
// Remove articles that we cant fetch after x attempts. |
1070
|
|
|
DB::transaction(function () use ($groupArr) { |
1071
|
|
|
DB::delete( |
1072
|
|
|
sprintf( |
1073
|
|
|
'DELETE FROM missed_parts WHERE attempts >= %d AND groups_id = %d', |
1074
|
|
|
$this->_partRepairMaxTries, |
1075
|
|
|
$groupArr['id'] |
1076
|
|
|
) |
1077
|
|
|
); |
1078
|
|
|
}, 10); |
1079
|
|
|
} |
1080
|
|
|
|
1081
|
|
|
/** |
1082
|
|
|
* Returns unix time for an article number. |
1083
|
|
|
* |
1084
|
|
|
* @param int $post The article number to get the time from. |
1085
|
|
|
* @param array $groupData Usenet group info from NNTP selectGroup method. |
1086
|
|
|
* @return int Timestamp. |
1087
|
|
|
* |
1088
|
|
|
* @throws \Exception |
1089
|
|
|
*/ |
1090
|
|
|
public function postdate(int $post, array $groupData): int |
1091
|
|
|
{ |
1092
|
|
|
$currentPost = $post; |
1093
|
|
|
|
1094
|
|
|
$attempts = $date = 0; |
1095
|
|
|
do { |
1096
|
|
|
// Try to get the article date locally first. |
1097
|
|
|
// Try to get locally. |
1098
|
|
|
$local = DB::select( |
1099
|
|
|
sprintf( |
1100
|
|
|
' |
1101
|
|
|
SELECT c.date AS date |
1102
|
|
|
FROM collections c |
1103
|
|
|
INNER JOIN binaries b ON(c.id=b.collections_id) |
1104
|
|
|
INNER JOIN parts p ON(b.id=p.binaries_id) |
1105
|
|
|
WHERE p.number = %s', |
1106
|
|
|
$currentPost |
1107
|
|
|
) |
1108
|
|
|
); |
1109
|
|
|
if (! empty($local) && \count($local) > 0) { |
1110
|
|
|
$date = $local[0]->date; |
1111
|
|
|
break; |
1112
|
|
|
} |
1113
|
|
|
|
1114
|
|
|
// If we could not find it locally, try usenet. |
1115
|
|
|
$header = $this->_nntp->getXOVER($currentPost); |
1116
|
|
|
if (! $this->_nntp::isError($header) && isset($header[0]['Date']) && $header[0]['Date'] !== '') { |
1117
|
|
|
$date = $header[0]['Date']; |
1118
|
|
|
break; |
1119
|
|
|
} |
1120
|
|
|
|
1121
|
|
|
// Try to get a different article number. |
1122
|
|
|
if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) { |
1123
|
|
|
$tempPost = round($currentPost / (random_int(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP); |
1124
|
|
|
if ($tempPost < $groupData['first']) { |
1125
|
|
|
$tempPost = $groupData['first']; |
1126
|
|
|
} |
1127
|
|
|
} else { |
1128
|
|
|
$tempPost = round((random_int(1005, 1012) / 1000) * $currentPost, 0, PHP_ROUND_HALF_UP); |
1129
|
|
|
if ($tempPost > $groupData['last']) { |
1130
|
|
|
$tempPost = $groupData['last']; |
1131
|
|
|
} |
1132
|
|
|
} |
1133
|
|
|
// If we got the same article number as last time, give up. |
1134
|
|
|
if ($tempPost === $currentPost) { |
1135
|
|
|
break; |
1136
|
|
|
} |
1137
|
|
|
$currentPost = $tempPost; |
1138
|
|
|
} while ($attempts++ <= 20); |
1139
|
|
|
|
1140
|
|
|
// If we didn't get a date, set it to now. |
1141
|
|
|
if (! $date) { |
1142
|
|
|
$date = time(); |
1143
|
|
|
} else { |
1144
|
|
|
$date = strtotime($date); |
1145
|
|
|
} |
1146
|
|
|
|
1147
|
|
|
return $date; |
1148
|
|
|
} |
1149
|
|
|
|
1150
|
|
|
/** |
1151
|
|
|
* Returns article number based on # of days. |
1152
|
|
|
* |
1153
|
|
|
* @param int $days How many days back we want to go. |
1154
|
|
|
* @param array $data Group data from usenet. |
1155
|
|
|
* |
1156
|
|
|
* @throws \Exception |
1157
|
|
|
*/ |
1158
|
|
|
public function daytopost(int $days, array $data): string |
1159
|
|
|
{ |
1160
|
|
|
$goalTime = now()->subDays($days)->timestamp; |
1161
|
|
|
// The time we want = current unix time (ex. 1395699114) - minus 86400 (seconds in a day) |
1162
|
|
|
// times days wanted. (ie 1395699114 - 2592000 (30days)) = 1393107114 |
1163
|
|
|
|
1164
|
|
|
// The servers oldest date. |
1165
|
|
|
$firstDate = $this->postdate($data['first'], $data); |
1166
|
|
|
if ($goalTime < $firstDate) { |
1167
|
|
|
// If the date we want is older than the oldest date in the group return the groups oldest article. |
1168
|
|
|
return $data['first']; |
1169
|
|
|
} |
1170
|
|
|
|
1171
|
|
|
// The servers newest date. |
1172
|
|
|
$lastDate = $this->postdate($data['last'], $data); |
1173
|
|
|
if ($goalTime > $lastDate) { |
1174
|
|
|
// If the date we want is newer than the groups newest date, return the groups newest article. |
1175
|
|
|
return $data['last']; |
1176
|
|
|
} |
1177
|
|
|
|
1178
|
|
|
if ($this->_echoCLI) { |
1179
|
|
|
$this->colorCli->primary( |
1180
|
|
|
'Searching for an approximate article number for group '.$data['group'].' '.$days.' days back.' |
1181
|
|
|
); |
1182
|
|
|
} |
1183
|
|
|
|
1184
|
|
|
// Pick the middle to start with |
1185
|
|
|
$wantedArticle = round(($data['last'] + $data['first']) / 2); |
1186
|
|
|
$aMax = $data['last']; |
1187
|
|
|
$aMin = $data['first']; |
1188
|
|
|
$oldArticle = $articleTime = null; |
1189
|
|
|
|
1190
|
|
|
while (true) { |
1191
|
|
|
// Article exists outside available range, this shouldn't happen |
1192
|
|
|
if ($wantedArticle <= $data['first'] || $wantedArticle >= $data['last']) { |
1193
|
|
|
break; |
1194
|
|
|
} |
1195
|
|
|
|
1196
|
|
|
// Keep a note of the last articles we checked |
1197
|
|
|
$reallyOldArticle = $oldArticle; |
1198
|
|
|
$oldArticle = $wantedArticle; |
1199
|
|
|
|
1200
|
|
|
// Get the date of this article |
1201
|
|
|
$articleTime = $this->postdate($wantedArticle, $data); |
|
|
|
|
1202
|
|
|
|
1203
|
|
|
// Article doesn't exist, start again with something random |
1204
|
|
|
if (! $articleTime) { |
1205
|
|
|
$wantedArticle = random_int($aMin, $aMax); |
1206
|
|
|
$articleTime = $this->postdate($wantedArticle, $data); |
1207
|
|
|
} |
1208
|
|
|
|
1209
|
|
|
if ($articleTime < $goalTime) { |
1210
|
|
|
// Article is older than we want |
1211
|
|
|
$aMin = $oldArticle; |
1212
|
|
|
$wantedArticle = round(($aMax + $oldArticle) / 2); |
1213
|
|
|
if ($this->_echoCLI) { |
1214
|
|
|
echo '-'; |
1215
|
|
|
} |
1216
|
|
|
} elseif ($articleTime > $goalTime) { |
1217
|
|
|
// Article is newer than we want |
1218
|
|
|
$aMax = $oldArticle; |
1219
|
|
|
$wantedArticle = round(($aMin + $oldArticle) / 2); |
1220
|
|
|
if ($this->_echoCLI) { |
1221
|
|
|
echo '+'; |
1222
|
|
|
} |
1223
|
|
|
} elseif ($articleTime === $goalTime) { |
1224
|
|
|
// Exact match. We did it! (this will likely never happen though) |
1225
|
|
|
break; |
1226
|
|
|
} |
1227
|
|
|
|
1228
|
|
|
// We seem to be flip-flopping between 2 articles, assume we're out of articles to check. |
1229
|
|
|
// End on an article more recent than our oldest so that we don't miss any releases. |
1230
|
|
|
if ($reallyOldArticle === $wantedArticle && ($goalTime - $articleTime) <= 0) { |
1231
|
|
|
break; |
1232
|
|
|
} |
1233
|
|
|
} |
1234
|
|
|
|
1235
|
|
|
$wantedArticle = (int) $wantedArticle; |
1236
|
|
|
if ($this->_echoCLI) { |
1237
|
|
|
$this->colorCli->primary( |
1238
|
|
|
PHP_EOL.'Found article #'.$wantedArticle.' which has a date of '.date('r', $articleTime). |
1239
|
|
|
', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.Carbon::createFromTimestamp($goalTime)->diffInDays(Carbon::createFromTimestamp($articleTime)).'days.' |
|
|
|
|
1240
|
|
|
); |
1241
|
|
|
} |
1242
|
|
|
|
1243
|
|
|
return $wantedArticle; |
1244
|
|
|
} |
1245
|
|
|
|
1246
|
|
|
/** |
1247
|
|
|
* Add article numbers from missing headers to DB. |
1248
|
|
|
* |
1249
|
|
|
* @param array $numbers The article numbers of the missing headers. |
1250
|
|
|
* @param int $groupID The ID of this groups. |
1251
|
|
|
*/ |
1252
|
|
|
private function addMissingParts(array $numbers, int $groupID): void |
1253
|
|
|
{ |
1254
|
|
|
$insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES '; |
1255
|
|
|
foreach ($numbers as $number) { |
1256
|
|
|
$insertStr .= '('.$number.','.$groupID.'),'; |
1257
|
|
|
} |
1258
|
|
|
|
1259
|
|
|
DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1'); |
1260
|
|
|
} |
1261
|
|
|
|
1262
|
|
|
/** |
1263
|
|
|
* Clean up part repair table. |
1264
|
|
|
* |
1265
|
|
|
* @param array $numbers The article numbers. |
1266
|
|
|
* @param int $groupID The ID of the group. |
1267
|
|
|
* |
1268
|
|
|
* @throws \Throwable |
1269
|
|
|
*/ |
1270
|
|
|
private function removeRepairedParts(array $numbers, int $groupID): void |
1271
|
|
|
{ |
1272
|
|
|
$sql = 'DELETE FROM missed_parts WHERE numberid in ('; |
1273
|
|
|
foreach ($numbers as $number) { |
1274
|
|
|
$sql .= $number.','; |
1275
|
|
|
} |
1276
|
|
|
DB::transaction(static function () use ($groupID, $sql) { |
1277
|
|
|
DB::delete(rtrim($sql, ',').') AND groups_id = '.$groupID); |
1278
|
|
|
}, 10); |
1279
|
|
|
} |
1280
|
|
|
|
1281
|
|
|
/** |
1282
|
|
|
* Are white or black lists loaded for a group name? |
1283
|
|
|
*/ |
1284
|
|
|
protected array $_listsFound = []; |
1285
|
|
|
|
1286
|
|
|
/** |
1287
|
|
|
* Get blacklist and cache it. Return if already cached. |
1288
|
|
|
*/ |
1289
|
|
|
protected function _retrieveBlackList(string $groupName): void |
1290
|
|
|
{ |
1291
|
|
|
if (! isset($this->blackList[$groupName])) { |
1292
|
|
|
$this->blackList[$groupName] = $this->getBlacklist(true, self::OPTYPE_BLACKLIST, $groupName, true); |
1293
|
|
|
} |
1294
|
|
|
if (! isset($this->whiteList[$groupName])) { |
1295
|
|
|
$this->whiteList[$groupName] = $this->getBlacklist(true, self::OPTYPE_WHITELIST, $groupName, true); |
1296
|
|
|
} |
1297
|
|
|
$this->_listsFound[$groupName] = ($this->blackList[$groupName] || $this->whiteList[$groupName]); |
1298
|
|
|
} |
1299
|
|
|
|
1300
|
|
|
/** |
1301
|
|
|
* Check if an article is blacklisted. |
1302
|
|
|
* |
1303
|
|
|
* @param array $msg The article header (OVER format). |
1304
|
|
|
* @param string $groupName The group name. |
1305
|
|
|
*/ |
1306
|
|
|
public function isBlackListed(array $msg, string $groupName): bool |
1307
|
|
|
{ |
1308
|
|
|
if (! isset($this->_listsFound[$groupName])) { |
1309
|
|
|
$this->_retrieveBlackList($groupName); |
1310
|
|
|
} |
1311
|
|
|
if (! $this->_listsFound[$groupName]) { |
1312
|
|
|
return false; |
1313
|
|
|
} |
1314
|
|
|
|
1315
|
|
|
$blackListed = false; |
1316
|
|
|
|
1317
|
|
|
$field = [ |
1318
|
|
|
self::BLACKLIST_FIELD_SUBJECT => $msg['Subject'], |
1319
|
|
|
self::BLACKLIST_FIELD_FROM => $msg['From'], |
1320
|
|
|
self::BLACKLIST_FIELD_MESSAGEID => $msg['Message-ID'], |
1321
|
|
|
]; |
1322
|
|
|
|
1323
|
|
|
// Try white lists first. |
1324
|
|
|
if ($this->whiteList[$groupName]) { |
1325
|
|
|
// There are white lists for this group, so anything that doesn't match a white list should be considered black listed. |
1326
|
|
|
$blackListed = true; |
1327
|
|
|
foreach ($this->whiteList[$groupName] as $whiteList) { |
1328
|
|
|
if (preg_match('/'.$whiteList['regex'].'/i', $field[$whiteList['msgcol']])) { |
1329
|
|
|
// This field matched a white list, so it might not be black listed. |
1330
|
|
|
$blackListed = false; |
1331
|
|
|
$this->_binaryBlacklistIdsToUpdate[$whiteList['id']] = $whiteList['id']; |
1332
|
|
|
break; |
1333
|
|
|
} |
1334
|
|
|
} |
1335
|
|
|
} |
1336
|
|
|
|
1337
|
|
|
// Check if the field is blacklisted. |
1338
|
|
|
|
1339
|
|
|
if (! $blackListed && $this->blackList[$groupName]) { |
1340
|
|
|
foreach ($this->blackList[$groupName] as $blackList) { |
1341
|
|
|
if (preg_match('/'.$blackList->regex.'/i', $field[$blackList->msgcol])) { |
1342
|
|
|
$blackListed = true; |
1343
|
|
|
$this->_binaryBlacklistIdsToUpdate[$blackList->id] = $blackList->id; |
1344
|
|
|
break; |
1345
|
|
|
} |
1346
|
|
|
} |
1347
|
|
|
} |
1348
|
|
|
|
1349
|
|
|
return $blackListed; |
1350
|
|
|
} |
1351
|
|
|
|
1352
|
|
|
/** |
1353
|
|
|
* Return all blacklists. |
1354
|
|
|
* |
1355
|
|
|
* @param bool $activeOnly Only display active blacklists ? |
1356
|
|
|
* @param int|string $opType Optional, get white or black lists (use Binaries constants). |
1357
|
|
|
* @param string $groupName Optional, group. |
1358
|
|
|
* @param bool $groupRegex Optional Join groups / binary blacklist using regexp for equals. |
1359
|
|
|
*/ |
1360
|
|
|
public function getBlacklist(bool $activeOnly = true, int|string $opType = -1, string $groupName = '', bool $groupRegex = false): array |
1361
|
|
|
{ |
1362
|
|
|
$opType = match ($opType) { |
1363
|
|
|
self::OPTYPE_BLACKLIST => 'AND bb.optype = '.self::OPTYPE_BLACKLIST, |
1364
|
|
|
self::OPTYPE_WHITELIST => 'AND bb.optype = '.self::OPTYPE_WHITELIST, |
1365
|
|
|
default => '', |
1366
|
|
|
}; |
1367
|
|
|
|
1368
|
|
|
return DB::select( |
1369
|
|
|
sprintf( |
1370
|
|
|
' |
1371
|
|
|
SELECT |
1372
|
|
|
bb.id, bb.optype, bb.status, bb.description, |
1373
|
|
|
bb.groupname AS groupname, bb.regex, g.id AS group_id, bb.msgcol, |
1374
|
|
|
bb.last_activity as last_activity |
1375
|
|
|
FROM binaryblacklist bb |
1376
|
|
|
LEFT OUTER JOIN usenet_groups g ON g.name %s bb.groupname |
1377
|
|
|
WHERE 1=1 %s %s %s |
1378
|
|
|
ORDER BY coalesce(groupname,\'zzz\')', |
1379
|
|
|
($groupRegex ? 'REGEXP' : '='), |
1380
|
|
|
($activeOnly ? 'AND bb.status = 1' : ''), |
1381
|
|
|
$opType, |
1382
|
|
|
($groupName ? ('AND g.name REGEXP '.escapeString($groupName)) : '') |
1383
|
|
|
) |
1384
|
|
|
); |
1385
|
|
|
} |
1386
|
|
|
|
1387
|
|
|
/** |
1388
|
|
|
* Return a blacklist by ID. |
1389
|
|
|
* |
1390
|
|
|
* @param int $id The ID of the blacklist. |
1391
|
|
|
*/ |
1392
|
|
|
public function getBlacklistByID(int $id) |
1393
|
|
|
{ |
1394
|
|
|
return BinaryBlacklist::query()->where('id', $id)->first(); |
1395
|
|
|
} |
1396
|
|
|
|
1397
|
|
|
/** |
1398
|
|
|
* Delete a blacklist. |
1399
|
|
|
* |
1400
|
|
|
* @param int $id The ID of the blacklist. |
1401
|
|
|
*/ |
1402
|
|
|
public function deleteBlacklist(int $id): void |
1403
|
|
|
{ |
1404
|
|
|
BinaryBlacklist::query()->where('id', $id)->delete(); |
1405
|
|
|
} |
1406
|
|
|
|
1407
|
|
|
public function updateBlacklist(array $blacklistArray): void |
1408
|
|
|
{ |
1409
|
|
|
BinaryBlacklist::query()->where('id', $blacklistArray['id'])->update( |
1410
|
|
|
[ |
1411
|
|
|
'groupname' => $blacklistArray['groupname'] === '' ? 'null' : preg_replace('/a\.b\./i', 'alt.binaries.', $blacklistArray['groupname']), |
1412
|
|
|
'regex' => $blacklistArray['regex'], |
1413
|
|
|
'status' => $blacklistArray['status'], |
1414
|
|
|
'description' => $blacklistArray['description'], |
1415
|
|
|
'optype' => $blacklistArray['optype'], |
1416
|
|
|
'msgcol' => $blacklistArray['msgcol'], |
1417
|
|
|
] |
1418
|
|
|
); |
1419
|
|
|
} |
1420
|
|
|
|
1421
|
|
|
/** |
1422
|
|
|
* Adds a new blacklist from binary blacklist edit admin web page. |
1423
|
|
|
*/ |
1424
|
|
|
public function addBlacklist(array $blacklistArray): void |
1425
|
|
|
{ |
1426
|
|
|
BinaryBlacklist::query()->insert( |
1427
|
|
|
[ |
1428
|
|
|
'groupname' => $blacklistArray['groupname'] === '' ? 'null' : preg_replace('/a\.b\./i', 'alt.binaries.', $blacklistArray['groupname']), |
1429
|
|
|
'regex' => $blacklistArray['regex'], |
1430
|
|
|
'status' => $blacklistArray['status'], |
1431
|
|
|
'description' => $blacklistArray['description'], |
1432
|
|
|
'optype' => $blacklistArray['optype'], |
1433
|
|
|
'msgcol' => $blacklistArray['msgcol'], |
1434
|
|
|
] |
1435
|
|
|
); |
1436
|
|
|
} |
1437
|
|
|
|
1438
|
|
|
/** |
1439
|
|
|
* Delete Collections/Binaries/Parts for a Collection ID. |
1440
|
|
|
* |
1441
|
|
|
* @param int $collectionID Collections table ID |
1442
|
|
|
* |
1443
|
|
|
* @note A trigger automatically deletes the parts/binaries. |
1444
|
|
|
* |
1445
|
|
|
* @throws \Throwable |
1446
|
|
|
*/ |
1447
|
|
|
public function delete(int $collectionID): void |
1448
|
|
|
{ |
1449
|
|
|
DB::transaction(static function () use ($collectionID) { |
1450
|
|
|
DB::delete(sprintf('DELETE FROM collections WHERE id = %d', $collectionID)); |
1451
|
|
|
}, 10); |
1452
|
|
|
|
1453
|
|
|
Collection::query()->where('id', $collectionID)->delete(); |
1454
|
|
|
} |
1455
|
|
|
|
1456
|
|
|
/** |
1457
|
|
|
* Log / Echo message. |
1458
|
|
|
* |
1459
|
|
|
* @param string $message Message to log. |
1460
|
|
|
* @param string $method Method that called this. |
1461
|
|
|
* @param string $color ColorCLI method name. |
1462
|
|
|
*/ |
1463
|
|
|
private function log(string $message, string $method, string $color): void |
1464
|
|
|
{ |
1465
|
|
|
if ($this->_echoCLI) { |
1466
|
|
|
$this->colorCli->$color($message.' ['.__CLASS__."::$method]"); |
1467
|
|
|
} |
1468
|
|
|
} |
1469
|
|
|
|
1470
|
|
|
protected function runQuery($query): bool |
1471
|
|
|
{ |
1472
|
|
|
try { |
1473
|
|
|
return DB::insert($query); |
1474
|
|
|
} catch (QueryException $e) { |
1475
|
|
|
if (config('app.debug') === true) { |
1476
|
|
|
Log::error($e->getMessage()); |
1477
|
|
|
} |
1478
|
|
|
$this->colorCli->debug('Query error occurred.'); |
1479
|
|
|
} catch (\PDOException $e) { |
1480
|
|
|
if (config('app.debug') === true) { |
1481
|
|
|
Log::error($e->getMessage()); |
1482
|
|
|
} |
1483
|
|
|
$this->colorCli->debug('Query error occurred.'); |
1484
|
|
|
} catch (\Throwable $e) { |
1485
|
|
|
if (config('app.debug') === true) { |
1486
|
|
|
Log::error($e->getMessage()); |
1487
|
|
|
} |
1488
|
|
|
$this->colorCli->debug('Query error occurred.'); |
1489
|
|
|
} |
1490
|
|
|
|
1491
|
|
|
return false; |
1492
|
|
|
} |
1493
|
|
|
|
1494
|
|
|
private function getFileCount($subject): array |
1495
|
|
|
{ |
1496
|
|
|
if (! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])\s$:]/i', $subject, $fileCount)) { |
1497
|
|
|
$fileCount[1] = $fileCount[3] = 0; |
1498
|
|
|
} |
1499
|
|
|
|
1500
|
|
|
return $fileCount; |
1501
|
|
|
} |
1502
|
|
|
} |
1503
|
|
|
|