Total Complexity | 206 |
Total Lines | 1482 |
Duplicated Lines | 0 % |
Changes | 2 | ||
Bugs | 1 | Features | 0 |
Complex classes like Binaries often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use Binaries, and based on these observations, apply Extract Interface, too.
1 | <?php |
||
19 | class Binaries |
||
20 | { |
||
21 | public const OPTYPE_BLACKLIST = 1; |
||
22 | |||
23 | public const OPTYPE_WHITELIST = 2; |
||
24 | |||
25 | public const BLACKLIST_DISABLED = 0; |
||
26 | |||
27 | public const BLACKLIST_ENABLED = 1; |
||
28 | |||
29 | public const BLACKLIST_FIELD_SUBJECT = 1; |
||
30 | |||
31 | public const BLACKLIST_FIELD_FROM = 2; |
||
32 | |||
33 | public const BLACKLIST_FIELD_MESSAGEID = 3; |
||
34 | |||
35 | public array $blackList = []; |
||
36 | |||
37 | public array $whiteList = []; |
||
38 | |||
39 | public int $messageBuffer; |
||
40 | |||
41 | /** |
||
42 | * @var \Blacklight\ColorCLI |
||
43 | */ |
||
44 | protected mixed $colorCli; |
||
45 | |||
46 | /** |
||
47 | * @var \Blacklight\CollectionsCleaning |
||
48 | */ |
||
49 | protected mixed $_collectionsCleaning; |
||
50 | |||
51 | /** |
||
52 | * @var \Blacklight\NNTP |
||
53 | */ |
||
54 | protected mixed $_nntp; |
||
55 | |||
56 | /** |
||
57 | * Should we use header compression? |
||
58 | */ |
||
59 | protected bool $_compressedHeaders; |
||
60 | |||
61 | /** |
||
62 | * Should we use part repair? |
||
63 | */ |
||
64 | protected bool $_partRepair; |
||
65 | |||
66 | protected \Closure|\PDO $_pdo; |
||
67 | |||
68 | /** |
||
69 | * How many days to go back on a new group? |
||
70 | */ |
||
71 | protected bool $_newGroupScanByDays; |
||
72 | |||
73 | /** |
||
74 | * How many headers to download on new groups? |
||
75 | */ |
||
76 | protected int $_newGroupMessagesToScan; |
||
77 | |||
78 | /** |
||
79 | * How many days to go back on new groups? |
||
80 | */ |
||
81 | protected int $_newGroupDaysToScan; |
||
82 | |||
83 | /** |
||
84 | * How many headers to download per run of part repair? |
||
85 | */ |
||
86 | protected int $_partRepairLimit; |
||
87 | |||
88 | /** |
||
89 | * Echo to cli? |
||
90 | */ |
||
91 | protected bool $_echoCLI; |
||
92 | |||
93 | /** |
||
94 | * Max tries to download headers. |
||
95 | */ |
||
96 | protected int $_partRepairMaxTries; |
||
97 | |||
98 | /** |
||
99 | * An array of BinaryBlacklist IDs that should have their activity date updated. |
||
100 | */ |
||
101 | protected array $_binaryBlacklistIdsToUpdate = []; |
||
102 | |||
103 | protected \DateTime $startCleaning; |
||
104 | |||
105 | protected \DateTime $startLoop; |
||
106 | |||
107 | /** |
||
108 | * @var int How long it took in seconds to download headers |
||
109 | */ |
||
110 | protected int $timeHeaders; |
||
111 | |||
112 | /** |
||
113 | * @var int How long it took in seconds to clean/parse headers |
||
114 | */ |
||
115 | protected int $timeCleaning; |
||
116 | |||
117 | protected \DateTime $startPR; |
||
118 | |||
119 | protected \DateTime $startUpdate; |
||
120 | |||
121 | /** |
||
122 | * @var int The time it took to insert the headers |
||
123 | */ |
||
124 | protected int $timeInsert; |
||
125 | |||
126 | /** |
||
127 | * @var array the header currently being scanned |
||
128 | */ |
||
129 | protected array $header; |
||
130 | |||
131 | /** |
||
132 | * @var bool Should we add parts to part repair queue? |
||
133 | */ |
||
134 | protected bool $addToPartRepair; |
||
135 | |||
136 | /** |
||
137 | * @var array Numbers of Headers received from the USP |
||
138 | */ |
||
139 | protected array $headersReceived; |
||
140 | |||
141 | /** |
||
142 | * @var array The current newsgroup information being updated |
||
143 | */ |
||
144 | protected array $groupMySQL; |
||
145 | |||
146 | /** |
||
147 | * @var int the last article number in the range |
||
148 | */ |
||
149 | protected int $last; |
||
150 | |||
151 | /** |
||
152 | * @var int the first article number in the range |
||
153 | */ |
||
154 | protected int $first; |
||
155 | |||
156 | /** |
||
157 | * @var int How many received headers were not yEnc encoded |
||
158 | */ |
||
159 | protected int $notYEnc; |
||
160 | |||
161 | /** |
||
162 | * @var int How many received headers were blacklist matched |
||
163 | */ |
||
164 | protected int $headersBlackListed; |
||
165 | |||
166 | /** |
||
167 | * @var array Header numbers that were not inserted |
||
168 | */ |
||
169 | protected array $headersNotInserted; |
||
170 | |||
171 | public function __construct() |
||
172 | { |
||
173 | $this->startUpdate = now(); |
||
174 | $this->timeCleaning = 0; |
||
175 | |||
176 | $this->_echoCLI = config('nntmux.echocli'); |
||
177 | |||
178 | $this->_pdo = DB::connection()->getPdo(); |
||
179 | $this->colorCli = new ColorCLI(); |
||
180 | $this->_nntp = new NNTP(); |
||
181 | $this->_collectionsCleaning = new CollectionsCleaning(); |
||
182 | |||
183 | $this->messageBuffer = Settings::settingValue('..maxmssgs') !== '' ? |
||
|
|||
184 | (int) Settings::settingValue('..maxmssgs') : 20000; |
||
185 | $this->_compressedHeaders = (int) Settings::settingValue('..compressedheaders') === 1; |
||
186 | $this->_partRepair = (int) Settings::settingValue('..partrepair') === 1; |
||
187 | $this->_newGroupScanByDays = (int) Settings::settingValue('..newgroupscanmethod') === 1; |
||
188 | $this->_newGroupMessagesToScan = Settings::settingValue('..newgroupmsgstoscan') !== '' ? (int) Settings::settingValue('..newgroupmsgstoscan') : 50000; |
||
189 | $this->_newGroupDaysToScan = Settings::settingValue('..newgroupdaystoscan') !== '' ? (int) Settings::settingValue('..newgroupdaystoscan') : 3; |
||
190 | $this->_partRepairLimit = Settings::settingValue('..maxpartrepair') !== '' ? (int) Settings::settingValue('..maxpartrepair') : 15000; |
||
191 | $this->_partRepairMaxTries = (Settings::settingValue('..partrepairmaxtries') !== '' ? (int) Settings::settingValue('..partrepairmaxtries') : 3); |
||
192 | |||
193 | $this->blackList = $this->whiteList = []; |
||
194 | } |
||
195 | |||
196 | /** |
||
197 | * Download new headers for all active groups. |
||
198 | * |
||
199 | * @param int $maxHeaders (Optional) How many headers to download max. |
||
200 | * |
||
201 | * @throws \Exception |
||
202 | * @throws \Throwable |
||
203 | */ |
||
204 | public function updateAllGroups(int $maxHeaders = 100000): void |
||
205 | { |
||
206 | $groups = UsenetGroup::getActive()->toArray(); |
||
207 | |||
208 | $groupCount = \count($groups); |
||
209 | if ($groupCount > 0) { |
||
210 | $counter = 1; |
||
211 | $allTime = now(); |
||
212 | |||
213 | $this->log( |
||
214 | 'Updating: '.$groupCount.' group(s) - Using compression? '.($this->_compressedHeaders ? 'Yes' : 'No'), |
||
215 | __FUNCTION__, |
||
216 | 'header' |
||
217 | ); |
||
218 | |||
219 | // Loop through groups. |
||
220 | foreach ($groups as $group) { |
||
221 | $this->log( |
||
222 | 'Starting group '.$counter.' of '.$groupCount, |
||
223 | __FUNCTION__, |
||
224 | 'header' |
||
225 | ); |
||
226 | $this->updateGroup($group, $maxHeaders); |
||
227 | $counter++; |
||
228 | } |
||
229 | |||
230 | $endTime = now()->diffInSeconds($allTime); |
||
231 | $this->log( |
||
232 | 'Updating completed in '.$endTime.Str::plural(' second', $endTime), |
||
233 | __FUNCTION__, |
||
234 | 'primary' |
||
235 | ); |
||
236 | } else { |
||
237 | $this->log( |
||
238 | 'No groups specified. Ensure groups are added to NNTmux\'s database for updating.', |
||
239 | __FUNCTION__, |
||
240 | 'warning' |
||
241 | ); |
||
242 | } |
||
243 | } |
||
244 | |||
245 | /** |
||
246 | * When the indexer is started, log the date/time. |
||
247 | */ |
||
248 | public function logIndexerStart(): void |
||
249 | { |
||
250 | Settings::query()->where('setting', '=', 'last_run_time')->update(['value' => now()]); |
||
251 | } |
||
252 | |||
253 | /** |
||
254 | * Download new headers for a single group. |
||
255 | * |
||
256 | * @param array $groupMySQL Array of MySQL results for a single group. |
||
257 | * @param int $maxHeaders (Optional) How many headers to download max. |
||
258 | * |
||
259 | * @throws \Exception |
||
260 | * @throws \Throwable |
||
261 | */ |
||
262 | public function updateGroup(array $groupMySQL, int $maxHeaders = 0): void |
||
263 | { |
||
264 | $startGroup = now(); |
||
265 | $this->logIndexerStart(); |
||
266 | |||
267 | // Select the group on the NNTP server, gets the latest info on it. |
||
268 | $groupNNTP = $this->_nntp->selectGroup($groupMySQL['name']); |
||
269 | if ($this->_nntp::isError($groupNNTP)) { |
||
270 | $groupNNTP = $this->_nntp->dataError($this->_nntp, $groupMySQL['name']); |
||
271 | |||
272 | if (isset($groupNNTP['code']) && (int) $groupNNTP['code'] === 411) { |
||
273 | UsenetGroup::disableIfNotExist($groupMySQL['id']); |
||
274 | } |
||
275 | if ($this->_nntp::isError($groupNNTP)) { |
||
276 | return; |
||
277 | } |
||
278 | } |
||
279 | |||
280 | if ($this->_echoCLI) { |
||
281 | $this->colorCli->primary('Processing '.$groupMySQL['name']); |
||
282 | } |
||
283 | |||
284 | // Attempt to repair any missing parts before grabbing new ones. |
||
285 | if ((int) $groupMySQL['last_record'] !== 0) { |
||
286 | if ($this->_partRepair) { |
||
287 | if ($this->_echoCLI) { |
||
288 | $this->colorCli->primary('Part repair enabled. Checking for missing parts.'); |
||
289 | } |
||
290 | $this->partRepair($groupMySQL); |
||
291 | } elseif ($this->_echoCLI) { |
||
292 | $this->colorCli->primary('Part repair disabled by user.'); |
||
293 | } |
||
294 | } |
||
295 | |||
296 | // Generate postdate for first record, for those that upgraded. |
||
297 | if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] !== 0) { |
||
298 | $groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); |
||
299 | UsenetGroup::query()->where('id', $groupMySQL['id'])->update(['first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'])]); |
||
300 | } |
||
301 | |||
302 | // Get first article we want aka the oldest. |
||
303 | if ((int) $groupMySQL['last_record'] === 0) { |
||
304 | if ($this->_newGroupScanByDays) { |
||
305 | // For new newsgroups - determine here how far we want to go back using date. |
||
306 | $first = $this->daytopost($this->_newGroupDaysToScan, $groupNNTP); |
||
307 | } elseif ($groupNNTP['first'] >= ($groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer))) { |
||
308 | // If what we want is lower than the groups first article, set the wanted first to the first. |
||
309 | $first = $groupNNTP['first']; |
||
310 | } else { |
||
311 | // Or else, use the newest article minus how much we should get for new groups. |
||
312 | $first = (string) ($groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer)); |
||
313 | } |
||
314 | |||
315 | // We will use this to subtract so we leave articles for the next time (in case the server doesn't have them yet) |
||
316 | $leaveOver = $this->messageBuffer; |
||
317 | |||
318 | // If this is not a new group, go from our newest to the servers newest. |
||
319 | } else { |
||
320 | // Set our oldest wanted to our newest local article. |
||
321 | $first = $groupMySQL['last_record']; |
||
322 | |||
323 | // This is how many articles we will grab. (the servers newest minus our newest). |
||
324 | $totalCount = (string) ($groupNNTP['last'] - $first); |
||
325 | |||
326 | // Check if the server has more articles than our loop limit x 2. |
||
327 | if ($totalCount > ($this->messageBuffer * 2)) { |
||
328 | // Get the remainder of $totalCount / $this->message buffer |
||
329 | $leaveOver = round($totalCount % $this->messageBuffer, 0, PHP_ROUND_HALF_DOWN) + $this->messageBuffer; |
||
330 | } else { |
||
331 | // Else get half of the available. |
||
332 | $leaveOver = round($totalCount / 2, 0, PHP_ROUND_HALF_DOWN); |
||
333 | } |
||
334 | } |
||
335 | |||
336 | // The last article we want, aka the newest. |
||
337 | $last = $groupLast = (string) ($groupNNTP['last'] - $leaveOver); |
||
338 | |||
339 | // If the newest we want is older than the oldest we want somehow.. set them equal. |
||
340 | if ($last < $first) { |
||
341 | $last = $groupLast = $first; |
||
342 | } |
||
343 | |||
344 | // This is how many articles we are going to get. |
||
345 | $total = (string) ($groupLast - $first); |
||
346 | // This is how many articles are available (without $leaveOver). |
||
347 | $realTotal = (string) ($groupNNTP['last'] - $first); |
||
348 | |||
349 | // Check if we should limit the amount of fetched new headers. |
||
350 | if ($maxHeaders > 0) { |
||
351 | if ($maxHeaders < ($groupLast - $first)) { |
||
352 | $groupLast = $last = (string) ($first + $maxHeaders); |
||
353 | } |
||
354 | $total = (string) ($groupLast - $first); |
||
355 | } |
||
356 | |||
357 | // If total is bigger than 0 it means we have new parts in the newsgroup. |
||
358 | if ($total > 0) { |
||
359 | if ($this->_echoCLI) { |
||
360 | $this->colorCli->primary( |
||
361 | ( |
||
362 | (int) $groupMySQL['last_record'] === 0 |
||
363 | ? 'New group '.$groupNNTP['group'].' starting with '. |
||
364 | ( |
||
365 | $this->_newGroupScanByDays |
||
366 | ? $this->_newGroupDaysToScan.' days' |
||
367 | : number_format($this->_newGroupMessagesToScan).' messages' |
||
368 | ).' worth.' |
||
369 | : 'Group '.$groupNNTP['group'].' has '.number_format($realTotal).' new articles.' |
||
370 | ). |
||
371 | ' Leaving '.number_format($leaveOver). |
||
372 | " for next pass.\nServer oldest: ".number_format($groupNNTP['first']). |
||
373 | ' Server newest: '.number_format($groupNNTP['last']). |
||
374 | ' Local newest: '.number_format($groupMySQL['last_record']) |
||
375 | ); |
||
376 | } |
||
377 | |||
378 | $done = false; |
||
379 | // Get all the parts (in portions of $this->messageBuffer to not use too much memory). |
||
380 | while (! $done) { |
||
381 | // Increment last until we reach $groupLast (group newest article). |
||
382 | if ($total > $this->messageBuffer) { |
||
383 | if ((string) ($first + $this->messageBuffer) > $groupLast) { |
||
384 | $last = $groupLast; |
||
385 | } else { |
||
386 | $last = (string) ($first + $this->messageBuffer); |
||
387 | } |
||
388 | } |
||
389 | // Increment first so we don't get an article we already had. |
||
390 | $first++; |
||
391 | |||
392 | if ($this->_echoCLI) { |
||
393 | $this->colorCli->header( |
||
394 | PHP_EOL.'Getting '.number_format($last - $first + 1).' articles ('.number_format($first). |
||
395 | ' to '.number_format($last).') from '.$groupMySQL['name'].' - ('. |
||
396 | number_format($groupLast - $last).' articles in queue).' |
||
397 | ); |
||
398 | } |
||
399 | |||
400 | // Get article headers from newsgroup. |
||
401 | $scanSummary = $this->scan($groupMySQL, $first, $last); |
||
402 | |||
403 | // Check if we fetched headers. |
||
404 | if (! empty($scanSummary)) { |
||
405 | // If new group, update first record & postdate |
||
406 | if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] === 0) { |
||
407 | $groupMySQL['first_record'] = $scanSummary['firstArticleNumber']; |
||
408 | |||
409 | if (isset($scanSummary['firstArticleDate'])) { |
||
410 | $groupMySQL['first_record_postdate'] = strtotime($scanSummary['firstArticleDate']); |
||
411 | } else { |
||
412 | $groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); |
||
413 | } |
||
414 | |||
415 | UsenetGroup::query() |
||
416 | ->where('id', $groupMySQL['id']) |
||
417 | ->update( |
||
418 | [ |
||
419 | 'first_record' => $scanSummary['firstArticleNumber'], |
||
420 | 'first_record_postdate' => Carbon::createFromTimestamp( |
||
421 | $groupMySQL['first_record_postdate'] |
||
422 | ), |
||
423 | ] |
||
424 | ); |
||
425 | } |
||
426 | |||
427 | $scanSummary['lastArticleDate'] = (isset($scanSummary['lastArticleDate']) ? strtotime($scanSummary['lastArticleDate']) : false); |
||
428 | if (! is_numeric($scanSummary['lastArticleDate'])) { |
||
429 | $scanSummary['lastArticleDate'] = $this->postdate($scanSummary['lastArticleNumber'], $groupNNTP); |
||
430 | } |
||
431 | |||
432 | UsenetGroup::query() |
||
433 | ->where('id', $groupMySQL['id']) |
||
434 | ->update( |
||
435 | [ |
||
436 | 'last_record' => $scanSummary['lastArticleNumber'], |
||
437 | 'last_record_postdate' => Carbon::createFromTimestamp($scanSummary['lastArticleDate']), |
||
438 | 'last_updated' => now(), |
||
439 | ] |
||
440 | ); |
||
441 | } else { |
||
442 | // If we didn't fetch headers, update the record still. |
||
443 | UsenetGroup::query() |
||
444 | ->where('id', $groupMySQL['id']) |
||
445 | ->update( |
||
446 | [ |
||
447 | 'last_record' => $last, |
||
448 | 'last_updated' => now(), |
||
449 | ] |
||
450 | ); |
||
451 | } |
||
452 | |||
453 | if ((int) $last === (int) $groupLast) { |
||
454 | $done = true; |
||
455 | } else { |
||
456 | $first = $last; |
||
457 | } |
||
458 | } |
||
459 | |||
460 | if ($this->_echoCLI) { |
||
461 | $endGroup = now()->diffInSeconds($startGroup); |
||
462 | $this->colorCli->primary( |
||
463 | PHP_EOL.'Group '.$groupMySQL['name'].' processed in '. |
||
464 | $endGroup.Str::plural(' second', $endGroup) |
||
465 | ); |
||
466 | } |
||
467 | } elseif ($this->_echoCLI) { |
||
468 | $this->colorCli->primary( |
||
469 | 'No new articles for '.$groupMySQL['name'].' (first '.number_format($first). |
||
470 | ', last '.number_format($last).', grouplast '.number_format($groupMySQL['last_record']). |
||
471 | ', total '.number_format($total).")\n".'Server oldest: '.number_format($groupNNTP['first']). |
||
472 | ' Server newest: '.number_format($groupNNTP['last']).' Local newest: '.number_format($groupMySQL['last_record']) |
||
473 | ); |
||
474 | } |
||
475 | } |
||
476 | |||
477 | /** |
||
478 | * Loop over range of wanted headers, insert headers into DB. |
||
479 | * |
||
480 | * @param array $groupMySQL The group info from mysql. |
||
481 | * @param int $first The oldest wanted header. |
||
482 | * @param int $last The newest wanted header. |
||
483 | * @param string $type Is this part repair or update or backfill? |
||
484 | * @param array|null $missingParts If we are running in part repair, the list of missing article numbers. |
||
485 | * @return array Empty on failure. |
||
486 | * |
||
487 | * @throws \Exception |
||
488 | * @throws \Throwable |
||
489 | */ |
||
490 | public function scan(array $groupMySQL, int $first, int $last, string $type = 'update', array $missingParts = null): array |
||
684 | } |
||
685 | |||
686 | /** |
||
687 | * Parse headers into collections/binaries and store header data as parts. |
||
688 | * |
||
689 | * |
||
690 | * |
||
691 | * @throws \Exception |
||
692 | * @throws \Throwable |
||
693 | */ |
||
694 | protected function storeHeaders(array $headers = []): void |
||
695 | { |
||
696 | $binariesUpdate = $collectionIDs = $articles = []; |
||
697 | |||
698 | DB::beginTransaction(); |
||
699 | |||
700 | $partsQuery = $partsCheck = 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '; |
||
701 | |||
702 | // Loop articles, figure out files/parts. |
||
703 | foreach ($headers as $this->header) { |
||
704 | // Set up the info for inserting into parts/binaries/collections tables. |
||
705 | if (! isset($articles[$this->header['matches'][1]])) { |
||
706 | // Attempt to find the file count. If it is not found, set it to 0. |
||
707 | $fileCount = $this->getFileCount($this->header['matches'][1]); |
||
708 | if ($fileCount[1] === 0 && $fileCount[3] === 0) { |
||
709 | $fileCount = $this->getFileCount($this->header['matches'][0]); |
||
710 | } |
||
711 | |||
712 | $collMatch = $this->_collectionsCleaning->collectionsCleaner( |
||
713 | $this->header['matches'][1], $this->groupMySQL['name'] |
||
714 | ); |
||
715 | |||
716 | // Used to group articles together when forming the release. |
||
717 | $this->header['CollectionKey'] = $collMatch['name'].$fileCount[3]; |
||
718 | |||
719 | // If this header's collection key isn't in memory, attempt to insert the collection |
||
720 | if (! isset($collectionIDs[$this->header['CollectionKey']])) { |
||
721 | /* Date from header should be a string this format: |
||
722 | * 31 Mar 2014 15:36:04 GMT or 6 Oct 1998 04:38:40 -0500 |
||
723 | * Still make sure it's not unix time, convert it to unix time if it is. |
||
724 | */ |
||
725 | $this->header['Date'] = (is_numeric($this->header['Date']) ? $this->header['Date'] : strtotime($this->header['Date'])); |
||
726 | |||
727 | // Get the current unixtime from PHP. |
||
728 | $now = now()->timestamp; |
||
729 | |||
730 | $xrefsData = Collection::whereCollectionhash(sha1($this->header['CollectionKey']))->value('xref'); |
||
731 | |||
732 | $tempHeaderXrefs = []; |
||
733 | foreach (explode(' ', $this->header['Xref']) as $headerXref) { |
||
734 | if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $headerXref, $hit) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $headerXref, $hit)) { |
||
735 | $tempHeaderXrefs[] = $hit[0]; |
||
736 | } |
||
737 | } |
||
738 | |||
739 | $tempXrefsData = []; |
||
740 | |||
741 | if ($xrefsData !== null) { |
||
742 | foreach (explode(' ', $xrefsData) as $xrefData) { |
||
743 | if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $xrefData, $match1) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $xrefData, $match1)) { |
||
744 | $tempXrefsData[] = $match1[0]; |
||
745 | } |
||
746 | } |
||
747 | } |
||
748 | |||
749 | $finalXrefArray = []; |
||
750 | foreach ($tempHeaderXrefs as $tempHeaderXref) { |
||
751 | if (! in_array($tempHeaderXref, $tempXrefsData, false)) { |
||
752 | $finalXrefArray[] = $tempHeaderXref; |
||
753 | } |
||
754 | } |
||
755 | |||
756 | $finaXref = implode(' ', $finalXrefArray); |
||
757 | |||
758 | $xref = sprintf('xref = CONCAT(xref, "\\n"%s ),', escapeString($finaXref)); |
||
759 | |||
760 | $date = $this->header['Date'] > $now ? $now : $this->header['Date']; |
||
761 | $unixtime = is_numeric($this->header['Date']) ? $date : $now; |
||
762 | |||
763 | $random = random_bytes(16); |
||
764 | $number = random_int(3, 9999999); |
||
765 | $special = Str::random(5); |
||
766 | $string = htmlspecialchars(str_shuffle('trusin_ @'.$number.$special), ENT_QUOTES); |
||
767 | |||
768 | $collectionID = false; |
||
769 | |||
770 | try { |
||
771 | DB::insert(sprintf(" |
||
772 | INSERT INTO collections (subject, fromname, date, xref, groups_id, |
||
773 | totalfiles, collectionhash, collection_regexes_id, dateadded) |
||
774 | VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', %d, NOW()) |
||
775 | ON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'", escapeString(substr(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings()), 0, 255)), escapeString(mb_convert_encoding($this->header['From'], 'UTF-8', mb_list_encodings())), $unixtime, escapeString(implode(' ', $tempHeaderXrefs)), $this->groupMySQL['id'], $fileCount[3], sha1($this->header['CollectionKey']), $collMatch['id'], $xref, sodium_bin2hex($random))); |
||
776 | $collectionID = $this->_pdo->lastInsertId(); |
||
777 | DB::commit(); |
||
778 | } catch (\Throwable $e) { |
||
779 | if (config('app.debug') === true) { |
||
780 | Log::error($e->getMessage()); |
||
781 | } |
||
782 | DB::rollBack(); |
||
783 | } |
||
784 | |||
785 | if ($collectionID === false) { |
||
786 | if ($this->addToPartRepair) { |
||
787 | $this->headersNotInserted[] = $this->header['Number']; |
||
788 | } |
||
789 | DB::rollBack(); |
||
790 | DB::beginTransaction(); |
||
791 | |||
792 | continue; |
||
793 | } |
||
794 | $collectionIDs[$this->header['CollectionKey']] = $collectionID; |
||
795 | } else { |
||
796 | $collectionID = $collectionIDs[$this->header['CollectionKey']]; |
||
797 | } |
||
798 | |||
799 | // Binary Hash should be unique to the group |
||
800 | $hash = md5($this->header['matches'][1].$this->header['From'].$this->groupMySQL['id']); |
||
801 | |||
802 | $binaryID = false; |
||
803 | |||
804 | try { |
||
805 | DB::insert(sprintf(" |
||
806 | INSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) |
||
807 | VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d) |
||
808 | ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $hash, escapeString(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings())), $collectionID, $this->header['matches'][3], $fileCount[1], $this->header['Bytes'], $this->header['Bytes'])); |
||
809 | $binaryID = $this->_pdo->lastInsertId(); |
||
810 | DB::commit(); |
||
811 | } catch (\Throwable $e) { |
||
812 | if (config('app.debug') === true) { |
||
813 | Log::error($e->getMessage()); |
||
814 | } |
||
815 | DB::rollBack(); |
||
816 | } |
||
817 | |||
818 | if ($binaryID === false) { |
||
819 | if ($this->addToPartRepair) { |
||
820 | $this->headersNotInserted[] = $this->header['Number']; |
||
821 | } |
||
822 | DB::rollBack(); |
||
823 | DB::beginTransaction(); |
||
824 | |||
825 | continue; |
||
826 | } |
||
827 | |||
828 | $binariesUpdate[$binaryID]['Size'] = 0; |
||
829 | $binariesUpdate[$binaryID]['Parts'] = 0; |
||
830 | |||
831 | $articles[$this->header['matches'][1]]['CollectionID'] = $collectionID; |
||
832 | $articles[$this->header['matches'][1]]['BinaryID'] = $binaryID; |
||
833 | } else { |
||
834 | $binaryID = $articles[$this->header['matches'][1]]['BinaryID']; |
||
835 | $binariesUpdate[$binaryID]['Size'] += $this->header['Bytes']; |
||
836 | $binariesUpdate[$binaryID]['Parts']++; |
||
837 | } |
||
838 | |||
839 | // In case there are quotes in the message id |
||
840 | $this->header['Message-ID'] = addslashes($this->header['Message-ID']); |
||
841 | |||
842 | // Strip the < and >, saves space in DB. |
||
843 | $this->header['Message-ID'][0] = "'"; |
||
844 | |||
845 | $partsQuery .= |
||
846 | '('.$binaryID.','.$this->header['Number'].','.rtrim($this->header['Message-ID'], '>')."',". |
||
847 | $this->header['matches'][2].','.$this->header['Bytes'].'),'; |
||
848 | } |
||
849 | |||
850 | unset($headers); // Reclaim memory. |
||
851 | |||
852 | // Start of inserting into SQL. |
||
853 | $this->startUpdate = now(); |
||
854 | |||
855 | // End of processing headers. |
||
856 | $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning); |
||
857 | $binariesQuery = $binariesCheck = 'INSERT INTO binaries (id, partsize, currentparts) VALUES '; |
||
858 | foreach ($binariesUpdate as $binaryID => $binary) { |
||
859 | $binariesQuery .= '('.$binaryID.','.$binary['Size'].','.$binary['Parts'].'),'; |
||
860 | } |
||
861 | $binariesEnd = ' ON DUPLICATE KEY UPDATE partsize = VALUES(partsize) + partsize, currentparts = VALUES(currentparts) + currentparts'; |
||
862 | $binariesQuery = rtrim($binariesQuery, ',').$binariesEnd; |
||
863 | |||
864 | // Check if we got any binaries. If we did, try to insert them. |
||
865 | if (\strlen($binariesCheck.$binariesEnd) === \strlen($binariesQuery) || $this->runQuery($binariesQuery)) { |
||
866 | if (\strlen($partsQuery) === \strlen($partsCheck) || $this->runQuery(rtrim($partsQuery, ','))) { |
||
867 | DB::commit(); |
||
868 | } else { |
||
869 | if ($this->addToPartRepair) { |
||
870 | $this->headersNotInserted += $this->headersReceived; |
||
871 | } |
||
872 | DB::rollBack(); |
||
873 | } |
||
874 | } else { |
||
875 | if ($this->addToPartRepair) { |
||
876 | $this->headersNotInserted += $this->headersReceived; |
||
877 | } |
||
878 | DB::rollBack(); |
||
879 | } |
||
880 | } |
||
881 | |||
882 | /** |
||
883 | * Gets the First and Last Article Number and Date for the received headers. |
||
884 | */ |
||
885 | protected function getHighLowArticleInfo(array &$returnArray, array $headers, int $msgCount): void |
||
886 | { |
||
887 | // Get highest and lowest article numbers/dates. |
||
888 | $iterator1 = 0; |
||
889 | $iterator2 = $msgCount - 1; |
||
890 | while (true) { |
||
891 | if (! isset($returnArray['firstArticleNumber']) && isset($headers[$iterator1]['Number'])) { |
||
892 | $returnArray['firstArticleNumber'] = $headers[$iterator1]['Number']; |
||
893 | $returnArray['firstArticleDate'] = $headers[$iterator1]['Date']; |
||
894 | } |
||
895 | |||
896 | if (! isset($returnArray['lastArticleNumber']) && isset($headers[$iterator2]['Number'])) { |
||
897 | $returnArray['lastArticleNumber'] = $headers[$iterator2]['Number']; |
||
898 | $returnArray['lastArticleDate'] = $headers[$iterator2]['Date']; |
||
899 | } |
||
900 | |||
901 | // Break if we found non empty articles. |
||
902 | if (isset($returnArray['firstArticleNumber, lastArticleNumber'])) { |
||
903 | break; |
||
904 | } |
||
905 | |||
906 | // Break out if we couldn't find anything. |
||
907 | if ($iterator1++ >= $msgCount - 1 || $iterator2-- <= 0) { |
||
908 | break; |
||
909 | } |
||
910 | } |
||
911 | } |
||
912 | |||
913 | /** |
||
914 | * Updates Blacklist Regex Timers in DB to reflect last usage. |
||
915 | */ |
||
916 | protected function updateBlacklistUsage(): void |
||
917 | { |
||
918 | BinaryBlacklist::query()->whereIn('id', $this->_binaryBlacklistIdsToUpdate)->update(['last_activity' => now()]); |
||
919 | $this->_binaryBlacklistIdsToUpdate = []; |
||
920 | } |
||
921 | |||
922 | /** |
||
923 | * Outputs the initial header scan results after yEnc check and blacklist routines. |
||
924 | */ |
||
925 | protected function outputHeaderInitial(): void |
||
926 | { |
||
927 | $this->colorCli->primary( |
||
928 | 'Received '.\count($this->headersReceived). |
||
929 | ' articles of '.number_format($this->last - $this->first + 1).' requested, '. |
||
930 | $this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.' |
||
931 | ); |
||
932 | } |
||
933 | |||
934 | /** |
||
935 | * Outputs speed metrics of the scan function to CLI. |
||
936 | */ |
||
937 | protected function outputHeaderDuration(): void |
||
938 | { |
||
939 | $currentMicroTime = now(); |
||
940 | if ($this->_echoCLI) { |
||
941 | $this->colorCli->alternateOver($this->timeHeaders.'s'). |
||
942 | $this->colorCli->primaryOver(' to download articles, '). |
||
943 | $this->colorCli->alternateOver($this->timeCleaning.'s'). |
||
944 | $this->colorCli->primaryOver(' to process collections, '). |
||
945 | $this->colorCli->alternateOver($this->timeInsert.'s'). |
||
946 | $this->colorCli->primaryOver(' to insert binaries/parts, '). |
||
947 | $this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startPR).'s'). |
||
948 | $this->colorCli->primaryOver(' for part repair, '). |
||
949 | $this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startLoop).'s'). |
||
950 | $this->colorCli->primary(' total.'); |
||
951 | } |
||
952 | } |
||
953 | |||
954 | /** |
||
955 | * Attempt to get missing article headers. |
||
956 | * |
||
957 | * @param array $groupArr The info for this group from mysql. |
||
958 | * |
||
959 | * @throws \Exception |
||
960 | * @throws \Throwable |
||
961 | */ |
||
962 | public function partRepair(array $groupArr): void |
||
963 | { |
||
964 | // Get all parts in part repair table. |
||
965 | $missingParts = []; |
||
966 | try { |
||
967 | $missingParts = DB::select(sprintf(' |
||
968 | SELECT * FROM missed_parts |
||
969 | WHERE groups_id = %d AND attempts < %d |
||
970 | ORDER BY numberid ASC LIMIT %d', $groupArr['id'], $this->_partRepairMaxTries, $this->_partRepairLimit)); |
||
971 | } catch (\PDOException $e) { |
||
972 | if ($e->getMessage() === 'SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction') { |
||
973 | $this->colorCli->notice('Deadlock occurred'); |
||
974 | DB::rollBack(); |
||
975 | } |
||
976 | } |
||
977 | |||
978 | $missingCount = \count($missingParts); |
||
979 | if ($missingCount > 0) { |
||
980 | if ($this->_echoCLI) { |
||
981 | $this->colorCli->primary( |
||
982 | 'Attempting to repair '. |
||
983 | number_format($missingCount). |
||
984 | ' parts.' |
||
985 | ); |
||
986 | } |
||
987 | |||
988 | // Loop through each part to group into continuous ranges with a maximum range of messagebuffer/4. |
||
989 | $ranges = $partList = []; |
||
990 | $firstPart = $lastNum = $missingParts[0]->numberid; |
||
991 | |||
992 | foreach ($missingParts as $part) { |
||
993 | if (($part->numberid - $firstPart) > ($this->messageBuffer / 4)) { |
||
994 | $ranges[] = [ |
||
995 | 'partfrom' => $firstPart, |
||
996 | 'partto' => $lastNum, |
||
997 | 'partlist' => $partList, |
||
998 | ]; |
||
999 | |||
1000 | $firstPart = $part->numberid; |
||
1001 | $partList = []; |
||
1002 | } |
||
1003 | $partList[] = $part->numberid; |
||
1004 | $lastNum = $part->numberid; |
||
1005 | } |
||
1006 | |||
1007 | $ranges[] = [ |
||
1008 | 'partfrom' => $firstPart, |
||
1009 | 'partto' => $lastNum, |
||
1010 | 'partlist' => $partList, |
||
1011 | ]; |
||
1012 | |||
1013 | // Download missing parts in ranges. |
||
1014 | foreach ($ranges as $range) { |
||
1015 | $partFrom = $range['partfrom']; |
||
1016 | $partTo = $range['partto']; |
||
1017 | $partList = $range['partlist']; |
||
1018 | |||
1019 | if ($this->_echoCLI) { |
||
1020 | echo \chr(random_int(45, 46)).PHP_EOL; |
||
1021 | } |
||
1022 | |||
1023 | // Get article headers from newsgroup. |
||
1024 | $this->scan($groupArr, $partFrom, $partTo, 'partrepair', $partList); |
||
1025 | } |
||
1026 | |||
1027 | // Calculate parts repaired |
||
1028 | $result = DB::select( |
||
1029 | sprintf( |
||
1030 | ' |
||
1031 | SELECT COUNT(id) AS num |
||
1032 | FROM missed_parts |
||
1033 | WHERE groups_id = %d |
||
1034 | AND numberid <= %d', |
||
1035 | $groupArr['id'], |
||
1036 | $missingParts[$missingCount - 1]->numberid |
||
1037 | ) |
||
1038 | ); |
||
1039 | |||
1040 | $partsRepaired = 0; |
||
1041 | if ($result > 0) { |
||
1042 | $partsRepaired = ($missingCount - $result[0]->num); |
||
1043 | } |
||
1044 | |||
1045 | // Update attempts on remaining parts for active group |
||
1046 | if (isset($missingParts[$missingCount - 1]->id)) { |
||
1047 | DB::update( |
||
1048 | sprintf( |
||
1049 | ' |
||
1050 | UPDATE missed_parts |
||
1051 | SET attempts = attempts + 1 |
||
1052 | WHERE groups_id = %d |
||
1053 | AND numberid <= %d', |
||
1054 | $groupArr['id'], |
||
1055 | $missingParts[$missingCount - 1]->numberid |
||
1056 | ) |
||
1057 | ); |
||
1058 | } |
||
1059 | |||
1060 | if ($this->_echoCLI) { |
||
1061 | $this->colorCli->primary( |
||
1062 | PHP_EOL. |
||
1063 | number_format($partsRepaired). |
||
1064 | ' parts repaired.' |
||
1065 | ); |
||
1066 | } |
||
1067 | } |
||
1068 | |||
1069 | // Remove articles that we cant fetch after x attempts. |
||
1070 | DB::transaction(function () use ($groupArr) { |
||
1071 | DB::delete( |
||
1072 | sprintf( |
||
1073 | 'DELETE FROM missed_parts WHERE attempts >= %d AND groups_id = %d', |
||
1074 | $this->_partRepairMaxTries, |
||
1075 | $groupArr['id'] |
||
1076 | ) |
||
1077 | ); |
||
1078 | }, 10); |
||
1079 | } |
||
1080 | |||
1081 | /** |
||
1082 | * Returns unix time for an article number. |
||
1083 | * |
||
1084 | * @param int $post The article number to get the time from. |
||
1085 | * @param array $groupData Usenet group info from NNTP selectGroup method. |
||
1086 | * @return int Timestamp. |
||
1087 | * |
||
1088 | * @throws \Exception |
||
1089 | */ |
||
1090 | public function postdate(int $post, array $groupData): int |
||
1091 | { |
||
1092 | $currentPost = $post; |
||
1093 | |||
1094 | $attempts = $date = 0; |
||
1095 | do { |
||
1096 | // Try to get the article date locally first. |
||
1097 | // Try to get locally. |
||
1098 | $local = DB::select( |
||
1099 | sprintf( |
||
1100 | ' |
||
1101 | SELECT c.date AS date |
||
1102 | FROM collections c |
||
1103 | INNER JOIN binaries b ON(c.id=b.collections_id) |
||
1104 | INNER JOIN parts p ON(b.id=p.binaries_id) |
||
1105 | WHERE p.number = %s', |
||
1106 | $currentPost |
||
1107 | ) |
||
1108 | ); |
||
1109 | if (! empty($local) && \count($local) > 0) { |
||
1110 | $date = $local[0]->date; |
||
1111 | break; |
||
1112 | } |
||
1113 | |||
1114 | // If we could not find it locally, try usenet. |
||
1115 | $header = $this->_nntp->getXOVER($currentPost); |
||
1116 | if (! $this->_nntp::isError($header) && isset($header[0]['Date']) && $header[0]['Date'] !== '') { |
||
1117 | $date = $header[0]['Date']; |
||
1118 | break; |
||
1119 | } |
||
1120 | |||
1121 | // Try to get a different article number. |
||
1122 | if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) { |
||
1123 | $tempPost = round($currentPost / (random_int(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP); |
||
1124 | if ($tempPost < $groupData['first']) { |
||
1125 | $tempPost = $groupData['first']; |
||
1126 | } |
||
1127 | } else { |
||
1128 | $tempPost = round((random_int(1005, 1012) / 1000) * $currentPost, 0, PHP_ROUND_HALF_UP); |
||
1129 | if ($tempPost > $groupData['last']) { |
||
1130 | $tempPost = $groupData['last']; |
||
1131 | } |
||
1132 | } |
||
1133 | // If we got the same article number as last time, give up. |
||
1134 | if ($tempPost === $currentPost) { |
||
1135 | break; |
||
1136 | } |
||
1137 | $currentPost = $tempPost; |
||
1138 | } while ($attempts++ <= 20); |
||
1139 | |||
1140 | // If we didn't get a date, set it to now. |
||
1141 | if (! $date) { |
||
1142 | $date = time(); |
||
1143 | } else { |
||
1144 | $date = strtotime($date); |
||
1145 | } |
||
1146 | |||
1147 | return $date; |
||
1148 | } |
||
1149 | |||
1150 | /** |
||
1151 | * Returns article number based on # of days. |
||
1152 | * |
||
1153 | * @param int $days How many days back we want to go. |
||
1154 | * @param array $data Group data from usenet. |
||
1155 | * |
||
1156 | * @throws \Exception |
||
1157 | */ |
||
1158 | public function daytopost(int $days, array $data): string |
||
1159 | { |
||
1160 | $goalTime = now()->subDays($days)->timestamp; |
||
1161 | // The time we want = current unix time (ex. 1395699114) - minus 86400 (seconds in a day) |
||
1162 | // times days wanted. (ie 1395699114 - 2592000 (30days)) = 1393107114 |
||
1163 | |||
1164 | // The servers oldest date. |
||
1165 | $firstDate = $this->postdate($data['first'], $data); |
||
1166 | if ($goalTime < $firstDate) { |
||
1167 | // If the date we want is older than the oldest date in the group return the groups oldest article. |
||
1168 | return $data['first']; |
||
1169 | } |
||
1170 | |||
1171 | // The servers newest date. |
||
1172 | $lastDate = $this->postdate($data['last'], $data); |
||
1173 | if ($goalTime > $lastDate) { |
||
1174 | // If the date we want is newer than the groups newest date, return the groups newest article. |
||
1175 | return $data['last']; |
||
1176 | } |
||
1177 | |||
1178 | if ($this->_echoCLI) { |
||
1179 | $this->colorCli->primary( |
||
1180 | 'Searching for an approximate article number for group '.$data['group'].' '.$days.' days back.' |
||
1181 | ); |
||
1182 | } |
||
1183 | |||
1184 | // Pick the middle to start with |
||
1185 | $wantedArticle = round(($data['last'] + $data['first']) / 2); |
||
1186 | $aMax = $data['last']; |
||
1187 | $aMin = $data['first']; |
||
1188 | $oldArticle = $articleTime = null; |
||
1189 | |||
1190 | while (true) { |
||
1191 | // Article exists outside available range, this shouldn't happen |
||
1192 | if ($wantedArticle <= $data['first'] || $wantedArticle >= $data['last']) { |
||
1193 | break; |
||
1194 | } |
||
1195 | |||
1196 | // Keep a note of the last articles we checked |
||
1197 | $reallyOldArticle = $oldArticle; |
||
1198 | $oldArticle = $wantedArticle; |
||
1199 | |||
1200 | // Get the date of this article |
||
1201 | $articleTime = $this->postdate($wantedArticle, $data); |
||
1202 | |||
1203 | // Article doesn't exist, start again with something random |
||
1204 | if (! $articleTime) { |
||
1205 | $wantedArticle = random_int($aMin, $aMax); |
||
1206 | $articleTime = $this->postdate($wantedArticle, $data); |
||
1207 | } |
||
1208 | |||
1209 | if ($articleTime < $goalTime) { |
||
1210 | // Article is older than we want |
||
1211 | $aMin = $oldArticle; |
||
1212 | $wantedArticle = round(($aMax + $oldArticle) / 2); |
||
1213 | if ($this->_echoCLI) { |
||
1214 | echo '-'; |
||
1215 | } |
||
1216 | } elseif ($articleTime > $goalTime) { |
||
1217 | // Article is newer than we want |
||
1218 | $aMax = $oldArticle; |
||
1219 | $wantedArticle = round(($aMin + $oldArticle) / 2); |
||
1220 | if ($this->_echoCLI) { |
||
1221 | echo '+'; |
||
1222 | } |
||
1223 | } elseif ($articleTime === $goalTime) { |
||
1224 | // Exact match. We did it! (this will likely never happen though) |
||
1225 | break; |
||
1226 | } |
||
1227 | |||
1228 | // We seem to be flip-flopping between 2 articles, assume we're out of articles to check. |
||
1229 | // End on an article more recent than our oldest so that we don't miss any releases. |
||
1230 | if ($reallyOldArticle === $wantedArticle && ($goalTime - $articleTime) <= 0) { |
||
1231 | break; |
||
1232 | } |
||
1233 | } |
||
1234 | |||
1235 | $wantedArticle = (int) $wantedArticle; |
||
1236 | if ($this->_echoCLI) { |
||
1237 | $this->colorCli->primary( |
||
1238 | PHP_EOL.'Found article #'.$wantedArticle.' which has a date of '.date('r', $articleTime). |
||
1239 | ', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.Carbon::createFromTimestamp($goalTime)->diffInDays(Carbon::createFromTimestamp($articleTime)).'days.' |
||
1240 | ); |
||
1241 | } |
||
1242 | |||
1243 | return $wantedArticle; |
||
1244 | } |
||
1245 | |||
1246 | /** |
||
1247 | * Add article numbers from missing headers to DB. |
||
1248 | * |
||
1249 | * @param array $numbers The article numbers of the missing headers. |
||
1250 | * @param int $groupID The ID of this groups. |
||
1251 | */ |
||
1252 | private function addMissingParts(array $numbers, int $groupID): void |
||
1253 | { |
||
1254 | $insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES '; |
||
1255 | foreach ($numbers as $number) { |
||
1256 | $insertStr .= '('.$number.','.$groupID.'),'; |
||
1257 | } |
||
1258 | |||
1259 | DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1'); |
||
1260 | } |
||
1261 | |||
1262 | /** |
||
1263 | * Clean up part repair table. |
||
1264 | * |
||
1265 | * @param array $numbers The article numbers. |
||
1266 | * @param int $groupID The ID of the group. |
||
1267 | * |
||
1268 | * @throws \Throwable |
||
1269 | */ |
||
1270 | private function removeRepairedParts(array $numbers, int $groupID): void |
||
1271 | { |
||
1272 | $sql = 'DELETE FROM missed_parts WHERE numberid in ('; |
||
1273 | foreach ($numbers as $number) { |
||
1274 | $sql .= $number.','; |
||
1275 | } |
||
1276 | DB::transaction(static function () use ($groupID, $sql) { |
||
1277 | DB::delete(rtrim($sql, ',').') AND groups_id = '.$groupID); |
||
1278 | }, 10); |
||
1279 | } |
||
1280 | |||
1281 | /** |
||
1282 | * Are white or black lists loaded for a group name? |
||
1283 | */ |
||
1284 | protected array $_listsFound = []; |
||
1285 | |||
1286 | /** |
||
1287 | * Get blacklist and cache it. Return if already cached. |
||
1288 | */ |
||
1289 | protected function _retrieveBlackList(string $groupName): void |
||
1290 | { |
||
1291 | if (! isset($this->blackList[$groupName])) { |
||
1292 | $this->blackList[$groupName] = $this->getBlacklist(true, self::OPTYPE_BLACKLIST, $groupName, true); |
||
1293 | } |
||
1294 | if (! isset($this->whiteList[$groupName])) { |
||
1295 | $this->whiteList[$groupName] = $this->getBlacklist(true, self::OPTYPE_WHITELIST, $groupName, true); |
||
1296 | } |
||
1297 | $this->_listsFound[$groupName] = ($this->blackList[$groupName] || $this->whiteList[$groupName]); |
||
1298 | } |
||
1299 | |||
1300 | /** |
||
1301 | * Check if an article is blacklisted. |
||
1302 | * |
||
1303 | * @param array $msg The article header (OVER format). |
||
1304 | * @param string $groupName The group name. |
||
1305 | */ |
||
1306 | public function isBlackListed(array $msg, string $groupName): bool |
||
1307 | { |
||
1308 | if (! isset($this->_listsFound[$groupName])) { |
||
1309 | $this->_retrieveBlackList($groupName); |
||
1310 | } |
||
1311 | if (! $this->_listsFound[$groupName]) { |
||
1312 | return false; |
||
1313 | } |
||
1314 | |||
1315 | $blackListed = false; |
||
1316 | |||
1317 | $field = [ |
||
1318 | self::BLACKLIST_FIELD_SUBJECT => $msg['Subject'], |
||
1319 | self::BLACKLIST_FIELD_FROM => $msg['From'], |
||
1320 | self::BLACKLIST_FIELD_MESSAGEID => $msg['Message-ID'], |
||
1321 | ]; |
||
1322 | |||
1323 | // Try white lists first. |
||
1324 | if ($this->whiteList[$groupName]) { |
||
1325 | // There are white lists for this group, so anything that doesn't match a white list should be considered black listed. |
||
1326 | $blackListed = true; |
||
1327 | foreach ($this->whiteList[$groupName] as $whiteList) { |
||
1328 | if (preg_match('/'.$whiteList['regex'].'/i', $field[$whiteList['msgcol']])) { |
||
1329 | // This field matched a white list, so it might not be black listed. |
||
1330 | $blackListed = false; |
||
1331 | $this->_binaryBlacklistIdsToUpdate[$whiteList['id']] = $whiteList['id']; |
||
1332 | break; |
||
1333 | } |
||
1334 | } |
||
1335 | } |
||
1336 | |||
1337 | // Check if the field is blacklisted. |
||
1338 | |||
1339 | if (! $blackListed && $this->blackList[$groupName]) { |
||
1340 | foreach ($this->blackList[$groupName] as $blackList) { |
||
1341 | if (preg_match('/'.$blackList->regex.'/i', $field[$blackList->msgcol])) { |
||
1342 | $blackListed = true; |
||
1343 | $this->_binaryBlacklistIdsToUpdate[$blackList->id] = $blackList->id; |
||
1344 | break; |
||
1345 | } |
||
1346 | } |
||
1347 | } |
||
1348 | |||
1349 | return $blackListed; |
||
1350 | } |
||
1351 | |||
1352 | /** |
||
1353 | * Return all blacklists. |
||
1354 | * |
||
1355 | * @param bool $activeOnly Only display active blacklists ? |
||
1356 | * @param int|string $opType Optional, get white or black lists (use Binaries constants). |
||
1357 | * @param string $groupName Optional, group. |
||
1358 | * @param bool $groupRegex Optional Join groups / binary blacklist using regexp for equals. |
||
1359 | */ |
||
1360 | public function getBlacklist(bool $activeOnly = true, int|string $opType = -1, string $groupName = '', bool $groupRegex = false): array |
||
1361 | { |
||
1362 | $opType = match ($opType) { |
||
1363 | self::OPTYPE_BLACKLIST => 'AND bb.optype = '.self::OPTYPE_BLACKLIST, |
||
1364 | self::OPTYPE_WHITELIST => 'AND bb.optype = '.self::OPTYPE_WHITELIST, |
||
1365 | default => '', |
||
1366 | }; |
||
1367 | |||
1368 | return DB::select( |
||
1369 | sprintf( |
||
1370 | ' |
||
1371 | SELECT |
||
1372 | bb.id, bb.optype, bb.status, bb.description, |
||
1373 | bb.groupname AS groupname, bb.regex, g.id AS group_id, bb.msgcol, |
||
1374 | bb.last_activity as last_activity |
||
1375 | FROM binaryblacklist bb |
||
1376 | LEFT OUTER JOIN usenet_groups g ON g.name %s bb.groupname |
||
1377 | WHERE 1=1 %s %s %s |
||
1378 | ORDER BY coalesce(groupname,\'zzz\')', |
||
1379 | ($groupRegex ? 'REGEXP' : '='), |
||
1380 | ($activeOnly ? 'AND bb.status = 1' : ''), |
||
1381 | $opType, |
||
1382 | ($groupName ? ('AND g.name REGEXP '.escapeString($groupName)) : '') |
||
1383 | ) |
||
1384 | ); |
||
1385 | } |
||
1386 | |||
1387 | /** |
||
1388 | * Return a blacklist by ID. |
||
1389 | * |
||
1390 | * @param int $id The ID of the blacklist. |
||
1391 | */ |
||
1392 | public function getBlacklistByID(int $id) |
||
1395 | } |
||
1396 | |||
1397 | /** |
||
1398 | * Delete a blacklist. |
||
1399 | * |
||
1400 | * @param int $id The ID of the blacklist. |
||
1401 | */ |
||
1402 | public function deleteBlacklist(int $id): void |
||
1403 | { |
||
1404 | BinaryBlacklist::query()->where('id', $id)->delete(); |
||
1405 | } |
||
1406 | |||
1407 | public function updateBlacklist(array $blacklistArray): void |
||
1408 | { |
||
1409 | BinaryBlacklist::query()->where('id', $blacklistArray['id'])->update( |
||
1410 | [ |
||
1411 | 'groupname' => $blacklistArray['groupname'] === '' ? 'null' : preg_replace('/a\.b\./i', 'alt.binaries.', $blacklistArray['groupname']), |
||
1412 | 'regex' => $blacklistArray['regex'], |
||
1413 | 'status' => $blacklistArray['status'], |
||
1414 | 'description' => $blacklistArray['description'], |
||
1415 | 'optype' => $blacklistArray['optype'], |
||
1416 | 'msgcol' => $blacklistArray['msgcol'], |
||
1417 | ] |
||
1418 | ); |
||
1419 | } |
||
1420 | |||
1421 | /** |
||
1422 | * Adds a new blacklist from binary blacklist edit admin web page. |
||
1423 | */ |
||
1424 | public function addBlacklist(array $blacklistArray): void |
||
1425 | { |
||
1426 | BinaryBlacklist::query()->insert( |
||
1427 | [ |
||
1428 | 'groupname' => $blacklistArray['groupname'] === '' ? 'null' : preg_replace('/a\.b\./i', 'alt.binaries.', $blacklistArray['groupname']), |
||
1429 | 'regex' => $blacklistArray['regex'], |
||
1430 | 'status' => $blacklistArray['status'], |
||
1431 | 'description' => $blacklistArray['description'], |
||
1432 | 'optype' => $blacklistArray['optype'], |
||
1433 | 'msgcol' => $blacklistArray['msgcol'], |
||
1434 | ] |
||
1435 | ); |
||
1436 | } |
||
1437 | |||
1438 | /** |
||
1439 | * Delete Collections/Binaries/Parts for a Collection ID. |
||
1440 | * |
||
1441 | * @param int $collectionID Collections table ID |
||
1442 | * |
||
1443 | * @note A trigger automatically deletes the parts/binaries. |
||
1444 | * |
||
1445 | * @throws \Throwable |
||
1446 | */ |
||
1447 | public function delete(int $collectionID): void |
||
1448 | { |
||
1449 | DB::transaction(static function () use ($collectionID) { |
||
1450 | DB::delete(sprintf('DELETE FROM collections WHERE id = %d', $collectionID)); |
||
1451 | }, 10); |
||
1452 | |||
1453 | Collection::query()->where('id', $collectionID)->delete(); |
||
1454 | } |
||
1455 | |||
1456 | /** |
||
1457 | * Log / Echo message. |
||
1458 | * |
||
1459 | * @param string $message Message to log. |
||
1460 | * @param string $method Method that called this. |
||
1461 | * @param string $color ColorCLI method name. |
||
1462 | */ |
||
1463 | private function log(string $message, string $method, string $color): void |
||
1464 | { |
||
1465 | if ($this->_echoCLI) { |
||
1466 | $this->colorCli->$color($message.' ['.__CLASS__."::$method]"); |
||
1467 | } |
||
1468 | } |
||
1469 | |||
1470 | protected function runQuery($query): bool |
||
1471 | { |
||
1472 | try { |
||
1473 | return DB::insert($query); |
||
1474 | } catch (QueryException $e) { |
||
1475 | if (config('app.debug') === true) { |
||
1476 | Log::error($e->getMessage()); |
||
1477 | } |
||
1478 | $this->colorCli->debug('Query error occurred.'); |
||
1479 | } catch (\PDOException $e) { |
||
1480 | if (config('app.debug') === true) { |
||
1481 | Log::error($e->getMessage()); |
||
1482 | } |
||
1483 | $this->colorCli->debug('Query error occurred.'); |
||
1484 | } catch (\Throwable $e) { |
||
1485 | if (config('app.debug') === true) { |
||
1486 | Log::error($e->getMessage()); |
||
1487 | } |
||
1488 | $this->colorCli->debug('Query error occurred.'); |
||
1489 | } |
||
1490 | |||
1491 | return false; |
||
1492 | } |
||
1493 | |||
1494 | private function getFileCount($subject): array |
||
1501 | } |
||
1502 | } |
||
1503 |