NNTmux /
newznab-tmux
| Total Complexity | 128 |
| Total Lines | 877 |
| Duplicated Lines | 0 % |
| Changes | 0 | ||
Complex classes like BinariesService often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use BinariesService, and based on these observations, apply Extract Interface, too.
| 1 | <?php |
||
| 25 | class BinariesService |
||
| 26 | { |
||
| 27 | private BinariesConfig $config; |
||
|
|
|||
| 28 | |||
| 29 | private HeaderParser $headerParser; |
||
| 30 | |||
| 31 | private HeaderStorageService $headerStorage; |
||
| 32 | |||
| 33 | private MissedPartHandler $missedPartHandler; |
||
| 34 | |||
| 35 | private ColorCLI $colorCli; |
||
| 36 | |||
| 37 | private ?NNTP $nntp = null; |
||
| 38 | |||
| 39 | // Timing metrics |
||
| 40 | private float $timeHeaders = 0; |
||
| 41 | |||
| 42 | private float $timeCleaning = 0; |
||
| 43 | |||
| 44 | private float $timeInsert = 0; |
||
| 45 | |||
| 46 | private \DateTime $startLoop; |
||
| 47 | |||
| 48 | private \DateTime $startCleaning; |
||
| 49 | |||
| 50 | private \DateTime $startPR; |
||
| 51 | |||
| 52 | private \DateTime $startUpdate; |
||
| 53 | |||
| 54 | // Scan state |
||
| 55 | private array $groupMySQL = []; |
||
| 56 | |||
| 57 | private int $first = 0; |
||
| 58 | |||
| 59 | private int $last = 0; |
||
| 60 | |||
| 61 | private int $notYEnc = 0; |
||
| 62 | |||
| 63 | private int $headersBlackListed = 0; |
||
| 64 | |||
| 65 | private array $headersReceived = []; |
||
| 66 | |||
| 67 | public function __construct( |
||
| 68 | ?BinariesConfig $config = null, |
||
| 69 | ?HeaderParser $headerParser = null, |
||
| 70 | ?HeaderStorageService $headerStorage = null, |
||
| 71 | ?MissedPartHandler $missedPartHandler = null, |
||
| 72 | ?ColorCLI $colorCli = null, |
||
| 73 | ?NNTP $nntp = null |
||
| 74 | ) { |
||
| 75 | $this->config = $config ?? BinariesConfig::fromSettings(); |
||
| 76 | $this->headerParser = $headerParser ?? new HeaderParser; |
||
| 77 | $this->headerStorage = $headerStorage ?? new HeaderStorageService(config: $this->config); |
||
| 78 | $this->missedPartHandler = $missedPartHandler ?? new MissedPartHandler( |
||
| 79 | $this->config->partRepairLimit, |
||
| 80 | $this->config->partRepairMaxTries |
||
| 81 | ); |
||
| 82 | $this->colorCli = $colorCli ?? new ColorCLI; |
||
| 83 | $this->nntp = $nntp; |
||
| 84 | $this->startUpdate = now(); |
||
| 85 | } |
||
| 86 | |||
| 87 | /** |
||
| 88 | * Set NNTP connection (for external injection). |
||
| 89 | */ |
||
| 90 | public function setNntp(NNTP $nntp): void |
||
| 91 | { |
||
| 92 | $this->nntp = $nntp; |
||
| 93 | } |
||
| 94 | |||
| 95 | /** |
||
| 96 | * Get the NNTP connection, creating one if needed. |
||
| 97 | */ |
||
| 98 | public function getNntp(): NNTP |
||
| 99 | { |
||
| 100 | if ($this->nntp === null) { |
||
| 101 | $this->nntp = new NNTP; |
||
| 102 | } |
||
| 103 | |||
| 104 | return $this->nntp; |
||
| 105 | } |
||
| 106 | |||
| 107 | /** |
||
| 108 | * Get the configuration object. |
||
| 109 | */ |
||
| 110 | public function getConfig(): BinariesConfig |
||
| 111 | { |
||
| 112 | return $this->config; |
||
| 113 | } |
||
| 114 | |||
| 115 | /** |
||
| 116 | * Get the message buffer size. |
||
| 117 | */ |
||
| 118 | public function getMessageBuffer(): int |
||
| 119 | { |
||
| 120 | return $this->config->messageBuffer; |
||
| 121 | } |
||
| 122 | |||
| 123 | /** |
||
| 124 | * Download new headers for all active groups. |
||
| 125 | * |
||
| 126 | * @param int $maxHeaders (Optional) How many headers to download max. |
||
| 127 | * |
||
| 128 | * @throws \Exception |
||
| 129 | * @throws \Throwable |
||
| 130 | */ |
||
| 131 | public function updateAllGroups(int $maxHeaders = 100000): void |
||
| 132 | { |
||
| 133 | $groups = UsenetGroup::getActive()->toArray(); |
||
| 134 | $groupCount = \count($groups); |
||
| 135 | |||
| 136 | if ($groupCount === 0) { |
||
| 137 | $this->log( |
||
| 138 | 'No groups specified. Ensure groups are added to NNTmux\'s database for updating.', |
||
| 139 | __FUNCTION__, |
||
| 140 | 'warning' |
||
| 141 | ); |
||
| 142 | |||
| 143 | return; |
||
| 144 | } |
||
| 145 | |||
| 146 | $counter = 1; |
||
| 147 | $allTime = now(); |
||
| 148 | |||
| 149 | $this->log( |
||
| 150 | 'Updating: '.$groupCount.' group(s) - Using compression? '.($this->config->compressedHeaders ? 'Yes' : 'No'), |
||
| 151 | __FUNCTION__, |
||
| 152 | 'header' |
||
| 153 | ); |
||
| 154 | |||
| 155 | foreach ($groups as $group) { |
||
| 156 | $this->log( |
||
| 157 | 'Starting group '.$counter.' of '.$groupCount, |
||
| 158 | __FUNCTION__, |
||
| 159 | 'header' |
||
| 160 | ); |
||
| 161 | |||
| 162 | try { |
||
| 163 | $this->updateGroup($group, $maxHeaders); |
||
| 164 | } catch (\Throwable $e) { |
||
| 165 | $this->logError('Error updating group '.$group['name'].': '.$e->getMessage()); |
||
| 166 | } |
||
| 167 | |||
| 168 | $counter++; |
||
| 169 | } |
||
| 170 | |||
| 171 | $endTime = now()->diffInSeconds($allTime, true); |
||
| 172 | $this->log( |
||
| 173 | 'Updating completed in '.$endTime.Str::plural(' second', $endTime), |
||
| 174 | __FUNCTION__, |
||
| 175 | 'primary' |
||
| 176 | ); |
||
| 177 | } |
||
| 178 | |||
| 179 | /** |
||
| 180 | * Log the indexer start time. |
||
| 181 | */ |
||
| 182 | public function logIndexerStart(): void |
||
| 183 | { |
||
| 184 | Settings::query()->where('name', '=', 'last_run_time')->update(['value' => now()]); |
||
| 185 | } |
||
| 186 | |||
| 187 | /** |
||
| 188 | * Download new headers for a single group. |
||
| 189 | * |
||
| 190 | * @param array $groupMySQL Array of MySQL results for a single group. |
||
| 191 | * @param int $maxHeaders (Optional) How many headers to download max. |
||
| 192 | * |
||
| 193 | * @throws \Exception |
||
| 194 | * @throws \Throwable |
||
| 195 | */ |
||
| 196 | public function updateGroup(array $groupMySQL, int $maxHeaders = 0): void |
||
| 197 | { |
||
| 198 | $startGroup = now(); |
||
| 199 | $this->logIndexerStart(); |
||
| 200 | |||
| 201 | $nntp = $this->getNntp(); |
||
| 202 | |||
| 203 | // Select the group on the NNTP server |
||
| 204 | $groupNNTP = $this->selectNntpGroup($groupMySQL, $nntp); |
||
| 205 | if ($groupNNTP === null) { |
||
| 206 | return; |
||
| 207 | } |
||
| 208 | |||
| 209 | if ($this->config->echoCli) { |
||
| 210 | $this->colorCli->primary('Processing '.$groupMySQL['name']); |
||
| 211 | } |
||
| 212 | |||
| 213 | // Attempt to repair any missing parts before grabbing new ones |
||
| 214 | if ((int) $groupMySQL['last_record'] !== 0 && $this->config->partRepair) { |
||
| 215 | if ($this->config->echoCli) { |
||
| 216 | $this->colorCli->primary('Part repair enabled. Checking for missing parts.'); |
||
| 217 | } |
||
| 218 | $this->partRepair($groupMySQL); |
||
| 219 | } elseif ($this->config->echoCli && (int) $groupMySQL['last_record'] !== 0) { |
||
| 220 | $this->colorCli->primary('Part repair disabled by user.'); |
||
| 221 | } |
||
| 222 | |||
| 223 | // Generate postdate for first record, for those that upgraded |
||
| 224 | if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] !== 0) { |
||
| 225 | $groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); |
||
| 226 | UsenetGroup::query()->where('id', $groupMySQL['id'])->update([ |
||
| 227 | 'first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'], date_default_timezone_get()), |
||
| 228 | ]); |
||
| 229 | } |
||
| 230 | |||
| 231 | // Calculate article range |
||
| 232 | $range = $this->calculateArticleRange($groupMySQL, $groupNNTP, $maxHeaders); |
||
| 233 | |||
| 234 | if ($range['total'] <= 0) { |
||
| 235 | $this->outputNoNewArticles($groupMySQL, $groupNNTP, $range); |
||
| 236 | |||
| 237 | return; |
||
| 238 | } |
||
| 239 | |||
| 240 | $this->outputNewArticlesInfo($groupMySQL, $groupNNTP, $range); |
||
| 241 | $this->processArticleRange($groupMySQL, $groupNNTP, $range); |
||
| 242 | |||
| 243 | if ($this->config->echoCli) { |
||
| 244 | $endGroup = now()->diffInSeconds($startGroup, true); |
||
| 245 | $this->colorCli->primary( |
||
| 246 | PHP_EOL.'Group '.$groupMySQL['name'].' processed in '.$endGroup.Str::plural(' second', $endGroup) |
||
| 247 | ); |
||
| 248 | } |
||
| 249 | } |
||
| 250 | |||
| 251 | /** |
||
| 252 | * Loop over range of wanted headers, insert headers into DB. |
||
| 253 | * |
||
| 254 | * @param array $groupMySQL The group info from mysql. |
||
| 255 | * @param int $first The oldest wanted header. |
||
| 256 | * @param int $last The newest wanted header. |
||
| 257 | * @param string $type Is this part repair or update or backfill? |
||
| 258 | * @param array|null $missingParts If we are running in part repair, the list of missing article numbers. |
||
| 259 | * @return array Empty on failure. |
||
| 260 | * |
||
| 261 | * @throws \Exception |
||
| 262 | * @throws \Throwable |
||
| 263 | */ |
||
| 264 | public function scan(array $groupMySQL, int $first, int $last, string $type = 'update', ?array $missingParts = null): array |
||
| 265 | { |
||
| 266 | $this->startLoop = now(); |
||
| 267 | $this->groupMySQL = $groupMySQL; |
||
| 268 | $this->last = $last; |
||
| 269 | $this->first = $first; |
||
| 270 | $this->notYEnc = $this->headersBlackListed = 0; |
||
| 271 | $this->headersReceived = []; |
||
| 272 | |||
| 273 | $returnArray = []; |
||
| 274 | $partRepair = ($type === 'partrepair'); |
||
| 275 | $addToPartRepair = ($type === 'update' && $this->config->partRepair); |
||
| 276 | |||
| 277 | // Download headers from NNTP |
||
| 278 | $headers = $this->downloadHeaders($partRepair); |
||
| 279 | if ($headers === null) { |
||
| 280 | if ($partRepair) { |
||
| 281 | $this->missedPartHandler->incrementRangeAttempts($groupMySQL['id'], $first, $last); |
||
| 282 | } |
||
| 283 | |||
| 284 | return $returnArray; |
||
| 285 | } |
||
| 286 | |||
| 287 | $this->startCleaning = now(); |
||
| 288 | $this->timeHeaders = $this->startCleaning->diffInSeconds($this->startLoop, true); |
||
| 289 | |||
| 290 | $msgCount = \count($headers); |
||
| 291 | if ($msgCount < 1) { |
||
| 292 | return $returnArray; |
||
| 293 | } |
||
| 294 | |||
| 295 | // Extract article range info |
||
| 296 | $returnArray = $this->headerParser->getArticleRange($headers); |
||
| 297 | |||
| 298 | // Parse and filter headers |
||
| 299 | $this->headerParser->reset(); |
||
| 300 | $parseResult = $this->headerParser->parse($headers, $groupMySQL['name'], $partRepair, $missingParts); |
||
| 301 | |||
| 302 | $this->headersReceived = array_column($headers, 'Number'); |
||
| 303 | $this->headersReceived = array_filter($this->headersReceived); |
||
| 304 | $this->notYEnc = $parseResult['notYEnc']; |
||
| 305 | $this->headersBlackListed = $parseResult['blacklisted']; |
||
| 306 | |||
| 307 | // Update blacklist last_activity |
||
| 308 | $this->headerParser->flushBlacklistUpdates(); |
||
| 309 | |||
| 310 | unset($headers); |
||
| 311 | |||
| 312 | if ($this->config->echoCli && ! $partRepair) { |
||
| 313 | $this->outputHeaderInitial(); |
||
| 314 | } |
||
| 315 | |||
| 316 | // Store headers |
||
| 317 | $this->startUpdate = now(); // Reset before storage begins |
||
| 318 | $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning, true); |
||
| 319 | |||
| 320 | $headersNotInserted = []; |
||
| 321 | if (! empty($parseResult['headers'])) { |
||
| 322 | try { |
||
| 323 | $headersNotInserted = $this->headerStorage->store($parseResult['headers'], $groupMySQL, $addToPartRepair); |
||
| 324 | } catch (\Throwable $e) { |
||
| 325 | $this->logError('storeHeaders failed: '.$e->getMessage()); |
||
| 326 | } |
||
| 327 | } |
||
| 328 | |||
| 329 | $this->startPR = now(); |
||
| 330 | $this->timeInsert = $this->startPR->diffInSeconds($this->startUpdate, true); |
||
| 331 | |||
| 332 | // Handle repaired parts |
||
| 333 | if ($partRepair && ! empty($parseResult['repaired'])) { |
||
| 334 | $this->missedPartHandler->removeRepairedParts($parseResult['repaired'], $groupMySQL['id']); |
||
| 335 | } |
||
| 336 | |||
| 337 | // Handle part repair tracking |
||
| 338 | if ($addToPartRepair) { |
||
| 339 | $this->handlePartRepairTracking($headersNotInserted, $parseResult['headers']); |
||
| 340 | } |
||
| 341 | |||
| 342 | $this->outputHeaderDuration(); |
||
| 343 | |||
| 344 | return $returnArray; |
||
| 345 | } |
||
| 346 | |||
| 347 | /** |
||
| 348 | * Attempt to get missing article headers. |
||
| 349 | * |
||
| 350 | * @param array $groupArr The info for this group from mysql. |
||
| 351 | * |
||
| 352 | * @throws \Exception |
||
| 353 | * @throws \Throwable |
||
| 354 | */ |
||
| 355 | public function partRepair(array $groupArr): void |
||
| 356 | { |
||
| 357 | $missingParts = $this->missedPartHandler->getMissingParts($groupArr['id']); |
||
| 358 | $missingCount = \count($missingParts); |
||
| 359 | |||
| 360 | if ($missingCount === 0) { |
||
| 361 | $this->missedPartHandler->cleanupExhaustedParts($groupArr['id']); |
||
| 362 | |||
| 363 | return; |
||
| 364 | } |
||
| 365 | |||
| 366 | if ($this->config->echoCli) { |
||
| 367 | $this->colorCli->primary('Attempting to repair '.number_format($missingCount).' parts.'); |
||
| 368 | } |
||
| 369 | |||
| 370 | // Group into continuous ranges |
||
| 371 | $ranges = $this->groupMissingPartsIntoRanges($missingParts); |
||
| 372 | |||
| 373 | // Download missing parts in ranges |
||
| 374 | foreach ($ranges as $range) { |
||
| 375 | if ($this->config->echoCli) { |
||
| 376 | echo \chr(random_int(45, 46)).PHP_EOL; |
||
| 377 | } |
||
| 378 | |||
| 379 | $this->scan($groupArr, $range['partfrom'], $range['partto'], 'partrepair', $range['partlist']); |
||
| 380 | } |
||
| 381 | |||
| 382 | // Calculate parts repaired |
||
| 383 | $lastPartNumber = $missingParts[$missingCount - 1]->numberid; |
||
| 384 | $remainingCount = $this->missedPartHandler->getCount($groupArr['id'], $lastPartNumber); |
||
| 385 | $partsRepaired = $missingCount - $remainingCount; |
||
| 386 | |||
| 387 | // Update attempts on remaining parts |
||
| 388 | if (isset($missingParts[$missingCount - 1]->id)) { |
||
| 389 | $this->missedPartHandler->incrementAttempts($groupArr['id'], $lastPartNumber); |
||
| 390 | } |
||
| 391 | |||
| 392 | if ($this->config->echoCli) { |
||
| 393 | $this->colorCli->primary(PHP_EOL.number_format($partsRepaired).' parts repaired.'); |
||
| 394 | } |
||
| 395 | |||
| 396 | // Remove articles that exceeded max tries |
||
| 397 | $this->missedPartHandler->cleanupExhaustedParts($groupArr['id']); |
||
| 398 | } |
||
| 399 | |||
| 400 | /** |
||
| 401 | * Returns unix time for an article number. |
||
| 402 | * |
||
| 403 | * @param int|string $post The article number to get the time from. |
||
| 404 | * @param array $groupData Usenet group info from NNTP selectGroup method. |
||
| 405 | * @return int Timestamp. |
||
| 406 | * |
||
| 407 | * @throws \Exception |
||
| 408 | */ |
||
| 409 | public function postdate(int|string $post, array $groupData): int |
||
| 410 | { |
||
| 411 | $nntp = $this->getNntp(); |
||
| 412 | $currentPost = (int) $post; |
||
| 413 | $attempts = 0; |
||
| 414 | $date = 0; |
||
| 415 | |||
| 416 | do { |
||
| 417 | // Try to get the article date locally first |
||
| 418 | $local = DB::select( |
||
| 419 | sprintf( |
||
| 420 | 'SELECT c.date AS date FROM collections c |
||
| 421 | INNER JOIN binaries b ON(c.id=b.collections_id) |
||
| 422 | INNER JOIN parts p ON(b.id=p.binaries_id) |
||
| 423 | WHERE p.number = %s', |
||
| 424 | $currentPost |
||
| 425 | ) |
||
| 426 | ); |
||
| 427 | |||
| 428 | if (! empty($local)) { |
||
| 429 | $date = $local[0]->date; |
||
| 430 | break; |
||
| 431 | } |
||
| 432 | |||
| 433 | // Try usenet |
||
| 434 | $header = $nntp->getXOVER((string) $currentPost); |
||
| 435 | if (! NNTP::isError($header) && isset($header[0]['Date']) && $header[0]['Date'] !== '') { |
||
| 436 | $date = $header[0]['Date']; |
||
| 437 | break; |
||
| 438 | } |
||
| 439 | |||
| 440 | // Try a different article number |
||
| 441 | $currentPost = $this->getNextArticleToTry($currentPost, $groupData); |
||
| 442 | if ($currentPost === null) { |
||
| 443 | break; |
||
| 444 | } |
||
| 445 | } while ($attempts++ <= 20); |
||
| 446 | |||
| 447 | if (! $date) { |
||
| 448 | return time(); |
||
| 449 | } |
||
| 450 | |||
| 451 | return strtotime($date); |
||
| 452 | } |
||
| 453 | |||
| 454 | /** |
||
| 455 | * Returns article number based on # of days. |
||
| 456 | * |
||
| 457 | * @param int $days How many days back we want to go. |
||
| 458 | * @param array $data Group data from usenet. |
||
| 459 | * |
||
| 460 | * @throws \Exception |
||
| 461 | */ |
||
| 462 | public function daytopost(int $days, array $data): string |
||
| 463 | { |
||
| 464 | $goalTime = now()->subDays($days)->timestamp; |
||
| 465 | |||
| 466 | $firstDate = $this->postdate($data['first'], $data); |
||
| 467 | if ($goalTime < $firstDate) { |
||
| 468 | return $data['first']; |
||
| 469 | } |
||
| 470 | |||
| 471 | $lastDate = $this->postdate($data['last'], $data); |
||
| 472 | if ($goalTime > $lastDate) { |
||
| 473 | return $data['last']; |
||
| 474 | } |
||
| 475 | |||
| 476 | if ($this->config->echoCli) { |
||
| 477 | $this->colorCli->primary( |
||
| 478 | 'Searching for an approximate article number for group '.$data['group'].' '.$days.' days back.' |
||
| 479 | ); |
||
| 480 | } |
||
| 481 | |||
| 482 | return $this->binarySearchArticleByDate($goalTime, $data); |
||
| 483 | } |
||
| 484 | |||
| 485 | // ==================== Private Helper Methods ==================== |
||
| 486 | |||
| 487 | private function selectNntpGroup(array &$groupMySQL, NNTP $nntp): ?array |
||
| 488 | { |
||
| 489 | $groupNNTP = $nntp->selectGroup($groupMySQL['name']); |
||
| 490 | |||
| 491 | if (NNTP::isError($groupNNTP)) { |
||
| 492 | $groupNNTP = $nntp->dataError($nntp, $groupMySQL['name']); |
||
| 493 | |||
| 494 | if (isset($groupNNTP['code']) && (int) $groupNNTP['code'] === 411) { |
||
| 495 | UsenetGroup::disableIfNotExist($groupMySQL['id']); |
||
| 496 | } |
||
| 497 | |||
| 498 | if (NNTP::isError($groupNNTP)) { |
||
| 499 | return null; |
||
| 500 | } |
||
| 501 | } |
||
| 502 | |||
| 503 | return $groupNNTP; |
||
| 504 | } |
||
| 505 | |||
| 506 | private function calculateArticleRange(array $groupMySQL, array $groupNNTP, int $maxHeaders): array |
||
| 507 | { |
||
| 508 | if ((int) $groupMySQL['last_record'] === 0) { |
||
| 509 | return $this->calculateNewGroupRange($groupNNTP); |
||
| 510 | } |
||
| 511 | |||
| 512 | return $this->calculateExistingGroupRange($groupMySQL, $groupNNTP, $maxHeaders); |
||
| 513 | } |
||
| 514 | |||
| 515 | private function calculateNewGroupRange(array $groupNNTP): array |
||
| 516 | { |
||
| 517 | if ($this->config->newGroupScanByDays) { |
||
| 518 | $first = (int) $this->daytopost($this->config->newGroupDaysToScan, $groupNNTP); |
||
| 519 | } elseif ($groupNNTP['first'] >= ($groupNNTP['last'] - ($this->config->newGroupMessagesToScan + $this->config->messageBuffer))) { |
||
| 520 | $first = (int) $groupNNTP['first']; |
||
| 521 | } else { |
||
| 522 | $first = (int) ($groupNNTP['last'] - ($this->config->newGroupMessagesToScan + $this->config->messageBuffer)); |
||
| 523 | } |
||
| 524 | |||
| 525 | $leaveOver = $this->config->messageBuffer; |
||
| 526 | $last = $groupLast = (int) ($groupNNTP['last'] - $leaveOver); |
||
| 527 | |||
| 528 | if ($last < $first) { |
||
| 529 | $last = $groupLast = $first; |
||
| 530 | } |
||
| 531 | |||
| 532 | $total = (int) ($groupLast - $first); |
||
| 533 | $realTotal = (int) ($groupNNTP['last'] - $first); |
||
| 534 | |||
| 535 | return [ |
||
| 536 | 'first' => $first, |
||
| 537 | 'last' => $last, |
||
| 538 | 'groupLast' => $groupLast, |
||
| 539 | 'total' => $total, |
||
| 540 | 'realTotal' => $realTotal, |
||
| 541 | 'leaveOver' => $leaveOver, |
||
| 542 | 'isNew' => true, |
||
| 543 | ]; |
||
| 544 | } |
||
| 545 | |||
| 546 | private function calculateExistingGroupRange(array $groupMySQL, array $groupNNTP, int $maxHeaders): array |
||
| 547 | { |
||
| 548 | $first = (int) $groupMySQL['last_record']; |
||
| 549 | $totalCount = (int) ($groupNNTP['last'] - $first); |
||
| 550 | |||
| 551 | if ($totalCount > ($this->config->messageBuffer * 2)) { |
||
| 552 | $leaveOver = (int) round($totalCount % $this->config->messageBuffer, 0, PHP_ROUND_HALF_DOWN) + $this->config->messageBuffer; |
||
| 553 | } else { |
||
| 554 | $leaveOver = (int) round($totalCount / 2, 0, PHP_ROUND_HALF_DOWN); |
||
| 555 | } |
||
| 556 | |||
| 557 | $last = $groupLast = (int) ($groupNNTP['last'] - $leaveOver); |
||
| 558 | |||
| 559 | if ($last < $first) { |
||
| 560 | $last = $groupLast = $first; |
||
| 561 | } |
||
| 562 | |||
| 563 | $total = (int) ($groupLast - $first); |
||
| 564 | $realTotal = (int) ($groupNNTP['last'] - $first); |
||
| 565 | |||
| 566 | // Apply max headers limit |
||
| 567 | if ($maxHeaders > 0 && $maxHeaders < ($groupLast - $first)) { |
||
| 568 | $groupLast = $last = (int) ($first + $maxHeaders); |
||
| 569 | $total = (int) ($groupLast - $first); |
||
| 570 | } |
||
| 571 | |||
| 572 | return [ |
||
| 573 | 'first' => $first, |
||
| 574 | 'last' => $last, |
||
| 575 | 'groupLast' => $groupLast, |
||
| 576 | 'total' => $total, |
||
| 577 | 'realTotal' => $realTotal, |
||
| 578 | 'leaveOver' => $leaveOver, |
||
| 579 | 'isNew' => false, |
||
| 580 | ]; |
||
| 581 | } |
||
| 582 | |||
| 583 | private function processArticleRange(array &$groupMySQL, array $groupNNTP, array $range): void |
||
| 616 | } |
||
| 617 | } |
||
| 618 | } |
||
| 619 | |||
| 620 | private function updateGroupAfterScan(array &$groupMySQL, array $groupNNTP, array $scanSummary, int $last): void |
||
| 621 | { |
||
| 622 | if (! empty($scanSummary)) { |
||
| 623 | // New group - update first record |
||
| 624 | if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] === 0) { |
||
| 625 | $groupMySQL['first_record'] = $scanSummary['firstArticleNumber']; |
||
| 626 | $groupMySQL['first_record_postdate'] = isset($scanSummary['firstArticleDate']) |
||
| 627 | ? strtotime($scanSummary['firstArticleDate']) |
||
| 628 | : $this->postdate($groupMySQL['first_record'], $groupNNTP); |
||
| 629 | |||
| 630 | UsenetGroup::query()->where('id', $groupMySQL['id'])->update([ |
||
| 631 | 'first_record' => $scanSummary['firstArticleNumber'], |
||
| 632 | 'first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'], date_default_timezone_get()), |
||
| 633 | ]); |
||
| 634 | } |
||
| 635 | |||
| 636 | $lastArticleDate = isset($scanSummary['lastArticleDate']) |
||
| 637 | ? strtotime($scanSummary['lastArticleDate']) |
||
| 638 | : $this->postdate($scanSummary['lastArticleNumber'], $groupNNTP); |
||
| 639 | |||
| 640 | UsenetGroup::query()->where('id', $groupMySQL['id'])->update([ |
||
| 641 | 'last_record' => $scanSummary['lastArticleNumber'], |
||
| 642 | 'last_record_postdate' => Carbon::createFromTimestamp($lastArticleDate, date_default_timezone_get()), |
||
| 643 | 'last_updated' => now(), |
||
| 644 | ]); |
||
| 645 | } else { |
||
| 646 | UsenetGroup::query()->where('id', $groupMySQL['id'])->update([ |
||
| 647 | 'last_record' => $last, |
||
| 648 | 'last_updated' => now(), |
||
| 649 | ]); |
||
| 650 | } |
||
| 651 | } |
||
| 652 | |||
| 653 | private function downloadHeaders(bool $partRepair): ?array |
||
| 654 | { |
||
| 655 | $nntp = $this->getNntp(); |
||
| 656 | |||
| 657 | if ($partRepair) { |
||
| 658 | $headers = $nntp->getOverview($this->first.'-'.$this->last, true, false); |
||
| 659 | } else { |
||
| 660 | $headers = $nntp->getXOVER($this->first.'-'.$this->last); |
||
| 661 | } |
||
| 662 | |||
| 663 | if (NNTP::isError($headers)) { |
||
| 664 | if ($partRepair) { |
||
| 665 | return null; |
||
| 666 | } |
||
| 667 | |||
| 668 | // Retry without compression |
||
| 669 | $nntp->doQuit(); |
||
| 670 | if ($nntp->doConnect(false) !== true) { |
||
| 671 | return null; |
||
| 672 | } |
||
| 673 | |||
| 674 | $nntp->selectGroup($this->groupMySQL['name']); |
||
| 675 | $headers = $nntp->getXOVER($this->first.'-'.$this->last); |
||
| 676 | $nntp->enableCompression(); |
||
| 677 | |||
| 678 | if (NNTP::isError($headers)) { |
||
| 679 | $message = ((int) $headers->code === 0 ? 'Unknown error' : $headers->message); |
||
| 680 | $this->log("Code {$headers->code}: $message\nSkipping group: {$this->groupMySQL['name']}", __FUNCTION__, 'error'); |
||
| 681 | |||
| 682 | return null; |
||
| 683 | } |
||
| 684 | } |
||
| 685 | |||
| 686 | return $headers; |
||
| 687 | } |
||
| 688 | |||
| 689 | private function handlePartRepairTracking(array $headersNotInserted, array $parsedHeaders): void |
||
| 690 | { |
||
| 691 | $notInsertedCount = \count($headersNotInserted); |
||
| 692 | if ($notInsertedCount > 0) { |
||
| 693 | $this->missedPartHandler->addMissingParts($headersNotInserted, $this->groupMySQL['id']); |
||
| 694 | $this->log($notInsertedCount.' articles failed to insert!', __FUNCTION__, 'warning'); |
||
| 695 | } |
||
| 696 | |||
| 697 | // Check for missing headers in range |
||
| 698 | $expectedCount = $this->last - $this->first - $this->notYEnc - $this->headersBlackListed + 1; |
||
| 699 | if ($expectedCount > \count($this->headersReceived)) { |
||
| 700 | $rangeNotReceived = array_diff(range($this->first, $this->last), $this->headersReceived); |
||
| 701 | $notReceivedCount = \count($rangeNotReceived); |
||
| 702 | |||
| 703 | if ($notReceivedCount > 0) { |
||
| 704 | $this->missedPartHandler->addMissingParts($rangeNotReceived, $this->groupMySQL['id']); |
||
| 705 | |||
| 706 | if ($this->config->echoCli) { |
||
| 707 | $this->colorCli->alternate( |
||
| 708 | 'Server did not return '.$notReceivedCount.' articles from '.$this->groupMySQL['name'].'.' |
||
| 709 | ); |
||
| 710 | } |
||
| 711 | } |
||
| 712 | } |
||
| 713 | } |
||
| 714 | |||
| 715 | private function groupMissingPartsIntoRanges(array $missingParts): array |
||
| 716 | { |
||
| 717 | $ranges = []; |
||
| 718 | $partList = []; |
||
| 719 | $firstPart = $lastNum = $missingParts[0]->numberid; |
||
| 720 | |||
| 721 | foreach ($missingParts as $part) { |
||
| 722 | if (($part->numberid - $firstPart) > ($this->config->messageBuffer / 4)) { |
||
| 723 | $ranges[] = [ |
||
| 724 | 'partfrom' => $firstPart, |
||
| 725 | 'partto' => $lastNum, |
||
| 726 | 'partlist' => $partList, |
||
| 727 | ]; |
||
| 728 | $firstPart = $part->numberid; |
||
| 729 | $partList = []; |
||
| 730 | } |
||
| 731 | $partList[] = $part->numberid; |
||
| 732 | $lastNum = $part->numberid; |
||
| 733 | } |
||
| 734 | |||
| 735 | $ranges[] = [ |
||
| 736 | 'partfrom' => $firstPart, |
||
| 737 | 'partto' => $lastNum, |
||
| 738 | 'partlist' => $partList, |
||
| 739 | ]; |
||
| 740 | |||
| 741 | return $ranges; |
||
| 742 | } |
||
| 743 | |||
| 744 | private function getNextArticleToTry(int $currentPost, array $groupData): ?int |
||
| 745 | { |
||
| 746 | if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) { |
||
| 747 | $tempPost = (int) round($currentPost / (random_int(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP); |
||
| 748 | if ($tempPost < $groupData['first']) { |
||
| 749 | $tempPost = $groupData['first']; |
||
| 750 | } |
||
| 751 | } else { |
||
| 752 | $tempPost = (int) round((random_int(1005, 1012) / 1000) * $currentPost, 0, PHP_ROUND_HALF_UP); |
||
| 753 | if ($tempPost > $groupData['last']) { |
||
| 754 | $tempPost = $groupData['last']; |
||
| 755 | } |
||
| 756 | } |
||
| 757 | |||
| 758 | // If we got the same article number, give up |
||
| 759 | if ($tempPost === $currentPost) { |
||
| 760 | return null; |
||
| 761 | } |
||
| 762 | |||
| 763 | return $tempPost; |
||
| 764 | } |
||
| 765 | |||
| 766 | private function binarySearchArticleByDate(int $goalTime, array $data): string |
||
| 767 | { |
||
| 768 | $wantedArticle = (int) round(($data['last'] + $data['first']) / 2); |
||
| 769 | $aMax = $data['last']; |
||
| 770 | $aMin = $data['first']; |
||
| 771 | $oldArticle = $articleTime = null; |
||
| 772 | |||
| 773 | while (true) { |
||
| 774 | if ($wantedArticle <= $data['first'] || $wantedArticle >= $data['last']) { |
||
| 775 | break; |
||
| 776 | } |
||
| 777 | |||
| 778 | $reallyOldArticle = $oldArticle; |
||
| 779 | $oldArticle = $wantedArticle; |
||
| 780 | |||
| 781 | $articleTime = $this->postdate($wantedArticle, $data); |
||
| 782 | |||
| 783 | if (! $articleTime) { |
||
| 784 | $wantedArticle = random_int($aMin, $aMax); |
||
| 785 | $articleTime = $this->postdate($wantedArticle, $data); |
||
| 786 | } |
||
| 787 | |||
| 788 | if ($articleTime < $goalTime) { |
||
| 789 | $aMin = $oldArticle; |
||
| 790 | $wantedArticle = (int) round(($aMax + $oldArticle) / 2); |
||
| 791 | if ($this->config->echoCli) { |
||
| 792 | echo '-'; |
||
| 793 | } |
||
| 794 | } elseif ($articleTime > $goalTime) { |
||
| 795 | $aMax = $oldArticle; |
||
| 796 | $wantedArticle = (int) round(($aMin + $oldArticle) / 2); |
||
| 797 | if ($this->config->echoCli) { |
||
| 798 | echo '+'; |
||
| 799 | } |
||
| 800 | } else { |
||
| 801 | break; |
||
| 802 | } |
||
| 803 | |||
| 804 | if ($reallyOldArticle === $wantedArticle && ($goalTime - $articleTime) <= 0) { |
||
| 805 | break; |
||
| 806 | } |
||
| 807 | } |
||
| 808 | |||
| 809 | if ($this->config->echoCli) { |
||
| 810 | $goalCarbon = Carbon::createFromTimestamp($goalTime, date_default_timezone_get()); |
||
| 811 | $articleCarbon = Carbon::createFromTimestamp($articleTime, date_default_timezone_get()); |
||
| 812 | $diffDays = $goalCarbon->diffInDays($articleCarbon, true); |
||
| 813 | $this->colorCli->primary( |
||
| 814 | PHP_EOL.'Found article #'.$wantedArticle.' which has a date of '.date('r', $articleTime). |
||
| 815 | ', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.$diffDays.' days.' |
||
| 816 | ); |
||
| 817 | } |
||
| 818 | |||
| 819 | return (string) $wantedArticle; |
||
| 820 | } |
||
| 821 | |||
| 822 | // ==================== Output Methods ==================== |
||
| 823 | |||
| 824 | private function outputNoNewArticles(array $groupMySQL, array $groupNNTP, array $range): void |
||
| 825 | { |
||
| 826 | if ($this->config->echoCli) { |
||
| 827 | $this->colorCli->primary( |
||
| 828 | 'No new articles for '.$groupMySQL['name'].' (first '.number_format((int) $range['first']). |
||
| 829 | ', last '.number_format((int) $range['last']).', grouplast '.number_format((int) $groupMySQL['last_record']). |
||
| 830 | ', total '.number_format((int) $range['total']).")\n".'Server oldest: '.number_format((int) $groupNNTP['first']). |
||
| 831 | ' Server newest: '.number_format((int) $groupNNTP['last']).' Local newest: '.number_format((int) $groupMySQL['last_record']) |
||
| 832 | ); |
||
| 833 | } |
||
| 834 | } |
||
| 835 | |||
| 836 | private function outputNewArticlesInfo(array $groupMySQL, array $groupNNTP, array $range): void |
||
| 837 | { |
||
| 838 | if (! $this->config->echoCli) { |
||
| 839 | return; |
||
| 840 | } |
||
| 841 | |||
| 842 | $message = $range['isNew'] |
||
| 843 | ? 'New group '.$groupNNTP['group'].' starting with '. |
||
| 844 | ($this->config->newGroupScanByDays |
||
| 845 | ? $this->config->newGroupDaysToScan.' days' |
||
| 846 | : number_format($this->config->newGroupMessagesToScan).' messages').' worth.' |
||
| 847 | : 'Group '.$groupNNTP['group'].' has '.number_format((int) $range['realTotal']).' new articles.'; |
||
| 848 | |||
| 849 | $this->colorCli->primary( |
||
| 850 | $message. |
||
| 851 | ' Leaving '.number_format((int) $range['leaveOver']). |
||
| 852 | " for next pass.\nServer oldest: ".number_format((int) $groupNNTP['first']). |
||
| 853 | ' Server newest: '.number_format((int) $groupNNTP['last']). |
||
| 854 | ' Local newest: '.number_format((int) $groupMySQL['last_record']) |
||
| 855 | ); |
||
| 856 | } |
||
| 857 | |||
| 858 | private function outputHeaderInitial(): void |
||
| 864 | ); |
||
| 865 | } |
||
| 866 | |||
| 867 | private function outputHeaderDuration(): void |
||
| 868 | { |
||
| 869 | if (! $this->config->echoCli) { |
||
| 870 | return; |
||
| 871 | } |
||
| 872 | |||
| 873 | $currentMicroTime = now(); |
||
| 874 | $this->colorCli->alternateOver(number_format($this->timeHeaders, 2).'s'). |
||
| 884 | } |
||
| 885 | |||
| 886 | // ==================== Logging Methods ==================== |
||
| 887 | |||
| 888 | private function log(string $message, string $method, string $color): void |
||
| 889 | { |
||
| 890 | if ($this->config->echoCli) { |
||
| 891 | $this->colorCli->$color($message.' ['.__CLASS__."::$method]"); |
||
| 892 | } |
||
| 893 | } |
||
| 894 | |||
| 895 | private function logError(string $message): void |
||
| 902 | } |
||
| 903 | } |
||
| 904 | } |
||
| 905 | |||
| 906 |
Loading history...
The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g.
excluded_paths: ["lib/*"], you can move it to the dependency path list as follows:For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths