Total Complexity | 152 |
Total Lines | 1094 |
Duplicated Lines | 0 % |
Changes | 0 |
Complex classes like Forking often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
While breaking up the class, it is a good idea to analyze how other classes use Forking, and based on these observations, apply Extract Interface, too.
1 | <?php |
||
22 | class Forking extends \fork_daemon |
||
23 | { |
||
24 | private const OUTPUT_NONE = 0; // Don't display child output. |
||
25 | private const OUTPUT_REALTIME = 1; // Display child output in real time. |
||
26 | private const OUTPUT_SERIALLY = 2; // Display child output when child is done. |
||
27 | |||
28 | /** |
||
29 | * @var \Blacklight\ColorCLI |
||
30 | */ |
||
31 | public $_colorCLI; |
||
32 | |||
33 | /** |
||
34 | * @var int The type of output |
||
35 | */ |
||
36 | protected $outputType; |
||
37 | |||
38 | /** |
||
39 | * Path to do not run folder. |
||
40 | * |
||
41 | * @var string |
||
42 | */ |
||
43 | private $dnr_path; |
||
44 | |||
45 | /** |
||
46 | * Work to work on. |
||
47 | * |
||
48 | * @var array |
||
49 | */ |
||
50 | private $work = []; |
||
51 | |||
52 | /** |
||
53 | * How much work do we have to do? |
||
54 | * |
||
55 | * @var int |
||
56 | */ |
||
57 | public $_workCount = 0; |
||
58 | |||
59 | /** |
||
60 | * The type of work we want to work on. |
||
61 | * |
||
62 | * @var string |
||
63 | */ |
||
64 | private $workType = ''; |
||
65 | |||
66 | /** |
||
67 | * List of passed in options for the current work type. |
||
68 | * |
||
69 | * @var array |
||
70 | */ |
||
71 | private $workTypeOptions = []; |
||
72 | |||
73 | /** |
||
74 | * Max amount of child processes to do work at a time. |
||
75 | * |
||
76 | * @var int |
||
77 | */ |
||
78 | private $maxProcesses = 1; |
||
79 | |||
80 | /** |
||
81 | * Group used for safe backfill. |
||
82 | * |
||
83 | * @var string |
||
84 | */ |
||
85 | private $safeBackfillGroup = ''; |
||
86 | |||
87 | /** |
||
88 | * @var \Blacklight\db\DB |
||
89 | */ |
||
90 | public $pdo; |
||
91 | |||
92 | /** |
||
93 | * @var int |
||
94 | */ |
||
95 | protected $maxSize; |
||
96 | |||
97 | /** |
||
98 | * @var int |
||
99 | */ |
||
100 | protected $minSize; |
||
101 | |||
102 | /** |
||
103 | * @var int |
||
104 | */ |
||
105 | protected $maxRetries; |
||
106 | |||
107 | /** |
||
108 | * @var int |
||
109 | */ |
||
110 | protected $dummy; |
||
111 | |||
112 | /** |
||
113 | * @var bool |
||
114 | */ |
||
115 | private $processAdditional = false; // Should we process additional? |
||
116 | private $processNFO = false; // Should we process NFOs? |
||
117 | private $processMovies = false; // Should we process Movies? |
||
118 | private $processTV = false; // Should we process TV? |
||
119 | |||
120 | /** |
||
121 | * Setup required parent / self vars. |
||
122 | * |
||
123 | * @throws \Exception |
||
124 | */ |
||
125 | public function __construct() |
||
126 | { |
||
127 | parent::__construct(); |
||
128 | |||
129 | $this->_colorCLI = new ColorCLI(); |
||
130 | |||
131 | $this->register_logging( |
||
132 | [0 => $this, 1 => 'logger'], |
||
|
|||
133 | (\defined('NN_MULTIPROCESSING_LOG_TYPE') ? NN_MULTIPROCESSING_LOG_TYPE : \fork_daemon::LOG_LEVEL_INFO) |
||
134 | ); |
||
135 | |||
136 | if (\defined('NN_MULTIPROCESSING_MAX_CHILD_WORK')) { |
||
137 | $this->max_work_per_child_set(NN_MULTIPROCESSING_MAX_CHILD_WORK); |
||
138 | } else { |
||
139 | $this->max_work_per_child_set(1); |
||
140 | } |
||
141 | |||
142 | if (\defined('NN_MULTIPROCESSING_MAX_CHILD_TIME')) { |
||
143 | $this->child_max_run_time_set(NN_MULTIPROCESSING_MAX_CHILD_TIME); |
||
144 | } else { |
||
145 | $this->child_max_run_time_set(1800); |
||
146 | } |
||
147 | |||
148 | // Use a single exit method for all children, makes things easier. |
||
149 | $this->register_parent_child_exit([0 => $this, 1 => 'childExit']); |
||
150 | |||
151 | if (\defined('NN_MULTIPROCESSING_CHILD_OUTPUT_TYPE')) { |
||
152 | switch (NN_MULTIPROCESSING_CHILD_OUTPUT_TYPE) { |
||
153 | case 0: |
||
154 | $this->outputType = self::OUTPUT_NONE; |
||
155 | break; |
||
156 | case 1: |
||
157 | $this->outputType = self::OUTPUT_REALTIME; |
||
158 | break; |
||
159 | case 2: |
||
160 | $this->outputType = self::OUTPUT_SERIALLY; |
||
161 | break; |
||
162 | default: |
||
163 | $this->outputType = self::OUTPUT_REALTIME; |
||
164 | } |
||
165 | } else { |
||
166 | $this->outputType = self::OUTPUT_REALTIME; |
||
167 | } |
||
168 | |||
169 | $this->dnr_path = PHP_BINARY.' '.NN_MULTIPROCESSING.'.do_not_run/switch.php "php '; |
||
170 | |||
171 | $this->maxSize = (int) Settings::settingValue('..maxsizetoprocessnfo'); |
||
172 | $this->minSize = (int) Settings::settingValue('..minsizetoprocessnfo'); |
||
173 | $this->maxRetries = (int) Settings::settingValue('..maxnforetries') >= 0 ? -((int) Settings::settingValue('..maxnforetries') + 1) : Nfo::NFO_UNPROC; |
||
174 | $this->maxRetries = $this->maxRetries < -8 ? -8 : $this->maxRetries; |
||
175 | } |
||
176 | |||
177 | /** |
||
178 | * Setup the class to work on a type of work, then process the work. |
||
179 | * Valid work types:. |
||
180 | * |
||
181 | * @param string $type The type of multiProcessing to do : backfill, binaries, releases, postprocess |
||
182 | * @param array $options Array containing arguments for the type of work. |
||
183 | * |
||
184 | * @throws \Exception |
||
185 | */ |
||
186 | public function processWorkType($type, array $options = []) |
||
187 | { |
||
188 | // Set/reset some variables. |
||
189 | $startTime = microtime(true); |
||
190 | $this->workType = $type; |
||
191 | $this->workTypeOptions = $options; |
||
192 | $this->processAdditional = $this->processNFO = $this->processTV = $this->processMovies = $this->ppRenamedOnly = false; |
||
193 | $this->work = []; |
||
194 | |||
195 | // Init Settings here, as forking causes errors when it's destroyed. |
||
196 | $this->pdo = new DB(); |
||
197 | |||
198 | // Process extra work that should not be forked and done before forking. |
||
199 | $this->processStartWork(); |
||
200 | |||
201 | // Get work to fork. |
||
202 | $this->getWork(); |
||
203 | |||
204 | // Now we destroy settings, to prevent errors from forking. |
||
205 | unset($this->pdo); |
||
206 | |||
207 | // Process the work we got. |
||
208 | $this->processWork(); |
||
209 | |||
210 | // Process extra work that should not be forked and done after. |
||
211 | $this->processEndWork(); |
||
212 | |||
213 | if (config('nntmux.echocli')) { |
||
214 | ColorCLI::doEcho( |
||
215 | ColorCLI::header( |
||
216 | 'Multi-processing for '.$this->workType.' finished in '.(microtime(true) - $startTime). |
||
217 | ' seconds at '.date(DATE_RFC2822).'.'.PHP_EOL |
||
218 | ), true |
||
219 | ); |
||
220 | } |
||
221 | } |
||
222 | |||
223 | /** |
||
224 | * Only post process renamed movie / tv releases? |
||
225 | * |
||
226 | * @var bool |
||
227 | */ |
||
228 | private $ppRenamedOnly; |
||
229 | |||
230 | /** |
||
231 | * Get work for our workers to work on, set the max child processes here. |
||
232 | * |
||
233 | * @throws \Exception |
||
234 | */ |
||
235 | private function getWork() |
||
236 | { |
||
237 | $maxProcesses = 0; |
||
238 | |||
239 | switch ($this->workType) { |
||
240 | |||
241 | case 'backfill': |
||
242 | $maxProcesses = $this->backfillMainMethod(); |
||
243 | break; |
||
244 | |||
245 | case 'binaries': |
||
246 | $maxProcesses = $this->binariesMainMethod(); |
||
247 | break; |
||
248 | |||
249 | case 'fixRelNames_standard': |
||
250 | case 'fixRelNames_predbft': |
||
251 | $maxProcesses = $this->fixRelNamesMainMethod(); |
||
252 | break; |
||
253 | |||
254 | case 'releases': |
||
255 | $maxProcesses = $this->releasesMainMethod(); |
||
256 | break; |
||
257 | |||
258 | case 'postProcess_ama': |
||
259 | $this->processSingle(); |
||
260 | break; |
||
261 | |||
262 | case 'postProcess_add': |
||
263 | $maxProcesses = $this->postProcessAddMainMethod(); |
||
264 | break; |
||
265 | |||
266 | case 'postProcess_mov': |
||
267 | $this->ppRenamedOnly = (isset($this->workTypeOptions[0]) && $this->workTypeOptions[0] === true); |
||
268 | $maxProcesses = $this->postProcessMovMainMethod(); |
||
269 | break; |
||
270 | |||
271 | case 'postProcess_nfo': |
||
272 | $maxProcesses = $this->postProcessNfoMainMethod(); |
||
273 | break; |
||
274 | |||
275 | case 'postProcess_sha': |
||
276 | $this->processSharing(); |
||
277 | break; |
||
278 | |||
279 | case 'postProcess_tv': |
||
280 | $this->ppRenamedOnly = (isset($this->workTypeOptions[0]) && $this->workTypeOptions[0] === true); |
||
281 | $maxProcesses = $this->postProcessTvMainMethod(); |
||
282 | break; |
||
283 | |||
284 | case 'safe_backfill': |
||
285 | $maxProcesses = $this->safeBackfillMainMethod(); |
||
286 | break; |
||
287 | |||
288 | case 'safe_binaries': |
||
289 | $maxProcesses = $this->safeBinariesMainMethod(); |
||
290 | break; |
||
291 | |||
292 | case 'update_per_group': |
||
293 | $maxProcesses = $this->updatePerGroupMainMethod(); |
||
294 | break; |
||
295 | } |
||
296 | |||
297 | $this->setMaxProcesses($maxProcesses); |
||
298 | } |
||
299 | |||
300 | /** |
||
301 | * Process work if we have any. |
||
302 | */ |
||
303 | private function processWork() |
||
304 | { |
||
305 | $this->_workCount = \count($this->work); |
||
306 | if ($this->_workCount > 0) { |
||
307 | if (config('nntmux.echocli')) { |
||
308 | ColorCLI::doEcho( |
||
309 | ColorCLI::header( |
||
310 | 'Multi-processing started at '.date(DATE_RFC2822).' for '.$this->workType.' with '.$this->_workCount. |
||
311 | ' job(s) to do using a max of '.$this->maxProcesses.' child process(es).' |
||
312 | ), true |
||
313 | ); |
||
314 | } |
||
315 | |||
316 | $this->addwork($this->work); |
||
317 | $this->process_work(true); |
||
318 | } else { |
||
319 | if (config('nntmux.echocli')) { |
||
320 | ColorCLI::doEcho( |
||
321 | ColorCLI::header('No work to do!'), true |
||
322 | ); |
||
323 | } |
||
324 | } |
||
325 | } |
||
326 | |||
327 | /** |
||
328 | * Process any work that does not need to be forked, but needs to run at the end. |
||
329 | */ |
||
330 | private function processStartWork() |
||
331 | { |
||
332 | switch ($this->workType) { |
||
333 | case 'safe_backfill': |
||
334 | case 'safe_binaries': |
||
335 | $this->_executeCommand( |
||
336 | PHP_BINARY.' '.NN_UPDATE.'tmux/bin/update_groups.php' |
||
337 | ); |
||
338 | break; |
||
339 | } |
||
340 | } |
||
341 | |||
342 | /** |
||
343 | * Process any work that does not need to be forked, but needs to run at the end. |
||
344 | */ |
||
345 | private function processEndWork() |
||
346 | { |
||
347 | switch ($this->workType) { |
||
348 | case 'releases': |
||
349 | $this->_executeCommand( |
||
350 | $this->dnr_path.'releases '.\count($this->work).'_"' |
||
351 | ); |
||
352 | break; |
||
353 | case 'update_per_group': |
||
354 | $this->_executeCommand( |
||
355 | $this->dnr_path.'releases '.\count($this->work).'_"' |
||
356 | ); |
||
357 | break; |
||
358 | } |
||
359 | } |
||
360 | |||
361 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
362 | //////////////////////////////////////// All backFill code here //////////////////////////////////////////////////// |
||
363 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
364 | |||
365 | /** |
||
366 | * @return int |
||
367 | * @throws \Exception |
||
368 | */ |
||
369 | private function backfillMainMethod(): int |
||
370 | { |
||
371 | $this->register_child_run([0 => $this, 1 => 'backFillChildWorker']); |
||
372 | // The option for backFill is for doing up to x articles. Else it's done by date. |
||
373 | $this->work = $this->pdo->query( |
||
374 | sprintf( |
||
375 | 'SELECT name %s FROM groups WHERE backfill = 1', |
||
376 | ($this->workTypeOptions[0] === false ? '' : (', '.$this->workTypeOptions[0].' AS max')) |
||
377 | ) |
||
378 | ); |
||
379 | |||
380 | return (int) Settings::settingValue('..backfillthreads'); |
||
381 | } |
||
382 | |||
383 | /** |
||
384 | * @param $groups |
||
385 | */ |
||
386 | public function backFillChildWorker($groups) |
||
387 | { |
||
388 | foreach ($groups as $group) { |
||
389 | $this->_executeCommand( |
||
390 | PHP_BINARY.' '.NN_UPDATE.'backfill.php '. |
||
391 | $group['name'].(isset($group['max']) ? (' '.$group['max']) : '') |
||
392 | ); |
||
393 | } |
||
394 | } |
||
395 | |||
396 | /** |
||
397 | * @return int |
||
398 | * @throws \Exception |
||
399 | */ |
||
400 | private function safeBackfillMainMethod(): int |
||
401 | { |
||
402 | $this->register_child_run([0 => $this, 1 => 'safeBackfillChildWorker']); |
||
403 | |||
404 | $backfill_qty = Settings::settingValue('site.tmux.backfill_qty'); |
||
405 | $backfill_order = Settings::settingValue('site.tmux.backfill_order'); |
||
406 | $backfill_days = Settings::settingValue('site.tmux.backfill_days'); |
||
407 | $maxmssgs = Settings::settingValue('maxmssgs'); |
||
408 | $threads = Settings::settingValue('..backfillthreads'); |
||
409 | |||
410 | $orderby = 'ORDER BY a.last_record ASC'; |
||
411 | switch ((int) $backfill_order) { |
||
412 | case 1: |
||
413 | $orderby = 'ORDER BY first_record_postdate DESC'; |
||
414 | break; |
||
415 | |||
416 | case 2: |
||
417 | $orderby = 'ORDER BY first_record_postdate ASC'; |
||
418 | break; |
||
419 | |||
420 | case 3: |
||
421 | $orderby = 'ORDER BY name ASC'; |
||
422 | break; |
||
423 | |||
424 | case 4: |
||
425 | $orderby = 'ORDER BY name DESC'; |
||
426 | break; |
||
427 | |||
428 | case 5: |
||
429 | $orderby = 'ORDER BY a.last_record DESC'; |
||
430 | break; |
||
431 | } |
||
432 | |||
433 | $backfilldays = ''; |
||
434 | if ((int) $backfill_days === 1) { |
||
435 | $days = 'backfill_target'; |
||
436 | $backfilldays = Carbon::now()->subDays($days); |
||
437 | } elseif ((int) $backfill_days === 2) { |
||
438 | $backfilldays = Settings::settingValue('..safebackfilldate'); |
||
439 | } |
||
440 | |||
441 | $data = $this->pdo->queryOneRow( |
||
442 | sprintf( |
||
443 | 'SELECT g.name, |
||
444 | g.first_record AS our_first, |
||
445 | MAX(a.first_record) AS their_first, |
||
446 | MAX(a.last_record) AS their_last |
||
447 | FROM groups g |
||
448 | INNER JOIN short_groups a ON g.name = a.name |
||
449 | WHERE g.first_record IS NOT NULL |
||
450 | AND g.first_record_postdate IS NOT NULL |
||
451 | AND g.backfill = 1 |
||
452 | AND %s < g.first_record_postdate |
||
453 | GROUP BY a.name, a.last_record, g.name, g.first_record |
||
454 | %s', |
||
455 | $backfilldays, |
||
456 | $orderby |
||
457 | ) |
||
458 | ); |
||
459 | |||
460 | $count = 0; |
||
461 | if ($data['name']) { |
||
462 | $this->safeBackfillGroup = $data['name']; |
||
463 | |||
464 | $count = ($data['our_first'] - $data['their_first']); |
||
465 | } |
||
466 | |||
467 | if ($count > 0) { |
||
468 | if ($count > ($backfill_qty * $threads)) { |
||
469 | $geteach = ceil(($backfill_qty * $threads) / $maxmssgs); |
||
470 | } else { |
||
471 | $geteach = $count / $maxmssgs; |
||
472 | } |
||
473 | |||
474 | $queue = []; |
||
475 | for ($i = 0; $i <= $geteach - 1; $i++) { |
||
476 | $queue[$i] = sprintf('get_range backfill %s %s %s %s', $data['name'], $data['our_first'] - $i * $maxmssgs - $maxmssgs, $data['our_first'] - $i * $maxmssgs - 1, $i + 1); |
||
477 | } |
||
478 | $this->work = $queue; |
||
479 | } |
||
480 | |||
481 | return $threads; |
||
482 | } |
||
483 | |||
484 | /** |
||
485 | * @param $ranges |
||
486 | * @param string $identifier |
||
487 | */ |
||
488 | public function safeBackfillChildWorker($ranges, $identifier = '') |
||
489 | { |
||
490 | foreach ($ranges as $range) { |
||
491 | $this->_executeCommand( |
||
492 | $this->dnr_path.$range.'"' |
||
493 | ); |
||
494 | } |
||
495 | } |
||
496 | |||
497 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
498 | //////////////////////////////////////// All binaries code here //////////////////////////////////////////////////// |
||
499 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
500 | |||
501 | /** |
||
502 | * @return null|string |
||
503 | * @throws \Exception |
||
504 | */ |
||
505 | private function binariesMainMethod() |
||
506 | { |
||
507 | $this->register_child_run([0 => $this, 1 => 'binariesChildWorker']); |
||
508 | $this->work = $this->pdo->query( |
||
509 | sprintf( |
||
510 | 'SELECT name, %d AS max FROM groups WHERE active = 1', |
||
511 | $this->workTypeOptions[0] |
||
512 | ) |
||
513 | ); |
||
514 | |||
515 | return Settings::settingValue('..binarythreads'); |
||
516 | } |
||
517 | |||
518 | /** |
||
519 | * @param $groups |
||
520 | * @param string $identifier |
||
521 | */ |
||
522 | public function binariesChildWorker($groups, $identifier = '') |
||
523 | { |
||
524 | foreach ($groups as $group) { |
||
525 | $this->_executeCommand( |
||
526 | PHP_BINARY.' '.NN_UPDATE.'update_binaries.php '.$group['name'].' '.$group['max'] |
||
527 | ); |
||
528 | } |
||
529 | } |
||
530 | |||
531 | /** |
||
532 | * @return int |
||
533 | * @throws \Exception |
||
534 | */ |
||
535 | private function safeBinariesMainMethod() |
||
536 | { |
||
537 | $this->register_child_run([0 => $this, 1 => 'safeBinariesChildWorker']); |
||
538 | |||
539 | $maxheaders = Settings::settingValue('..max_headers_iteration') ?: 1000000; |
||
540 | $maxmssgs = Settings::settingValue('..maxmssgs'); |
||
541 | $threads = Settings::settingValue('..binarythreads'); |
||
542 | |||
543 | $groups = $this->pdo->query( |
||
544 | ' |
||
545 | SELECT g.name AS groupname, g.last_record AS our_last, |
||
546 | a.last_record AS their_last |
||
547 | FROM groups g |
||
548 | INNER JOIN short_groups a ON g.active = 1 AND g.name = a.name |
||
549 | ORDER BY a.last_record DESC' |
||
550 | ); |
||
551 | |||
552 | if (! empty($groups)) { |
||
553 | $i = 1; |
||
554 | $queue = []; |
||
555 | foreach ($groups as $group) { |
||
556 | if ((int) $group['our_last'] === 0) { |
||
557 | $queue[$i] = sprintf('update_group_headers %s', $group['groupname']); |
||
558 | $i++; |
||
559 | } else { |
||
560 | //only process if more than 20k headers available and skip the first 20k |
||
561 | $count = $group['their_last'] - $group['our_last'] - 20000; |
||
562 | //echo "count: " . $count . "maxmsgs x2: " . ($maxmssgs * 2) . PHP_EOL; |
||
563 | if ($count <= $maxmssgs * 2) { |
||
564 | $queue[$i] = sprintf('update_group_headers %s', $group['groupname']); |
||
565 | $i++; |
||
566 | } else { |
||
567 | $queue[$i] = sprintf('part_repair %s', $group['groupname']); |
||
568 | $i++; |
||
569 | $geteach = floor(min($count, $maxheaders) / $maxmssgs); |
||
570 | $remaining = min($count, $maxheaders) - $geteach * $maxmssgs; |
||
571 | //echo "maxmssgs: " . $maxmssgs . " geteach: " . $geteach . " remaining: " . $remaining . PHP_EOL; |
||
572 | for ($j = 0; $j < $geteach; $j++) { |
||
573 | $queue[$i] = sprintf('get_range binaries %s %s %s %s', $group['groupname'], $group['our_last'] + $j * $maxmssgs + 1, $group['our_last'] + $j * $maxmssgs + $maxmssgs, $i); |
||
574 | $i++; |
||
575 | } |
||
576 | //add remainder to queue |
||
577 | $queue[$i] = sprintf('get_range binaries %s %s %s %s', $group['groupname'], $group['our_last'] + ($j + 1) * $maxmssgs + 1, $group['our_last'] + ($j + 1) * $maxmssgs + $remaining + 1, $i); |
||
578 | $i++; |
||
579 | } |
||
580 | } |
||
581 | } |
||
582 | //var_dump($queue); |
||
583 | $this->work = $queue; |
||
584 | } |
||
585 | |||
586 | return $threads; |
||
587 | } |
||
588 | |||
589 | /** |
||
590 | * @param $ranges |
||
591 | * @param string $identifier |
||
592 | */ |
||
593 | public function safeBinariesChildWorker($ranges, $identifier = '') |
||
594 | { |
||
595 | foreach ($ranges as $range) { |
||
596 | $this->_executeCommand( |
||
597 | $this->dnr_path.$range.'"' |
||
598 | ); |
||
599 | } |
||
600 | } |
||
601 | |||
602 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
603 | //////////////////////////////////// All fix release names code here /////////////////////////////////////////////// |
||
604 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
605 | |||
606 | /** |
||
607 | * @return int|null|string |
||
608 | * @throws \Exception |
||
609 | */ |
||
610 | private function fixRelNamesMainMethod() |
||
611 | { |
||
612 | $this->register_child_run([0 => $this, 1 => 'fixRelNamesChildWorker']); |
||
613 | |||
614 | $threads = (int) Settings::settingValue('..fixnamethreads'); |
||
615 | $maxperrun = (int) Settings::settingValue('..fixnamesperrun'); |
||
616 | |||
617 | if ($threads > 16) { |
||
618 | $threads = 16; |
||
619 | } elseif ($threads === 0) { |
||
620 | $threads = 1; |
||
621 | } |
||
622 | |||
623 | $leftguids = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; |
||
624 | |||
625 | // Prevent PreDB FT from always running |
||
626 | if ($this->workTypeOptions[0] === 'predbft') { |
||
627 | $preCount = $this->pdo->queryOneRow( |
||
628 | sprintf( |
||
629 | " |
||
630 | SELECT COUNT(p.id) AS num |
||
631 | FROM predb p |
||
632 | WHERE LENGTH(p.title) >= 15 |
||
633 | AND p.title NOT REGEXP '[\"\<\> ]' |
||
634 | AND p.searched = 0 |
||
635 | AND p.predate < (NOW() - INTERVAL 1 DAY)" |
||
636 | ) |
||
637 | ); |
||
638 | if ($preCount['num'] > 0) { |
||
639 | $leftguids = \array_slice($leftguids, 0, (int) ceil($preCount['num'] / $maxperrun)); |
||
640 | } else { |
||
641 | $leftguids = []; |
||
642 | } |
||
643 | } |
||
644 | |||
645 | $count = 0; |
||
646 | $queue = []; |
||
647 | foreach ($leftguids as $leftguid) { |
||
648 | $count++; |
||
649 | if ($maxperrun > 0) { |
||
650 | $queue[$count] = sprintf('%s %s %s %s', $this->workTypeOptions[0], $leftguid, $maxperrun, $count); |
||
651 | } |
||
652 | } |
||
653 | $this->work = $queue; |
||
654 | |||
655 | return $threads; |
||
656 | } |
||
657 | |||
658 | /** |
||
659 | * @param $guids |
||
660 | * @param string $identifier |
||
661 | */ |
||
662 | public function fixRelNamesChildWorker($guids, $identifier = '') |
||
663 | { |
||
664 | foreach ($guids as $guid) { |
||
665 | $this->_executeCommand( |
||
666 | PHP_BINARY.' '.NN_UPDATE.'tmux/bin/groupfixrelnames.php "'.$guid.'"'.' true' |
||
667 | ); |
||
668 | } |
||
669 | } |
||
670 | |||
671 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
672 | //////////////////////////////////////// All releases code here //////////////////////////////////////////////////// |
||
673 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
674 | |||
675 | /** |
||
676 | * @return null|string |
||
677 | * @throws \Exception |
||
678 | */ |
||
679 | private function releasesMainMethod() |
||
680 | { |
||
681 | $this->register_child_run([0 => $this, 1 => 'releasesChildWorker']); |
||
682 | |||
683 | $groups = $this->pdo->queryDirect('SELECT id FROM groups WHERE (active = 1 OR backfill = 1)'); |
||
684 | |||
685 | if ($groups instanceof \Traversable) { |
||
686 | foreach ($groups as $group) { |
||
687 | try { |
||
688 | if ($this->pdo->queryOneRow(sprintf('SELECT id FROM collections_%d LIMIT 1', $group['id'])) !== false) { |
||
689 | $this->work[] = ['id' => $group['id']]; |
||
690 | } |
||
691 | } catch (\PDOException $e) { |
||
692 | $e->getMessage(); |
||
693 | } |
||
694 | } |
||
695 | } |
||
696 | |||
697 | return (int) Settings::settingValue('..releasethreads'); |
||
698 | } |
||
699 | |||
700 | /** |
||
701 | * @param $groups |
||
702 | * @param string $identifier |
||
703 | */ |
||
704 | public function releasesChildWorker($groups, $identifier = '') |
||
705 | { |
||
706 | foreach ($groups as $group) { |
||
707 | $this->_executeCommand($this->dnr_path.'releases '.$group['id'].'"'); |
||
708 | } |
||
709 | } |
||
710 | |||
711 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
712 | /////////////////////////////////////// All post process code here ///////////////////////////////////////////////// |
||
713 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
714 | |||
715 | /** |
||
716 | * Only 1 exit method is used for post process, since they are all similar. |
||
717 | * |
||
718 | * |
||
719 | * @param array $groups |
||
720 | * @param string $identifier |
||
721 | */ |
||
722 | public function postProcessChildWorker($groups, $identifier = '') |
||
723 | { |
||
724 | $type = ''; |
||
725 | if ($this->processAdditional) { |
||
726 | $type = 'pp_additional '; |
||
727 | } elseif ($this->processNFO) { |
||
728 | $type = 'pp_nfo '; |
||
729 | } elseif ($this->processMovies) { |
||
730 | $type = 'pp_movie '; |
||
731 | } elseif ($this->processTV) { |
||
732 | $type = 'pp_tv '; |
||
733 | } |
||
734 | foreach ($groups as $group) { |
||
735 | if ($type !== '') { |
||
736 | $this->_executeCommand( |
||
737 | $this->dnr_path.$type.$group['id'].(isset($group['renamed']) ? (' '.$group['renamed']) : '').'"' |
||
738 | ); |
||
739 | } |
||
740 | } |
||
741 | } |
||
742 | |||
743 | private $ppAddMinSize; |
||
744 | private $ppAddMaxSize; |
||
745 | |||
746 | /** |
||
747 | * Check if we should process Additional's. |
||
748 | * @return bool |
||
749 | * @throws \Exception |
||
750 | */ |
||
751 | private function checkProcessAdditional() |
||
752 | { |
||
753 | $this->ppAddMinSize = |
||
754 | (Settings::settingValue('..minsizetopostprocess') !== '') ? (int) Settings::settingValue('..minsizetopostprocess') : 1; |
||
755 | $this->ppAddMinSize = ($this->ppAddMinSize > 0 ? ('AND r.size > '.($this->ppAddMinSize * 1048576)) : ''); |
||
756 | $this->ppAddMaxSize = |
||
757 | (Settings::settingValue('..maxsizetopostprocess') !== '') ? (int) Settings::settingValue('..maxsizetopostprocess') : 100; |
||
758 | $this->ppAddMaxSize = ($this->ppAddMaxSize > 0 ? ('AND r.size < '.($this->ppAddMaxSize * 1073741824)) : ''); |
||
759 | |||
760 | return $this->pdo->queryOneRow(sprintf(' |
||
761 | SELECT r.id |
||
762 | FROM releases r |
||
763 | LEFT JOIN categories c ON c.id = r.categories_id |
||
764 | WHERE r.nzbstatus = %d |
||
765 | AND r.passwordstatus BETWEEN -6 AND -1 |
||
766 | AND r.haspreview = -1 |
||
767 | AND c.disablepreview = 0 |
||
768 | %s %s |
||
769 | LIMIT 1', NZB::NZB_ADDED, $this->ppAddMaxSize, $this->ppAddMinSize)) !== false; |
||
770 | } |
||
771 | |||
772 | /** |
||
773 | * @return int|null|string |
||
774 | * @throws \Exception |
||
775 | */ |
||
776 | private function postProcessAddMainMethod() |
||
777 | { |
||
778 | $maxProcesses = 1; |
||
779 | if ($this->checkProcessAdditional() === true) { |
||
780 | $this->processAdditional = true; |
||
781 | $this->register_child_run([0 => $this, 1 => 'postProcessChildWorker']); |
||
782 | $this->work = $this->pdo->query( |
||
783 | sprintf( |
||
784 | ' |
||
785 | SELECT leftguid AS id |
||
786 | FROM releases r |
||
787 | LEFT JOIN categories c ON c.id = r.categories_id |
||
788 | WHERE r.nzbstatus = %d |
||
789 | AND r.passwordstatus BETWEEN -6 AND -1 |
||
790 | AND r.haspreview = -1 |
||
791 | AND c.disablepreview = 0 |
||
792 | %s %s |
||
793 | GROUP BY leftguid |
||
794 | LIMIT 16', |
||
795 | NZB::NZB_ADDED, |
||
796 | $this->ppAddMaxSize, |
||
797 | $this->ppAddMinSize |
||
798 | ) |
||
799 | ); |
||
800 | $maxProcesses = (int) Settings::settingValue('..postthreads'); |
||
801 | } |
||
802 | |||
803 | return $maxProcesses; |
||
804 | } |
||
805 | |||
806 | private $nfoQueryString = ''; |
||
807 | |||
808 | /** |
||
809 | * Check if we should process NFO's. |
||
810 | * |
||
811 | * @return bool |
||
812 | * @throws \Exception |
||
813 | */ |
||
814 | private function checkProcessNfo(): bool |
||
815 | { |
||
816 | if ((int) Settings::settingValue('..lookupnfo') === 1) { |
||
817 | $this->nfoQueryString = Nfo::NfoQueryString($this->pdo); |
||
818 | |||
819 | return $this->pdo->queryOneRow(sprintf('SELECT r.id FROM releases r WHERE 1=1 %s LIMIT 1', $this->nfoQueryString)) !== false; |
||
820 | } |
||
821 | |||
822 | return false; |
||
823 | } |
||
824 | |||
825 | /** |
||
826 | * @return int |
||
827 | * @throws \Exception |
||
828 | */ |
||
829 | private function postProcessNfoMainMethod(): int |
||
849 | } |
||
850 | |||
851 | /** |
||
852 | * @return bool |
||
853 | * @throws \Exception |
||
854 | */ |
||
855 | private function checkProcessMovies(): bool |
||
869 | } |
||
870 | |||
871 | /** |
||
872 | * @return int |
||
873 | * @throws \Exception |
||
874 | */ |
||
875 | private function postProcessMovMainMethod(): int |
||
876 | { |
||
877 | $maxProcesses = 1; |
||
878 | if ($this->checkProcessMovies() === true) { |
||
879 | $this->processMovies = true; |
||
880 | $this->register_child_run([0 => $this, 1 => 'postProcessChildWorker']); |
||
881 | $this->work = $this->pdo->query( |
||
882 | sprintf( |
||
883 | ' |
||
884 | SELECT leftguid AS id, %d AS renamed |
||
885 | FROM releases |
||
886 | WHERE categories_id BETWEEN 5000 AND 5999 |
||
887 | AND nzbstatus = %d |
||
888 | AND imdbid IS NULL |
||
889 | %s %s |
||
890 | GROUP BY leftguid |
||
891 | LIMIT 16', |
||
892 | ($this->ppRenamedOnly ? 2 : 1), |
||
893 | NZB::NZB_ADDED, |
||
894 | ((int) Settings::settingValue('..lookupimdb') === 2 ? 'AND isrenamed = 1' : ''), |
||
895 | ($this->ppRenamedOnly ? 'AND isrenamed = 1' : '') |
||
896 | ) |
||
897 | ); |
||
898 | $maxProcesses = (int) Settings::settingValue('..postthreadsnon'); |
||
899 | } |
||
900 | |||
901 | return $maxProcesses; |
||
902 | } |
||
903 | |||
904 | /** |
||
905 | * Check if we should process TV's. |
||
906 | * @return bool |
||
907 | * @throws \Exception |
||
908 | */ |
||
909 | private function checkProcessTV() |
||
910 | { |
||
911 | if ((int) Settings::settingValue('..lookuptvrage') > 0) { |
||
912 | return $this->pdo->queryOneRow(sprintf(' |
||
913 | SELECT id |
||
914 | FROM releases |
||
915 | WHERE categories_id BETWEEN 5000 AND 5999 |
||
916 | AND nzbstatus = %d |
||
917 | AND size > 1048576 |
||
918 | AND tv_episodes_id BETWEEN -2 AND 0 |
||
919 | %s %s |
||
920 | ', NZB::NZB_ADDED, (int) Settings::settingValue('..lookuptvrage') === 2 ? 'AND isrenamed = 1' : '', $this->ppRenamedOnly ? 'AND isrenamed = 1' : '')) !== false; |
||
921 | } |
||
922 | |||
923 | return false; |
||
924 | } |
||
925 | |||
926 | /** |
||
927 | * @return int |
||
928 | * @throws \Exception |
||
929 | */ |
||
930 | private function postProcessTvMainMethod() |
||
931 | { |
||
932 | $maxProcesses = 1; |
||
933 | if ($this->checkProcessTV() === true) { |
||
934 | $this->processTV = true; |
||
935 | $this->register_child_run([0 => $this, 1 => 'postProcessChildWorker']); |
||
936 | $this->work = $this->pdo->query( |
||
937 | sprintf( |
||
938 | ' |
||
939 | SELECT leftguid AS id, %d AS renamed |
||
940 | FROM releases |
||
941 | WHERE categories_id BETWEEN 3000 AND 3999 |
||
942 | AND nzbstatus = %d |
||
943 | AND tv_episodes_id BETWEEN -2 AND 0 |
||
944 | AND size > 1048576 |
||
945 | %s %s |
||
946 | GROUP BY leftguid |
||
947 | LIMIT 16', |
||
948 | ($this->ppRenamedOnly ? 2 : 1), |
||
949 | NZB::NZB_ADDED, |
||
950 | (int) Settings::settingValue('..lookuptvrage') === 2 ? 'AND isrenamed = 1' : '', |
||
951 | ($this->ppRenamedOnly ? 'AND isrenamed = 1' : '') |
||
952 | ) |
||
953 | ); |
||
954 | $maxProcesses = (int) Settings::settingValue('..postthreadsnon'); |
||
955 | } |
||
956 | |||
957 | return $maxProcesses; |
||
958 | } |
||
959 | |||
960 | /** |
||
961 | * Process sharing. |
||
962 | * @return bool |
||
963 | * @throws \Exception |
||
964 | */ |
||
965 | private function processSharing() |
||
966 | { |
||
967 | $sharing = $this->pdo->queryOneRow('SELECT enabled FROM sharing'); |
||
968 | if ($sharing !== false && (int) $sharing['enabled'] === 1) { |
||
969 | $nntp = new NNTP(['Settings' => $this->pdo]); |
||
970 | if ((int) (Settings::settingValue('..alternate_nntp') === 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) === true) { |
||
971 | (new PostProcess(['Settings' => $this->pdo, 'ColorCLI' => $this->_colorCLI]))->processSharing($nntp); |
||
972 | } |
||
973 | |||
974 | return true; |
||
975 | } |
||
976 | |||
977 | return false; |
||
978 | } |
||
979 | |||
980 | /** |
||
981 | * Process all that require a single thread. |
||
982 | * |
||
983 | * @throws \Exception |
||
984 | */ |
||
985 | private function processSingle() |
||
986 | { |
||
987 | $postProcess = new PostProcess(['Settings' => $this->pdo, 'ColorCLI' => $this->_colorCLI]); |
||
988 | //$postProcess->processAnime(); |
||
989 | $postProcess->processBooks(); |
||
990 | $postProcess->processConsoles(); |
||
991 | $postProcess->processGames(); |
||
992 | $postProcess->processMusic(); |
||
993 | $postProcess->processXXX(); |
||
994 | } |
||
995 | |||
996 | /** |
||
997 | * @param $groups |
||
998 | * @param string $identifier |
||
999 | */ |
||
1000 | public function requestIDChildWorker($groups, $identifier = '') |
||
1001 | { |
||
1002 | foreach ($groups as $group) { |
||
1003 | $this->_executeCommand($this->dnr_path.'requestid '.$group['id'].'"'); |
||
1004 | } |
||
1005 | } |
||
1006 | |||
1007 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
1008 | ///////////////////////////////// All "update_per_Group" code goes here //////////////////////////////////////////// |
||
1009 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
1010 | |||
1011 | /** |
||
1012 | * @return int |
||
1013 | * @throws \Exception |
||
1014 | */ |
||
1015 | private function updatePerGroupMainMethod() |
||
1016 | { |
||
1017 | $this->register_child_run([0 => $this, 1 => 'updatePerGroupChildWorker']); |
||
1018 | $this->work = $this->pdo->query('SELECT id FROM groups WHERE (active = 1 OR backfill = 1)'); |
||
1019 | |||
1020 | return (int) Settings::settingValue('..releasethreads'); |
||
1021 | } |
||
1022 | |||
1023 | /** |
||
1024 | * @param $groups |
||
1025 | * @param string $identifier |
||
1026 | */ |
||
1027 | public function updatePerGroupChildWorker($groups, $identifier = '') |
||
1032 | ); |
||
1033 | } |
||
1034 | } |
||
1035 | |||
1036 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
1037 | //////////////////////////////////////////// Various methods /////////////////////////////////////////////////////// |
||
1038 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// |
||
1039 | |||
1040 | /** |
||
1041 | * Execute a shell command, use the appropriate PHP function based on user setting. |
||
1042 | * |
||
1043 | * @param string $command |
||
1044 | */ |
||
1045 | protected function _executeCommand($command) |
||
1057 | } |
||
1058 | } |
||
1059 | |||
1060 | /** |
||
1061 | * Set the amount of max child processes. |
||
1062 | * |
||
1063 | * @param int $maxProcesses |
||
1064 | */ |
||
1065 | private function setMaxProcesses($maxProcesses) |
||
1066 | { |
||
1067 | // Check if override setting is on. |
||
1068 | if (\defined('NN_MULTIPROCESSING_MAX_CHILDREN_OVERRIDE') && NN_MULTIPROCESSING_MAX_CHILDREN_OVERRIDE > 0) { |
||
1069 | $maxProcesses = NN_MULTIPROCESSING_MAX_CHILDREN_OVERRIDE; |
||
1070 | } |
||
1071 | |||
1072 | if (is_numeric($maxProcesses) && $maxProcesses > 0) { |
||
1073 | switch ($this->workType) { |
||
1074 | case 'postProcess_tv': |
||
1075 | case 'postProcess_mov': |
||
1076 | case 'postProcess_nfo': |
||
1077 | case 'postProcess_add': |
||
1078 | if ($maxProcesses > 16) { |
||
1079 | $maxProcesses = 16; |
||
1080 | } |
||
1081 | } |
||
1082 | $this->maxProcesses = (int) $maxProcesses; |
||
1083 | $this->max_children_set($this->maxProcesses); |
||
1084 | } else { |
||
1085 | $this->max_children_set(1); |
||
1086 | } |
||
1087 | } |
||
1088 | |||
1089 | /** |
||
1090 | * Echo a message to CLI. |
||
1091 | * |
||
1092 | * @param string $message |
||
1093 | */ |
||
1094 | public function logger($message) |
||
1098 | } |
||
1099 | } |
||
1100 | |||
1101 | /** |
||
1102 | * This method is executed whenever a child is finished doing work. |
||
1103 | * |
||
1104 | * @param string $pid The PID numbers. |
||
1105 | * @param string $identifier Optional identifier to give a PID a name. |
||
1106 | */ |
||
1107 | public function childExit($pid, $identifier = '') |
||
1116 | ); |
||
1117 | } |
||
1120 |