AOEpeople /
crawler
| 1 | <?php |
||
| 59 | * @noRector \Rector\DeadCode\Rector\Property\RemoveUnusedPrivatePropertyRector |
||
| 60 | * @noRector \Rector\DeadCode\Rector\Property\RemoveSetterOnlyPropertyAndMethodCallRector |
||
| 61 | */ |
||
| 62 | private $deprecatedPublicProperties = [ |
||
|
introduced
by
|
|||
| 63 | 'queueRepository' => 'Using queueRepository is deprecated since 9.0.1 and will be removed in v11.x', |
||
| 64 | 'crawlerController' => 'Using crawlerController is deprecated since 9.0.1 and will be removed in v11.x', |
||
| 65 | 'countInARun' => 'Using countInARun is deprecated since 9.0.1 and will be removed in v11.x', |
||
| 73 | * @noRector \Rector\DeadCode\Rector\Property\RemoveUnusedPrivatePropertyRector |
||
| 74 | * @noRector \Rector\DeadCode\Rector\Property\RemoveSetterOnlyPropertyAndMethodCallRector |
||
| 75 | */ |
||
| 76 | private $deprecatedPublicMethods = [ |
||
| 77 | 'multiProcess' => 'Using ProcessService::multiProcess() is deprecated since 9.0.1 and will be removed in v11.x', |
||
| 78 | 'reportItemStatus' => 'Using ProcessService::reportItemStatus() is deprecated since 9.0.1 and will be removed in v11.x', |
||
| 79 | 'startRequiredProcesses' => 'Using ProcessService::startRequiredProcesses() is deprecated since 9.0.1 and will be removed in v11.x', |
||
| 128 | { |
||
| 129 | 8 | $objectManager = GeneralUtility::makeInstance(ObjectManager::class); |
|
| 130 | 8 | $this->processRepository = $objectManager->get(ProcessRepository::class); |
|
| 131 | 8 | $this->queueRepository = $objectManager->get(QueueRepository::class); |
|
| 132 | 8 | $this->extensionSettings = GeneralUtility::makeInstance(ExtensionConfigurationProvider::class)->getExtensionConfiguration(); |
|
| 133 | 8 | $this->timeToLive = (int) $this->extensionSettings['processMaxRunTime']; |
|
| 134 | 8 | $this->countInARun = (int) $this->extensionSettings['countInARun']; |
|
| 135 | 8 | $this->processLimit = (int) $this->extensionSettings['processLimit']; |
|
| 136 | 8 | $this->verbose = (bool) $this->extensionSettings['processVerbose']; |
|
| 137 | 8 | } |
|
| 138 | |||
| 139 | /** |
||
| 147 | */ |
||
| 148 | public function multiProcess($timeout): void |
||
| 149 | { |
||
| 150 | if ($this->processLimit <= 1) { |
||
| 151 | throw new \RuntimeException('To run crawler in multi process mode you have to configure the processLimit > 1.' . PHP_EOL); |
||
| 152 | } |
||
| 153 | |||
| 154 | $pendingItemsStart = $this->queueRepository->countAllPendingItems(); |
||
| 155 | $itemReportLimit = 20; |
||
| 156 | $reportItemCount = $pendingItemsStart - $itemReportLimit; |
||
| 157 | if ($this->verbose) { |
||
| 158 | $this->reportItemStatus(); |
||
| 159 | } |
||
| 160 | $this->startRequiredProcesses(); |
||
| 161 | $nextTimeOut = time() + $this->timeToLive; |
||
| 162 | $currentPendingItems = ''; |
||
| 163 | for ($i = 0; $i < $timeout; $i++) { |
||
| 164 | $currentPendingItems = $this->queueRepository->countAllPendingItems(); |
||
| 165 | if ($this->startRequiredProcesses()) { |
||
| 166 | $nextTimeOut = time() + $this->timeToLive; |
||
| 167 | } |
||
| 168 | if ($currentPendingItems === 0) { |
||
| 169 | if ($this->verbose) { |
||
| 170 | echo 'Finished...' . chr(10); |
||
| 171 | } |
||
| 172 | break; |
||
| 173 | } |
||
| 174 | if ($currentPendingItems < $reportItemCount) { |
||
| 175 | if ($this->verbose) { |
||
| 176 | $this->reportItemStatus(); |
||
| 177 | } |
||
| 178 | $reportItemCount = $currentPendingItems - $itemReportLimit; |
||
| 179 | } |
||
| 181 | if ($nextTimeOut < time()) { |
||
| 182 | $timedOutProcesses = $this->processRepository->findAll(); |
||
| 183 | $nextTimeOut = time() + $this->timeToLive; |
||
| 184 | if ($this->verbose) { |
||
| 185 | echo 'Cleanup' . implode(',', $timedOutProcesses->getProcessIds()) . chr(10); |
||
| 186 | } |
||
| 187 | $this->crawlerController->CLI_releaseProcesses($timedOutProcesses->getProcessIds()); |
||
| 188 | } |
||
| 189 | } |
||
| 190 | if ($currentPendingItems > 0 && $this->verbose) { |
||
| 191 | echo 'Stop with timeout' . chr(10); |
||
| 192 | } |
||
| 193 | } |
||
| 248 | */ |
||
| 249 | protected function reportItemStatus(): void |
||
| 250 | { |
||
| 251 | echo 'Pending:' . $this->queueRepository->countAllPendingItems() . ' / Assigned:' . $this->queueRepository->countAllAssignedPendingItems() . chr(10); |
||
| 252 | } |
||
| 253 | |||
| 254 | /** |
||
| 264 | { |
||
| 265 | $ret = false; |
||
| 266 | $currentProcesses = $this->processRepository->findAllActive()->count(); |
||
| 267 | $availableProcessesCount = $this->processLimit - $currentProcesses; |
||
| 268 | $requiredProcessesCount = ceil($this->queueRepository->countAllUnassignedPendingItems() / $this->countInARun); |
||
| 269 | $startProcessCount = min([$availableProcessesCount, $requiredProcessesCount]); |
||
| 270 | if ($startProcessCount <= 0) { |
||
| 271 | return $ret; |
||
| 272 | } |
||
| 273 | if ($startProcessCount && $this->verbose) { |
||
| 274 | echo 'Start ' . $startProcessCount . ' new processes (Running:' . $currentProcesses . ')'; |
||
| 275 | } |
||
| 276 | for ($i = 0; $i < $startProcessCount; $i++) { |
||
| 277 | usleep(100); |
||
| 278 | if ($this->startProcess() && $this->verbose) { |
||
| 279 | echo '.'; |
||
| 280 | $ret = true; |
||
| 281 | } |
||
| 282 | } |
||
| 283 | if ($this->verbose) { |
||
| 284 | echo chr(10); |
||
| 285 | } |
||
| 286 | return $ret; |
||
| 287 | } |
||
| 288 | } |
||
| 289 |
Stefan Rotsch
Tomas Norre Mikkelsen
Loading history...