Completed
Push — master ( c702e0...af42cc )
by Matthew
07:24
created

BaseJobManager::resetStalledJobs()   B

Complexity

Conditions 6
Paths 5

Size

Total Lines 32
Code Lines 22

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 20
CRAP Score 6.1666

Importance

Changes 0
Metric Value
dl 0
loc 32
ccs 20
cts 24
cp 0.8333
rs 8.439
c 0
b 0
f 0
cc 6
eloc 22
nc 5
nop 2
crap 6.1666
1
<?php
2
3
namespace Dtc\QueueBundle\Doctrine;
4
5
use Doctrine\Common\Persistence\ObjectManager;
6
use Doctrine\Common\Persistence\ObjectRepository;
7
use Doctrine\ODM\MongoDB\DocumentRepository;
8
use Doctrine\ORM\EntityRepository;
9
use Dtc\QueueBundle\Model\BaseJob;
10
use Dtc\QueueBundle\Model\Job;
11
use Dtc\QueueBundle\Model\PriorityJobManager;
12
use Dtc\QueueBundle\Model\RetryableJob;
13
use Dtc\QueueBundle\Model\Run;
14
use Dtc\QueueBundle\Util\Util;
15
16
abstract class BaseJobManager extends PriorityJobManager
17
{
18
    /** Number of jobs to prune / reset / gather at a time */
19
    const FETCH_COUNT = 100;
20
21
    /** Number of seconds before a job is considered stalled if the runner is no longer active */
22
    const STALLED_SECONDS = 1800;
23
    protected $objectManager;
24
    protected $objectName;
25
    protected $archiveObjectName;
26
    protected $runClass;
27
    protected $runArchiveClass;
28
29
    /**
30
     * @param string $objectName
31
     * @param string $archiveObjectName
32
     * @param string $runClass
33
     * @param string $runArchiveClass
34
     */
35 11
    public function __construct(ObjectManager $objectManager,
36
        $objectName,
37
        $archiveObjectName,
38
        $runClass,
39
        $runArchiveClass)
40
    {
41 11
        $this->objectManager = $objectManager;
42 11
        $this->objectName = $objectName;
43 11
        $this->archiveObjectName = $archiveObjectName;
44 11
        $this->runClass = $runClass;
45 11
        $this->runArchiveClass = $runArchiveClass;
46 11
    }
47
48
    /**
49
     * @return ObjectManager
50
     */
51 36
    public function getObjectManager()
52
    {
53 36
        return $this->objectManager;
54
    }
55
56
    /**
57
     * @return string
58
     */
59 34
    public function getObjectName()
60
    {
61 34
        return $this->objectName;
62
    }
63
64
    /**
65
     * @return string
66
     */
67 25
    public function getArchiveObjectName()
68
    {
69 25
        return $this->archiveObjectName;
70
    }
71
72
    /**
73
     * @return string
74
     */
75 4
    public function getRunClass()
76
    {
77 4
        return $this->runClass;
78
    }
79
80
    /**
81
     * @return string
82
     */
83 5
    public function getRunArchiveClass()
84
    {
85 5
        return $this->runArchiveClass;
86
    }
87
88
    /**
89
     * @return ObjectRepository
90
     */
91 27
    public function getRepository()
92
    {
93 27
        return $this->getObjectManager()->getRepository($this->getObjectName());
94
    }
95
96
    /**
97
     * @param string $objectName
98
     */
99
    abstract protected function countJobsByStatus($objectName, $status, $workerName = null, $method = null);
100
101 2
    public function resetErroneousJobs($workerName = null, $method = null)
102
    {
103 2
        $count = $this->countJobsByStatus($this->getArchiveObjectName(), Job::STATUS_ERROR, $workerName, $method);
104
105 2
        $criterion = ['status' => Job::STATUS_ERROR];
106 2
        $this->addWorkerNameMethod($criterion, $workerName, $method);
107
108 2
        $countProcessed = 0;
109 2
        for ($i = 0; $i < $count; $i += static::FETCH_COUNT) {
110 2
            $countProcessed += $this->resetJobsByCriterion(
111 2
                $criterion, static::FETCH_COUNT, $i);
112 2
        }
113
114 2
        return $countProcessed;
115
    }
116
117
    /**
118
     * Sets the status to Job::STATUS_EXPIRED for those jobs that are expired.
119
     *
120
     * @param null $workerName
121
     * @param null $method
122
     *
123
     * @return mixed
124
     */
125
    abstract protected function updateExpired($workerName = null, $method = null);
126
127 9
    protected function addWorkerNameMethod(array &$criterion, $workerName = null, $method = null)
128
    {
129 9
        if (null !== $workerName) {
130 4
            $criterion['workerName'] = $workerName;
131 4
        }
132 9
        if (null !== $method) {
133 4
            $criterion['method'] = $method;
134 4
        }
135 9
    }
136
137 3
    public function pruneExpiredJobs($workerName = null, $method = null)
138
    {
139 3
        $count = $this->updateExpired($workerName, $method);
140 3
        $criterion = ['status' => Job::STATUS_EXPIRED];
141 3
        $this->addWorkerNameMethod($criterion, $workerName, $method);
142 3
        $objectManager = $this->getObjectManager();
143 3
        $repository = $this->getRepository();
144 3
        $finalCount = 0;
145 3
        for ($i = 0; $i < $count; $i += static::FETCH_COUNT) {
146 3
            $expiredJobs = $repository->findBy($criterion, null, static::FETCH_COUNT, $i);
147 3
            if (!empty($expiredJobs)) {
148 3
                foreach ($expiredJobs as $expiredJob) {
149
                    /* @var Job $expiredJob */
150 3
                    $expiredJob->setStatus(Job::STATUS_EXPIRED);
151 3
                    $objectManager->remove($expiredJob);
152 3
                    ++$finalCount;
153 3
                }
154 3
            }
155 3
            $objectManager->flush();
156 3
        }
157
158 3
        return $finalCount;
159
    }
160
161 4
    protected function getStalledJobs($workerName = null, $method = null)
162
    {
163 4
        $count = $this->countJobsByStatus($this->getObjectName(), Job::STATUS_RUNNING, $workerName, $method);
164
165 4
        $criterion = ['status' => BaseJob::STATUS_RUNNING];
166 4
        $this->addWorkerNameMethod($criterion, $workerName, $method);
167
168 4
        $runningJobs = $this->findRunningJobs($criterion, $count);
169
170 4
        return $this->extractStalledJobs($runningJobs);
171
    }
172
173 4
    protected function findRunningJobs($criterion, $count)
174
    {
175 4
        $repository = $this->getRepository();
176 4
        $runningJobsById = [];
177
178 4
        for ($i = 0; $i < $count; $i += static::FETCH_COUNT) {
179 4
            $runningJobs = $repository->findBy($criterion, null, static::FETCH_COUNT, $i);
180 4
            if (!empty($runningJobs)) {
181 4
                foreach ($runningJobs as $job) {
182
                    /** @var RetryableJob $job */
183 4
                    if (null !== $runId = $job->getRunId()) {
184 4
                        $runningJobsById[$runId][] = $job;
185 4
                    }
186 4
                }
187 4
            }
188 4
        }
189
190 4
        return $runningJobsById;
191
    }
192
193 4
    protected function extractStalledJobs(array $runningJobsById)
194
    {
195 4
        $objectManager = $this->getObjectManager();
196 4
        $runRepository = $objectManager->getRepository($this->runClass);
197
        /** @var EntityRepository|DocumentRepository $runArchiveRepository */
198 4
        $runArchiveRepository = $objectManager->getRepository($this->runArchiveClass);
199
200 4
        $stalledJobs = [];
201 4
        foreach (array_keys($runningJobsById) as $runId) {
202 4
            if ($runRepository->find($runId)) {
203
                continue;
204
            }
205
            /** @var Run $run */
206 4
            if ($run = $runArchiveRepository->find($runId)) {
207 4
                if ($endTime = $run->getEndedAt()) {
208
                    // Did it end over an hour ago
209 4
                    if ((time() - $endTime->getTimestamp()) > static::STALLED_SECONDS) {
210 4
                        $stalledJobs = array_merge($stalledJobs, $runningJobsById[$runId]);
211 4
                    }
212 4
                }
213 4
            }
214 4
        }
215
216 4
        return $stalledJobs;
217
    }
218
219 6
    protected function updateMaxStatus(RetryableJob $job, $status, $max = null, $count = 0)
220
    {
221 6
        if (null !== $max && $count >= $max) {
222
            $job->setStatus($status);
223
224
            return true;
225
        }
226
227 6
        return false;
228
    }
229
230 2
    public function resetStalledJobs($workerName = null, $method = null)
231
    {
232 2
        $stalledJobs = $this->getStalledJobs($workerName, $method);
233
234 2
        $objectManager = $this->getObjectManager();
235
236 2
        $countProcessed = 0;
237 2
        for ($i = 0, $count = count($stalledJobs); $i < $count; $i += static::FETCH_COUNT) {
238 2
            for ($j = $i, $max = $i + static::FETCH_COUNT; $j < $max && $j < $count; ++$j) {
239 2
                $job = $stalledJobs[$j];
240
                /* RetryableJob $job */
241 2
                $job->setStalledCount($job->getStalledCount() + 1);
242 2
                if ($this->updateMaxStatus($job, RetryableJob::STATUS_MAX_STALLED, $job->getMaxStalled(), $job->getStalledCount())) {
243
                    $objectManager->remove($job);
244
                    continue;
245 2
                } elseif ($this->updateMaxStatus($job, RetryableJob::STATUS_MAX_RETRIES, $job->getMaxRetries(), $job->getRetries())) {
246
                    $objectManager->remove($job);
247
                    continue;
248
                }
249
250 2
                $job->setRetries($job->getRetries() + 1);
251 2
                $job->setStatus(BaseJob::STATUS_NEW);
252 2
                $job->setLocked(null);
253 2
                $job->setLockedAt(null);
254 2
                $objectManager->persist($job);
255 2
                ++$countProcessed;
256 2
            }
257 2
            $objectManager->flush();
258 2
        }
259
260 2
        return $countProcessed;
261
    }
262
263
    /**
264
     * @param string $workerName
265
     * @param string $method
266
     */
267 2
    public function pruneStalledJobs($workerName = null, $method = null)
268
    {
269 2
        $stalledJobs = $this->getStalledJobs($workerName, $method);
270 2
        $objectManager = $this->getObjectManager();
271
272 2
        $countProcessed = 0;
273 2
        for ($i = 0, $count = count($stalledJobs); $i < $count; $i += static::FETCH_COUNT) {
274 2
            for ($j = $i, $max = $i + static::FETCH_COUNT; $j < $max && $j < $count; ++$j) {
275
                /** @var RetryableJob $job */
276 2
                $job = $stalledJobs[$j];
277 2
                $job->setStalledCount($job->getStalledCount() + 1);
278 2
                $job->setStatus(BaseJob::STATUS_ERROR);
279 2
                $job->setMessage('stalled');
280 2
                $this->updateMaxStatus($job, RetryableJob::STATUS_MAX_STALLED, $job->getMaxStalled(), $job->getStalledCount());
281 2
                $objectManager->remove($job);
282 2
                ++$countProcessed;
283 2
            }
284 2
            $objectManager->flush();
285 2
        }
286
287 2
        return $countProcessed;
288
    }
289
290 10
    public function deleteJob(\Dtc\QueueBundle\Model\Job $job)
291
    {
292 10
        $objectManager = $this->getObjectManager();
293 10
        $objectManager->remove($job);
294 10
        $objectManager->flush();
295 10
    }
296
297 2
    public function saveHistory(\Dtc\QueueBundle\Model\Job $job)
298
    {
299 2
        $this->deleteJob($job); // Should cause job to be archived
300 2
    }
301
302 31
    protected function prioritySave(\Dtc\QueueBundle\Model\Job $job)
303
    {
304
        // Generate crc hash for the job
305 31
        $hashValues = array($job->getClassName(), $job->getMethod(), $job->getWorkerName(), $job->getArgs());
306 31
        $crcHash = hash('sha256', serialize($hashValues));
307 31
        $job->setCrcHash($crcHash);
308 31
        $objectManager = $this->getObjectManager();
309
310 31
        if (true === $job->getBatch()) {
311
            // See if similar job that hasn't run exists
312 2
            $criteria = array('crcHash' => $crcHash, 'status' => BaseJob::STATUS_NEW);
313 2
            $oldJob = $this->getRepository()->findOneBy($criteria);
314
315 2
            if ($oldJob) {
316
                // Old job exists - just override fields Set higher priority
317 2
                $oldJob->setPriority($this->findHigherPriority($job->getPriority(), $oldJob->getPriority()));
318 2
                $oldJob->setWhenAt(min($job->getWhenAt(), $oldJob->getWhenAt()));
319 2
                $oldJob->setBatch(true);
320 2
                $objectManager->persist($oldJob);
321 2
                $objectManager->flush();
322
323 2
                return $oldJob;
324
            }
325
        }
326
327
        // Just save a new job
328 31
        $this->resetSaveOk(__FUNCTION__);
329 31
        $objectManager->persist($job);
330 31
        $objectManager->flush();
331
332 31
        return $job;
333
    }
334
335
    /**
336
     * @param string $objectName
337
     */
338
    abstract protected function stopIdGenerator($objectName);
339
340
    abstract protected function restoreIdGenerator($objectName);
341
342
    /**
343
     * @param array $criterion
344
     * @param int   $limit
345
     * @param int   $offset
346
     */
347 2
    private function resetJobsByCriterion(
348
        array $criterion,
349
        $limit,
350
        $offset)
351
    {
352 2
        $objectManager = $this->getObjectManager();
353 2
        $this->resetSaveOk(__FUNCTION__);
354 2
        $objectName = $this->getObjectName();
355 2
        $archiveObjectName = $this->getArchiveObjectName();
356 2
        $jobRepository = $objectManager->getRepository($objectName);
357 2
        $jobArchiveRepository = $objectManager->getRepository($archiveObjectName);
358 2
        $className = $jobRepository->getClassName();
359 2
        $metadata = $objectManager->getClassMetadata($className);
360 2
        $this->stopIdGenerator($objectName);
361 2
        $identifierData = $metadata->getIdentifier();
362 2
        $idColumn = isset($identifierData[0]) ? $identifierData[0] : 'id';
363 2
        $results = $jobArchiveRepository->findBy($criterion, [$idColumn => 'ASC'], $limit, $offset);
364 2
        $countProcessed = 0;
365
366 2
        foreach ($results as $jobArchive) {
367 2
            $this->resetJob($jobArchive, $className, $countProcessed);
368 2
        }
369 2
        $objectManager->flush();
370
371 2
        $this->restoreIdGenerator($objectName);
372
373 2
        return $countProcessed;
374
    }
375
376 17
    protected function resetSaveOk($function)
377
    {
378 17
    }
379
380
    /**
381
     * @param RetryableJob $jobArchive
382
     * @param $className
383
     * @param $countProcessed
384
     */
385 2
    protected function resetJob(RetryableJob $jobArchive, $className, &$countProcessed)
386
    {
387 2
        $objectManager = $this->getObjectManager();
388 2
        if ($this->updateMaxStatus($jobArchive, RetryableJob::STATUS_MAX_RETRIES, $jobArchive->getMaxRetries(), $jobArchive->getRetries())) {
389
            $objectManager->persist($jobArchive);
390
391
            return;
392
        }
393
394
        /** @var RetryableJob $job */
395 2
        $job = new $className();
396
397 2
        Util::copy($jobArchive, $job);
398 2
        $job->setStatus(BaseJob::STATUS_NEW);
399 2
        $job->setLocked(null);
400 2
        $job->setLockedAt(null);
401 2
        $job->setMessage(null);
402 2
        $job->setFinishedAt(null);
403 2
        $job->setStartedAt(null);
404 2
        $job->setElapsed(null);
405 2
        $job->setRetries($job->getRetries() + 1);
406
407 2
        $objectManager->persist($job);
408 2
        $objectManager->remove($jobArchive);
409 2
        ++$countProcessed;
410 2
    }
411
}
412