Completed
Push — master ( 122dc2...fa7a2a )
by Matthew
09:15
created

BaseJobManager::updateMaxStatus()   A

Complexity

Conditions 3
Paths 2

Size

Total Lines 10
Code Lines 5

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 3
CRAP Score 3.576

Importance

Changes 0
Metric Value
dl 0
loc 10
ccs 3
cts 5
cp 0.6
rs 9.4285
c 0
b 0
f 0
cc 3
eloc 5
nc 2
nop 4
crap 3.576
1
<?php
2
3
namespace Dtc\QueueBundle\Doctrine;
4
5
use Doctrine\Common\Persistence\ObjectManager;
6
use Doctrine\Common\Persistence\ObjectRepository;
7
use Doctrine\ODM\MongoDB\DocumentRepository;
8
use Doctrine\ORM\EntityRepository;
9
use Dtc\QueueBundle\Model\BaseJob;
10
use Dtc\QueueBundle\Model\Job;
11
use Dtc\QueueBundle\Model\PriorityJobManager;
12
use Dtc\QueueBundle\Model\RetryableJob;
13
use Dtc\QueueBundle\Model\Run;
14
use Dtc\QueueBundle\Util\Util;
15
16
abstract class BaseJobManager extends PriorityJobManager
17
{
18
    /** Number of jobs to prune / reset / gather at a time */
19
    const FETCH_COUNT = 100;
20
21
    /** Number of seconds before a job is considered stalled if the runner is no longer active */
22
    const STALLED_SECONDS = 1800;
23
    protected $objectManager;
24
    protected $objectName;
25
    protected $archiveObjectName;
26
    protected $runClass;
27
    protected $runArchiveClass;
28
29
    /**
30
     * @param string $objectName
31
     * @param string $archiveObjectName
32
     * @param string $runClass
33
     * @param string $runArchiveClass
34
     */
35 11
    public function __construct(ObjectManager $objectManager,
36
        $objectName,
37
        $archiveObjectName,
38
        $runClass,
39
        $runArchiveClass)
40
    {
41 11
        $this->objectManager = $objectManager;
42 11
        $this->objectName = $objectName;
43 11
        $this->archiveObjectName = $archiveObjectName;
44 11
        $this->runClass = $runClass;
45 11
        $this->runArchiveClass = $runArchiveClass;
46 11
    }
47
48
    /**
49
     * @return ObjectManager
50
     */
51 36
    public function getObjectManager()
52
    {
53 36
        return $this->objectManager;
54
    }
55
56
    /**
57
     * @return string
58
     */
59 34
    public function getObjectName()
60
    {
61 34
        return $this->objectName;
62
    }
63
64
    /**
65
     * @return string
66
     */
67 27
    public function getArchiveObjectName()
68
    {
69 27
        return $this->archiveObjectName;
70
    }
71
72
    /**
73
     * @return string
74
     */
75 4
    public function getRunClass()
76
    {
77 4
        return $this->runClass;
78
    }
79
80
    /**
81
     * @return string
82
     */
83 5
    public function getRunArchiveClass()
84
    {
85 5
        return $this->runArchiveClass;
86
    }
87
88
    /**
89
     * @return ObjectRepository
90
     */
91 27
    public function getRepository()
92
    {
93 27
        return $this->getObjectManager()->getRepository($this->getObjectName());
94
    }
95
96
    /**
97
     * @param string $objectName
98
     */
99
    abstract protected function countJobsByStatus($objectName, $status, $workerName = null, $method = null);
100
101 2
    public function resetErroneousJobs($workerName = null, $method = null)
102
    {
103 2
        $count = $this->countJobsByStatus($this->getArchiveObjectName(), Job::STATUS_ERROR, $workerName, $method);
104
105 2
        $criterion = ['status' => Job::STATUS_ERROR];
106 2
        $this->addWorkerNameMethod($criterion, $workerName, $method);
107
108 2
        $countProcessed = 0;
109 2
        for ($i = 0; $i < $count; $i += static::FETCH_COUNT) {
110 2
            $countProcessed += $this->resetJobsByCriterion(
111 2
                $criterion, static::FETCH_COUNT, $i);
112
        }
113
114 2
        return $countProcessed;
115
    }
116
117
    /**
118
     * Sets the status to Job::STATUS_EXPIRED for those jobs that are expired.
119
     *
120
     * @param null $workerName
121
     * @param null $method
122
     *
123
     * @return mixed
124
     */
125
    abstract protected function updateExpired($workerName = null, $method = null);
126
127 9
    protected function addWorkerNameMethod(array &$criterion, $workerName = null, $method = null)
128
    {
129 9
        if (null !== $workerName) {
130 4
            $criterion['workerName'] = $workerName;
131
        }
132 9
        if (null !== $method) {
133 4
            $criterion['method'] = $method;
134
        }
135 9
    }
136
137 3
    public function pruneExpiredJobs($workerName = null, $method = null)
138
    {
139 3
        $count = $this->updateExpired($workerName, $method);
140 3
        $criterion = ['status' => Job::STATUS_EXPIRED];
141 3
        $this->addWorkerNameMethod($criterion, $workerName, $method);
142 3
        $objectManager = $this->getObjectManager();
143 3
        $repository = $this->getRepository();
144 3
        $finalCount = 0;
145 3
        for ($i = 0; $i < $count; $i += static::FETCH_COUNT) {
146 3
            $expiredJobs = $repository->findBy($criterion, null, static::FETCH_COUNT, $i);
147 3
            if (!empty($expiredJobs)) {
148 3
                foreach ($expiredJobs as $expiredJob) {
149
                    /* @var Job $expiredJob */
150 3
                    $expiredJob->setStatus(Job::STATUS_EXPIRED);
151 3
                    $objectManager->remove($expiredJob);
152 3
                    ++$finalCount;
153
                }
154
            }
155 3
            $objectManager->flush();
156
        }
157
158 3
        return $finalCount;
159
    }
160
161 4
    protected function getStalledJobs($workerName = null, $method = null)
162
    {
163 4
        $count = $this->countJobsByStatus($this->getObjectName(), Job::STATUS_RUNNING, $workerName, $method);
164
165 4
        $criterion = ['status' => BaseJob::STATUS_RUNNING];
166 4
        $this->addWorkerNameMethod($criterion, $workerName, $method);
167
168 4
        $runningJobs = $this->findRunningJobs($criterion, $count);
169
170 4
        return $this->extractStalledJobs($runningJobs);
171
    }
172
173 4
    protected function findRunningJobs($criterion, $count)
174
    {
175 4
        $repository = $this->getRepository();
176 4
        $runningJobsById = [];
177
178 4
        for ($i = 0; $i < $count; $i += static::FETCH_COUNT) {
179 4
            $runningJobs = $repository->findBy($criterion, null, static::FETCH_COUNT, $i);
180 4
            if (!empty($runningJobs)) {
181 4
                foreach ($runningJobs as $job) {
182
                    /** @var RetryableJob $job */
183 4
                    if (null !== $runId = $job->getRunId()) {
184 4
                        $runningJobsById[$runId][] = $job;
185
                    }
186
                }
187
            }
188
        }
189
190 4
        return $runningJobsById;
191
    }
192
193 4
    protected function extractStalledJobs(array $runningJobsById)
194
    {
195 4
        $objectManager = $this->getObjectManager();
196 4
        $runRepository = $objectManager->getRepository($this->runClass);
197
        /** @var EntityRepository|DocumentRepository $runArchiveRepository */
198 4
        $runArchiveRepository = $objectManager->getRepository($this->runArchiveClass);
199
200 4
        $stalledJobs = [];
201 4
        foreach (array_keys($runningJobsById) as $runId) {
202 4
            if ($runRepository->find($runId)) {
203
                continue;
204
            }
205
            /** @var Run $run */
206 4
            if ($run = $runArchiveRepository->find($runId)) {
207 4
                if ($endTime = $run->getEndedAt()) {
208
                    // Did it end over an hour ago
209 4
                    if ((time() - $endTime->getTimestamp()) > static::STALLED_SECONDS) {
210 4
                        $stalledJobs = array_merge($stalledJobs, $runningJobsById[$runId]);
211
                    }
212
                }
213
            }
214
        }
215
216 4
        return $stalledJobs;
217
    }
218
219 6
    protected function updateMaxStatus(RetryableJob $job, $status, $max = null, $count = 0)
220
    {
221 6
        if (null !== $max && $count >= $max) {
222
            $job->setStatus($status);
223
224
            return true;
225
        }
226
227 6
        return false;
228
    }
229
230 2
    public function resetStalledJobs($workerName = null, $method = null)
231
    {
232 2
        $stalledJobs = $this->getStalledJobs($workerName, $method);
233
234 2
        $objectManager = $this->getObjectManager();
235
236 2
        $countProcessed = 0;
237 2
        for ($i = 0, $count = count($stalledJobs); $i < $count; $i += static::FETCH_COUNT) {
238 2
            for ($j = $i, $max = $i + static::FETCH_COUNT; $j < $max && $j < $count; ++$j) {
239 2
                $job = $stalledJobs[$j];
240
                /* RetryableJob $job */
241 2
                $job->setStalledCount($job->getStalledCount() + 1);
242 2
                if ($this->updateMaxStatus($job, RetryableJob::STATUS_MAX_STALLED, $job->getMaxStalled(), $job->getStalledCount())) {
243
                    $objectManager->remove($job);
244
                    continue;
245 2
                } elseif ($this->updateMaxStatus($job, RetryableJob::STATUS_MAX_RETRIES, $job->getMaxRetries(), $job->getRetries())) {
246
                    $objectManager->remove($job);
247
                    continue;
248
                }
249
250 2
                $job->setRetries($job->getRetries() + 1);
251 2
                $job->setStatus(BaseJob::STATUS_NEW);
252 2
                $job->setLocked(null);
253 2
                $job->setLockedAt(null);
254 2
                $objectManager->persist($job);
255 2
                ++$countProcessed;
256
            }
257 2
            $objectManager->flush();
258
        }
259
260 2
        return $countProcessed;
261
    }
262
263
    /**
264
     * @param string $workerName
265
     * @param string $method
266
     */
267 2
    public function pruneStalledJobs($workerName = null, $method = null)
268
    {
269 2
        $stalledJobs = $this->getStalledJobs($workerName, $method);
270 2
        $objectManager = $this->getObjectManager();
271
272 2
        $countProcessed = 0;
273 2
        for ($i = 0, $count = count($stalledJobs); $i < $count; $i += static::FETCH_COUNT) {
274 2
            for ($j = $i, $max = $i + static::FETCH_COUNT; $j < $max && $j < $count; ++$j) {
275
                /** @var RetryableJob $job */
276 2
                $job = $stalledJobs[$j];
277 2
                $job->setStalledCount($job->getStalledCount() + 1);
278 2
                $job->setStatus(BaseJob::STATUS_ERROR);
279 2
                $job->setMessage('stalled');
280 2
                $this->updateMaxStatus($job, RetryableJob::STATUS_MAX_STALLED, $job->getMaxStalled(), $job->getStalledCount());
281 2
                $objectManager->remove($job);
282 2
                ++$countProcessed;
283
            }
284 2
            $objectManager->flush();
285
        }
286
287 2
        return $countProcessed;
288
    }
289
290 10
    public function deleteJob(\Dtc\QueueBundle\Model\Job $job)
291
    {
292 10
        $objectManager = $this->getObjectManager();
293 10
        $objectManager->remove($job);
294 10
        $objectManager->flush();
295 10
    }
296
297 2
    public function saveHistory(\Dtc\QueueBundle\Model\Job $job)
298
    {
299 2
        $this->deleteJob($job); // Should cause job to be archived
300 2
    }
301
302 31
    protected function prioritySave(\Dtc\QueueBundle\Model\Job $job)
303
    {
304
        // Generate crc hash for the job
305 31
        $hashValues = array($job->getClassName(), $job->getMethod(), $job->getWorkerName(), $job->getArgs());
306 31
        $crcHash = hash('sha256', serialize($hashValues));
307 31
        $job->setCrcHash($crcHash);
308 31
        $objectManager = $this->getObjectManager();
309
310 31
        if (true === $job->getBatch()) {
311
            // See if similar job that hasn't run exists
312
            //  Bug: what if there are multiple - shouldn't we batch up to the soonest or
313
            //     do we batch to the latest - does it matter which we batch to - unless priority shift
314
            //     or whenAt shift happens, no, correct?
315
            //   Bug: Atomicity - shouldn't this be in a transaction for ORM and a findAndModify for ODM?
316
            //  Option: Include STATUS_RUNNING as well?
317 2
            $oldJob = $this->updateNearestBatch($job);
318 2
            if ($oldJob) {
319 2
                return $oldJob;
320
            }
321
        }
322
323
        // Just save a new job
324 31
        $this->resetSaveOk(__FUNCTION__);
325 31
        $objectManager->persist($job);
326 31
        $objectManager->flush();
327
328 31
        return $job;
329
    }
330
331
    abstract protected function updateNearestBatch(Job $job);
332
333
    /**
334
     * @param string $objectName
335
     */
336
    abstract protected function stopIdGenerator($objectName);
337
338
    abstract protected function restoreIdGenerator($objectName);
339
340
    /**
341
     * @param array $criterion
342
     * @param int   $limit
343
     * @param int   $offset
344
     */
345 2
    private function resetJobsByCriterion(
346
        array $criterion,
347
        $limit,
348
        $offset)
349
    {
350 2
        $objectManager = $this->getObjectManager();
351 2
        $this->resetSaveOk(__FUNCTION__);
352 2
        $objectName = $this->getObjectName();
353 2
        $archiveObjectName = $this->getArchiveObjectName();
354 2
        $jobRepository = $objectManager->getRepository($objectName);
355 2
        $jobArchiveRepository = $objectManager->getRepository($archiveObjectName);
356 2
        $className = $jobRepository->getClassName();
357 2
        $metadata = $objectManager->getClassMetadata($className);
358 2
        $this->stopIdGenerator($objectName);
359 2
        $identifierData = $metadata->getIdentifier();
360 2
        $idColumn = isset($identifierData[0]) ? $identifierData[0] : 'id';
361 2
        $results = $jobArchiveRepository->findBy($criterion, [$idColumn => 'ASC'], $limit, $offset);
362 2
        $countProcessed = 0;
363
364 2
        foreach ($results as $jobArchive) {
365 2
            $this->resetJob($jobArchive, $className, $countProcessed);
366
        }
367 2
        $objectManager->flush();
368
369 2
        $this->restoreIdGenerator($objectName);
370
371 2
        return $countProcessed;
372
    }
373
374 17
    protected function resetSaveOk($function)
375
    {
376 17
    }
377
378
    /**
379
     * @param RetryableJob $jobArchive
380
     * @param $className
381
     * @param $countProcessed
382
     */
383 2
    protected function resetJob(RetryableJob $jobArchive, $className, &$countProcessed)
384
    {
385 2
        $objectManager = $this->getObjectManager();
386 2
        if ($this->updateMaxStatus($jobArchive, RetryableJob::STATUS_MAX_RETRIES, $jobArchive->getMaxRetries(), $jobArchive->getRetries())) {
387
            $objectManager->persist($jobArchive);
388
389
            return;
390
        }
391
392
        /** @var RetryableJob $job */
393 2
        $job = new $className();
394
395 2
        Util::copy($jobArchive, $job);
396 2
        $job->setStatus(BaseJob::STATUS_NEW);
397 2
        $job->setLocked(null);
398 2
        $job->setLockedAt(null);
399 2
        $job->setMessage(null);
400 2
        $job->setFinishedAt(null);
401 2
        $job->setStartedAt(null);
402 2
        $job->setElapsed(null);
403 2
        $job->setRetries($job->getRetries() + 1);
404
405 2
        $objectManager->persist($job);
406 2
        $objectManager->remove($jobArchive);
407 2
        ++$countProcessed;
408 2
    }
409
}
410