1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
namespace Dtc\QueueBundle\Doctrine; |
4
|
|
|
|
5
|
|
|
use Doctrine\Common\Persistence\ObjectManager; |
6
|
|
|
use Doctrine\Common\Persistence\ObjectRepository; |
7
|
|
|
use Doctrine\ODM\MongoDB\DocumentRepository; |
8
|
|
|
use Doctrine\ORM\EntityManager; |
9
|
|
|
use Doctrine\ORM\EntityRepository; |
10
|
|
|
use Dtc\QueueBundle\Model\AbstractJobManager; |
11
|
|
|
use Dtc\QueueBundle\Model\BaseJob; |
12
|
|
|
use Dtc\QueueBundle\Model\Job; |
13
|
|
|
use Dtc\QueueBundle\Model\RetryableJob; |
14
|
|
|
use Dtc\QueueBundle\Model\Run; |
15
|
|
|
use Dtc\QueueBundle\Util\Util; |
16
|
|
|
|
17
|
|
|
abstract class BaseJobManager extends AbstractJobManager |
18
|
|
|
{ |
19
|
|
|
/** Number of jobs to prune / reset / gather at a time */ |
20
|
|
|
const FETCH_COUNT = 100; |
21
|
|
|
|
22
|
|
|
/** Number of seconds before a job is considered stalled if the runner is no longer active */ |
23
|
|
|
const STALLED_SECONDS = 1800; |
24
|
|
|
protected $objectManager; |
25
|
|
|
protected $objectName; |
26
|
|
|
protected $archiveObjectName; |
27
|
|
|
protected $runClass; |
28
|
|
|
protected $runArchiveClass; |
29
|
|
|
|
30
|
|
|
/** |
31
|
|
|
* @param string $objectName |
32
|
|
|
* @param string $archiveObjectName |
33
|
|
|
* @param string $runClass |
34
|
|
|
* @param string $runArchiveClass |
35
|
|
|
*/ |
36
|
3 |
|
public function __construct(ObjectManager $objectManager, |
37
|
|
|
$objectName, |
38
|
|
|
$archiveObjectName, |
39
|
|
|
$runClass, |
40
|
|
|
$runArchiveClass) |
41
|
|
|
{ |
42
|
3 |
|
$this->objectManager = $objectManager; |
43
|
3 |
|
$this->objectName = $objectName; |
44
|
3 |
|
$this->archiveObjectName = $archiveObjectName; |
45
|
3 |
|
$this->runClass = $runClass; |
46
|
3 |
|
$this->runArchiveClass = $runArchiveClass; |
47
|
3 |
|
} |
48
|
|
|
|
49
|
|
|
/** |
50
|
|
|
* @return ObjectManager |
51
|
|
|
*/ |
52
|
22 |
|
public function getObjectManager() |
53
|
|
|
{ |
54
|
22 |
|
return $this->objectManager; |
55
|
|
|
} |
56
|
|
|
|
57
|
|
|
/** |
58
|
|
|
* @return string |
59
|
|
|
*/ |
60
|
20 |
|
public function getObjectName() |
61
|
|
|
{ |
62
|
20 |
|
return $this->objectName; |
63
|
|
|
} |
64
|
|
|
|
65
|
|
|
/** |
66
|
|
|
* @return string |
67
|
|
|
*/ |
68
|
16 |
|
public function getArchiveObjectName() |
69
|
|
|
{ |
70
|
16 |
|
return $this->archiveObjectName; |
71
|
|
|
} |
72
|
|
|
|
73
|
|
|
/** |
74
|
|
|
* @return string |
75
|
|
|
*/ |
76
|
4 |
|
public function getRunClass() |
77
|
|
|
{ |
78
|
4 |
|
return $this->runClass; |
79
|
|
|
} |
80
|
|
|
|
81
|
|
|
/** |
82
|
|
|
* @return string |
83
|
|
|
*/ |
84
|
4 |
|
public function getRunArchiveClass() |
85
|
|
|
{ |
86
|
4 |
|
return $this->runArchiveClass; |
87
|
|
|
} |
88
|
|
|
|
89
|
|
|
/** |
90
|
|
|
* @return ObjectRepository |
91
|
|
|
*/ |
92
|
16 |
|
public function getRepository() |
93
|
|
|
{ |
94
|
16 |
|
return $this->getObjectManager()->getRepository($this->getObjectName()); |
95
|
|
|
} |
96
|
|
|
|
97
|
|
|
/** |
98
|
|
|
* @param string $objectName |
99
|
|
|
*/ |
100
|
|
|
abstract protected function countJobsByStatus($objectName, $status, $workerName = null, $method = null); |
101
|
|
|
|
102
|
2 |
|
public function resetErroneousJobs($workerName = null, $method = null) |
103
|
|
|
{ |
104
|
2 |
|
$count = $this->countJobsByStatus($this->getArchiveObjectName(), Job::STATUS_ERROR, $workerName, $method); |
105
|
|
|
|
106
|
2 |
|
$criterion = ['status' => Job::STATUS_ERROR]; |
107
|
2 |
|
$this->addWorkerNameMethod($criterion, $workerName, $method); |
108
|
|
|
|
109
|
2 |
|
$countProcessed = 0; |
110
|
2 |
|
for ($i = 0; $i < $count; $i += static::FETCH_COUNT) { |
111
|
2 |
|
$countProcessed += $this->resetJobsByCriterion( |
112
|
2 |
|
$criterion, static::FETCH_COUNT, $i); |
113
|
|
|
} |
114
|
|
|
|
115
|
2 |
|
return $countProcessed; |
116
|
|
|
} |
117
|
|
|
|
118
|
|
|
/** |
119
|
|
|
* Sets the status to Job::STATUS_EXPIRED for those jobs that are expired. |
120
|
|
|
* |
121
|
|
|
* @param null $workerName |
122
|
|
|
* @param null $method |
123
|
|
|
* |
124
|
|
|
* @return mixed |
125
|
|
|
*/ |
126
|
|
|
abstract protected function updateExpired($workerName = null, $method = null); |
127
|
|
|
|
128
|
8 |
|
protected function addWorkerNameMethod(array &$criterion, $workerName = null, $method = null) |
129
|
|
|
{ |
130
|
8 |
|
if (null !== $workerName) { |
131
|
4 |
|
$criterion['workerName'] = $workerName; |
132
|
|
|
} |
133
|
8 |
|
if (null !== $method) { |
134
|
4 |
|
$criterion['method'] = $method; |
135
|
|
|
} |
136
|
8 |
|
} |
137
|
|
|
|
138
|
2 |
|
public function pruneExpiredJobs($workerName = null, $method = null) |
139
|
|
|
{ |
140
|
2 |
|
$count = $this->updateExpired($workerName, $method); |
141
|
2 |
|
$criterion = ['status' => Job::STATUS_EXPIRED]; |
142
|
2 |
|
$this->addWorkerNameMethod($criterion, $workerName, $method); |
143
|
2 |
|
$objectManager = $this->getObjectManager(); |
144
|
2 |
|
$repository = $this->getRepository(); |
145
|
2 |
|
$finalCount = 0; |
146
|
2 |
|
for ($i = 0; $i < $count; $i += static::FETCH_COUNT) { |
147
|
2 |
|
$expiredJobs = $repository->findBy($criterion, null, static::FETCH_COUNT, $i); |
148
|
2 |
|
if (!empty($expiredJobs)) { |
149
|
2 |
|
foreach ($expiredJobs as $expiredJob) { |
150
|
|
|
/* @var Job $expiredJob */ |
151
|
2 |
|
$expiredJob->setStatus(Job::STATUS_EXPIRED); |
152
|
2 |
|
$objectManager->remove($expiredJob); |
153
|
2 |
|
++$finalCount; |
154
|
|
|
} |
155
|
|
|
} |
156
|
2 |
|
$objectManager->flush(); |
157
|
|
|
} |
158
|
|
|
|
159
|
2 |
|
return $finalCount; |
160
|
|
|
} |
161
|
|
|
|
162
|
4 |
|
protected function getStalledJobs($workerName = null, $method = null) |
163
|
|
|
{ |
164
|
4 |
|
$count = $this->countJobsByStatus($this->getObjectName(), Job::STATUS_RUNNING, $workerName, $method); |
165
|
|
|
|
166
|
4 |
|
$criterion = ['status' => BaseJob::STATUS_RUNNING]; |
167
|
4 |
|
$this->addWorkerNameMethod($criterion, $workerName, $method); |
168
|
|
|
|
169
|
4 |
|
$runningJobs = $this->findRunningJobs($criterion, $count); |
170
|
|
|
|
171
|
4 |
|
return $this->extractStalledJobs($runningJobs); |
172
|
|
|
} |
173
|
|
|
|
174
|
4 |
|
protected function findRunningJobs($criterion, $count) |
175
|
|
|
{ |
176
|
4 |
|
$repository = $this->getRepository(); |
177
|
4 |
|
$runningJobsById = []; |
178
|
|
|
|
179
|
4 |
|
for ($i = 0; $i < $count; $i += static::FETCH_COUNT) { |
180
|
4 |
|
$runningJobs = $repository->findBy($criterion, null, static::FETCH_COUNT, $i); |
181
|
4 |
|
if (!empty($runningJobs)) { |
182
|
4 |
|
foreach ($runningJobs as $job) { |
183
|
|
|
/** @var RetryableJob $job */ |
184
|
4 |
|
if (null !== $runId = $job->getRunId()) { |
185
|
4 |
|
$runningJobsById[$runId][] = $job; |
186
|
|
|
} |
187
|
|
|
} |
188
|
|
|
} |
189
|
|
|
} |
190
|
|
|
|
191
|
4 |
|
return $runningJobsById; |
192
|
|
|
} |
193
|
|
|
|
194
|
4 |
|
protected function extractStalledJobs(array $runningJobsById) |
195
|
|
|
{ |
196
|
4 |
|
$objectManager = $this->getObjectManager(); |
197
|
4 |
|
$runRepository = $objectManager->getRepository($this->runClass); |
198
|
|
|
/** @var EntityRepository|DocumentRepository $runArchiveRepository */ |
199
|
4 |
|
$runArchiveRepository = $objectManager->getRepository($this->runArchiveClass); |
200
|
|
|
|
201
|
4 |
|
$stalledJobs = []; |
202
|
4 |
|
foreach (array_keys($runningJobsById) as $runId) { |
203
|
4 |
|
if ($runRepository->find($runId)) { |
204
|
|
|
continue; |
205
|
|
|
} |
206
|
|
|
/** @var Run $run */ |
207
|
4 |
|
if ($run = $runArchiveRepository->find($runId)) { |
208
|
4 |
|
if ($endTime = $run->getEndedAt()) { |
209
|
|
|
// Did it end over an hour ago |
210
|
4 |
|
if ((time() - $endTime->getTimestamp()) > static::STALLED_SECONDS) { |
211
|
4 |
|
$stalledJobs = array_merge($stalledJobs, $runningJobsById[$runId]); |
212
|
|
|
} |
213
|
|
|
} |
214
|
|
|
} |
215
|
|
|
} |
216
|
|
|
|
217
|
4 |
|
return $stalledJobs; |
218
|
|
|
} |
219
|
|
|
|
220
|
6 |
|
protected function updateMaxStatus(RetryableJob $job, $status, $max = null, $count = 0) { |
221
|
6 |
|
if (null !== $max && $count >= $max) { |
222
|
|
|
$job->setStatus($status); |
223
|
|
|
return true; |
224
|
|
|
} |
225
|
6 |
|
return false; |
226
|
|
|
} |
227
|
|
|
|
228
|
2 |
|
public function resetStalledJobs($workerName = null, $method = null) |
229
|
|
|
{ |
230
|
2 |
|
$stalledJobs = $this->getStalledJobs($workerName, $method); |
231
|
|
|
|
232
|
2 |
|
$objectManager = $this->getObjectManager(); |
233
|
|
|
|
234
|
2 |
|
$countProcessed = 0; |
235
|
2 |
|
for ($i = 0, $count = count($stalledJobs); $i < $count; $i += static::FETCH_COUNT) { |
236
|
2 |
|
for ($j = $i, $max = $i + static::FETCH_COUNT; $j < $max && $j < $count; ++$j) { |
237
|
2 |
|
$job = $stalledJobs[$j]; |
238
|
|
|
/* RetryableJob $job */ |
239
|
2 |
|
$job->setStalledCount($job->getStalledCount() + 1); |
240
|
2 |
|
if ($this->updateMaxStatus($job, RetryableJob::STATUS_MAX_STALLED, $job->getMaxStalled(), $job->getStalledCount())) { |
241
|
|
|
$objectManager->remove($job); |
242
|
|
|
continue; |
243
|
|
|
} |
244
|
2 |
|
else if ($this->updateMaxStatus($job, RetryableJob::STATUS_MAX_RETRIES, $job->getMaxRetries(), $job->getRetries())) { |
245
|
|
|
$objectManager->remove($job); |
246
|
|
|
continue; |
247
|
|
|
} |
248
|
|
|
|
249
|
2 |
|
$job->setRetries($job->getRetries() + 1); |
250
|
2 |
|
$job->setStatus(BaseJob::STATUS_NEW); |
251
|
2 |
|
$job->setLocked(null); |
252
|
2 |
|
$job->setLockedAt(null); |
253
|
2 |
|
$objectManager->persist($job); |
254
|
2 |
|
++$countProcessed; |
255
|
|
|
} |
256
|
2 |
|
$objectManager->flush(); |
257
|
|
|
} |
258
|
|
|
|
259
|
2 |
|
return $countProcessed; |
260
|
|
|
} |
261
|
|
|
|
262
|
|
|
/** |
263
|
|
|
* @param string $workerName |
264
|
|
|
* @param string $method |
265
|
|
|
*/ |
266
|
2 |
|
public function pruneStalledJobs($workerName = null, $method = null) |
267
|
|
|
{ |
268
|
2 |
|
$stalledJobs = $this->getStalledJobs($workerName, $method); |
269
|
2 |
|
$objectManager = $this->getObjectManager(); |
270
|
|
|
|
271
|
2 |
|
$countProcessed = 0; |
272
|
2 |
|
for ($i = 0, $count = count($stalledJobs); $i < $count; $i += static::FETCH_COUNT) { |
273
|
2 |
|
for ($j = $i, $max = $i + static::FETCH_COUNT; $j < $max && $j < $count; ++$j) { |
274
|
|
|
/** @var RetryableJob $job */ |
275
|
2 |
|
$job = $stalledJobs[$j]; |
276
|
2 |
|
$job->setStalledCount($job->getStalledCount() + 1); |
277
|
2 |
|
$job->setStatus(BaseJob::STATUS_ERROR); |
278
|
2 |
|
$job->setMessage('stalled'); |
279
|
2 |
|
$this->updateMaxStatus($job, RetryableJob::STATUS_MAX_STALLED, $job->getMaxStalled(), $job->getStalledCount()); |
280
|
2 |
|
$objectManager->remove($job); |
281
|
2 |
|
++$countProcessed; |
282
|
|
|
} |
283
|
2 |
|
$objectManager->flush(); |
284
|
|
|
} |
285
|
|
|
|
286
|
2 |
|
return $countProcessed; |
287
|
|
|
} |
288
|
|
|
|
289
|
8 |
|
public function deleteJob(\Dtc\QueueBundle\Model\Job $job) |
290
|
|
|
{ |
291
|
8 |
|
$objectManager = $this->getObjectManager(); |
292
|
8 |
|
$objectManager->remove($job); |
293
|
8 |
|
$objectManager->flush(); |
294
|
8 |
|
} |
295
|
|
|
|
296
|
|
|
public function saveHistory(\Dtc\QueueBundle\Model\Job $job) |
297
|
|
|
{ |
298
|
|
|
$this->deleteJob($job); // Should cause job to be archived |
299
|
|
|
} |
300
|
|
|
|
301
|
22 |
|
public function save(\Dtc\QueueBundle\Model\Job $job) |
302
|
|
|
{ |
303
|
|
|
// Todo: Serialize args |
304
|
|
|
|
305
|
|
|
// Generate crc hash for the job |
306
|
22 |
|
$hashValues = array($job->getClassName(), $job->getMethod(), $job->getWorkerName(), $job->getArgs()); |
307
|
22 |
|
$crcHash = hash('sha256', serialize($hashValues)); |
308
|
22 |
|
$job->setCrcHash($crcHash); |
309
|
22 |
|
$objectManager = $this->getObjectManager(); |
310
|
|
|
|
311
|
22 |
|
if (true === $job->getBatch()) { |
312
|
|
|
// See if similar job that hasn't run exists |
313
|
|
|
$criteria = array('crcHash' => $crcHash, 'status' => BaseJob::STATUS_NEW); |
314
|
|
|
$oldJob = $this->getRepository()->findOneBy($criteria); |
315
|
|
|
|
316
|
|
|
if ($oldJob) { |
317
|
|
|
// Old job exists - just override fields Set higher priority |
318
|
|
|
$oldJob->setPriority(max($job->getPriority(), $oldJob->getPriority())); |
319
|
|
|
$oldJob->setWhenAt(min($job->getWhenAt(), $oldJob->getWhenAt())); |
320
|
|
|
$oldJob->setBatch(true); |
321
|
|
|
$objectManager->persist($oldJob); |
322
|
|
|
$objectManager->flush(); |
323
|
|
|
|
324
|
|
|
return $oldJob; |
325
|
|
|
} |
326
|
|
|
} |
327
|
|
|
|
328
|
|
|
// Just save a new job |
329
|
22 |
|
$this->resetSaveOk(__FUNCTION__); |
330
|
22 |
|
$objectManager->persist($job); |
331
|
22 |
|
$objectManager->flush(); |
332
|
|
|
|
333
|
22 |
|
return $job; |
334
|
|
|
} |
335
|
|
|
|
336
|
|
|
/** |
337
|
|
|
* @param string $objectName |
338
|
|
|
*/ |
339
|
|
|
abstract protected function stopIdGenerator($objectName); |
340
|
|
|
|
341
|
|
|
abstract protected function restoreIdGenerator($objectName); |
342
|
|
|
|
343
|
|
|
/** |
344
|
|
|
* @param array $criterion |
345
|
|
|
* @param int $limit |
346
|
|
|
* @param int $offset |
347
|
|
|
*/ |
348
|
2 |
|
private function resetJobsByCriterion( |
349
|
|
|
array $criterion, |
350
|
|
|
$limit, |
351
|
|
|
$offset) |
352
|
|
|
{ |
353
|
2 |
|
$objectManager = $this->getObjectManager(); |
354
|
2 |
|
$this->resetSaveOk(__FUNCTION__); |
355
|
2 |
|
$objectName = $this->getObjectName(); |
356
|
2 |
|
$archiveObjectName = $this->getArchiveObjectName(); |
357
|
2 |
|
$jobRepository = $objectManager->getRepository($objectName); |
358
|
2 |
|
$jobArchiveRepository = $objectManager->getRepository($archiveObjectName); |
359
|
2 |
|
$className = $jobRepository->getClassName(); |
360
|
2 |
|
$metadata = $objectManager->getClassMetadata($className); |
361
|
2 |
|
$this->stopIdGenerator($objectName); |
362
|
2 |
|
$identifierData = $metadata->getIdentifier(); |
363
|
2 |
|
$idColumn = isset($identifierData[0]) ? $identifierData[0] : 'id'; |
364
|
2 |
|
$results = $jobArchiveRepository->findBy($criterion, [$idColumn => 'ASC'], $limit, $offset); |
365
|
2 |
|
$countProcessed = 0; |
366
|
|
|
|
367
|
2 |
|
foreach ($results as $jobArchive) { |
368
|
2 |
|
$this->resetJob($jobArchive, $className, $countProcessed); |
369
|
|
|
} |
370
|
2 |
|
$objectManager->flush(); |
371
|
|
|
|
372
|
2 |
|
$this->restoreIdGenerator($objectName); |
373
|
|
|
|
374
|
2 |
|
return $countProcessed; |
375
|
|
|
} |
376
|
|
|
|
377
|
|
|
protected function resetSaveOk($function) {} |
378
|
|
|
|
379
|
|
|
/** |
380
|
|
|
* @param RetryableJob $jobArchive |
381
|
|
|
* @param $className |
382
|
|
|
* @param $countProcessed |
383
|
|
|
*/ |
384
|
2 |
|
protected function resetJob(RetryableJob $jobArchive, $className, &$countProcessed) { |
385
|
2 |
|
$objectManager = $this->getObjectManager(); |
386
|
2 |
|
if ($this->updateMaxStatus($jobArchive, RetryableJob::STATUS_MAX_RETRIES, $jobArchive->getMaxRetries(), $jobArchive->getRetries())) { |
387
|
|
|
$objectManager->persist($jobArchive); |
388
|
|
|
return; |
389
|
|
|
} |
390
|
|
|
|
391
|
|
|
/** @var RetryableJob $job */ |
392
|
2 |
|
$job = new $className(); |
393
|
|
|
|
394
|
2 |
|
Util::copy($jobArchive, $job); |
395
|
2 |
|
$job->setStatus(BaseJob::STATUS_NEW); |
396
|
2 |
|
$job->setLocked(null); |
397
|
2 |
|
$job->setLockedAt(null); |
398
|
2 |
|
$job->setMessage(null); |
399
|
2 |
|
$job->setFinishedAt(null); |
400
|
2 |
|
$job->setStartedAt(null); |
401
|
2 |
|
$job->setElapsed(null); |
402
|
2 |
|
$job->setRetries($job->getRetries() + 1); |
403
|
|
|
|
404
|
2 |
|
$objectManager->persist($job); |
405
|
2 |
|
$objectManager->remove($jobArchive); |
406
|
2 |
|
++$countProcessed; |
407
|
2 |
|
} |
408
|
|
|
} |
409
|
|
|
|