1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
namespace Symbiote\QueuedJobs\Services; |
4
|
|
|
|
5
|
|
|
use Exception; |
6
|
|
|
use Monolog\Handler\BufferHandler; |
7
|
|
|
use Monolog\Logger; |
8
|
|
|
use Psr\Log\LoggerInterface; |
9
|
|
|
use SilverStripe\Control\Controller; |
10
|
|
|
use SilverStripe\Control\Director; |
11
|
|
|
use SilverStripe\Control\Email\Email; |
12
|
|
|
use SilverStripe\Core\Config\Config; |
13
|
|
|
use SilverStripe\Core\Config\Configurable; |
14
|
|
|
use SilverStripe\Core\Extensible; |
15
|
|
|
use SilverStripe\Core\Injector\Injectable; |
16
|
|
|
use SilverStripe\Core\Injector\Injector; |
17
|
|
|
use SilverStripe\ORM\DataList; |
18
|
|
|
use SilverStripe\ORM\DataObject; |
19
|
|
|
use SilverStripe\ORM\DB; |
20
|
|
|
use SilverStripe\ORM\FieldType\DBDatetime; |
21
|
|
|
use SilverStripe\ORM\ValidationException; |
22
|
|
|
use SilverStripe\Security\Member; |
23
|
|
|
use SilverStripe\Security\Security; |
24
|
|
|
use SilverStripe\Subsites\Model\Subsite; |
25
|
|
|
use Symbiote\QueuedJobs\DataObjects\QueuedJobDescriptor; |
26
|
|
|
use Symbiote\QueuedJobs\QJUtils; |
27
|
|
|
use Symbiote\QueuedJobs\Tasks\Engines\TaskRunnerEngine; |
28
|
|
|
|
29
|
|
|
/** |
30
|
|
|
* A service that can be used for starting, stopping and listing queued jobs. |
31
|
|
|
* |
32
|
|
|
* When a job is first added, it is initialised, its job type determined, then persisted to the database |
33
|
|
|
* |
34
|
|
|
* When the queues are scanned, a job is reloaded and processed. Ignoring the persistence and reloading, it looks |
35
|
|
|
* something like |
36
|
|
|
* |
37
|
|
|
* job->getJobType(); |
38
|
|
|
* job->getJobData(); |
39
|
|
|
* data->write(); |
40
|
|
|
* job->setup(); |
41
|
|
|
* while !job->isComplete |
42
|
|
|
* job->process(); |
43
|
|
|
* job->getJobData(); |
44
|
|
|
* data->write(); |
45
|
|
|
* |
46
|
|
|
* |
47
|
|
|
* @author Marcus Nyeholt <[email protected]> |
48
|
|
|
* @license BSD http://silverstripe.org/bsd-license/ |
49
|
|
|
*/ |
50
|
|
|
class QueuedJobService |
51
|
|
|
{ |
52
|
|
|
use Configurable; |
53
|
|
|
use Injectable; |
54
|
|
|
use Extensible; |
55
|
|
|
|
56
|
|
|
/** |
57
|
|
|
* @config |
58
|
|
|
* @var int |
59
|
|
|
*/ |
60
|
|
|
private static $stall_threshold = 3; |
61
|
|
|
|
62
|
|
|
/** |
63
|
|
|
* How much ram will we allow before pausing and releasing the memory? |
64
|
|
|
* |
65
|
|
|
* For instance, set to 268435456 (256MB) to pause this process if used memory exceeds |
66
|
|
|
* this value. This needs to be set to a value lower than the php_ini max_memory as |
67
|
|
|
* the system will otherwise crash before shutdown can be handled gracefully. |
68
|
|
|
* |
69
|
|
|
* This was increased to 256MB for SilverStripe 4.x as framework uses more memory than 3.x |
70
|
|
|
* |
71
|
|
|
* @var int |
72
|
|
|
* @config |
73
|
|
|
*/ |
74
|
|
|
private static $memory_limit = 268435456; |
75
|
|
|
|
76
|
|
|
/** |
77
|
|
|
* Optional time limit (in seconds) to run the service before restarting to release resources. |
78
|
|
|
* |
79
|
|
|
* Defaults to no limit. |
80
|
|
|
* |
81
|
|
|
* @var int |
82
|
|
|
* @config |
83
|
|
|
*/ |
84
|
|
|
private static $time_limit = 0; |
85
|
|
|
|
86
|
|
|
/** |
87
|
|
|
* Disable health checks that usually occur when a runner first picks up a queue. Note that completely disabling |
88
|
|
|
* health checks could result in many jobs that are always marked as running - that will never be restarted. If |
89
|
|
|
* this option is disabled you may alternatively use the build task |
90
|
|
|
* |
91
|
|
|
* @see \Symbiote\QueuedJobs\Tasks\CheckJobHealthTask |
92
|
|
|
* |
93
|
|
|
* @var bool |
94
|
|
|
* @config |
95
|
|
|
*/ |
96
|
|
|
private static $disable_health_check = false; |
97
|
|
|
|
98
|
|
|
/** |
99
|
|
|
* Maximum number of jobs that can be initialised at any one time. |
100
|
|
|
* |
101
|
|
|
* Prevents too many jobs getting into this state in case something goes wrong with the child processes. |
102
|
|
|
* We shouldn't have too many jobs in the initialising state anyway. |
103
|
|
|
* |
104
|
|
|
* Valid values: |
105
|
|
|
* 0 - unlimited (default) |
106
|
|
|
* greater than 0 - maximum number of jobs in initialised state |
107
|
|
|
* |
108
|
|
|
* @var int |
109
|
|
|
* @config |
110
|
|
|
*/ |
111
|
|
|
private static $max_init_jobs = 0; |
112
|
|
|
|
113
|
|
|
/** |
114
|
|
|
* Timestamp (in seconds) when the queue was started |
115
|
|
|
* |
116
|
|
|
* @var int |
117
|
|
|
*/ |
118
|
|
|
protected $startedAt = 0; |
119
|
|
|
|
120
|
|
|
/** |
121
|
|
|
* Should "immediate" jobs be managed using the shutdown function? |
122
|
|
|
* |
123
|
|
|
* It is recommended you set up an inotify watch and use that for |
124
|
|
|
* triggering immediate jobs. See the wiki for more information |
125
|
|
|
* |
126
|
|
|
* @var boolean |
127
|
|
|
* @config |
128
|
|
|
*/ |
129
|
|
|
private static $use_shutdown_function = true; |
130
|
|
|
|
131
|
|
|
/** |
132
|
|
|
* The location for immediate jobs to be stored in |
133
|
|
|
* |
134
|
|
|
* @var string |
135
|
|
|
* @config |
136
|
|
|
*/ |
137
|
|
|
private static $cache_dir = 'queuedjobs'; |
138
|
|
|
|
139
|
|
|
/** |
140
|
|
|
* Maintenance lock file feature enabled / disable setting |
141
|
|
|
* |
142
|
|
|
* @config |
143
|
|
|
* @var bool |
144
|
|
|
*/ |
145
|
|
|
private static $lock_file_enabled = false; |
146
|
|
|
|
147
|
|
|
/** |
148
|
|
|
* Maintenance lock file name |
149
|
|
|
* |
150
|
|
|
* @config |
151
|
|
|
* @var string |
152
|
|
|
*/ |
153
|
|
|
private static $lock_file_name = 'maintenance-lock.txt'; |
154
|
|
|
|
155
|
|
|
/** |
156
|
|
|
* Maintenance lock path (relative path starting at the base folder) |
157
|
|
|
* Note that this path needs to point to a folder on a shared drive if multiple instances are used |
158
|
|
|
* |
159
|
|
|
* @config |
160
|
|
|
* @var string |
161
|
|
|
*/ |
162
|
|
|
private static $lock_file_path = ''; |
163
|
|
|
|
164
|
|
|
/** |
165
|
|
|
* @var DefaultQueueHandler |
166
|
|
|
*/ |
167
|
|
|
public $queueHandler; |
168
|
|
|
|
169
|
|
|
/** |
170
|
|
|
* |
171
|
|
|
* @var TaskRunnerEngine |
172
|
|
|
*/ |
173
|
|
|
public $queueRunner; |
174
|
|
|
|
175
|
|
|
/** |
176
|
|
|
* Config controlled list of default/required jobs |
177
|
|
|
* |
178
|
|
|
* @var array |
179
|
|
|
*/ |
180
|
|
|
public $defaultJobs = []; |
181
|
|
|
|
182
|
|
|
/** |
183
|
|
|
* Register our shutdown handler |
184
|
|
|
*/ |
185
|
|
|
public function __construct() |
186
|
|
|
{ |
187
|
|
|
// bind a shutdown function to process all 'immediate' queued jobs if needed, but only in CLI mode |
188
|
|
|
if (static::config()->get('use_shutdown_function') && Director::is_cli()) { |
189
|
|
|
register_shutdown_function([$this, 'onShutdown']); |
190
|
|
|
} |
191
|
|
|
|
192
|
|
|
$queuedEmail = Config::inst()->get(Email::class, 'queued_job_admin_email'); |
193
|
|
|
|
194
|
|
|
// if not set (and not explictly set to false), fallback to the admin email. |
195
|
|
|
if (!$queuedEmail && $queuedEmail !== false) { |
196
|
|
|
Config::modify()->set( |
197
|
|
|
Email::class, |
198
|
|
|
'queued_job_admin_email', |
199
|
|
|
Config::inst()->get(Email::class, 'admin_email') |
200
|
|
|
); |
201
|
|
|
} |
202
|
|
|
} |
203
|
|
|
|
204
|
|
|
/** |
205
|
|
|
* Adds a job to the queue to be started |
206
|
|
|
* |
207
|
|
|
* @param QueuedJob $job The job to start. |
208
|
|
|
* @param null|string $startAfter The date (in Y-m-d H:i:s format) to start execution after |
209
|
|
|
* @param null|int $userId The ID of a user to execute the job as. Defaults to the current user |
210
|
|
|
* @param null|int $queueName |
211
|
|
|
* @return int |
212
|
|
|
* @throws ValidationException |
213
|
|
|
*/ |
214
|
|
|
public function queueJob(QueuedJob $job, $startAfter = null, $userId = null, $queueName = null) |
215
|
|
|
{ |
216
|
|
|
$signature = $job->getSignature(); |
217
|
|
|
|
218
|
|
|
// see if we already have this job in a queue |
219
|
|
|
$filter = [ |
220
|
|
|
'Signature' => $signature, |
221
|
|
|
'JobStatus' => [ |
222
|
|
|
QueuedJob::STATUS_NEW, |
223
|
|
|
QueuedJob::STATUS_INIT, |
224
|
|
|
], |
225
|
|
|
]; |
226
|
|
|
|
227
|
|
|
$existing = QueuedJobDescriptor::get() |
228
|
|
|
->filter($filter) |
229
|
|
|
->first(); |
230
|
|
|
|
231
|
|
|
if ($existing && $existing->ID) { |
232
|
|
|
return $existing->ID; |
233
|
|
|
} |
234
|
|
|
|
235
|
|
|
$jobDescriptor = new QueuedJobDescriptor(); |
236
|
|
|
$jobDescriptor->JobTitle = $job->getTitle(); |
237
|
|
|
$jobDescriptor->JobType = $queueName ? $queueName : $job->getJobType(); |
238
|
|
|
$jobDescriptor->Signature = $signature; |
239
|
|
|
$jobDescriptor->Implementation = get_class($job); |
240
|
|
|
$jobDescriptor->StartAfter = $startAfter; |
241
|
|
|
|
242
|
|
|
// no user provided - fallback to job user default |
243
|
|
|
if ($userId === null) { |
244
|
|
|
$userId = $job->getRunAsMemberID(); |
245
|
|
|
} |
246
|
|
|
|
247
|
|
|
// still no user - fallback to current user |
248
|
|
|
if ($userId === null) { |
249
|
|
|
if (Security::getCurrentUser() && Security::getCurrentUser()->exists()) { |
250
|
|
|
// current user available |
251
|
|
|
$runAsID = Security::getCurrentUser()->ID; |
252
|
|
|
} else { |
253
|
|
|
// current user unavailable |
254
|
|
|
$runAsID = 0; |
255
|
|
|
} |
256
|
|
|
} else { |
257
|
|
|
$runAsID = $userId; |
258
|
|
|
} |
259
|
|
|
|
260
|
|
|
$jobDescriptor->RunAsID = $runAsID; |
|
|
|
|
261
|
|
|
|
262
|
|
|
// use this to populate custom data columns before job is queued |
263
|
|
|
// note: you can pass arbitrary data to your job and then move it to job descriptor |
264
|
|
|
// this is useful if you need some data that needs to be exposed as a separate |
265
|
|
|
// DB column as opposed to serialised data |
266
|
|
|
$this->extend('updateJobDescriptorBeforeQueued', $jobDescriptor, $job); |
267
|
|
|
|
268
|
|
|
// copy data |
269
|
|
|
$this->copyJobToDescriptor($job, $jobDescriptor); |
270
|
|
|
|
271
|
|
|
$jobDescriptor->write(); |
272
|
|
|
|
273
|
|
|
$this->startJob($jobDescriptor, $startAfter); |
274
|
|
|
|
275
|
|
|
return $jobDescriptor->ID; |
276
|
|
|
} |
277
|
|
|
|
278
|
|
|
/** |
279
|
|
|
* Start a job (or however the queue handler determines it should be started) |
280
|
|
|
* |
281
|
|
|
* @param QueuedJobDescriptor $jobDescriptor |
282
|
|
|
* @param string $startAfter |
283
|
|
|
*/ |
284
|
|
|
public function startJob($jobDescriptor, $startAfter = null) |
285
|
|
|
{ |
286
|
|
|
if ($startAfter && strtotime($startAfter) > DBDatetime::now()->getTimestamp()) { |
|
|
|
|
287
|
|
|
$this->queueHandler->scheduleJob($jobDescriptor, $startAfter); |
288
|
|
|
} else { |
289
|
|
|
// immediately start it on the queue, however that works |
290
|
|
|
$this->queueHandler->startJobOnQueue($jobDescriptor); |
291
|
|
|
} |
292
|
|
|
} |
293
|
|
|
|
294
|
|
|
/** |
295
|
|
|
* Check if maximum number of jobs are currently initialised. |
296
|
|
|
* |
297
|
|
|
* @return bool |
298
|
|
|
*/ |
299
|
|
|
public function isAtMaxJobs() |
300
|
|
|
{ |
301
|
|
|
$initJobsMax = $this->config()->get('max_init_jobs'); |
302
|
|
|
if (!$initJobsMax) { |
303
|
|
|
return false; |
304
|
|
|
} |
305
|
|
|
|
306
|
|
|
$initJobsCount = QueuedJobDescriptor::get() |
307
|
|
|
->filter(['JobStatus' => QueuedJob::STATUS_INIT]) |
308
|
|
|
->count(); |
309
|
|
|
|
310
|
|
|
if ($initJobsCount >= $initJobsMax) { |
311
|
|
|
return true; |
312
|
|
|
} |
313
|
|
|
|
314
|
|
|
return false; |
315
|
|
|
} |
316
|
|
|
|
317
|
|
|
/** |
318
|
|
|
* Copies data from a job into a descriptor for persisting |
319
|
|
|
* |
320
|
|
|
* @param QueuedJob $job |
321
|
|
|
* @param QueuedJobDescriptor $jobDescriptor |
322
|
|
|
*/ |
323
|
|
|
protected function copyJobToDescriptor($job, $jobDescriptor) |
324
|
|
|
{ |
325
|
|
|
$data = $job->getJobData(); |
326
|
|
|
|
327
|
|
|
$jobDescriptor->TotalSteps = $data->totalSteps; |
328
|
|
|
$jobDescriptor->StepsProcessed = $data->currentStep; |
329
|
|
|
if ($data->isComplete) { |
330
|
|
|
$jobDescriptor->JobStatus = QueuedJob::STATUS_COMPLETE; |
331
|
|
|
$jobDescriptor->JobFinished = DBDatetime::now()->Rfc2822(); |
332
|
|
|
} |
333
|
|
|
|
334
|
|
|
$jobDescriptor->SavedJobData = serialize($data->jobData); |
335
|
|
|
$jobDescriptor->SavedJobMessages = serialize($data->messages); |
336
|
|
|
} |
337
|
|
|
|
338
|
|
|
/** |
339
|
|
|
* @param QueuedJobDescriptor $jobDescriptor |
340
|
|
|
* @param QueuedJob $job |
341
|
|
|
*/ |
342
|
|
|
protected function copyDescriptorToJob($jobDescriptor, $job) |
343
|
|
|
{ |
344
|
|
|
$jobData = null; |
|
|
|
|
345
|
|
|
$messages = null; |
|
|
|
|
346
|
|
|
|
347
|
|
|
// switching to php's serialize methods... not sure why this wasn't done from the start! |
348
|
|
|
$jobData = @unserialize($jobDescriptor->SavedJobData); |
349
|
|
|
$messages = @unserialize($jobDescriptor->SavedJobMessages); |
350
|
|
|
|
351
|
|
|
// try decoding as json if null |
352
|
|
|
$jobData = $jobData ?: json_decode($jobDescriptor->SavedJobData); |
353
|
|
|
$messages = $messages ?: json_decode($jobDescriptor->SavedJobMessages); |
354
|
|
|
|
355
|
|
|
$job->setJobData( |
356
|
|
|
$jobDescriptor->TotalSteps, |
357
|
|
|
$jobDescriptor->StepsProcessed, |
358
|
|
|
$jobDescriptor->JobStatus == QueuedJob::STATUS_COMPLETE, |
359
|
|
|
$jobData, |
360
|
|
|
$messages |
361
|
|
|
); |
362
|
|
|
} |
363
|
|
|
|
364
|
|
|
/** |
365
|
|
|
* Check the current job queues and see if any of the jobs currently in there should be started. If so, |
366
|
|
|
* return the next job that should be executed |
367
|
|
|
* |
368
|
|
|
* @param string $type Job type |
369
|
|
|
* |
370
|
|
|
* @return QueuedJobDescriptor|false |
371
|
|
|
*/ |
372
|
|
|
public function getNextPendingJob($type = null) |
373
|
|
|
{ |
374
|
|
|
// Filter jobs by type |
375
|
|
|
$type = $type ?: QueuedJob::QUEUED; |
376
|
|
|
$list = QueuedJobDescriptor::get() |
377
|
|
|
->filter('JobType', $type) |
378
|
|
|
->sort('ID', 'ASC'); |
379
|
|
|
|
380
|
|
|
// see if there's any blocked jobs that need to be resumed |
381
|
|
|
/** @var QueuedJobDescriptor $waitingJob */ |
382
|
|
|
$waitingJob = $list |
383
|
|
|
->filter('JobStatus', QueuedJob::STATUS_WAIT) |
384
|
|
|
->first(); |
385
|
|
|
if ($waitingJob) { |
386
|
|
|
return $waitingJob; |
387
|
|
|
} |
388
|
|
|
|
389
|
|
|
// If there's an existing job either running or pending, the lets just return false to indicate |
390
|
|
|
// that we're still executing |
391
|
|
|
$runningJob = $list |
|
|
|
|
392
|
|
|
->filter('JobStatus', [QueuedJob::STATUS_INIT, QueuedJob::STATUS_RUN]) |
393
|
|
|
->first(); |
394
|
|
|
if ($runningJob) { |
395
|
|
|
return false; |
396
|
|
|
} |
397
|
|
|
|
398
|
|
|
// Otherwise, lets find any 'new' jobs that are waiting to execute |
399
|
|
|
/** @var QueuedJobDescriptor $newJob */ |
400
|
|
|
$newJob = $list |
401
|
|
|
->filter('JobStatus', QueuedJob::STATUS_NEW) |
402
|
|
|
->where(sprintf( |
403
|
|
|
'"StartAfter" < \'%s\' OR "StartAfter" IS NULL', |
404
|
|
|
DBDatetime::now()->getValue() |
405
|
|
|
)) |
406
|
|
|
->first(); |
407
|
|
|
|
408
|
|
|
return $newJob; |
409
|
|
|
} |
410
|
|
|
|
411
|
|
|
/** |
412
|
|
|
* Runs an explicit check on all currently running jobs to make sure their "processed" count is incrementing |
413
|
|
|
* between each run. If it's not, then we need to flag it as paused due to an error. |
414
|
|
|
* |
415
|
|
|
* This typically happens when a PHP fatal error is thrown, which can't be picked up by the error |
416
|
|
|
* handler or exception checker; in this case, we detect these stalled jobs later and fix (try) to |
417
|
|
|
* fix them |
418
|
|
|
* |
419
|
|
|
* @param int $queue The queue to check against |
420
|
|
|
*/ |
421
|
|
|
public function checkJobHealth($queue = null) |
422
|
|
|
{ |
423
|
|
|
$queue = $queue ?: QueuedJob::QUEUED; |
424
|
|
|
// Select all jobs currently marked as running |
425
|
|
|
$runningJobs = QueuedJobDescriptor::get() |
426
|
|
|
->filter([ |
427
|
|
|
'JobStatus' => [ |
428
|
|
|
QueuedJob::STATUS_RUN, |
429
|
|
|
QueuedJob::STATUS_INIT, |
430
|
|
|
], |
431
|
|
|
'JobType' => $queue, |
432
|
|
|
]); |
433
|
|
|
|
434
|
|
|
// If no steps have been processed since the last run, consider it a broken job |
435
|
|
|
// Only check jobs that have been viewed before. LastProcessedCount defaults to -1 on new jobs. |
436
|
|
|
$stalledJobs = $runningJobs |
437
|
|
|
->filter('LastProcessedCount:GreaterThanOrEqual', 0) |
438
|
|
|
->where('"StepsProcessed" = "LastProcessedCount"'); |
439
|
|
|
foreach ($stalledJobs as $stalledJob) { |
440
|
|
|
$this->restartStalledJob($stalledJob); |
441
|
|
|
} |
442
|
|
|
|
443
|
|
|
// now, find those that need to be marked before the next check |
444
|
|
|
// foreach job, mark it as having been incremented |
445
|
|
|
foreach ($runningJobs as $job) { |
446
|
|
|
$job->LastProcessedCount = $job->StepsProcessed; |
447
|
|
|
$job->write(); |
448
|
|
|
} |
449
|
|
|
|
450
|
|
|
// finally, find the list of broken jobs and send an email if there's some found |
451
|
|
|
$brokenJobs = QueuedJobDescriptor::get()->filter('JobStatus', QueuedJob::STATUS_BROKEN); |
452
|
|
View Code Duplication |
if ($brokenJobs && $brokenJobs->count()) { |
|
|
|
|
453
|
|
|
$this->getLogger()->error( |
454
|
|
|
print_r( |
455
|
|
|
[ |
456
|
|
|
'errno' => 0, |
457
|
|
|
'errstr' => 'Broken jobs were found in the job queue', |
458
|
|
|
'errfile' => __FILE__, |
459
|
|
|
'errline' => __LINE__, |
460
|
|
|
'errcontext' => [], |
461
|
|
|
], |
462
|
|
|
true |
463
|
|
|
), |
464
|
|
|
[ |
465
|
|
|
'file' => __FILE__, |
466
|
|
|
'line' => __LINE__, |
467
|
|
|
] |
468
|
|
|
); |
469
|
|
|
} |
470
|
|
|
|
471
|
|
|
return $stalledJobs->count(); |
472
|
|
|
} |
473
|
|
|
|
474
|
|
|
/** |
475
|
|
|
* Checks through ll the scheduled jobs that are expected to exist |
476
|
|
|
*/ |
477
|
|
|
public function checkDefaultJobs($queue = null) |
478
|
|
|
{ |
479
|
|
|
$queue = $queue ?: QueuedJob::QUEUED; |
|
|
|
|
480
|
|
|
if (count($this->defaultJobs)) { |
481
|
|
|
$activeJobs = QueuedJobDescriptor::get()->filter( |
482
|
|
|
'JobStatus', |
483
|
|
|
[ |
484
|
|
|
QueuedJob::STATUS_NEW, |
485
|
|
|
QueuedJob::STATUS_INIT, |
486
|
|
|
QueuedJob::STATUS_RUN, |
487
|
|
|
QueuedJob::STATUS_WAIT, |
488
|
|
|
QueuedJob::STATUS_PAUSED, |
489
|
|
|
] |
490
|
|
|
); |
491
|
|
|
foreach ($this->defaultJobs as $title => $jobConfig) { |
492
|
|
|
if (!isset($jobConfig['filter']) || !isset($jobConfig['type'])) { |
493
|
|
|
$this->getLogger()->error( |
494
|
|
|
"Default Job config: $title incorrectly set up. Please check the readme for examples", |
495
|
|
|
[ |
496
|
|
|
'file' => __FILE__, |
497
|
|
|
'line' => __LINE__, |
498
|
|
|
] |
499
|
|
|
); |
500
|
|
|
continue; |
501
|
|
|
} |
502
|
|
|
$job = $activeJobs->filter(array_merge( |
503
|
|
|
['Implementation' => $jobConfig['type']], |
504
|
|
|
$jobConfig['filter'] |
505
|
|
|
)); |
506
|
|
|
if (!$job->count()) { |
507
|
|
|
$this->getLogger()->info( |
508
|
|
|
"Default Job config: $title was missing from Queue", |
509
|
|
|
[ |
510
|
|
|
'file' => __FILE__, |
511
|
|
|
'line' => __LINE__, |
512
|
|
|
] |
513
|
|
|
); |
514
|
|
|
|
515
|
|
|
$to = isset($jobConfig['email']) |
516
|
|
|
? $jobConfig['email'] |
517
|
|
|
: Config::inst()->get('Email', 'queued_job_admin_email'); |
518
|
|
|
|
519
|
|
|
if ($to) { |
520
|
|
|
Email::create() |
521
|
|
|
->setTo($to) |
522
|
|
|
->setFrom(Config::inst()->get('Email', 'queued_job_admin_email')) |
523
|
|
|
->setSubject('Default Job "' . $title . '" missing') |
524
|
|
|
->setData($jobConfig) |
525
|
|
|
->addData('Title', $title) |
526
|
|
|
->addData('Site', Director::absoluteBaseURL()) |
|
|
|
|
527
|
|
|
->setHTMLTemplate('QueuedJobsDefaultJob') |
528
|
|
|
->send(); |
529
|
|
|
} |
530
|
|
|
|
531
|
|
|
if (isset($jobConfig['recreate']) && $jobConfig['recreate']) { |
532
|
|
|
if (!array_key_exists('construct', $jobConfig) |
533
|
|
|
|| !isset($jobConfig['startDateFormat']) |
534
|
|
|
|| !isset($jobConfig['startTimeString']) |
535
|
|
|
) { |
536
|
|
|
$this->getLogger()->error( |
537
|
|
|
"Default Job config: $title incorrectly set up. Please check the readme for examples", |
538
|
|
|
[ |
539
|
|
|
'file' => __FILE__, |
540
|
|
|
'line' => __LINE__, |
541
|
|
|
] |
542
|
|
|
); |
543
|
|
|
continue; |
544
|
|
|
} |
545
|
|
|
QueuedJobService::singleton()->queueJob( |
546
|
|
|
Injector::inst()->createWithArgs($jobConfig['type'], $jobConfig['construct']), |
547
|
|
|
date($jobConfig['startDateFormat'], strtotime($jobConfig['startTimeString'])) |
548
|
|
|
); |
549
|
|
|
$this->getLogger()->info( |
550
|
|
|
"Default Job config: $title has been re-added to the Queue", |
551
|
|
|
[ |
552
|
|
|
'file' => __FILE__, |
553
|
|
|
'line' => __LINE__, |
554
|
|
|
] |
555
|
|
|
); |
556
|
|
|
} |
557
|
|
|
} |
558
|
|
|
} |
559
|
|
|
} |
560
|
|
|
} |
561
|
|
|
|
562
|
|
|
/** |
563
|
|
|
* Attempt to restart a stalled job |
564
|
|
|
* |
565
|
|
|
* @param QueuedJobDescriptor $stalledJob |
566
|
|
|
* |
567
|
|
|
* @return bool True if the job was successfully restarted |
568
|
|
|
*/ |
569
|
|
|
protected function restartStalledJob($stalledJob) |
570
|
|
|
{ |
571
|
|
|
if ($stalledJob->ResumeCounts < static::config()->get('stall_threshold')) { |
572
|
|
|
$stalledJob->restart(); |
573
|
|
|
$logLevel = 'warning'; |
574
|
|
|
$message = _t( |
575
|
|
|
__CLASS__ . '.STALLED_JOB_RESTART_MSG', |
576
|
|
|
'A job named {name} (#{id}) appears to have stalled. It will be stopped and restarted, please ' |
577
|
|
|
. 'login to make sure it has continued', |
578
|
|
|
['name' => $stalledJob->JobTitle, 'id' => $stalledJob->ID] |
579
|
|
|
); |
580
|
|
|
} else { |
581
|
|
|
$stalledJob->pause(); |
582
|
|
|
$logLevel = 'error'; |
583
|
|
|
$message = _t( |
584
|
|
|
__CLASS__ . '.STALLED_JOB_MSG', |
585
|
|
|
'A job named {name} (#{id}) appears to have stalled. It has been paused, please login to check it', |
586
|
|
|
['name' => $stalledJob->JobTitle, 'id' => $stalledJob->ID] |
587
|
|
|
); |
588
|
|
|
} |
589
|
|
|
|
590
|
|
|
$this->getLogger()->log( |
591
|
|
|
$logLevel, |
592
|
|
|
$message, |
593
|
|
|
[ |
594
|
|
|
'file' => __FILE__, |
595
|
|
|
'line' => __LINE__, |
596
|
|
|
] |
597
|
|
|
); |
598
|
|
|
$from = Config::inst()->get(Email::class, 'admin_email'); |
599
|
|
|
$to = Config::inst()->get(Email::class, 'queued_job_admin_email'); |
600
|
|
|
$subject = _t(__CLASS__ . '.STALLED_JOB', 'Stalled job'); |
601
|
|
|
|
602
|
|
|
if ($to) { |
603
|
|
|
$mail = Email::create($from, $to, $subject) |
604
|
|
|
->setData([ |
605
|
|
|
'JobID' => $stalledJob->ID, |
606
|
|
|
'Message' => $message, |
607
|
|
|
'Site' => Director::absoluteBaseURL(), |
608
|
|
|
]) |
609
|
|
|
->setHTMLTemplate('QueuedJobsStalledJob'); |
610
|
|
|
$mail->send(); |
611
|
|
|
} |
612
|
|
|
} |
613
|
|
|
|
614
|
|
|
/** |
615
|
|
|
* Prepares the given jobDescriptor for execution. Returns the job that |
616
|
|
|
* will actually be run in a state ready for executing. |
617
|
|
|
* |
618
|
|
|
* Note that this is called each time a job is picked up to be executed from the cron |
619
|
|
|
* job - meaning that jobs that are paused and restarted will have 'setup()' called on them again, |
620
|
|
|
* so your job MUST detect that and act accordingly. |
621
|
|
|
* |
622
|
|
|
* @param QueuedJobDescriptor $jobDescriptor |
623
|
|
|
* The Job descriptor of a job to prepare for execution |
624
|
|
|
* |
625
|
|
|
* @return QueuedJob|boolean |
626
|
|
|
* @throws Exception |
627
|
|
|
*/ |
628
|
|
|
protected function initialiseJob(QueuedJobDescriptor $jobDescriptor) |
629
|
|
|
{ |
630
|
|
|
// create the job class |
631
|
|
|
$impl = $jobDescriptor->Implementation; |
632
|
|
|
/** @var QueuedJob $job */ |
633
|
|
|
$job = Injector::inst()->create($impl); |
634
|
|
|
/* @var $job QueuedJob */ |
635
|
|
|
if (!$job) { |
636
|
|
|
throw new Exception("Implementation $impl no longer exists"); |
637
|
|
|
} |
638
|
|
|
|
639
|
|
|
$jobDescriptor->JobStatus = QueuedJob::STATUS_INIT; |
640
|
|
|
$jobDescriptor->write(); |
641
|
|
|
|
642
|
|
|
// make sure the data is there |
643
|
|
|
$this->copyDescriptorToJob($jobDescriptor, $job); |
644
|
|
|
|
645
|
|
|
// see if it needs 'setup' or 'restart' called |
646
|
|
|
if ($jobDescriptor->StepsProcessed <= 0) { |
647
|
|
|
$job->setup(); |
648
|
|
|
} else { |
649
|
|
|
$job->prepareForRestart(); |
650
|
|
|
} |
651
|
|
|
|
652
|
|
|
// make sure the descriptor is up to date with anything changed |
653
|
|
|
$this->copyJobToDescriptor($job, $jobDescriptor); |
654
|
|
|
$jobDescriptor->write(); |
655
|
|
|
|
656
|
|
|
return $job; |
657
|
|
|
} |
658
|
|
|
|
659
|
|
|
/** |
660
|
|
|
* Given a {@link QueuedJobDescriptor} mark the job as initialised. Works sort of like a mutex. |
661
|
|
|
* Currently a database lock isn't entirely achievable, due to database adapters not supporting locks. |
662
|
|
|
* This may still have a race condition, but this should minimise the possibility. |
663
|
|
|
* Side effect is the job status will be changed to "Initialised". |
664
|
|
|
* |
665
|
|
|
* Assumption is the job has a status of "Queued" or "Wait". |
666
|
|
|
* |
667
|
|
|
* @param QueuedJobDescriptor $jobDescriptor |
668
|
|
|
* |
669
|
|
|
* @return boolean |
670
|
|
|
*/ |
671
|
|
|
protected function grabMutex(QueuedJobDescriptor $jobDescriptor) |
672
|
|
|
{ |
673
|
|
|
// write the status and determine if any rows were affected, for protection against a |
674
|
|
|
// potential race condition where two or more processes init the same job at once. |
675
|
|
|
// This deliberately does not use write() as that would always update LastEdited |
676
|
|
|
// and thus the row would always be affected. |
677
|
|
|
try { |
678
|
|
|
DB::query(sprintf( |
679
|
|
|
'UPDATE "QueuedJobDescriptor" SET "JobStatus" = \'%s\' WHERE "ID" = %s' |
680
|
|
|
. ' AND "JobFinished" IS NULL AND "JobStatus" NOT IN (%s)', |
681
|
|
|
QueuedJob::STATUS_INIT, |
682
|
|
|
$jobDescriptor->ID, |
683
|
|
|
"'" . QueuedJob::STATUS_RUN . "', '" . QueuedJob::STATUS_COMPLETE . "', '" |
684
|
|
|
. QueuedJob::STATUS_PAUSED . "', '" . QueuedJob::STATUS_CANCELLED . "'" |
685
|
|
|
)); |
686
|
|
|
} catch (Exception $e) { |
687
|
|
|
return false; |
688
|
|
|
} |
689
|
|
|
|
690
|
|
|
if (DB::get_conn()->affectedRows() === 0 && $jobDescriptor->JobStatus !== QueuedJob::STATUS_INIT) { |
691
|
|
|
return false; |
692
|
|
|
} |
693
|
|
|
|
694
|
|
|
return true; |
695
|
|
|
} |
696
|
|
|
|
697
|
|
|
/** |
698
|
|
|
* Start the actual execution of a job. |
699
|
|
|
* The assumption is the jobID refers to a {@link QueuedJobDescriptor} that is status set as "Queued". |
700
|
|
|
* |
701
|
|
|
* This method will continue executing until the job says it's completed |
702
|
|
|
* |
703
|
|
|
* @param int $jobId |
704
|
|
|
* The ID of the job to start executing |
705
|
|
|
* |
706
|
|
|
* @return boolean |
707
|
|
|
* @throws Exception |
708
|
|
|
*/ |
709
|
|
|
public function runJob($jobId) |
710
|
|
|
{ |
711
|
|
|
// first retrieve the descriptor |
712
|
|
|
/** @var QueuedJobDescriptor $jobDescriptor */ |
713
|
|
|
$jobDescriptor = DataObject::get_by_id( |
714
|
|
|
QueuedJobDescriptor::class, |
715
|
|
|
(int)$jobId |
716
|
|
|
); |
717
|
|
|
if (!$jobDescriptor) { |
718
|
|
|
throw new Exception("$jobId is invalid"); |
719
|
|
|
} |
720
|
|
|
|
721
|
|
|
// now lets see whether we have a current user to run as. Typically, if the job is executing via the CLI, |
722
|
|
|
// we want it to actually execute as the RunAs user - however, if running via the web (which is rare...), we |
723
|
|
|
// want to ensure that the current user has admin privileges before switching. Otherwise, we just run it |
724
|
|
|
// as the currently logged in user and hope for the best |
725
|
|
|
|
726
|
|
|
// We need to use $_SESSION directly because SS ties the session to a controller that no longer exists at |
727
|
|
|
// this point of execution in some circumstances |
728
|
|
|
$originalUserID = isset($_SESSION['loggedInAs']) ? $_SESSION['loggedInAs'] : 0; |
729
|
|
|
/** @var Member|null $originalUser */ |
730
|
|
|
$originalUser = $originalUserID |
731
|
|
|
? DataObject::get_by_id(Member::class, $originalUserID) |
732
|
|
|
: null; |
733
|
|
|
$runAsUser = null; |
734
|
|
|
|
735
|
|
|
// If the Job has requested that we run it as a particular user, then we should try and do that. |
736
|
|
|
if ($jobDescriptor->RunAs() !== null) { |
737
|
|
|
$runAsUser = $this->setRunAsUser($jobDescriptor->RunAs(), $originalUser); |
738
|
|
|
} |
739
|
|
|
|
740
|
|
|
// set up a custom error handler for this processing |
741
|
|
|
$errorHandler = new JobErrorHandler(); |
742
|
|
|
|
743
|
|
|
$job = null; |
744
|
|
|
|
745
|
|
|
$broken = false; |
746
|
|
|
|
747
|
|
|
// Push a config context onto the stack for the duration of this job run. |
748
|
|
|
Config::nest(); |
749
|
|
|
|
750
|
|
|
if ($this->grabMutex($jobDescriptor)) { |
751
|
|
|
try { |
752
|
|
|
$job = $this->initialiseJob($jobDescriptor); |
753
|
|
|
|
754
|
|
|
// get the job ready to begin. |
755
|
|
|
if (!$jobDescriptor->JobStarted) { |
756
|
|
|
$jobDescriptor->JobStarted = DBDatetime::now()->Rfc2822(); |
757
|
|
|
} else { |
758
|
|
|
$jobDescriptor->JobRestarted = DBDatetime::now()->Rfc2822(); |
|
|
|
|
759
|
|
|
} |
760
|
|
|
|
761
|
|
|
// Only write to job as "Running" if 'isComplete' was NOT set to true |
762
|
|
|
// during setup() or prepareForRestart() |
763
|
|
|
if (!$job->jobFinished()) { |
764
|
|
|
$jobDescriptor->JobStatus = QueuedJob::STATUS_RUN; |
765
|
|
|
$jobDescriptor->write(); |
766
|
|
|
} |
767
|
|
|
|
768
|
|
|
$lastStepProcessed = 0; |
769
|
|
|
// have we stalled at all? |
770
|
|
|
$stallCount = 0; |
771
|
|
|
|
772
|
|
|
if (class_exists(Subsite::class) && !empty($job->SubsiteID)) { |
773
|
|
|
Subsite::changeSubsite($job->SubsiteID); |
774
|
|
|
|
775
|
|
|
// lets set the base URL as far as Director is concerned so that our URLs are correct |
776
|
|
|
/** @var Subsite $subsite */ |
777
|
|
|
$subsite = DataObject::get_by_id(Subsite::class, $job->SubsiteID); |
778
|
|
|
if ($subsite && $subsite->exists()) { |
779
|
|
|
$domain = $subsite->domain(); |
780
|
|
|
$base = rtrim(Director::protocol() . $domain, '/') . '/'; |
781
|
|
|
|
782
|
|
|
Config::modify()->set(Director::class, 'alternate_base_url', $base); |
783
|
|
|
} |
784
|
|
|
} |
785
|
|
|
|
786
|
|
|
// while not finished |
787
|
|
|
while (!$job->jobFinished() && !$broken) { |
788
|
|
|
// see that we haven't been set to 'paused' or otherwise by another process |
789
|
|
|
/** @var QueuedJobDescriptor $jobDescriptor */ |
790
|
|
|
$jobDescriptor = DataObject::get_by_id( |
791
|
|
|
QueuedJobDescriptor::class, |
792
|
|
|
(int)$jobId |
793
|
|
|
); |
794
|
|
|
if (!$jobDescriptor || !$jobDescriptor->exists()) { |
795
|
|
|
$broken = true; |
796
|
|
|
$this->getLogger()->error( |
797
|
|
|
print_r( |
798
|
|
|
[ |
799
|
|
|
'errno' => 0, |
800
|
|
|
'errstr' => 'Job descriptor ' . $jobId . ' could not be found', |
801
|
|
|
'errfile' => __FILE__, |
802
|
|
|
'errline' => __LINE__, |
803
|
|
|
'errcontext' => [], |
804
|
|
|
], |
805
|
|
|
true |
806
|
|
|
), |
807
|
|
|
[ |
808
|
|
|
'file' => __FILE__, |
809
|
|
|
'line' => __LINE__, |
810
|
|
|
] |
811
|
|
|
); |
812
|
|
|
break; |
813
|
|
|
} |
814
|
|
View Code Duplication |
if ($jobDescriptor->JobStatus != QueuedJob::STATUS_RUN) { |
|
|
|
|
815
|
|
|
// we've been paused by something, so we'll just exit |
816
|
|
|
$job->addMessage(_t( |
817
|
|
|
__CLASS__ . '.JOB_PAUSED', |
818
|
|
|
'Job paused at {time}', |
819
|
|
|
['time' => DBDatetime::now()->Rfc2822()] |
820
|
|
|
)); |
821
|
|
|
$broken = true; |
822
|
|
|
} |
823
|
|
|
|
824
|
|
|
if (!$broken) { |
825
|
|
|
// Inject real-time log handler |
826
|
|
|
$logger = Injector::inst()->get(LoggerInterface::class); |
827
|
|
|
if ($logger instanceof Logger) { |
828
|
|
|
// Check if there is already a handler |
829
|
|
|
$exists = false; |
830
|
|
|
foreach ($logger->getHandlers() as $handler) { |
831
|
|
|
if ($handler instanceof QueuedJobHandler) { |
832
|
|
|
$exists = true; |
833
|
|
|
break; |
834
|
|
|
} |
835
|
|
|
} |
836
|
|
|
|
837
|
|
|
if (!$exists) { |
838
|
|
|
// Add the handler |
839
|
|
|
/** @var QueuedJobHandler $queuedJobHandler */ |
840
|
|
|
$queuedJobHandler = QueuedJobHandler::create($job, $jobDescriptor); |
841
|
|
|
|
842
|
|
|
// We only write for every 100 file |
843
|
|
|
$bufferHandler = new BufferHandler( |
844
|
|
|
$queuedJobHandler, |
845
|
|
|
100, |
846
|
|
|
Logger::DEBUG, |
847
|
|
|
true, |
848
|
|
|
true |
849
|
|
|
); |
850
|
|
|
|
851
|
|
|
$logger->pushHandler($bufferHandler); |
852
|
|
|
} |
853
|
|
|
} else { |
854
|
|
|
if ($logger instanceof LoggerInterface) { |
855
|
|
|
$logger->warning( |
856
|
|
|
'Monolog not found, messages will not output while the job is running' |
857
|
|
|
); |
858
|
|
|
} |
859
|
|
|
} |
860
|
|
|
|
861
|
|
|
// Collect output as job messages as well as sending it to the screen after processing |
862
|
|
|
$obLogger = function ($buffer, $phase) use ($job, $jobDescriptor) { |
|
|
|
|
863
|
|
|
$job->addMessage($buffer); |
864
|
|
|
if ($jobDescriptor) { |
865
|
|
|
$this->copyJobToDescriptor($job, $jobDescriptor); |
866
|
|
|
$jobDescriptor->write(); |
867
|
|
|
} |
868
|
|
|
return $buffer; |
869
|
|
|
}; |
870
|
|
|
ob_start($obLogger, 256); |
871
|
|
|
|
872
|
|
|
try { |
873
|
|
|
$job->process(); |
874
|
|
|
} catch (Exception $e) { |
875
|
|
|
// okay, we'll just catch this exception for now |
876
|
|
|
$job->addMessage( |
877
|
|
|
_t( |
878
|
|
|
__CLASS__ . '.JOB_EXCEPT', |
879
|
|
|
'Job caused exception {message} in {file} at line {line}', |
880
|
|
|
[ |
881
|
|
|
'message' => $e->getMessage(), |
882
|
|
|
'file' => $e->getFile(), |
883
|
|
|
'line' => $e->getLine(), |
884
|
|
|
] |
885
|
|
|
) |
886
|
|
|
); |
887
|
|
|
$this->getLogger()->error( |
888
|
|
|
$e->getMessage(), |
889
|
|
|
[ |
890
|
|
|
'exception' => $e, |
891
|
|
|
] |
892
|
|
|
); |
893
|
|
|
$jobDescriptor->JobStatus = QueuedJob::STATUS_BROKEN; |
894
|
|
|
$this->extend('updateJobDescriptorAndJobOnException', $jobDescriptor, $job, $e); |
895
|
|
|
} |
896
|
|
|
|
897
|
|
|
// Write any remaining batched messages at the end |
898
|
|
|
if (isset($bufferHandler)) { |
899
|
|
|
$bufferHandler->flush(); |
900
|
|
|
} |
901
|
|
|
|
902
|
|
|
ob_end_flush(); |
903
|
|
|
|
904
|
|
|
// now check the job state |
905
|
|
|
$data = $job->getJobData(); |
906
|
|
|
if ($data->currentStep == $lastStepProcessed) { |
907
|
|
|
$stallCount++; |
908
|
|
|
} |
909
|
|
|
|
910
|
|
View Code Duplication |
if ($stallCount > static::config()->get('stall_threshold')) { |
|
|
|
|
911
|
|
|
$broken = true; |
912
|
|
|
$job->addMessage( |
913
|
|
|
_t( |
914
|
|
|
__CLASS__ . '.JOB_STALLED', |
915
|
|
|
'Job stalled after {attempts} attempts - please check', |
916
|
|
|
['attempts' => $stallCount] |
917
|
|
|
) |
918
|
|
|
); |
919
|
|
|
$jobDescriptor->JobStatus = QueuedJob::STATUS_BROKEN; |
920
|
|
|
} |
921
|
|
|
|
922
|
|
|
// now we'll be good and check our memory usage. If it is too high, we'll set the job to |
923
|
|
|
// a 'Waiting' state, and let the next processing run pick up the job. |
924
|
|
|
if ($this->isMemoryTooHigh()) { |
925
|
|
|
$job->addMessage( |
926
|
|
|
_t( |
927
|
|
|
__CLASS__ . '.MEMORY_RELEASE', |
928
|
|
|
'Job releasing memory and waiting ({used} used)', |
929
|
|
|
['used' => $this->humanReadable($this->getMemoryUsage())] |
930
|
|
|
) |
931
|
|
|
); |
932
|
|
|
if ($jobDescriptor->JobStatus != QueuedJob::STATUS_BROKEN) { |
933
|
|
|
$jobDescriptor->JobStatus = QueuedJob::STATUS_WAIT; |
934
|
|
|
} |
935
|
|
|
$broken = true; |
936
|
|
|
} |
937
|
|
|
|
938
|
|
|
// Also check if we are running too long |
939
|
|
|
if ($this->hasPassedTimeLimit()) { |
940
|
|
|
$job->addMessage(_t( |
941
|
|
|
__CLASS__ . '.TIME_LIMIT', |
942
|
|
|
'Queue has passed time limit and will restart before continuing' |
943
|
|
|
)); |
944
|
|
|
if ($jobDescriptor->JobStatus != QueuedJob::STATUS_BROKEN) { |
945
|
|
|
$jobDescriptor->JobStatus = QueuedJob::STATUS_WAIT; |
946
|
|
|
} |
947
|
|
|
$broken = true; |
948
|
|
|
} |
949
|
|
|
} |
950
|
|
|
|
951
|
|
|
if ($jobDescriptor) { |
952
|
|
|
$this->copyJobToDescriptor($job, $jobDescriptor); |
953
|
|
|
$jobDescriptor->write(); |
954
|
|
View Code Duplication |
} else { |
|
|
|
|
955
|
|
|
$this->getLogger()->error( |
956
|
|
|
print_r( |
957
|
|
|
[ |
958
|
|
|
'errno' => 0, |
959
|
|
|
'errstr' => 'Job descriptor has been set to null', |
960
|
|
|
'errfile' => __FILE__, |
961
|
|
|
'errline' => __LINE__, |
962
|
|
|
'errcontext' => [], |
963
|
|
|
], |
964
|
|
|
true |
965
|
|
|
), |
966
|
|
|
[ |
967
|
|
|
'file' => __FILE__, |
968
|
|
|
'line' => __LINE__, |
969
|
|
|
] |
970
|
|
|
); |
971
|
|
|
$broken = true; |
972
|
|
|
} |
973
|
|
|
} |
974
|
|
|
|
975
|
|
|
// a last final save. The job is complete by now |
976
|
|
|
if ($jobDescriptor) { |
977
|
|
|
$jobDescriptor->write(); |
978
|
|
|
} |
979
|
|
|
|
980
|
|
|
if ($job->jobFinished()) { |
981
|
|
|
/** @var AbstractQueuedJob|QueuedJob $job */ |
982
|
|
|
$job->afterComplete(); |
983
|
|
|
$jobDescriptor->cleanupJob(); |
984
|
|
|
|
985
|
|
|
$this->extend('updateJobDescriptorAndJobOnCompletion', $jobDescriptor, $job); |
986
|
|
|
} |
987
|
|
|
} catch (Exception $e) { |
988
|
|
|
// PHP 5.6 exception handling |
989
|
|
|
$this->handleBrokenJobException($jobDescriptor, $job, $e); |
990
|
|
|
$broken = true; |
991
|
|
|
} catch (\Throwable $e) { |
|
|
|
|
992
|
|
|
// PHP 7 Error handling) |
993
|
|
|
$this->handleBrokenJobException($jobDescriptor, $job, $e); |
994
|
|
|
$broken = true; |
995
|
|
|
} |
996
|
|
|
} |
997
|
|
|
|
998
|
|
|
$errorHandler->clear(); |
999
|
|
|
|
1000
|
|
|
Config::unnest(); |
1001
|
|
|
|
1002
|
|
|
$this->unsetRunAsUser($runAsUser, $originalUser); |
1003
|
|
|
|
1004
|
|
|
return !$broken; |
1005
|
|
|
} |
1006
|
|
|
|
1007
|
|
|
/** |
1008
|
|
|
* @param QueuedJobDescriptor $jobDescriptor |
1009
|
|
|
* @param QueuedJob $job |
1010
|
|
|
* @param Exception|\Throwable $e |
1011
|
|
|
*/ |
1012
|
|
|
protected function handleBrokenJobException(QueuedJobDescriptor $jobDescriptor, QueuedJob $job, $e) |
1013
|
|
|
{ |
1014
|
|
|
// okay, we'll just catch this exception for now |
1015
|
|
|
$this->getLogger()->info( |
1016
|
|
|
$e->getMessage(), |
1017
|
|
|
[ |
1018
|
|
|
'exception' => $e, |
1019
|
|
|
] |
1020
|
|
|
); |
1021
|
|
|
$jobDescriptor->JobStatus = QueuedJob::STATUS_BROKEN; |
1022
|
|
|
$this->extend('updateJobDescriptorAndJobOnException', $jobDescriptor, $job, $e); |
1023
|
|
|
$jobDescriptor->write(); |
1024
|
|
|
} |
1025
|
|
|
|
1026
|
|
|
/** |
1027
|
|
|
* @param Member $runAsUser |
1028
|
|
|
* @param Member|null $originalUser |
1029
|
|
|
* @return null|Member |
1030
|
|
|
*/ |
1031
|
|
|
protected function setRunAsUser(Member $runAsUser, Member $originalUser = null) |
1032
|
|
|
{ |
1033
|
|
|
// Sanity check. Can't set the user if they don't exist. |
1034
|
|
|
if ($runAsUser === null || !$runAsUser->exists()) { |
1035
|
|
|
return null; |
1036
|
|
|
} |
1037
|
|
|
|
1038
|
|
|
// Don't need to set Security user if we're already logged in as that same user. |
1039
|
|
|
if ($originalUser && $originalUser->ID === $runAsUser->ID) { |
1040
|
|
|
return null; |
1041
|
|
|
} |
1042
|
|
|
|
1043
|
|
|
// We are currently either not logged in at all, or we're logged in as a different user. We should switch users |
1044
|
|
|
// so that the context within the Job is correct. |
1045
|
|
|
if (Controller::has_curr()) { |
1046
|
|
|
Security::setCurrentUser($runAsUser); |
1047
|
|
|
} else { |
1048
|
|
|
$_SESSION['loggedInAs'] = $runAsUser->ID; |
1049
|
|
|
} |
1050
|
|
|
|
1051
|
|
|
// This is an explicit coupling brought about by SS not having a nice way of mocking a user, as it requires |
1052
|
|
|
// session nastiness |
1053
|
|
|
if (class_exists('SecurityContext')) { |
1054
|
|
|
singleton('SecurityContext')->setMember($runAsUser); |
1055
|
|
|
} |
1056
|
|
|
|
1057
|
|
|
return $runAsUser; |
1058
|
|
|
} |
1059
|
|
|
|
1060
|
|
|
/** |
1061
|
|
|
* @param Member|null $runAsUser |
1062
|
|
|
* @param Member|null $originalUser |
1063
|
|
|
*/ |
1064
|
|
|
protected function unsetRunAsUser(Member $runAsUser = null, Member $originalUser = null) |
1065
|
|
|
{ |
1066
|
|
|
// No runAsUser was set, so we don't need to do anything. |
1067
|
|
|
if ($runAsUser === null) { |
1068
|
|
|
return; |
1069
|
|
|
} |
1070
|
|
|
|
1071
|
|
|
// There was no originalUser, so we should make sure that we set the user back to null. |
1072
|
|
|
if (!$originalUser) { |
1073
|
|
|
if (Controller::has_curr()) { |
1074
|
|
|
Security::setCurrentUser(null); |
1075
|
|
|
} else { |
1076
|
|
|
$_SESSION['loggedInAs'] = null; |
1077
|
|
|
} |
1078
|
|
|
|
1079
|
|
|
return; |
1080
|
|
|
} |
1081
|
|
|
|
1082
|
|
|
// Okay let's reset our user. |
1083
|
|
|
if (Controller::has_curr()) { |
1084
|
|
|
Security::setCurrentUser($originalUser); |
1085
|
|
|
} else { |
1086
|
|
|
$_SESSION['loggedInAs'] = $originalUser->ID; |
1087
|
|
|
} |
1088
|
|
|
} |
1089
|
|
|
|
1090
|
|
|
/** |
1091
|
|
|
* Start timer |
1092
|
|
|
*/ |
1093
|
|
|
protected function markStarted() |
1094
|
|
|
{ |
1095
|
|
|
if (!$this->startedAt) { |
1096
|
|
|
$this->startedAt = DBDatetime::now()->getTimestamp(); |
1097
|
|
|
} |
1098
|
|
|
} |
1099
|
|
|
|
1100
|
|
|
/** |
1101
|
|
|
* Is execution time too long? |
1102
|
|
|
* |
1103
|
|
|
* @return bool True if the script has passed the configured time_limit |
1104
|
|
|
*/ |
1105
|
|
|
protected function hasPassedTimeLimit() |
1106
|
|
|
{ |
1107
|
|
|
// Ensure a limit exists |
1108
|
|
|
$limit = static::config()->get('time_limit'); |
1109
|
|
|
if (!$limit) { |
1110
|
|
|
return false; |
1111
|
|
|
} |
1112
|
|
|
|
1113
|
|
|
// Ensure started date is set |
1114
|
|
|
$this->markStarted(); |
1115
|
|
|
|
1116
|
|
|
// Check duration |
1117
|
|
|
$now = DBDatetime::now()->getTimestamp(); |
1118
|
|
|
return $now > $this->startedAt + $limit; |
1119
|
|
|
} |
1120
|
|
|
|
1121
|
|
|
/** |
1122
|
|
|
* Is memory usage too high? |
1123
|
|
|
* |
1124
|
|
|
* @return bool |
1125
|
|
|
*/ |
1126
|
|
|
protected function isMemoryTooHigh() |
1127
|
|
|
{ |
1128
|
|
|
$used = $this->getMemoryUsage(); |
1129
|
|
|
$limit = $this->getMemoryLimit(); |
1130
|
|
|
return $limit && ($used > $limit); |
1131
|
|
|
} |
1132
|
|
|
|
1133
|
|
|
/** |
1134
|
|
|
* Get peak memory usage of this application |
1135
|
|
|
* |
1136
|
|
|
* @return float |
1137
|
|
|
*/ |
1138
|
|
|
protected function getMemoryUsage() |
1139
|
|
|
{ |
1140
|
|
|
// Note we use real_usage = false |
1141
|
|
|
// http://stackoverflow.com/questions/15745385/memory-get-peak-usage-with-real-usage |
1142
|
|
|
// Also we use the safer peak memory usage |
1143
|
|
|
return (float)memory_get_peak_usage(false); |
1144
|
|
|
} |
1145
|
|
|
|
1146
|
|
|
/** |
1147
|
|
|
* Determines the memory limit (in bytes) for this application |
1148
|
|
|
* Limits to the smaller of memory_limit configured via php.ini or silverstripe config |
1149
|
|
|
* |
1150
|
|
|
* @return float Memory limit in bytes |
1151
|
|
|
*/ |
1152
|
|
|
protected function getMemoryLimit() |
1153
|
|
|
{ |
1154
|
|
|
// Limit to smaller of explicit limit or php memory limit |
1155
|
|
|
$limit = $this->parseMemory(static::config()->get('memory_limit')); |
1156
|
|
|
if ($limit) { |
1157
|
|
|
return $limit; |
1158
|
|
|
} |
1159
|
|
|
|
1160
|
|
|
// Fallback to php memory limit |
1161
|
|
|
$phpLimit = $this->getPHPMemoryLimit(); |
1162
|
|
|
if ($phpLimit) { |
1163
|
|
|
return $phpLimit; |
1164
|
|
|
} |
1165
|
|
|
} |
1166
|
|
|
|
1167
|
|
|
/** |
1168
|
|
|
* Calculate the current memory limit of the server |
1169
|
|
|
* |
1170
|
|
|
* @return float |
1171
|
|
|
*/ |
1172
|
|
|
protected function getPHPMemoryLimit() |
1173
|
|
|
{ |
1174
|
|
|
return $this->parseMemory(trim(ini_get("memory_limit"))); |
1175
|
|
|
} |
1176
|
|
|
|
1177
|
|
|
/** |
1178
|
|
|
* Convert memory limit string to bytes. |
1179
|
|
|
* Based on implementation in install.php5 |
1180
|
|
|
* |
1181
|
|
|
* @param string $memString |
1182
|
|
|
* |
1183
|
|
|
* @return float |
1184
|
|
|
*/ |
1185
|
|
|
protected function parseMemory($memString) |
1186
|
|
|
{ |
1187
|
|
|
switch (strtolower(substr($memString, -1))) { |
1188
|
|
|
case "b": |
1189
|
|
|
return round(substr($memString, 0, -1)); |
1190
|
|
|
case "k": |
1191
|
|
|
return round(substr($memString, 0, -1) * 1024); |
1192
|
|
|
case "m": |
1193
|
|
|
return round(substr($memString, 0, -1) * 1024 * 1024); |
1194
|
|
|
case "g": |
1195
|
|
|
return round(substr($memString, 0, -1) * 1024 * 1024 * 1024); |
1196
|
|
|
default: |
1197
|
|
|
return round($memString); |
1198
|
|
|
} |
1199
|
|
|
} |
1200
|
|
|
|
1201
|
|
|
protected function humanReadable($size) |
1202
|
|
|
{ |
1203
|
|
|
$filesizename = [" Bytes", " KB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB"]; |
1204
|
|
|
return $size ? round($size / pow(1024, ($i = floor(log($size, 1024)))), 2) . $filesizename[$i] : '0 Bytes'; |
1205
|
|
|
} |
1206
|
|
|
|
1207
|
|
|
|
1208
|
|
|
/** |
1209
|
|
|
* Gets a list of all the current jobs (or jobs that have recently finished) |
1210
|
|
|
* |
1211
|
|
|
* @param string $type |
1212
|
|
|
* if we're after a particular job list |
1213
|
|
|
* @param int $includeUpUntil |
1214
|
|
|
* The number of seconds to include jobs that have just finished, allowing a job list to be built that |
1215
|
|
|
* includes recently finished jobs |
1216
|
|
|
* |
1217
|
|
|
* @return DataList|QueuedJobDescriptor[] |
1218
|
|
|
*/ |
1219
|
|
|
public function getJobList($type = null, $includeUpUntil = 0) |
1220
|
|
|
{ |
1221
|
|
|
return DataObject::get( |
1222
|
|
|
QueuedJobDescriptor::class, |
1223
|
|
|
$this->getJobListFilter($type, $includeUpUntil) |
1224
|
|
|
); |
1225
|
|
|
} |
1226
|
|
|
|
1227
|
|
|
/** |
1228
|
|
|
* Return the SQL filter used to get the job list - this is used by the UI for displaying the job list... |
1229
|
|
|
* |
1230
|
|
|
* @param string $type |
1231
|
|
|
* if we're after a particular job list |
1232
|
|
|
* @param int $includeUpUntil |
1233
|
|
|
* The number of seconds to include jobs that have just finished, allowing a job list to be built that |
1234
|
|
|
* includes recently finished jobs |
1235
|
|
|
* |
1236
|
|
|
* @return string |
1237
|
|
|
*/ |
1238
|
|
|
public function getJobListFilter($type = null, $includeUpUntil = 0) |
1239
|
|
|
{ |
1240
|
|
|
$util = singleton(QJUtils::class); |
1241
|
|
|
|
1242
|
|
|
$filter = ['JobStatus <>' => QueuedJob::STATUS_COMPLETE]; |
1243
|
|
|
if ($includeUpUntil) { |
1244
|
|
|
$filter['JobFinished > '] = DBDatetime::create()->setValue( |
1245
|
|
|
DBDatetime::now()->getTimestamp() - $includeUpUntil |
1246
|
|
|
)->Rfc2822(); |
1247
|
|
|
} |
1248
|
|
|
|
1249
|
|
|
$filter = $util->dbQuote($filter, ' OR '); |
1250
|
|
|
|
1251
|
|
|
if ($type) { |
|
|
|
|
1252
|
|
|
$filter = $util->dbQuote(['JobType =' => (string)$type]) . ' AND (' . $filter . ')'; |
1253
|
|
|
} |
1254
|
|
|
|
1255
|
|
|
return $filter; |
1256
|
|
|
} |
1257
|
|
|
|
1258
|
|
|
/** |
1259
|
|
|
* Process the job queue with the current queue runner |
1260
|
|
|
* |
1261
|
|
|
* @param string $queue |
1262
|
|
|
*/ |
1263
|
|
|
public function runQueue($queue) |
1264
|
|
|
{ |
1265
|
|
|
if (!self::config()->get('disable_health_check')) { |
1266
|
|
|
$this->checkJobHealth($queue); |
1267
|
|
|
} |
1268
|
|
|
$this->checkdefaultJobs($queue); |
1269
|
|
|
$this->queueRunner->runQueue($queue); |
1270
|
|
|
} |
1271
|
|
|
|
1272
|
|
|
/** |
1273
|
|
|
* Process all jobs from a given queue |
1274
|
|
|
* |
1275
|
|
|
* @param string $name The job queue to completely process |
1276
|
|
|
*/ |
1277
|
|
|
public function processJobQueue($name) |
1278
|
|
|
{ |
1279
|
|
|
// Start timer to measure lifetime |
1280
|
|
|
$this->markStarted(); |
1281
|
|
|
|
1282
|
|
|
// Begin main loop |
1283
|
|
|
do { |
1284
|
|
|
if (class_exists(Subsite::class)) { |
1285
|
|
|
// clear subsite back to default to prevent any subsite changes from leaking to |
1286
|
|
|
// subsequent actions |
1287
|
|
|
Subsite::changeSubsite(0); |
1288
|
|
|
} |
1289
|
|
|
|
1290
|
|
|
$job = $this->getNextPendingJob($name); |
1291
|
|
|
if ($job) { |
1292
|
|
|
$success = $this->runJob($job->ID); |
1293
|
|
|
if (!$success) { |
1294
|
|
|
// make sure job is null so it doesn't continue the current |
1295
|
|
|
// processing loop. Next queue executor can pick up where |
1296
|
|
|
// things left off |
1297
|
|
|
$job = null; |
1298
|
|
|
} |
1299
|
|
|
} |
1300
|
|
|
} while ($job); |
1301
|
|
|
} |
1302
|
|
|
|
1303
|
|
|
/** |
1304
|
|
|
* When PHP shuts down, we want to process all of the immediate queue items |
1305
|
|
|
* |
1306
|
|
|
* We use the 'getNextPendingJob' method, instead of just iterating the queue, to ensure |
1307
|
|
|
* we ignore paused or stalled jobs. |
1308
|
|
|
*/ |
1309
|
|
|
public function onShutdown() |
1310
|
|
|
{ |
1311
|
|
|
$this->processJobQueue(QueuedJob::IMMEDIATE); |
1312
|
|
|
} |
1313
|
|
|
|
1314
|
|
|
/** |
1315
|
|
|
* Get a logger |
1316
|
|
|
* |
1317
|
|
|
* @return LoggerInterface |
1318
|
|
|
*/ |
1319
|
|
|
public function getLogger() |
1320
|
|
|
{ |
1321
|
|
|
return Injector::inst()->get(LoggerInterface::class); |
1322
|
|
|
} |
1323
|
|
|
|
1324
|
|
|
public function enableMaintenanceLock() |
1325
|
|
|
{ |
1326
|
|
|
if (!$this->config()->get('lock_file_enabled')) { |
1327
|
|
|
return; |
1328
|
|
|
} |
1329
|
|
|
|
1330
|
|
|
$path = $this->lockFilePath(); |
1331
|
|
|
$contents = date('Y-m-d H:i:s'); |
1332
|
|
|
|
1333
|
|
|
file_put_contents($path, $contents); |
1334
|
|
|
} |
1335
|
|
|
|
1336
|
|
View Code Duplication |
public function disableMaintenanceLock() |
|
|
|
|
1337
|
|
|
{ |
1338
|
|
|
if (!$this->config()->get('lock_file_enabled')) { |
1339
|
|
|
return; |
1340
|
|
|
} |
1341
|
|
|
|
1342
|
|
|
$path = $this->lockFilePath(); |
1343
|
|
|
if (!file_exists($path)) { |
1344
|
|
|
return; |
1345
|
|
|
} |
1346
|
|
|
|
1347
|
|
|
unlink($path); |
1348
|
|
|
} |
1349
|
|
|
|
1350
|
|
|
/** |
1351
|
|
|
* @return bool |
1352
|
|
|
*/ |
1353
|
|
View Code Duplication |
public function isMaintenanceLockActive() |
|
|
|
|
1354
|
|
|
{ |
1355
|
|
|
if (!$this->config()->get('lock_file_enabled')) { |
1356
|
|
|
return false; |
1357
|
|
|
} |
1358
|
|
|
|
1359
|
|
|
$path = $this->lockFilePath(); |
1360
|
|
|
|
1361
|
|
|
return file_exists($path); |
1362
|
|
|
} |
1363
|
|
|
|
1364
|
|
|
/** |
1365
|
|
|
* @return string |
1366
|
|
|
*/ |
1367
|
|
|
private function lockFilePath() |
1368
|
|
|
{ |
1369
|
|
|
return sprintf( |
1370
|
|
|
'%s%s/%s', |
1371
|
|
|
Director::baseFolder(), |
1372
|
|
|
static::config()->get('lock_file_path'), |
1373
|
|
|
static::config()->get('lock_file_name') |
1374
|
|
|
); |
1375
|
|
|
} |
1376
|
|
|
} |
1377
|
|
|
|
Since your code implements the magic setter
_set
, this function will be called for any write access on an undefined variable. You can add the@property
annotation to your class or interface to document the existence of this variable.Since the property has write access only, you can use the @property-write annotation instead.
Of course, you may also just have mistyped another name, in which case you should fix the error.
See also the PhpDoc documentation for @property.