1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
namespace Dtc\QueueBundle\Controller; |
4
|
|
|
|
5
|
|
|
use Doctrine\ODM\MongoDB\DocumentManager; |
6
|
|
|
use Doctrine\ORM\EntityManager; |
7
|
|
|
use Dtc\QueueBundle\Doctrine\BaseJobTimingManager; |
8
|
|
|
use Dtc\QueueBundle\Exception\UnsupportedException; |
9
|
|
|
use Dtc\QueueBundle\Model\JobTiming; |
10
|
|
|
use Dtc\QueueBundle\Model\Worker; |
11
|
|
|
use Dtc\QueueBundle\ODM\JobManager; |
12
|
|
|
use Dtc\QueueBundle\ODM\JobTimingManager; |
13
|
|
|
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route; |
14
|
|
|
use Sensio\Bundle\FrameworkExtraBundle\Configuration\Template; |
15
|
|
|
use Symfony\Bundle\FrameworkBundle\Controller\Controller; |
16
|
|
|
use Symfony\Component\HttpFoundation\JsonResponse; |
17
|
|
|
use Zend\Stdlib\Request; |
18
|
|
|
|
19
|
|
|
class QueueController extends Controller |
20
|
|
|
{ |
21
|
|
|
/** |
22
|
|
|
* Summary stats. |
23
|
|
|
* |
24
|
|
|
* @Route("/") |
25
|
|
|
* @Route("/status/") |
26
|
|
|
* @Template("@DtcQueue/Queue/jobs.html.twig") |
27
|
|
|
*/ |
28
|
|
|
public function statusAction() |
29
|
|
|
{ |
30
|
|
|
$params = array(); |
31
|
|
|
$jobManager = $this->get('dtc_queue.job_manager'); |
32
|
|
|
|
33
|
|
|
$params['status'] = $jobManager->getStatus(); |
34
|
|
|
$this->addCssJs($params); |
35
|
|
|
|
36
|
|
|
return $params; |
37
|
|
|
} |
38
|
|
|
|
39
|
|
|
/** |
40
|
|
|
* List jobs in system by default. |
41
|
|
|
* |
42
|
|
|
* @Route("/jobs_all", name="dtc_queue_jobs_all") |
43
|
|
|
*/ |
44
|
|
View Code Duplication |
public function jobsAllAction() |
|
|
|
|
45
|
|
|
{ |
46
|
|
|
$this->validateManagerType('dtc_queue.default_manager'); |
47
|
|
|
$class1 = $this->container->getParameter('dtc_queue.class_job'); |
48
|
|
|
$class2 = $this->container->getParameter('dtc_queue.class_job_archive'); |
49
|
|
|
$label1 = 'Non-Archived Jobs'; |
50
|
|
|
$label2 = 'Archived Jobs'; |
51
|
|
|
|
52
|
|
|
$params = $this->getDualGridParams($class1, $class2, $label1, $label2); |
53
|
|
|
|
54
|
|
|
return $this->render('@DtcQueue/Queue/grid.html.twig', $params); |
55
|
|
|
} |
56
|
|
|
|
57
|
|
|
/** |
58
|
|
|
* List jobs in system by default. |
59
|
|
|
* |
60
|
|
|
* @Template("@DtcQueue/Queue/jobs.html.twig") |
61
|
|
|
* @Route("/jobs", name="dtc_queue_jobs") |
62
|
|
|
*/ |
63
|
|
|
public function jobsAction() |
64
|
|
|
{ |
65
|
|
|
$this->validateManagerType('dtc_queue.default_manager'); |
66
|
|
|
$managerType = $this->container->getParameter('dtc_queue.default_manager'); |
67
|
|
|
$rendererFactory = $this->get('dtc_grid.renderer.factory'); |
68
|
|
|
$renderer = $rendererFactory->create('datatables'); |
69
|
|
|
$gridSource = $this->get('dtc_queue.grid_source.live_jobs.'.('mongodb' === $managerType ? 'odm' : $managerType)); |
70
|
|
|
$renderer->bind($gridSource); |
71
|
|
|
$params = $renderer->getParams(); |
72
|
|
|
$this->addCssJs($params); |
73
|
|
|
|
74
|
|
|
return $params; |
75
|
|
|
} |
76
|
|
|
|
77
|
|
|
protected function validateManagerType($type) |
78
|
|
|
{ |
79
|
|
|
$managerType = $this->container->getParameter($type); |
80
|
|
|
if ('mongodb' !== $managerType && 'orm' != $managerType && 'odm' != $managerType) { |
81
|
|
|
throw new UnsupportedException("Unsupported manager type: $managerType"); |
82
|
|
|
} |
83
|
|
|
} |
84
|
|
|
|
85
|
|
|
/** |
86
|
|
|
* @param string $class1 |
87
|
|
|
* @param string $class2 |
88
|
|
|
* @param string $label1 |
89
|
|
|
* @param string $label2 |
90
|
|
|
* |
91
|
|
|
* @return \Symfony\Component\HttpFoundation\Response |
92
|
|
|
* |
93
|
|
|
* @throws \Exception |
94
|
|
|
*/ |
95
|
|
|
protected function getDualGridParams($class1, $class2, $label1, $label2) |
96
|
|
|
{ |
97
|
|
|
$rendererFactory = $this->get('dtc_grid.renderer.factory'); |
98
|
|
|
$renderer = $rendererFactory->create('datatables'); |
99
|
|
|
$gridSource = $this->get('dtc_grid.manager.source')->get($class1); |
100
|
|
|
$renderer->bind($gridSource); |
101
|
|
|
$params = $renderer->getParams(); |
102
|
|
|
|
103
|
|
|
$renderer2 = $rendererFactory->create('datatables'); |
104
|
|
|
$gridSource = $this->get('dtc_grid.manager.source')->get($class2); |
105
|
|
|
$renderer2->bind($gridSource); |
106
|
|
|
$params2 = $renderer2->getParams(); |
107
|
|
|
|
108
|
|
|
$params['archive_grid'] = $params2['dtc_grid']; |
109
|
|
|
|
110
|
|
|
$params['dtc_queue_grid_label1'] = $label1; |
111
|
|
|
$params['dtc_queue_grid_label2'] = $label2; |
112
|
|
|
$this->addCssJs($params); |
113
|
|
|
|
114
|
|
|
return $params; |
115
|
|
|
} |
116
|
|
|
|
117
|
|
|
protected function addCssJs(array &$params) |
118
|
|
|
{ |
119
|
|
|
$params['css'] = $this->container->getParameter('dtc_grid.theme.css'); |
120
|
|
|
$params['js'] = $this->container->getParameter('dtc_grid.theme.js'); |
121
|
|
|
$jQuery = $this->container->getParameter('dtc_grid.jquery'); |
122
|
|
|
array_unshift($params['js'], $jQuery['url']); |
123
|
|
|
$params['chartjs'] = $this->container->getParameter('dtc_queue.admin.chartjs'); |
124
|
|
|
} |
125
|
|
|
|
126
|
|
|
protected function validateJobTimingManager() |
127
|
|
|
{ |
128
|
|
|
if ($this->container->hasParameter('dtc_queue.job_timing_manager')) { |
129
|
|
|
$this->validateManagerType('dtc_queue.job_timing_manager'); |
130
|
|
|
} elseif ($this->container->hasParameter('dtc_queue.job_timing_manager')) { |
131
|
|
|
$this->validateManagerType('dtc_queue.run_manager'); |
132
|
|
|
} else { |
133
|
|
|
$this->validateManagerType('dtc_queue.default_manager'); |
134
|
|
|
} |
135
|
|
|
} |
136
|
|
|
|
137
|
|
|
protected function validateRunManager() |
138
|
|
|
{ |
139
|
|
|
if ($this->container->hasParameter('dtc_queue.job_timing_manager')) { |
140
|
|
|
$this->validateManagerType('dtc_queue.run_manager'); |
141
|
|
|
} else { |
142
|
|
|
$this->validateManagerType('dtc_queue.default_manager'); |
143
|
|
|
} |
144
|
|
|
} |
145
|
|
|
|
146
|
|
|
/** |
147
|
|
|
* List jobs in system by default. |
148
|
|
|
* |
149
|
|
|
* @Route("/runs", name="dtc_queue_runs") |
150
|
|
|
*/ |
151
|
|
View Code Duplication |
public function runsAction() |
|
|
|
|
152
|
|
|
{ |
153
|
|
|
$this->validateRunManager(); |
154
|
|
|
$class1 = $this->container->getParameter('dtc_queue.class_run'); |
155
|
|
|
$class2 = $this->container->getParameter('dtc_queue.class_run_archive'); |
156
|
|
|
$label1 = 'Live Runs'; |
157
|
|
|
$label2 = 'Archived Runs'; |
158
|
|
|
|
159
|
|
|
$params = $this->getDualGridParams($class1, $class2, $label1, $label2); |
160
|
|
|
|
161
|
|
|
return $this->render('@DtcQueue/Queue/grid.html.twig', $params); |
162
|
|
|
} |
163
|
|
|
|
164
|
|
|
/** |
165
|
|
|
* List registered workers in the system. |
166
|
|
|
* |
167
|
|
|
* @Route("/workers", name="dtc_queue_workers") |
168
|
|
|
* @Template("@DtcQueue/Queue/workers.html.twig") |
169
|
|
|
*/ |
170
|
|
|
public function workersAction() |
171
|
|
|
{ |
172
|
|
|
$workerManager = $this->get('dtc_queue.worker_manager'); |
173
|
|
|
$workers = $workerManager->getWorkers(); |
174
|
|
|
|
175
|
|
|
$workerList = []; |
176
|
|
|
foreach ($workers as $workerName => $worker) { |
177
|
|
|
/* @var Worker $worker */ |
178
|
|
|
$workerList[$workerName] = get_class($worker); |
179
|
|
|
} |
180
|
|
|
$params = ['workers' => $workerList]; |
181
|
|
|
$this->addCssJs($params); |
182
|
|
|
|
183
|
|
|
return $params; |
184
|
|
|
} |
185
|
|
|
|
186
|
|
|
/** |
187
|
|
|
* Show a graph of job trends. |
188
|
|
|
* |
189
|
|
|
* @Route("/trends", name="dtc_queue_trends") |
190
|
|
|
* @Template("@DtcQueue/Queue/trends.html.twig") |
191
|
|
|
*/ |
192
|
|
|
public function trendsAction() |
193
|
|
|
{ |
194
|
|
|
$recordTimings = $this->container->getParameter('dtc_queue.record_timings'); |
195
|
|
|
$params = ['record_timings' => $recordTimings, 'states' => JobTiming::getStates()]; |
196
|
|
|
$this->addCssJs($params); |
197
|
|
|
|
198
|
|
|
return $params; |
199
|
|
|
} |
200
|
|
|
|
201
|
|
|
/** |
202
|
|
|
* @Route("/timings", name="dtc_queue_timings") |
203
|
|
|
* |
204
|
|
|
* @param Request $request |
|
|
|
|
205
|
|
|
*/ |
206
|
|
|
public function getTimingsAction() |
207
|
|
|
{ |
208
|
|
|
$request = $this->get('request_stack')->getMasterRequest(); |
209
|
|
|
$begin = $request->query->get('begin'); |
210
|
|
|
$end = $request->query->get('end'); |
211
|
|
|
$type = $request->query->get('type', 'HOUR'); |
212
|
|
|
$beginDate = \DateTime::createFromFormat('Y-m-d\TH:i:s.uO', $begin) ?: null; |
213
|
|
|
$endDate = \DateTime::createFromFormat('Y-m-d\TH:i:s.uO', $end) ?: new \DateTime(); |
214
|
|
|
|
215
|
|
|
$recordTimings = $this->container->getParameter('dtc_queue.record_timings'); |
216
|
|
|
$params = []; |
217
|
|
|
if ($recordTimings) { |
218
|
|
|
$params = $this->calculateTimings($type, $beginDate, $endDate); |
219
|
|
|
} |
220
|
|
|
|
221
|
|
|
return new JsonResponse($params); |
222
|
|
|
} |
223
|
|
|
|
224
|
|
|
protected function calculateTimings($type, $beginDate, $endDate) |
225
|
|
|
{ |
226
|
|
|
$params = []; |
227
|
|
|
$this->validateJobTimingManager(); |
228
|
|
|
|
229
|
|
|
/** @var BaseJobTimingManager $jobTimingManager */ |
230
|
|
|
$jobTimingManager = $this->get('dtc_queue.job_timing_manager'); |
231
|
|
|
if ($jobTimingManager instanceof JobTimingManager) { |
232
|
|
|
$timings = $this->getJobTimingsOdm($type, $endDate, $beginDate); |
233
|
|
|
} else { |
234
|
|
|
$timings = $this->getJobTimingsOrm($type, $endDate, $beginDate); |
235
|
|
|
} |
236
|
|
|
|
237
|
|
|
$timingStates = JobTiming::getStates(); |
238
|
|
|
$timingsDates = []; |
239
|
|
|
foreach (array_keys($timingStates) as $state) { |
240
|
|
|
if (!isset($timings[$state])) { |
241
|
|
|
continue; |
242
|
|
|
} |
243
|
|
|
$timingsData = $timings[$state]; |
244
|
|
|
$timingsDates = array_unique(array_merge(array_keys($timingsData), $timingsDates)); |
245
|
|
|
} |
246
|
|
|
|
247
|
|
|
$format = $this->getDateFormat($type); |
248
|
|
|
usort($timingsDates, function ($date1str, $date2str) use ($format) { |
249
|
|
|
$date1 = \DateTime::createFromFormat($format, $date1str); |
250
|
|
|
$date2 = \DateTime::createFromFormat($format, $date2str); |
251
|
|
|
if (!$date2) { |
252
|
|
|
return false; |
253
|
|
|
} |
254
|
|
|
if (!$date1) { |
255
|
|
|
return false; |
256
|
|
|
} |
257
|
|
|
|
258
|
|
|
return $date1 > $date2; |
259
|
|
|
}); |
260
|
|
|
|
261
|
|
|
$timezoneOffset = $this->container->getParameter('dtc_queue.record_timings_timezone_offset'); |
262
|
|
|
$timingsDatesAdjusted = []; |
263
|
|
|
foreach ($timingsDates as $dateStr) { |
264
|
|
|
$date = \DateTime::createFromFormat($format, $dateStr); |
265
|
|
|
if ($timezoneOffset !== 0) { |
266
|
|
|
$date->setTimestamp($date->getTimestamp() + ($timezoneOffset * 3600)); |
267
|
|
|
} |
268
|
|
|
$timingsDatesAdjusted[] = $date->format(DATE_RFC3339); |
269
|
|
|
} |
270
|
|
|
|
271
|
|
|
foreach (array_keys($timingStates) as $state) { |
272
|
|
|
if (!isset($timings[$state])) { |
273
|
|
|
continue; |
274
|
|
|
} |
275
|
|
|
|
276
|
|
|
$timingsData = $timings[$state]; |
277
|
|
|
foreach ($timingsDates as $date) { |
278
|
|
|
$params['timings_data_'.$state][] = isset($timingsData[$date]) ? $timingsData[$date] : 0; |
279
|
|
|
} |
280
|
|
|
} |
281
|
|
|
$params['timings_dates'] = $timingsDates; |
282
|
|
|
$params['timings_dates_rfc3339'] = $timingsDatesAdjusted; |
283
|
|
|
return $params; |
284
|
|
|
} |
285
|
|
|
|
286
|
|
|
protected function getJobTimingsOdm($type, \DateTime $end, \DateTime $begin = null) |
287
|
|
|
{ |
288
|
|
|
/** @var JobTimingManager $runManager */ |
289
|
|
|
$jobTimingManager = $this->get('dtc_queue.job_timing_manager'); |
290
|
|
|
$jobTimingClass = $jobTimingManager->getJobTimingClass(); |
291
|
|
|
|
292
|
|
|
/** @var DocumentManager $documentManager */ |
293
|
|
|
$documentManager = $jobTimingManager->getObjectManager(); |
294
|
|
|
|
295
|
|
|
$regexInfo = $this->getRegexDate($type); |
296
|
|
|
if (!$begin) { |
297
|
|
|
$begin = clone $end; |
298
|
|
|
$begin->sub($regexInfo['interval']); |
299
|
|
|
} |
300
|
|
|
|
301
|
|
|
// Run a map reduce function get worker and status break down |
302
|
|
|
$mapFunc = "function() { |
303
|
|
|
var dateStr = this.finishedAt.toISOString(); |
304
|
|
|
dateStr = dateStr.replace(/{$regexInfo['regex']}/,'{$regexInfo['replacement']}'); |
305
|
|
|
var dateBegin = new Date('{$begin->format('c')}'); |
306
|
|
|
var dateEnd = new Date('{$end->format('c')}'); |
307
|
|
|
if (this.finishedAt >= dateBegin && this.finishedAt <= dateEnd) { |
308
|
|
|
var result = {}; |
309
|
|
|
result[dateStr] = 1; |
310
|
|
|
emit(this.status, result); |
311
|
|
|
} |
312
|
|
|
}"; |
313
|
|
|
$reduceFunc = JobManager::REDUCE_FUNCTION; |
314
|
|
|
$builder = $documentManager->createQueryBuilder($jobTimingClass); |
315
|
|
|
$builder->map($mapFunc) |
316
|
|
|
->reduce($reduceFunc); |
317
|
|
|
$query = $builder->getQuery(); |
318
|
|
|
$results = $query->execute(); |
319
|
|
|
$resultHash = []; |
320
|
|
|
foreach ($results as $info) { |
321
|
|
|
$resultHash[$info['_id']] = $info['value']; |
322
|
|
|
} |
323
|
|
|
|
324
|
|
|
return $resultHash; |
325
|
|
|
} |
326
|
|
|
|
327
|
|
|
protected function getDateFormat($type) { |
328
|
|
|
switch($type) { |
329
|
|
|
case 'YEAR': |
330
|
|
|
return 'Y'; |
331
|
|
|
case 'MONTH': |
332
|
|
|
return 'Y-m'; |
333
|
|
|
case 'DAY': |
334
|
|
|
return 'Y-m-d'; |
335
|
|
|
case 'HOUR': |
336
|
|
|
return 'Y-m-d H'; |
337
|
|
|
default: |
338
|
|
|
throw new \InvalidArgumentException("Invalid date format type '$type''"); |
339
|
|
|
} |
340
|
|
|
} |
341
|
|
|
|
342
|
|
|
protected function getRegexDate($type) |
343
|
|
|
{ |
344
|
|
|
switch ($type) { |
345
|
|
View Code Duplication |
case 'YEAR': |
|
|
|
|
346
|
|
|
return ['replace' => ['regex' => '(\d+)\-(\d+)\-(\d+)T(\d+):(\d+):(\d+).+$', 'replacement' => '$1'], |
347
|
|
|
'interval' => new \DateInterval('P10Y'), ]; |
348
|
|
View Code Duplication |
case 'MONTH': |
|
|
|
|
349
|
|
|
return ['replace' => ['regex' => '(\d+)\-(\d+)\-(\d+)T(\d+):(\d+):(\d+).+$', 'replacement' => '$1-$2'], |
350
|
|
|
'interval' => new \DateInterval('P12M'), ]; |
351
|
|
View Code Duplication |
case 'DAY': |
|
|
|
|
352
|
|
|
return ['replace' => ['regex' => '(\d+)\-(\d+)\-(\d+)T(\d+):(\d+):(\d+).+$', 'replacement' => '$1-$2-$3'], |
353
|
|
|
'interval' => new \DateInterval('P31D'), ]; |
354
|
|
View Code Duplication |
case 'HOUR': |
|
|
|
|
355
|
|
|
return ['replace' => ['regex' => '(\d+)\-(\d+)\-(\d+)T(\d+):(\d+):(\d+).+$', 'replacement' => '$1-$2-$3 $4'], |
356
|
|
|
'interval' => new \DateInterval('PT24H'), ]; |
357
|
|
View Code Duplication |
case 'MINUTE': |
|
|
|
|
358
|
|
|
return ['replace' => ['regex' => '(\d+)\-(\d+)\-(\d+)T(\d+):(\d+):(\d+).+$', 'replacement' => '$1-$2-$3 $4:$5'], |
359
|
|
|
'interval' => new \DateInterval('PT3600S'), ]; |
360
|
|
|
} |
361
|
|
|
throw new \InvalidArgumentException("Invalid type $type"); |
362
|
|
|
} |
363
|
|
|
|
364
|
|
|
protected function getOrmGroupBy($type) |
365
|
|
|
{ |
366
|
|
|
switch ($type) { |
367
|
|
|
case 'YEAR': |
368
|
|
|
return ['groupby' => 'YEAR(j.finishedAt)', |
369
|
|
|
'interval' => new \DateInterval('P10Y'), ]; |
370
|
|
|
case 'MONTH': |
371
|
|
|
return ['groupby' => 'CONCAT(YEAR(j.finishedAt),\'-\',MONTH(j.finishedAt))', |
372
|
|
|
'interval' => new \DateInterval('P12M'), ]; |
373
|
|
|
case 'DAY': |
374
|
|
|
return ['groupby' => 'CONCAT(YEAR(j.finishedAt),\'-\',MONTH(j.finishedAt),\'-\',DAY(j.finishedAt))', |
375
|
|
|
'interval' => new \DateInterval('P31D'), ]; |
376
|
|
|
case 'HOUR': |
377
|
|
|
return ['groupby' => 'CONCAT(YEAR(j.finishedAt),\'-\',MONTH(j.finishedAt),\'-\',DAY(j.finishedAt),\' \',HOUR(j.finishedAt))', |
378
|
|
|
'interval' => new \DateInterval('PT24H'), ]; |
379
|
|
|
case 'MINUTE': |
380
|
|
|
return ['groupby' => 'CONCAT(YEAR(j.finishedAt),\'-\',MONTH(j.finishedAt),\'-\',DAY(j.finishedAt),\' \',HOUR(j.finishedAt),\':\',MINUTE(j.finishedAt))', |
381
|
|
|
'interval' => new \DateInterval('PT3600S'), ]; |
382
|
|
|
} |
383
|
|
|
throw new \InvalidArgumentException("Invalid type $type"); |
384
|
|
|
} |
385
|
|
|
|
386
|
|
|
protected function getJobTimingsOrm($type, \DateTime $end, \DateTime $begin = null) |
387
|
|
|
{ |
388
|
|
|
/** @var JobTimingManager $jobTimingManager */ |
389
|
|
|
$jobTimingManager = $this->get('dtc_queue.job_timing_manager'); |
390
|
|
|
$jobTimingClass = $jobTimingManager->getJobTimingClass(); |
391
|
|
|
/** @var EntityManager $entityManager */ |
392
|
|
|
$entityManager = $jobTimingManager->getObjectManager(); |
393
|
|
|
|
394
|
|
|
$groupByInfo = $this->getOrmGroupBy($type); |
395
|
|
|
|
396
|
|
|
if (!$begin) { |
397
|
|
|
$begin = clone $end; |
398
|
|
|
$begin->sub($groupByInfo['interval']); |
399
|
|
|
} |
400
|
|
|
|
401
|
|
|
$queryBuilder = $entityManager->createQueryBuilder()->select("j.status as status, count(j.finishedAt) as thecount, {$groupByInfo['groupby']} as thedate") |
402
|
|
|
->from($jobTimingClass, 'j') |
403
|
|
|
->where('j.finishedAt <= :end') |
404
|
|
|
->andWhere('j.finishedAt >= :begin') |
405
|
|
|
->setParameter(':end', $end) |
406
|
|
|
->setParameter(':begin', $begin) |
407
|
|
|
->groupBy('status') |
408
|
|
|
->addGroupBy('thedate'); |
409
|
|
|
|
410
|
|
|
$result = $queryBuilder |
411
|
|
|
->getQuery()->getArrayResult(); |
412
|
|
|
|
413
|
|
|
$resultHash = []; |
414
|
|
|
foreach ($result as $row) { |
415
|
|
|
$resultHash[$row['status']][$row['thedate']] = intval($row['thecount']); |
416
|
|
|
} |
417
|
|
|
|
418
|
|
|
return $resultHash; |
419
|
|
|
} |
420
|
|
|
} |
421
|
|
|
|
Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation.
You can also find more detailed suggestions in the “Code” section of your repository.