1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
/* this file is part of pipelines */ |
4
|
|
|
|
5
|
|
|
namespace Ktomk\Pipelines; |
6
|
|
|
|
7
|
|
|
use Ktomk\Pipelines\Cli\Docker; |
8
|
|
|
use Ktomk\Pipelines\Cli\Exec; |
9
|
|
|
use Ktomk\Pipelines\Cli\Streams; |
10
|
|
|
use Ktomk\Pipelines\File\Image; |
11
|
|
|
use Ktomk\Pipelines\Runner\ArtifactSource; |
12
|
|
|
use Ktomk\Pipelines\Runner\Directories; |
13
|
|
|
use Ktomk\Pipelines\Runner\DockerLogin; |
14
|
|
|
use Ktomk\Pipelines\Runner\Env; |
15
|
|
|
|
16
|
|
|
/** |
17
|
|
|
* Pipeline runner with docker under the hood |
18
|
|
|
*/ |
19
|
|
|
class Runner |
20
|
|
|
{ |
21
|
|
|
const FLAGS = 19; |
22
|
|
|
const FLAG_DOCKER_REMOVE = 1; |
23
|
|
|
const FLAG_DOCKER_KILL = 2; |
24
|
|
|
const FLAG_DEPLOY_COPY = 4; # copy working dir into container |
25
|
|
|
const FLAG_KEEP_ON_ERROR = 8; |
26
|
|
|
const FLAG_SOCKET = 16; |
27
|
|
|
|
28
|
|
|
const STATUS_NO_STEPS = 1; |
29
|
|
|
const STATUS_RECURSION_DETECTED = 127; |
30
|
|
|
|
31
|
|
|
/** |
32
|
|
|
* @var string |
33
|
|
|
*/ |
34
|
|
|
private $prefix; |
35
|
|
|
|
36
|
|
|
/** |
37
|
|
|
* @var Directories |
38
|
|
|
*/ |
39
|
|
|
private $directories; |
40
|
|
|
|
41
|
|
|
/** |
42
|
|
|
* @var Exec |
43
|
|
|
*/ |
44
|
|
|
private $exec; |
45
|
|
|
|
46
|
|
|
/** |
47
|
|
|
* @var int |
48
|
|
|
*/ |
49
|
|
|
private $flags; |
50
|
|
|
|
51
|
|
|
/** |
52
|
|
|
* @var Env |
53
|
|
|
*/ |
54
|
|
|
private $env; |
55
|
|
|
/** |
56
|
|
|
* @var Streams |
57
|
|
|
*/ |
58
|
|
|
private $streams; |
59
|
|
|
|
60
|
|
|
/** |
61
|
|
|
* DockerSession constructor. |
62
|
|
|
* |
63
|
|
|
* @param string $prefix |
64
|
|
|
* @param Directories $directories source repository root directory based directories object |
65
|
|
|
* @param Exec $exec |
66
|
|
|
* @param int $flags [optional] |
67
|
|
|
* @param Env $env [optional] |
68
|
|
|
* @param Streams $streams [optional] |
69
|
|
|
*/ |
70
|
13 |
|
public function __construct( |
71
|
|
|
$prefix, |
72
|
|
|
Directories $directories, |
73
|
|
|
Exec $exec, |
74
|
|
|
$flags = null, |
75
|
|
|
Env $env = null, |
76
|
|
|
Streams $streams = null |
77
|
|
|
) |
78
|
|
|
{ |
79
|
13 |
|
$this->prefix = $prefix; |
80
|
13 |
|
$this->directories = $directories; |
81
|
13 |
|
$this->exec = $exec; |
82
|
13 |
|
$this->flags = null === $flags ? self::FLAGS : $flags; |
83
|
13 |
|
$this->env = null === $env ? Env::create() : $env; |
84
|
13 |
|
$this->streams = null === $streams ? Streams::create() : $streams; |
85
|
13 |
|
} |
86
|
|
|
|
87
|
|
|
/** |
88
|
|
|
* @param Pipeline $pipeline |
89
|
|
|
* @throws \RuntimeException |
90
|
|
|
* @return int status (as in exit status, 0 OK, !0 NOK) |
91
|
|
|
*/ |
92
|
13 |
|
public function run(Pipeline $pipeline) |
93
|
|
|
{ |
94
|
13 |
|
$hasId = $this->env->setPipelinesId($pipeline->getId()); # TODO give Env an addPipeline() method (compare addReference) |
95
|
13 |
|
if ($hasId) { |
96
|
1 |
|
$this->streams->err(sprintf( |
97
|
1 |
|
"pipelines: won't start pipeline '%s'; pipeline inside pipelines recursion detected\n", |
98
|
1 |
|
$pipeline->getId() |
99
|
|
|
)); |
100
|
|
|
|
101
|
1 |
|
return self::STATUS_RECURSION_DETECTED; |
102
|
|
|
} |
103
|
|
|
|
104
|
12 |
|
foreach ($pipeline->getSteps() as $step) { |
105
|
11 |
|
$status = $this->runStep($step); |
106
|
11 |
|
if (0 !== $status) { |
107
|
11 |
|
return $status; |
108
|
|
|
} |
109
|
|
|
} |
110
|
|
|
|
111
|
8 |
|
if (!isset($status)) { |
112
|
1 |
|
$this->streams->err("pipelines: pipeline with no step to execute\n"); |
113
|
|
|
|
114
|
1 |
|
return self::STATUS_NO_STEPS; |
115
|
|
|
} |
116
|
|
|
|
117
|
7 |
|
return $status; |
118
|
|
|
} |
119
|
|
|
|
120
|
|
|
/** |
121
|
|
|
* @param Step $step |
122
|
|
|
* @throws \RuntimeException |
123
|
|
|
* @return int exit status |
124
|
|
|
*/ |
125
|
11 |
|
public function runStep(Step $step) |
126
|
|
|
{ |
127
|
11 |
|
$dir = $this->directories->getProject(); |
128
|
11 |
|
$env = $this->env; |
129
|
11 |
|
$exec = $this->exec; |
130
|
11 |
|
$streams = $this->streams; |
131
|
|
|
|
132
|
11 |
|
$docker = new Docker($exec); |
133
|
|
|
|
134
|
11 |
|
$name = $this->generateContainerName($step); |
135
|
11 |
|
$this->zapContinerByName($name); |
136
|
11 |
|
$image = $step->getImage(); |
137
|
11 |
|
$env->setContainerName($name); |
138
|
|
|
|
139
|
|
|
# launch container |
140
|
11 |
|
$streams->out(sprintf( |
141
|
11 |
|
"\x1D+++ step #%d\n\n name...........: %s\n effective-image: %s\n container......: %s\n", |
142
|
11 |
|
$step->getIndex() + 1, |
143
|
11 |
|
$step->getName() ? '"' . $step->getName() . '"' : '(unnamed)', |
144
|
11 |
|
$image->getName(), |
145
|
11 |
|
$name |
146
|
|
|
)); |
147
|
|
|
|
148
|
|
|
# process docker login if image demands so, but continue on failure |
149
|
11 |
|
$this->imageLogin($image); |
150
|
|
|
|
151
|
11 |
|
$copy = (bool)($this->flags & self::FLAG_DEPLOY_COPY); |
152
|
|
|
|
153
|
|
|
// enable docker client inside docker by mounting docker socket |
154
|
|
|
// FIXME give controlling options, this is serious /!\ |
155
|
11 |
|
$socket = (bool)($this->flags & self::FLAG_SOCKET); |
156
|
11 |
|
$mountDockerSock = array(); |
157
|
11 |
|
if ($socket && file_exists('/var/run/docker.sock')) { |
158
|
|
|
$mountDockerSock = array( |
159
|
|
|
'-v', '/var/run/docker.sock:/var/run/docker.sock', |
160
|
|
|
); |
161
|
|
|
} |
162
|
|
|
|
163
|
11 |
|
$parentName = $env->getValue('PIPELINES_PARENT_CONTAINER_NAME'); |
164
|
11 |
|
$checkMount = $mountDockerSock && null !== $parentName; |
165
|
11 |
|
$deviceDir = $checkMount ? $docker->hostDevice($parentName, $dir) : $dir; |
166
|
|
|
|
167
|
11 |
|
$mountWorkingDirectory = $copy |
168
|
7 |
|
? array() |
169
|
11 |
|
: array('--volume', "${deviceDir}:/app"); // FIXME(tk): hard encoded /app |
170
|
|
|
|
171
|
|
|
$status = $exec->capture('docker', array( |
172
|
|
|
'run', '-i', '--name', $name, |
173
|
|
|
$env->getArgs('-e'), |
174
|
|
|
$mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app', |
175
|
|
|
$mountDockerSock, |
176
|
|
|
'--workdir', '/app', '--detach', $image->getName() |
177
|
|
|
), $out, $err); |
178
|
|
|
if (0 !== $status) { |
179
|
|
|
$streams->out(" container-id...: *failure*\n\n"); |
180
|
|
|
$streams->err("pipelines: setting up the container failed\n"); |
181
|
|
|
$streams->err("${err}\n"); |
182
|
|
|
$streams->out("${out}\n"); |
183
|
1 |
|
$streams->out(sprintf("exit status: %d\n", $status)); |
184
|
|
|
|
185
|
1 |
|
return $status; |
186
|
|
|
} |
187
|
10 |
|
$id = rtrim($out) ?: "*dry-run*"; # side-effect: internal exploit of no output with true exit status |
188
|
10 |
|
$streams->out(sprintf(" container-id...: %s\n\n", substr($id, 0, 12))); |
189
|
|
|
|
190
|
|
|
# TODO: different deployments, mount (default), mount-ro, copy |
191
|
10 |
|
if (null !== $result = $this->deployCopy($copy, $id, $this->directories->getProject())) { |
192
|
2 |
|
return $result; |
193
|
|
|
} |
194
|
|
|
|
195
|
8 |
|
$status = $this->runStepScript($step, $streams, $exec, $name); |
196
|
|
|
|
197
|
8 |
|
$this->captureStepArtifacts($step, $copy && 0 === $status, $id, $dir); |
198
|
|
|
|
199
|
8 |
|
$this->shutdownStepContainer($status, $id, $exec, $name); |
200
|
|
|
|
201
|
8 |
|
return $status; |
202
|
|
|
} |
203
|
|
|
|
204
|
|
|
/** |
205
|
|
|
* @param string $name |
206
|
|
|
*/ |
207
|
|
|
private function zapContinerByName($name) |
208
|
|
|
{ |
209
|
11 |
|
$ids = null; |
210
|
|
|
|
211
|
11 |
|
$status = $this->exec->capture( |
212
|
11 |
|
'docker', |
213
|
|
|
array( |
214
|
11 |
|
'ps', '-qa', '--filter', |
215
|
11 |
|
"name=^/${name}$" |
216
|
|
|
), |
217
|
11 |
|
$result |
218
|
|
|
); |
219
|
|
|
|
220
|
11 |
|
$status || $ids = Lib::lines($result); |
221
|
|
|
|
222
|
11 |
|
if ($status || !(is_array($ids) && 1 === count($ids))) { |
223
|
10 |
|
return; |
224
|
|
|
} |
225
|
|
|
|
226
|
1 |
|
$this->exec->capture('docker', Lib::merge('kill', $ids)); |
227
|
1 |
|
$this->exec->capture('docker', Lib::merge('rm', $ids)); |
228
|
1 |
|
} |
229
|
|
|
|
230
|
|
|
/** |
231
|
|
|
* @param Image $image |
232
|
|
|
* @throws \RuntimeException |
233
|
|
|
* @throws \InvalidArgumentException |
234
|
|
|
*/ |
235
|
|
|
private function imageLogin(Image $image) |
236
|
|
|
{ |
237
|
11 |
|
$login = new DockerLogin($this->exec, $this->env->getResolver()); |
238
|
11 |
|
$login->byImage($image); |
239
|
11 |
|
} |
240
|
|
|
|
241
|
|
|
/** |
242
|
|
|
* @param bool $copy |
243
|
|
|
* @param string $id container id |
244
|
|
|
* @param string $dir directory to copy contents into container |
245
|
|
|
* @throws \RuntimeException |
246
|
|
|
* @return null|int null if all clear, integer for exit status |
247
|
|
|
*/ |
248
|
|
|
private function deployCopy($copy, $id, $dir) |
249
|
|
|
{ |
250
|
10 |
|
if (!$copy) { |
251
|
3 |
|
return null; |
252
|
|
|
} |
253
|
|
|
|
254
|
7 |
|
$streams = $this->streams; |
255
|
7 |
|
$exec = $this->exec; |
256
|
|
|
|
257
|
7 |
|
$streams->out("\x1D+++ copying files into container...\n"); |
258
|
|
|
|
259
|
7 |
|
$tmpDir = sys_get_temp_dir() . '/pipelines/cp'; |
260
|
7 |
|
Lib::fsMkdir($tmpDir); |
261
|
7 |
|
Lib::fsSymlink($dir, $tmpDir . '/app'); |
262
|
7 |
|
$cd = Lib::cmd('cd', array($tmpDir . '/.')); |
263
|
7 |
|
$tar = Lib::cmd( |
264
|
7 |
|
'tar', |
265
|
|
|
array( |
266
|
7 |
|
'c', '-h', '-f', '-', '--no-recursion', 'app') |
267
|
|
|
); |
268
|
7 |
|
$dockerCp = Lib::cmd( |
269
|
7 |
|
'docker ', |
270
|
|
|
array( |
271
|
7 |
|
'cp', '-', $id . ':/.') |
272
|
|
|
); |
273
|
7 |
|
$status = $exec->pass("${cd} && echo 'app' | ${tar} | ${dockerCp}", array()); |
274
|
7 |
|
Lib::fsUnlink($tmpDir . '/app'); |
275
|
7 |
|
if (0 !== $status) { |
276
|
1 |
|
$streams->err('pipelines: deploy copy failure\n'); |
277
|
|
|
|
278
|
1 |
|
return $status; |
279
|
|
|
} |
280
|
|
|
|
281
|
6 |
|
$cd = Lib::cmd('cd', array($dir . '/.')); |
282
|
6 |
|
$tar = Lib::cmd( |
283
|
6 |
|
'tar', |
284
|
|
|
array( |
285
|
6 |
|
'c', '-f', '-', '.') |
286
|
|
|
); |
287
|
6 |
|
$dockerCp = Lib::cmd( |
288
|
6 |
|
'docker ', |
289
|
|
|
array( |
290
|
6 |
|
'cp', '-', $id . ':/app') |
291
|
|
|
); |
292
|
6 |
|
$status = $exec->pass("${cd} && ${tar} | ${dockerCp}", array()); |
293
|
6 |
|
if (0 !== $status) { |
294
|
1 |
|
$streams->err('pipelines: deploy copy failure\n'); |
295
|
|
|
|
296
|
1 |
|
return $status; |
297
|
|
|
} |
298
|
|
|
|
299
|
5 |
|
$streams(''); |
300
|
|
|
|
301
|
5 |
|
return null; |
302
|
|
|
} |
303
|
|
|
|
304
|
|
|
/** |
305
|
|
|
* @param Step $step |
306
|
|
|
* @param Streams $streams |
307
|
|
|
* @param Exec $exec |
308
|
|
|
* @param string $name container name |
309
|
|
|
* @return null|int should never be null, status, non-zero if a command failed |
310
|
|
|
*/ |
311
|
|
|
private function runStepScript(Step $step, Streams $streams, Exec $exec, $name) |
312
|
|
|
{ |
313
|
8 |
|
$script = $step->getScript(); |
314
|
8 |
|
$status = null; |
315
|
|
|
|
316
|
8 |
|
foreach ($script as $line => $command) { |
317
|
8 |
|
$streams->out(sprintf("\x1D+ %s\n", $command)); |
318
|
8 |
|
$status = $exec->pass('docker', array( |
319
|
8 |
|
'exec', '-i', $name, '/bin/sh', '-c', $command, |
320
|
|
|
)); |
321
|
8 |
|
$streams->out(sprintf("\n")); |
322
|
8 |
|
if (0 !== $status) { |
323
|
1 |
|
$this->streams->err(sprintf("script non-zero exit status: %d\n", $status)); |
324
|
|
|
|
325
|
8 |
|
break; |
326
|
|
|
} |
327
|
|
|
} |
328
|
|
|
|
329
|
8 |
|
return $status; |
330
|
|
|
} |
331
|
|
|
|
332
|
|
|
/** |
333
|
|
|
* @param Step $step |
334
|
|
|
* @param bool $copy |
335
|
|
|
* @param string $id container id |
336
|
|
|
* @param string $dir to put artifacts in (project directory) |
337
|
|
|
* @throws \RuntimeException |
338
|
|
|
*/ |
339
|
|
|
private function captureStepArtifacts(Step $step, $copy, $id, $dir) |
340
|
|
|
{ |
341
|
|
|
# capturing artifacts is only supported for deploy copy |
342
|
8 |
|
if (!$copy) { |
343
|
3 |
|
return; |
344
|
|
|
} |
345
|
|
|
|
346
|
5 |
|
$artifacts = $step->getArtifacts(); |
347
|
|
|
|
348
|
5 |
|
if (null === $artifacts) { |
349
|
2 |
|
return; |
350
|
|
|
} |
351
|
|
|
|
352
|
3 |
|
$exec = $this->exec; |
353
|
3 |
|
$streams = $this->streams; |
354
|
|
|
|
355
|
3 |
|
$streams->out("\x1D+++ copying artifacts from container...\n"); |
356
|
|
|
|
357
|
3 |
|
$source = new ArtifactSource($exec, $id, $dir); |
358
|
|
|
|
359
|
3 |
|
$patterns = $artifacts->getPatterns(); |
360
|
3 |
|
foreach ($patterns as $pattern) { |
361
|
3 |
|
$this->captureArtifactPattern($source, $pattern, $dir); |
362
|
|
|
} |
363
|
|
|
|
364
|
3 |
|
$streams(""); |
365
|
3 |
|
} |
366
|
|
|
|
367
|
|
|
/** |
368
|
|
|
* @see Runner::captureStepArtifacts() |
369
|
|
|
* |
370
|
|
|
* @param ArtifactSource $source |
371
|
|
|
* @param string $pattern |
372
|
|
|
* @param string $dir |
373
|
|
|
* @throws \RuntimeException |
374
|
|
|
*/ |
375
|
|
|
private function captureArtifactPattern(ArtifactSource $source, $pattern, $dir) |
376
|
|
|
{ |
377
|
3 |
|
$exec = $this->exec; |
378
|
3 |
|
$streams = $this->streams; |
379
|
|
|
|
380
|
3 |
|
$id = $source->getId(); |
381
|
3 |
|
$paths = $source->findByPattern($pattern); |
382
|
3 |
|
if (empty($paths)) { |
383
|
1 |
|
return; |
384
|
|
|
} |
385
|
|
|
|
386
|
2 |
|
$chunkSize = 1792; |
387
|
2 |
|
$chunks = array_chunk($paths, $chunkSize, true); |
388
|
|
|
|
389
|
2 |
|
foreach ($chunks as $paths) { |
390
|
2 |
|
$docker = Lib::cmd('docker', array('exec', '-w', '/app', $id)); |
391
|
2 |
|
$tar = Lib::cmd('tar', array('c', '-f', '-', $paths)); |
392
|
2 |
|
$unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir)); |
393
|
|
|
|
394
|
2 |
|
$status = $exec->pass($docker . ' ' . $tar . ' | ' . $unTar, array()); |
395
|
|
|
|
396
|
2 |
|
if (0 !== $status) { |
397
|
1 |
|
$streams->err( |
398
|
2 |
|
sprintf("pipelines: Artifact failure: '%s' (%d, %d paths)\n", $pattern, $status, count($paths)) |
399
|
|
|
); |
400
|
|
|
} |
401
|
|
|
} |
402
|
2 |
|
} |
403
|
|
|
|
404
|
|
|
/** |
405
|
|
|
* @param int $status |
406
|
|
|
* @param string $id container id |
407
|
|
|
* @param Exec $exec |
408
|
|
|
* @param string $name container name |
409
|
|
|
* @throws \RuntimeException |
410
|
|
|
*/ |
411
|
|
|
private function shutdownStepContainer($status, $id, Exec $exec, $name) |
412
|
|
|
{ |
413
|
8 |
|
$flags = $this->flags; |
414
|
|
|
|
415
|
|
|
# keep container on error |
416
|
8 |
|
if (0 !== $status && $flags & self::FLAG_KEEP_ON_ERROR) { |
417
|
1 |
|
$this->streams->err(sprintf( |
418
|
1 |
|
"keeping container id %s\n", |
419
|
1 |
|
substr($id, 0, 12) |
420
|
|
|
)); |
421
|
|
|
|
422
|
1 |
|
return; |
423
|
|
|
} |
424
|
|
|
|
425
|
|
|
# keep or remove container |
426
|
7 |
|
if ($flags & self::FLAG_DOCKER_KILL) { |
427
|
2 |
|
$exec->capture('docker', array('kill', $name)); |
428
|
|
|
} |
429
|
|
|
|
430
|
7 |
|
if ($flags & self::FLAG_DOCKER_REMOVE) { |
431
|
2 |
|
$exec->capture('docker', array('rm', $name)); |
432
|
|
|
} |
433
|
7 |
|
} |
434
|
|
|
|
435
|
|
|
/** |
436
|
|
|
* @param Step $step |
437
|
|
|
* @return string |
438
|
|
|
*/ |
439
|
|
|
private function generateContainerName(Step $step) |
440
|
|
|
{ |
441
|
11 |
|
$project = $this->directories->getName(); |
442
|
11 |
|
$idContainerSlug = preg_replace('([^a-zA-Z0-9_.-]+)', '', $step->getPipeline()->getId()); |
443
|
11 |
|
if ('' === $idContainerSlug) { |
444
|
11 |
|
$idContainerSlug = 'null'; |
445
|
|
|
} |
446
|
11 |
|
$nameSlug = preg_replace(array('( )', '([^a-zA-Z0-9_.-]+)'), array('-', ''), $step->getName()); |
447
|
11 |
|
if ('' === $nameSlug) { |
448
|
11 |
|
$nameSlug = 'no-name'; |
449
|
|
|
} |
450
|
|
|
|
451
|
11 |
|
return $this->prefix . '-' . implode( |
452
|
11 |
|
'.', |
453
|
11 |
|
array_reverse( |
454
|
|
|
array( |
455
|
11 |
|
$project, |
456
|
11 |
|
$idContainerSlug, |
457
|
11 |
|
$nameSlug, |
458
|
11 |
|
$step->getIndex() + 1, |
459
|
|
|
) |
460
|
|
|
) |
461
|
|
|
); |
462
|
|
|
} |
463
|
|
|
} |
464
|
|
|
|