Test Failed
Push — test ( e7f3db...44001a )
by Tom
02:25
created

StepRunner::deployDockerClient()   A

Complexity

Conditions 2
Paths 2

Size

Total Lines 11
Code Lines 5

Duplication

Lines 0
Ratio 0 %

Importance

Changes 1
Bugs 0 Features 0
Metric Value
cc 2
eloc 5
c 1
b 0
f 0
nc 2
nop 2
dl 0
loc 11
rs 10
1
<?php
2
3
/* this file is part of pipelines */
4
5
namespace Ktomk\Pipelines\Runner;
6
7
use Ktomk\Pipelines\Cli\Docker;
8
use Ktomk\Pipelines\Cli\Exec;
9
use Ktomk\Pipelines\Cli\Streams;
10
use Ktomk\Pipelines\DestructibleString;
11
use Ktomk\Pipelines\File\Image;
12
use Ktomk\Pipelines\File\Step;
13
use Ktomk\Pipelines\Lib;
14
use Ktomk\Pipelines\LibFs;
15
use Ktomk\Pipelines\LibTmp;
16
use Ktomk\Pipelines\Runner\Docker\Binary\Repository;
17
18
/**
19
 * Runner for a single step of a pipeline
20
 */
21
class StepRunner
22
{
23
    /**
24
     * @var string
25
     */
26
    private $prefix;
27
28
    /**
29
     * @var Directories
30
     */
31
    private $directories;
32
33
    /**
34
     * @var Exec
35
     */
36
    private $exec;
37
38
    /**
39
     * @var Flags
40
     */
41
    private $flags;
42
43
    /**
44
     * @var Env
45
     */
46
    private $env;
47
48
    /**
49
     * @var Streams
50
     */
51
    private $streams;
52
53
    /**
54
     * list of temporary directory destructible markers
55
     *
56
     * @var array
57
     */
58
    private $temporaryDirectories = array();
59
60
    /**
61
     * DockerSession constructor.
62
     *
63
     * @param string $prefix
64
     * @param Directories $directories source repository root directory based directories object
65
     * @param Exec $exec
66
     * @param Flags $flags [optional]
67
     * @param Env $env [optional]
68
     * @param Streams $streams [optional]
69
     */
70
    public function __construct(
71
        $prefix,
72
        Directories $directories,
73
        Exec $exec,
74
        Flags $flags,
75
        Env $env,
76
        Streams $streams
77
    )
78
    {
79
        $this->prefix = $prefix;
80
        $this->directories = $directories;
81
        $this->exec = $exec;
82
        $this->flags = $flags;
83
        $this->env = $env;
84
        $this->streams = $streams;
85
    }
86
87
    /**
88
     * method to wrap new to have a test-point
89
     *
90
     * @return Repository
91
     */
92
    public function getDockerBinaryRepository()
93
    {
94
        return new Repository($this->exec, $this->directories);
95
    }
96
97
    /**
98
     * @param Step $step
99
     * @return null|int exist status of step script or null if the run operation failed
100
     */
101
    public function runStep(Step $step)
102
    {
103
        $dir = $this->directories->getProjectDirectory();
104
        $env = $this->env;
105
        $exec = $this->exec;
106
        $streams = $this->streams;
107
        $reuseContainer = $this->flags->reuseContainer();
108
        $deployCopy = $this->flags->deployCopy();
109
110
        $name = $this->generateContainerName($step);
111
112
        if (false === $reuseContainer) {
113
            $this->zapContainerByName($name);
114
        }
115
        $image = $step->getImage();
116
        $env->setContainerName($name);
117
118
        # launch container
119
        $streams->out(sprintf(
120
            "\x1D+++ step #%d\n\n    name...........: %s\n    effective-image: %s\n    container......: %s\n",
121
            $step->getIndex() + 1,
122
            $step->getName() ? '"' . $step->getName() . '"' : '(unnamed)',
123
            $image->getName(),
124
            $name
125
        ));
126
127
        $id = null;
128
        if ($reuseContainer) {
129
            $id = $this->dockerGetContainerIdByName($name);
130
        }
131
132
        if (null === $id) {
133
            list($id, $status) = $this->runNewContainer($name, $dir, $deployCopy, $step);
134
            if (null === $id) {
135
                return $status;
136
            }
137
        }
138
139
        $streams->out(sprintf("    container-id...: %s\n\n", substr($id, 0, 12)));
140
141
        # TODO: different deployments, mount (default), mount-ro, copy
142
        if (null !== $result = $this->deployCopy($deployCopy, $id, $dir)) {
143
            return $result;
144
        }
145
146
        $this->deployDockerClient($step, $id);
147
148
        $status = $this->runStepScript($step, $streams, $exec, $name);
149
150
        $this->captureStepArtifacts($step, $deployCopy && 0 === $status, $id, $dir);
151
152
        $this->shutdownStepContainer($status, $id, $exec, $name);
153
154
        return $status;
155
    }
156
157
    /**
158
     * @param Step $step
159
     * @param bool $copy
160
     * @param string $id container id
161
     * @param string $dir to put artifacts in (project directory)
162
     * @throws \RuntimeException
163
     */
164
    private function captureStepArtifacts(Step $step, $copy, $id, $dir)
165
    {
166
        # capturing artifacts is only supported for deploy copy
167
        if (!$copy) {
168
            return;
169
        }
170
171
        $artifacts = $step->getArtifacts();
172
173
        if (null === $artifacts) {
174
            return;
175
        }
176
177
        $exec = $this->exec;
178
        $streams = $this->streams;
179
180
        $streams->out("\x1D+++ copying artifacts from container...\n");
181
182
        $source = new ArtifactSource($exec, $id, $dir);
183
184
        $patterns = $artifacts->getPatterns();
185
        foreach ($patterns as $pattern) {
186
            $this->captureArtifactPattern($source, $pattern, $dir);
187
        }
188
189
        $streams('');
190
    }
191
192
    /**
193
     * @see Runner::captureStepArtifacts()
194
     *
195
     * @param ArtifactSource $source
196
     * @param string $pattern
197
     * @param string $dir
198
     * @throws \RuntimeException
199
     */
200
    private function captureArtifactPattern(ArtifactSource $source, $pattern, $dir)
201
    {
202
        $exec = $this->exec;
203
        $streams = $this->streams;
204
205
        $id = $source->getId();
206
        $paths = $source->findByPattern($pattern);
207
        if (empty($paths)) {
208
            return;
209
        }
210
211
        $chunks = Lib::arrayChunkByStringLength($paths, 131072, 4);
212
213
        foreach ($chunks as $paths) {
214
            $docker = Lib::cmd('docker', array('exec', '-w', '/app', $id));
215
            $tar = Lib::cmd('tar', array('c', '-f', '-', $paths));
216
            $unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir));
217
218
            $command = $docker . ' ' . $tar . ' | ' . $unTar;
219
            $status = $exec->pass($command, array());
220
221
            if (0 !== $status) {
222
                $streams->err(sprintf(
223
                    "pipelines: Artifact failure: '%s' (%d, %d paths, %d bytes)\n",
224
                    $pattern,
225
                    $status,
226
                    count($paths),
227
                    strlen($command)
228
                ));
229
            }
230
        }
231
    }
232
233
    /**
234
     * @param bool $copy
235
     * @param string $id container id
236
     * @param string $dir directory to copy contents into container
237
     * @throws \RuntimeException
238
     * @return null|int null if all clear, integer for exit status
239
     */
240
    private function deployCopy($copy, $id, $dir)
241
    {
242
        if (!$copy) {
243
            return null;
244
        }
245
246
        $streams = $this->streams;
247
        $exec = $this->exec;
248
249
        $streams->out("\x1D+++ copying files into container...\n");
250
251
        $tmpDir = LibTmp::tmpDir('pipelines-cp.');
252
        $this->temporaryDirectories[] = DestructibleString::rmDir($tmpDir);
253
        LibFs::symlink($dir, $tmpDir . '/app');
254
        $cd = Lib::cmd('cd', array($tmpDir . '/.'));
255
        $tar = Lib::cmd('tar', array('c', '-h', '-f', '-', '--no-recursion', 'app'));
256
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/.'));
257
        $status = $exec->pass("${cd} && echo 'app' | ${tar} | ${dockerCp}", array());
258
        LibFs::unlink($tmpDir . '/app');
259
        if (0 !== $status) {
260
            $streams->err('pipelines: deploy copy failure\n');
261
262
            return $status;
263
        }
264
265
        $cd = Lib::cmd('cd', array($dir . '/.'));
266
        $tar = Lib::cmd('tar', array('c', '-f', '-', '.'));
267
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/app'));
268
        $status = $exec->pass("${cd} && ${tar} | ${dockerCp}", array());
269
        if (0 !== $status) {
270
            $streams->err('pipelines: deploy copy failure\n');
271
272
            return $status;
273
        }
274
275
        $streams('');
276
277
        return null;
278
    }
279
280
    /**
281
     * if there is the docker service in the step, deploy the
282
     * docker client
283
     *
284
     * @param Step $step
285
     * @param string $id
286
     */
287
    private function deployDockerClient(Step $step, $id)
288
    {
289
        if (!$step->getServices()->has('docker')) {
290
            return;
291
        }
292
293
        $this->streams->out(' +++ docker client install...: ');
294
295
        $this->getDockerBinaryRepository()->inject($id);
296
297
        $this->streams->out("\n");
298
    }
299
300
    /**
301
     * @param string $name
302
     * @return null|string
303
     */
304
    private function dockerGetContainerIdByName($name)
305
    {
306
        $ids = null;
307
308
        $status = $this->exec->capture(
309
            'docker',
310
            array(
311
                'ps', '-qa', '--filter',
312
                "name=^/${name}$"
313
            ),
314
            $result
315
        );
316
317
        $status || $ids = Lib::lines($result);
318
319
        if ($status || !(is_array($ids) && 1 === count($ids))) {
320
            return null;
321
        }
322
323
        return $ids[0];
324
    }
325
326
    /**
327
     * @param Step $step
328
     * @return string
329
     */
330
    private function generateContainerName(Step $step)
331
    {
332
        $project = $this->directories->getName();
333
        $idContainerSlug = preg_replace('([^a-zA-Z0-9_.-]+)', '-', $step->getPipeline()->getId());
334
        if ('' === $idContainerSlug) {
335
            $idContainerSlug = 'null';
336
        }
337
        $nameSlug = preg_replace(array('( )', '([^a-zA-Z0-9_.-]+)'), array('-', ''), $step->getName());
338
        if ('' === $nameSlug) {
339
            $nameSlug = 'no-name';
340
        }
341
342
        return $this->prefix . '-' . implode(
343
            '.',
344
            array_reverse(
345
                    array(
346
                        $project,
347
                        trim($idContainerSlug, '-'),
348
                        $nameSlug,
349
                        $step->getIndex() + 1,
350
                    )
351
                )
352
        );
353
    }
354
355
    /**
356
     * @param Image $image
357
     * @throws \RuntimeException
358
     * @throws \InvalidArgumentException
359
     */
360
    private function imageLogin(Image $image)
361
    {
362
        $login = new DockerLogin($this->exec, $this->env->getResolver());
363
        $login->byImage($image);
364
    }
365
366
    /**
367
     * @param string $name
368
     * @param string $dir
369
     * @param bool $copy
370
     * @param Step $step
371
     * @return array array(string|null $id, int $status)
372
     */
373
    private function runNewContainer($name, $dir, $copy, Step $step)
374
    {
375
        $env = $this->env;
376
        $exec = $this->exec;
377
        $streams = $this->streams;
378
379
        $image = $step->getImage();
380
        $docker = new Docker($exec);
381
382
        # process docker login if image demands so, but continue on failure
383
        $this->imageLogin($image);
384
385
        // enable docker client inside docker by mounting docker socket
386
        // FIXME give controlling options, this is serious /!\
387
        $mountDockerSock = array();
388
        if ($this->flags->useDockerSocket() && file_exists('/var/run/docker.sock')) {
389
            $mountDockerSock = array(
390
                '-v', '/var/run/docker.sock:/var/run/docker.sock',
391
            );
392
        }
393
394
        $parentName = $env->getValue('PIPELINES_PARENT_CONTAINER_NAME');
395
        $checkMount = $mountDockerSock && null !== $parentName;
396
        $deviceDir = $checkMount ? $docker->hostDevice($parentName, $dir) : $dir;
397
398
        $mountWorkingDirectory = $copy
399
            ? array()
400
            : array('--volume', "${deviceDir}:/app"); // FIXME(tk): hard encoded /app
401
402
        $status = $exec->capture('docker', array(
403
            'run', '-i', '--name', $name,
404
            $env->getArgs('-e'),
405
            $mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app',
406
            $mountDockerSock,
407
            '--workdir', '/app', '--detach', '--entrypoint=/bin/sh', $image->getName()
408
        ), $out, $err);
409
        if (0 !== $status) {
410
            $streams->out("    container-id...: *failure*\n\n");
411
            $streams->err("pipelines: setting up the container failed\n");
412
            $streams->err("${err}\n");
413
            $streams->out("${out}\n");
414
            $streams->out(sprintf("exit status: %d\n", $status));
415
416
            return array(null, $status);
417
        }
418
        $id = rtrim($out) ?: '*dry-run*'; # side-effect: internal exploit of no output with true exit status
419
420
        return array($id, 0);
421
    }
422
423
    /**
424
     * @param Step $step
425
     * @param Streams $streams
426
     * @param Exec $exec
427
     * @param string $name container name
428
     * @return null|int should never be null, status, non-zero if a command failed
429
     */
430
    private function runStepScript(Step $step, Streams $streams, Exec $exec, $name)
431
    {
432
        $script = $step->getScript();
433
434
        $buffer = Lib::cmd("<<'SCRIPT' docker", array(
435
            'exec', '-i', $name, '/bin/sh'
436
        ));
437
        $buffer .= "\n# this /bin/sh script is generated from a pipelines pipeline:\n";
438
        foreach ($script as $line => $command) {
439
            $buffer .= 'printf \'\\035+ %s\\n\' ' . Lib::quoteArg($command) . "\n";
440
            $buffer .= $command . "\n";
441
            $buffer .= 'ret=$?' . "\n";
442
            $buffer .= 'printf \'\\n\'' . "\n";
443
            $buffer .= 'if [ $ret -ne 0 ]; then exit $ret; fi' . "\n";
444
        }
445
        $buffer .= "SCRIPT\n";
446
447
        $status = $exec->pass($buffer, array());
448
449
        if (0 !== $status) {
450
            $streams->err(sprintf("script non-zero exit status: %d\n", $status));
451
        }
452
453
        return $status;
454
    }
455
456
    /**
457
     * @param int $status
458
     * @param string $id container id
459
     * @param Exec $exec
460
     * @param string $name container name
461
     * @throws \RuntimeException
462
     */
463
    private function shutdownStepContainer($status, $id, Exec $exec, $name)
464
    {
465
        $flags = $this->flags;
466
467
        # keep container on error
468
        if (0 !== $status && $flags->keepOnError()) {
469
            $this->streams->err(sprintf(
470
                "error, keeping container id %s\n",
471
                substr($id, 0, 12)
472
            ));
473
474
            return;
475
        }
476
477
        # keep or remove container
478
        if ($flags->killContainer()) {
479
            $exec->capture('docker', array('kill', $name));
480
        }
481
482
        if ($flags->removeContainer()) {
483
            $exec->capture('docker', array('rm', $name));
484
        }
485
486
        if ($flags->keep()) {
487
            $this->streams->out(sprintf(
488
                "keeping container id %s\n",
489
                substr($id, 0, 12)
490
            ));
491
        }
492
    }
493
494
    /**
495
     * @param string $name
496
     */
497
    private function zapContainerByName($name)
498
    {
499
        $ids = null;
500
501
        $status = $this->exec->capture(
502
            'docker',
503
            array(
504
                'ps', '-qa', '--filter',
505
                "name=^/${name}$"
506
            ),
507
            $result
508
        );
509
510
        $status || $ids = Lib::lines($result);
511
512
        if ($status || !(is_array($ids) && 1 === count($ids))) {
513
            return;
514
        }
515
516
        $this->exec->capture('docker', Lib::merge('kill', $ids));
517
        $this->exec->capture('docker', Lib::merge('rm', $ids));
518
    }
519
}
520