Passed
Push — test ( 870dbd...9d8e99 )
by Tom
03:05
created

StepRunner::captureStepArtifacts()   A

Complexity

Conditions 4
Paths 4

Size

Total Lines 26
Code Lines 13

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 14
CRAP Score 4

Importance

Changes 1
Bugs 0 Features 0
Metric Value
cc 4
eloc 13
c 1
b 0
f 0
nc 4
nop 4
dl 0
loc 26
ccs 14
cts 14
cp 1
crap 4
rs 9.8333
1
<?php
2
3
/* this file is part of pipelines */
4
5
namespace Ktomk\Pipelines\Runner;
6
7
use Ktomk\Pipelines\Cli\Docker;
8
use Ktomk\Pipelines\Cli\Exec;
9
use Ktomk\Pipelines\Cli\Streams;
10
use Ktomk\Pipelines\DestructibleString;
11
use Ktomk\Pipelines\File\Image;
12
use Ktomk\Pipelines\File\Step;
13
use Ktomk\Pipelines\Lib;
14
use Ktomk\Pipelines\LibFs;
15
use Ktomk\Pipelines\LibTmp;
16
17
/**
18
 * Runner for a single step of a pipeline
19
 */
20
class StepRunner
21
{
22
    /**
23
     * @var RunOpts
24
     */
25
    private $runOpts;
26
27
    /**
28
     * @var Directories
29
     */
30
    private $directories;
31
32
    /**
33
     * @var Exec
34
     */
35
    private $exec;
36
37
    /**
38
     * @var Flags
39
     */
40
    private $flags;
41
42
    /**
43
     * @var Env
44
     */
45
    private $env;
46
47
    /**
48
     * @var Streams
49
     */
50
    private $streams;
51
52
    /**
53
     * list of temporary directory destructible markers
54
     *
55
     * @var array
56
     */
57
    private $temporaryDirectories = array();
58
59
    /**
60
     * DockerSession constructor.
61
     *
62
     * @param RunOpts $runOpts
63
     * @param Directories $directories source repository root directory based directories object
64
     * @param Exec $exec
65
     * @param Flags $flags
66
     * @param Env $env
67
     * @param Streams $streams
68
     */
69 13
    public function __construct(
70
        RunOpts $runOpts,
71
        Directories $directories,
72
        Exec $exec,
73
        Flags $flags,
74
        Env $env,
75
        Streams $streams
76
    )
77
    {
78 13
        $this->runOpts = $runOpts;
79 13
        $this->directories = $directories;
80 13
        $this->exec = $exec;
81 13
        $this->flags = $flags;
82 13
        $this->env = $env;
83 13
        $this->streams = $streams;
84 13
    }
85
86
    /**
87
     * @param Step $step
88
     * @return null|int exist status of step script or null if the run operation failed
89
     */
90 13
    public function runStep(Step $step)
91
    {
92 13
        $dir = $this->directories->getProjectDirectory();
93 13
        $env = $this->env;
94 13
        $exec = $this->exec;
95 13
        $streams = $this->streams;
96 13
        $reuseContainer = $this->flags->reuseContainer();
97 13
        $deployCopy = $this->flags->deployCopy();
98
99 13
        $name = $this->generateContainerName($step);
100
101 13
        if (false === $reuseContainer) {
102 10
            $this->zapContainerByName($name);
103
        }
104 13
        $image = $step->getImage();
105 13
        $env->setContainerName($name);
106
107
        # launch container
108 13
        $streams->out(sprintf(
109 13
            "\x1D+++ step #%d\n\n    name...........: %s\n    effective-image: %s\n    container......: %s\n",
110 13
            $step->getIndex() + 1,
111 13
            $step->getName() ? '"' . $step->getName() . '"' : '(unnamed)',
112 13
            $image->getName(),
113 13
            $name
114
        ));
115
116 13
        $id = null;
117 13
        if ($reuseContainer) {
118 3
            $id = $this->dockerGetContainerIdByName($name);
119
        }
120
121 13
        if (null === $id) {
122 11
            list($id, $status) = $this->runNewContainer($name, $dir, $deployCopy, $step);
123 11
            if (null === $id) {
124 1
                return $status;
125
            }
126
        }
127
128 12
        $streams->out(sprintf("    container-id...: %s\n\n", substr($id, 0, 12)));
129
130
        # TODO: different deployments, mount (default), mount-ro, copy
131 12
        if (null !== $result = $this->deployCopy($deployCopy, $id, $dir)) {
132 2
            return $result;
133
        }
134
135 10
        $status = $this->runStepScript($step, $streams, $exec, $name);
136
137 10
        $this->captureStepArtifacts($step, $deployCopy && 0 === $status, $id, $dir);
138
139 10
        $this->shutdownStepContainer($status, $id, $exec, $name);
140
141 10
        return $status;
142
    }
143
144
    /**
145
     * @param Step $step
146
     * @param bool $copy
147
     * @param string $id container id
148
     * @param string $dir to put artifacts in (project directory)
149
     * @throws \RuntimeException
150
     */
151 10
    private function captureStepArtifacts(Step $step, $copy, $id, $dir)
152
    {
153
        # capturing artifacts is only supported for deploy copy
154 10
        if (!$copy) {
155 5
            return;
156
        }
157
158 5
        $artifacts = $step->getArtifacts();
159
160 5
        if (null === $artifacts) {
161 2
            return;
162
        }
163
164 3
        $exec = $this->exec;
165 3
        $streams = $this->streams;
166
167 3
        $streams->out("\x1D+++ copying artifacts from container...\n");
168
169 3
        $source = new ArtifactSource($exec, $id, $dir);
170
171 3
        $patterns = $artifacts->getPatterns();
172 3
        foreach ($patterns as $pattern) {
173 3
            $this->captureArtifactPattern($source, $pattern, $dir);
174
        }
175
176 3
        $streams('');
177 3
    }
178
179
    /**
180
     * @see Runner::captureStepArtifacts()
181
     *
182
     * @param ArtifactSource $source
183
     * @param string $pattern
184
     * @param string $dir
185
     * @throws \RuntimeException
186
     */
187 3
    private function captureArtifactPattern(ArtifactSource $source, $pattern, $dir)
188
    {
189 3
        $exec = $this->exec;
190 3
        $streams = $this->streams;
191
192 3
        $id = $source->getId();
193 3
        $paths = $source->findByPattern($pattern);
194 3
        if (empty($paths)) {
195 1
            return;
196
        }
197
198 2
        $chunks = Lib::arrayChunkByStringLength($paths, 131072, 4);
199
200 2
        foreach ($chunks as $paths) {
201 2
            $docker = Lib::cmd('docker', array('exec', '-w', '/app', $id));
202 2
            $tar = Lib::cmd('tar', array('c', '-f', '-', $paths));
203 2
            $unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir));
204
205 2
            $command = $docker . ' ' . $tar . ' | ' . $unTar;
206 2
            $status = $exec->pass($command, array());
207
208 2
            if (0 !== $status) {
209 1
                $streams->err(sprintf(
210 1
                    "pipelines: Artifact failure: '%s' (%d, %d paths, %d bytes)\n",
211 1
                    $pattern,
212 1
                    $status,
213 1
                    count($paths),
214 1
                    strlen($command)
215
                ));
216
            }
217
        }
218 2
    }
219
220
    /**
221
     * @param bool $copy
222
     * @param string $id container id
223
     * @param string $dir directory to copy contents into container
224
     * @throws \RuntimeException
225
     * @return null|int null if all clear, integer for exit status
226
     */
227 12
    private function deployCopy($copy, $id, $dir)
228
    {
229 12
        if (!$copy) {
230 5
            return null;
231
        }
232
233 7
        $streams = $this->streams;
234 7
        $exec = $this->exec;
235
236 7
        $streams->out("\x1D+++ copying files into container...\n");
237
238 7
        $tmpDir = LibTmp::tmpDir('pipelines-cp.');
239 7
        $this->temporaryDirectories[] = DestructibleString::rmDir($tmpDir);
240 7
        LibFs::symlink($dir, $tmpDir . '/app');
241 7
        $cd = Lib::cmd('cd', array($tmpDir . '/.'));
242 7
        $tar = Lib::cmd('tar', array('c', '-h', '-f', '-', '--no-recursion', 'app'));
243 7
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/.'));
244
        $status = $exec->pass("${cd} && echo 'app' | ${tar} | ${dockerCp}", array());
245
        LibFs::unlink($tmpDir . '/app');
246 7
        if (0 !== $status) {
247 1
            $streams->err('pipelines: deploy copy failure\n');
248
249 1
            return $status;
250
        }
251
252 6
        $cd = Lib::cmd('cd', array($dir . '/.'));
253 6
        $tar = Lib::cmd('tar', array('c', '-f', '-', '.'));
254 6
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/app'));
255 6
        $status = $exec->pass("${cd} && ${tar} | ${dockerCp}", array());
256 6
        if (0 !== $status) {
257 1
            $streams->err('pipelines: deploy copy failure\n');
258
259 1
            return $status;
260
        }
261
262 5
        $streams('');
263
264 5
        return null;
265
    }
266
267
    /**
268
     * @param string $name
269
     * @return null|string
270
     */
271
    private function dockerGetContainerIdByName($name)
272
    {
273 3
        $ids = null;
274
275 3
        $status = $this->exec->capture(
276 3
            'docker',
277
            array(
278 3
                'ps', '-qa', '--filter',
279 3
                "name=^/${name}$"
280
            ),
281 3
            $result
282
        );
283
284 3
        $status || $ids = Lib::lines($result);
285
286 3
        if ($status || !(is_array($ids) && 1 === count($ids))) {
287 1
            return null;
288
        }
289
290 2
        return $ids[0];
291
    }
292
293
    /**
294
     * @param Step $step
295
     * @return string
296
     */
297
    private function generateContainerName(Step $step)
298
    {
299 13
        $project = $this->directories->getName();
300 13
        $idContainerSlug = preg_replace('([^a-zA-Z0-9_.-]+)', '-', $step->getPipeline()->getId());
301 13
        if ('' === $idContainerSlug) {
302 13
            $idContainerSlug = 'null';
303
        }
304 13
        $nameSlug = preg_replace(array('( )', '([^a-zA-Z0-9_.-]+)'), array('-', ''), $step->getName());
305 13
        if ('' === $nameSlug) {
306 13
            $nameSlug = 'no-name';
307
        }
308
309 13
        return $this->runOpts->getPrefix() . '-' . implode(
310 13
            '.',
311 13
            array_reverse(
312
                array(
313 13
                        $project,
314 13
                        trim($idContainerSlug, '-'),
315 13
                        $nameSlug,
316 13
                        $step->getIndex() + 1,
317
                    )
318
            )
319
        );
320
    }
321
322
    /**
323
     * @param Image $image
324
     * @throws \RuntimeException
325
     * @throws \InvalidArgumentException
326
     */
327
    private function imageLogin(Image $image)
328
    {
329 11
        $login = new DockerLogin($this->exec, $this->env->getResolver());
330 11
        $login->byImage($image);
331 11
    }
332
333
    /**
334
     * @param string $name
335
     * @param string $dir
336
     * @param bool $copy
337
     * @param Step $step
338
     * @return array array(string|null $id, int $status)
339
     */
340
    private function runNewContainer($name, $dir, $copy, Step $step)
341
    {
342 11
        $env = $this->env;
343 11
        $exec = $this->exec;
344 11
        $streams = $this->streams;
345
346 11
        $image = $step->getImage();
347
348
        # process docker login if image demands so, but continue on failure
349 11
        $this->imageLogin($image);
350
351
        // enable docker client inside docker by mounting docker socket
352
        // FIXME give controlling options, this is serious /!\
353 11
        $mountDockerSock = array();
354 11
        if ($this->flags->useDockerSocket() && file_exists('/var/run/docker.sock')) {
355
            $mountDockerSock = array(
356
                '-v', '/var/run/docker.sock:/var/run/docker.sock',
357
            );
358
        }
359
360 11
        $parentName = $env->getValue('PIPELINES_PARENT_CONTAINER_NAME');
361 11
        $checkMount = $mountDockerSock && null !== $parentName;
362 11
        $deviceDir = $dir;
363 11
        if ($checkMount) {
364
            $docker = new Docker($exec);
365
            $deviceDir = $docker->hostDevice($parentName, $dir);
366
            unset($docker);
367
        }
368
369 11
        $mountWorkingDirectory = $copy
370 7
            ? array()
371 11
            : array('--volume', "${deviceDir}:/app"); // FIXME(tk): hard encoded /app
372
373 11
        $status = $exec->capture('docker', array(
374 11
            'run', '-i', '--name', $name,
375 11
            $env->getArgs('-e'),
376 11
            $mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app',
377 11
            $mountDockerSock,
378 11
            '--workdir', '/app', '--detach', '--entrypoint=/bin/sh', $image->getName()
379 11
        ), $out, $err);
380 11
        if (0 !== $status) {
381 1
            $streams->out("    container-id...: *failure*\n\n");
382 1
            $streams->err("pipelines: setting up the container failed\n");
383 1
            $streams->err("${err}\n");
384 1
            $streams->out("${out}\n");
385 1
            $streams->out(sprintf("exit status: %d\n", $status));
386
387 1
            return array(null, $status);
388
        }
389 10
        $id = rtrim($out) ?: '*dry-run*'; # side-effect: internal exploit of no output with true exit status
390
391 10
        return array($id, 0);
392
    }
393
394
    /**
395
     * @param Step $step
396
     * @param Streams $streams
397
     * @param Exec $exec
398
     * @param string $name container name
399
     * @return null|int should never be null, status, non-zero if a command failed
400
     */
401
    private function runStepScript(Step $step, Streams $streams, Exec $exec, $name)
402
    {
403 10
        $script = $step->getScript();
404
405 10
        $buffer = Lib::cmd("<<'SCRIPT' docker", array(
406 10
            'exec', '-i', $name, '/bin/sh'
407
        ));
408 10
        $buffer .= "\n# this /bin/sh script is generated from a pipelines pipeline:\n";
409 10
        $buffer .= "set -e\n";
410 10
        foreach ($script as $line => $command) {
411 10
            $line && $buffer .= 'printf \'\\n\'' . "\n";
412 10
            $buffer .= 'printf \'\\035+ %s\\n\' ' . Lib::quoteArg($command) . "\n";
413 10
            $buffer .= $command . "\n";
414
        }
415 10
        $buffer .= "SCRIPT\n";
416
417 10
        $status = $exec->pass($buffer, array());
418
419 10
        if (0 !== $status) {
420 2
            $streams->err(sprintf("script non-zero exit status: %d\n", $status));
421
        }
422
423 10
        return $status;
424
    }
425
426
    /**
427
     * @param int $status
428
     * @param string $id container id
429
     * @param Exec $exec
430
     * @param string $name container name
431
     * @throws \RuntimeException
432
     */
433
    private function shutdownStepContainer($status, $id, Exec $exec, $name)
434
    {
435 10
        $flags = $this->flags;
436
437
        # keep container on error
438 10
        if (0 !== $status && $flags->keepOnError()) {
439 2
            $this->streams->err(sprintf(
440 2
                "error, keeping container id %s\n",
441 2
                substr($id, 0, 12)
442
            ));
443
444 2
            return;
445
        }
446
447
        # keep or remove container
448 8
        if ($flags->killContainer()) {
449 7
            $exec->capture('docker', array('kill', $name));
450
        }
451
452 8
        if ($flags->removeContainer()) {
453 7
            $exec->capture('docker', array('rm', $name));
454
        }
455
456 8
        if ($flags->keep()) {
457 1
            $this->streams->out(sprintf(
458 1
                "keeping container id %s\n",
459 1
                substr($id, 0, 12)
460
            ));
461
        }
462 8
    }
463
464
    /**
465
     * @param string $name
466
     */
467
    private function zapContainerByName($name)
468
    {
469 10
        $ids = null;
470
471 10
        $status = $this->exec->capture(
472 10
            'docker',
473
            array(
474 10
                'ps', '-qa', '--filter',
475 10
                "name=^/${name}$"
476
            ),
477 10
            $result
478
        );
479
480 10
        $status || $ids = Lib::lines($result);
481
482 10
        if ($status || !(is_array($ids) && 1 === count($ids))) {
483 9
            return;
484
        }
485
486 1
        $this->exec->capture('docker', Lib::merge('kill', $ids));
487 1
        $this->exec->capture('docker', Lib::merge('rm', $ids));
488 1
    }
489
}
490