Passed
Push — test ( ce9d4d...85a148 )
by Tom
02:45
created

StepRunner::obtainWorkingDirMount()   B

Complexity

Conditions 9
Paths 7

Size

Total Lines 24
Code Lines 13

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 13
CRAP Score 9

Importance

Changes 1
Bugs 0 Features 0
Metric Value
cc 9
eloc 13
c 1
b 0
f 0
nc 7
nop 3
dl 0
loc 24
ccs 13
cts 13
cp 1
crap 9
rs 8.0555
1
<?php
2
3
/* this file is part of pipelines */
4
5
namespace Ktomk\Pipelines\Runner;
6
7
use Ktomk\Pipelines\Cli\Docker;
8
use Ktomk\Pipelines\DestructibleString;
9
use Ktomk\Pipelines\File\Pipeline\Step;
10
use Ktomk\Pipelines\Lib;
11
use Ktomk\Pipelines\LibFs;
12
use Ktomk\Pipelines\LibFsPath;
13
use Ktomk\Pipelines\LibTmp;
14
use Ktomk\Pipelines\Runner\Containers\StepContainer;
15
use Ktomk\Pipelines\Runner\Docker\ArgsBuilder;
16
use Ktomk\Pipelines\Runner\Docker\ArtifactSource;
17
use Ktomk\Pipelines\Runner\Docker\Binary\Repository;
18
use Ktomk\Pipelines\Runner\Docker\CacheIo;
19
use Ktomk\Pipelines\Runner\Docker\ImageLogin;
20
21
/**
22
 * Runner for a single step of a pipeline
23
 */
24
class StepRunner
25
{
26
    /**
27
     * list of temporary directory destructible markers
28
     *
29
     * @var array
30
     */
31
    private $temporaryDirectories = array();
32
33
    /**
34
     * @var Runner
35
     */
36
    private $runner;
37
38
    /**
39
     * DockerSession constructor.
40
     *
41
     * @param Runner $runner
42
     */
43 30
    public function __construct(Runner $runner)
44
    {
45 30
        $this->runner = $runner;
46 30
    }
47
48
    /**
49
     * @param Step $step
50
     *
51
     * @return null|int exist status of step script or null if the run operation failed
52
     */
53 28
    public function runStep(Step $step)
54
    {
55 28
        $dir = $this->runner->getDirectories()->getProjectDirectory();
56 28
        $env = $this->runner->getEnv();
57 28
        $streams = $this->runner->getStreams();
58
59 28
        $containers = new Containers($this->runner);
60
61 28
        $env->setPipelinesProjectPath($dir);
62
63 28
        $container = $containers->createStepContainer($step);
64
65 28
        $env->setContainerName($container->getName());
66
67 28
        $image = $step->getImage();
68
69
        # launch container
70 28
        $streams->out(sprintf(
71 28
            "\x1D+++ step #%d\n\n    name...........: %s\n    effective-image: %s\n    container......: %s\n",
72 28
            $step->getIndex() + 1,
73 28
            $step->getName() ? '"' . $step->getName() . '"' : '(unnamed)',
74 28
            $image->getName(),
75 28
            $container->getName()
76
        ));
77
78 28
        $id = $container->keepOrKill();
79
80 28
        $deployCopy = $this->runner->getFlags()->deployCopy();
81
82 28
        if (null === $id) {
83 26
            list($id, $status, $out, $err) = $this->runNewContainer($container, $dir, $deployCopy, $step);
84 25
            if (null === $id) {
85 3
                $streams->out("    container-id...: *failure*\n\n");
86 3
                $streams->err("pipelines: setting up the container failed\n");
87 3
                empty($err) || $streams->err("${err}\n");
88 3
                empty($out) || $streams->out("${out}\n");
89 3
                $streams->out(sprintf("exit status: %d\n", $status));
90
91 3
                return $status;
92
            }
93
        }
94
95 24
        $streams->out(sprintf("    container-id...: %s\n\n", substr($id, 0, 12)));
96
97
        # TODO: different deployments, mount (default), mount-ro, copy
98 24
        if (null !== $result = $this->deployCopy($deployCopy, $id, $dir)) {
99 2
            $streams->err('pipelines: deploy copy failure\n');
100
101 2
            return $result;
102
        }
103
104 22
        $deployCopy && $streams('');
105
106 22
        $cacheIo = CacheIo::createByRunner($this->runner, $id);
107 22
        $cacheIo->deployStepCaches($step);
108
109 22
        $status = StepScriptRunner::createRunStepScript($this->runner, $container->getName(), $step);
110
111 22
        $cacheIo->captureStepCaches($step, 0 === $status);
112
113 22
        $this->captureStepArtifacts($step, $deployCopy && 0 === $status, $id, $dir);
114
115 22
        $container->shutdown($status);
116
117 22
        return $status;
118
    }
119
120
    /**
121
     * method to wrap new to have a test-point
122
     *
123
     * @return Repository
124
     */
125
    public function getDockerBinaryRepository()
126
    {
127 2
        $repo = Repository::create($this->runner->getExec(), $this->runner->getDirectories());
128 2
        $repo->resolve($this->runner->getRunOpts()->getBinaryPackage());
129
130 1
        return $repo;
131
    }
132
133
    /**
134
     * @param Step $step
135
     * @param bool $copy
136
     * @param string $id container id
137
     * @param string $dir to put artifacts in (project directory)
138
     *
139
     * @throws \RuntimeException
140
     *
141
     * @return void
142
     */
143
    private function captureStepArtifacts(Step $step, $copy, $id, $dir)
144
    {
145
        # capturing artifacts is only supported for deploy copy
146 22
        if (!$copy) {
147 17
            return;
148
        }
149
150 5
        $artifacts = $step->getArtifacts();
151
152 5
        if (null === $artifacts) {
153 2
            return;
154
        }
155
156 3
        $exec = $this->runner->getExec();
157 3
        $streams = $this->runner->getStreams();
158
159 3
        $streams->out("\x1D+++ copying artifacts from container...\n");
160
161 3
        $source = new ArtifactSource($exec, $id, $dir);
162
163 3
        $patterns = $artifacts->getPatterns();
164 3
        foreach ($patterns as $pattern) {
165 3
            $this->captureArtifactPattern($source, $pattern, $dir);
166
        }
167
168 3
        $streams('');
169 3
    }
170
171
    /**
172
     * capture artifact pattern
173
     *
174
     * @param ArtifactSource $source
175
     * @param string $pattern
176
     * @param string $dir
177
     *
178
     * @throws \RuntimeException
179
     *
180
     * @return void
181
     *
182
     * @see Runner::captureStepArtifacts()
183
     *
184
     */
185
    private function captureArtifactPattern(ArtifactSource $source, $pattern, $dir)
186
    {
187 3
        $exec = $this->runner->getExec();
188 3
        $streams = $this->runner->getStreams();
189
190 3
        $id = $source->getId();
191 3
        $paths = $source->findByPattern($pattern);
192 3
        if (empty($paths)) {
193 1
            return;
194
        }
195
196 2
        $chunks = Lib::arrayChunkByStringLength($paths, 131072, 4);
197
198 2
        $clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
199
200 2
        foreach ($chunks as $paths) {
201 2
            $docker = Lib::cmd('docker', array('exec', '-w', $clonePath, $id));
202 2
            $tar = Lib::cmd('tar', array('c', '-f', '-', $paths));
203 2
            $unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir));
204
205 2
            $command = $docker . ' ' . $tar . ' | ' . $unTar;
206 2
            $status = $exec->pass($command, array());
207
208 2
            if (0 !== $status) {
209 1
                $streams->err(sprintf(
210 1
                    "pipelines: Artifact failure: '%s' (%d, %d paths, %d bytes)\n",
211
                    $pattern,
212
                    $status,
213 1
                    count($paths),
214 1
                    strlen($command)
215
                ));
216
            }
217
        }
218 2
    }
219
220
    /**
221
     * @param bool $copy
222
     * @param string $id container id
223
     * @param string $dir directory to copy contents into container
224
     *
225
     * @throws \RuntimeException
226
     *
227
     * @return null|int null if all clear, integer for exit status
228
     */
229
    private function deployCopy($copy, $id, $dir)
230
    {
231 24
        if (!$copy) {
232 17
            return null;
233
        }
234
235 7
        $streams = $this->runner->getStreams();
236 7
        $exec = $this->runner->getExec();
237
238 7
        $streams->out("\x1D+++ copying files into container...\n");
239
240 7
        $tmpDir = LibTmp::tmpDir('pipelines-cp.');
241 7
        $this->temporaryDirectories[] = DestructibleString::rmDir($tmpDir);
242
243 7
        $clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
244
245 7
        LibFs::symlink($dir, $tmpDir . $clonePath);
246 7
        $cd = Lib::cmd('cd', array($tmpDir . '/.'));
247
248 7
        $tar = Lib::cmd('tar', array('c', '-h', '-f', '-', '--no-recursion', '.' . $clonePath));
249 7
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/.'));
250 7
        $status = $exec->pass("${cd} && echo 'app' | ${tar} | ${dockerCp}", array());
251 7
        LibFs::unlink($tmpDir . $clonePath);
252 7
        if (0 !== $status) {
253 1
            return $status;
254
        }
255
256 6
        $cd = Lib::cmd('cd', array($dir . '/.'));
257 6
        $tar = Lib::cmd('tar', array('c', '-f', '-', '.'));
258 6
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':' . $clonePath));
259 6
        $status = $exec->pass("${cd} && ${tar} | ${dockerCp}", array());
260 6
        if (0 !== $status) {
261 1
            return $status;
262
        }
263
264 5
        return null;
265
    }
266
267
    /**
268
     * @param StepContainer $container
269
     * @param string $dir project directory (host)
270
     * @param bool $copy deployment
271
     * @param Step $step
272
     *
273
     * @return array array(string|null $id, int $status, string $out, string $err)
274
     */
275
    private function runNewContainer(StepContainer $container, $dir, $copy, Step $step)
276
    {
277 26
        $env = $this->runner->getEnv();
278
279 26
        $mountDockerSock = $this->obtainDockerSocketMount();
280
281 26
        $mountDockerClient = $this->obtainDockerClientMount($step);
282
283 25
        $mountWorkingDirectory = $this->obtainWorkingDirMount($copy, $dir, $mountDockerSock);
284 25
        if ($mountWorkingDirectory && is_int($mountWorkingDirectory[1])) {
285 2
            return $mountWorkingDirectory + array(2 => '', 3 => '');
286
        }
287
288 23
        $network = $container->getServiceContainers()->obtainNetwork();
289
290
        # process docker login if image demands so, but continue on failure
291 23
        $image = $step->getImage();
292 23
        ImageLogin::loginImage($this->runner->getExec(), $this->runner->getEnv()->getResolver(), null, $image);
293
294 23
        $clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
295
296 23
        list($status, $out, $err) = $container->run(
297
            array(
298 23
                $network,
299 23
                '-i', '--name', $container->getName(),
300 23
                $container->obtainLabelOptions(),
301 23
                $env->getArgs('-e'),
302 23
                ArgsBuilder::optMap('-e', $step->getEnv(), true),
303 23
                $mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app',
304 23
                $mountDockerSock,
305 23
                $mountDockerClient,
306 23
                $container->obtainUserOptions(),
307 23
                $container->obtainSshOptions(),
308 23
                '--workdir', $clonePath, '--detach', '--entrypoint=/bin/sh',
309 23
                $image->getName(),
310
            )
311
        );
312 23
        $id = $status ? null : $container->getDisplayId();
313
314 23
        return array($id, $status, $out, $err);
315
    }
316
317
    /**
318
     * @param Step $step
319
     *
320
     * @return string[]
321
     */
322
    private function obtainDockerClientMount(Step $step)
323
    {
324 26
        if (!$step->getServices()->has('docker')) {
325 23
            return array();
326
        }
327
328 3
        $path = $this->runner->getRunOpts()->getOption('docker.client.path');
329
330
        // prefer pip mount over package
331 3
        $hostPath = $this->pipHostConfigBind($path);
332 3
        if (null !== $hostPath) {
333 1
            return array('-v', sprintf('%s:%s:ro', $hostPath, $path));
334
        }
335
336 2
        $local = $this->getDockerBinaryRepository()->getBinaryPath();
337 1
        chmod($local, 0755);
338
339 1
        return array('-v', sprintf('%s:%s:ro', $local, $path));
340
    }
341
342
    /**
343
     * enable docker client inside docker by mounting docker socket
344
     *
345
     * @return array docker socket volume args for docker run, empty if not mounting
346
     */
347
    private function obtainDockerSocketMount()
348
    {
349 26
        $args = array();
350
351
        // FIXME give more controlling options, this is serious /!\
352 26
        if (!$this->runner->getFlags()->useDockerSocket()) {
353 1
            return $args;
354
        }
355
356 25
        $defaultSocketPath = $this->runner->getRunOpts()->getOption('docker.socket.path');
357 25
        $hostPathDockerSocket = $defaultSocketPath;
358
359
        // pipelines inside pipelines
360 25
        $hostPath = $this->pipHostConfigBind($defaultSocketPath);
361 25
        if (null !== $hostPath) {
362
            return array(
363 1
                '-v', sprintf('%s:%s', $hostPath, $defaultSocketPath),
364
            );
365
        }
366
367 24
        $dockerHost = $this->runner->getEnv()->getInheritValue('DOCKER_HOST');
368 24
        if (null !== $dockerHost && 0 === strpos($dockerHost, 'unix://')) {
369 1
            $hostPathDockerSocket = LibFsPath::normalize(substr($dockerHost, 7));
370
        }
371
372 24
        $pathDockerSock = $defaultSocketPath;
373
374 24
        if (file_exists($hostPathDockerSocket)) {
375
            $args = array(
376 21
                '-v', sprintf('%s:%s', $hostPathDockerSocket, $pathDockerSock),
377
            );
378
        }
379
380 24
        return $args;
381
    }
382
383
    /**
384
     * @param bool $copy deployment
385
     * @param string $dir project directory
386
     * @param array $mountDockerSock docker socket volume args for docker run, empty if not mounting
387
     *
388
     * @return array mount options or array(null, int $status) for error handling
389
     */
390
    private function obtainWorkingDirMount($copy, $dir, array $mountDockerSock)
391
    {
392 25
        if ($copy) {
393 7
            return array();
394
        }
395
396 18
        $parentName = $this->runner->getEnv()->getValue('PIPELINES_PARENT_CONTAINER_NAME');
397 18
        $hostDeviceDir = $this->pipHostConfigBind($dir);
398 18
        $checkMount = $mountDockerSock && null !== $parentName;
399 18
        $deviceDir = $hostDeviceDir ?: $dir;
400 18
        $clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
401 18
        if ($checkMount && $clonePath === $dir && null === $hostDeviceDir) {
402 2
            $deviceDir = $this->runner->getEnv()->getValue('PIPELINES_PROJECT_PATH');
403 2
            if ($deviceDir === $dir || null === $deviceDir) {
404 2
                $this->runner->getStreams()->err("pipelines: fatal: can not detect ${dir} mount point\n");
405
406 2
                return array(null, 1);
407
            }
408
        }
409
410
        // FIXME(tk): Never mount anything not matching /home/[a-zA-Z][a-zA-Z0-9]*/[^.].*/...
411
        //   + do realpath checking
412
        //   + prevent dot path injections (logical fix first)
413 16
        return array('-v', "${deviceDir}:${clonePath}");
414
    }
415
416
    /**
417
     * get host path from mount point if in pip level 2+
418
     *
419
     * @param mixed $mountPoint
420
     *
421
     * @return null|string
422
     */
423
    private function pipHostConfigBind($mountPoint)
424
    {
425
        // if there is a parent name, this is level 2+
426 26
        if (null === $pipName = $this->runner->getEnv()->getValue('PIPELINES_PIP_CONTAINER_NAME')) {
427 23
            return null;
428
        }
429
430 3
        return Docker::create($this->runner->getExec())->hostConfigBind($pipName, $mountPoint);
431
    }
432
}
433