Passed
Push — test ( c65f5a...b51793 )
by Tom
07:13
created

StepRunner::obtainServicesNetwork()   A

Complexity

Conditions 2
Paths 2

Size

Total Lines 17
Code Lines 10

Duplication

Lines 0
Ratio 0 %

Code Coverage

Tests 10
CRAP Score 2

Importance

Changes 0
Metric Value
cc 2
eloc 10
nc 2
nop 1
dl 0
loc 17
ccs 10
cts 10
cp 1
crap 2
rs 9.9332
c 0
b 0
f 0
1
<?php
2
3
/* this file is part of pipelines */
4
5
namespace Ktomk\Pipelines\Runner;
6
7
use Ktomk\Pipelines\Cli\Docker;
8
use Ktomk\Pipelines\DestructibleString;
9
use Ktomk\Pipelines\File\Pipeline\Step;
10
use Ktomk\Pipelines\Lib;
11
use Ktomk\Pipelines\LibFs;
12
use Ktomk\Pipelines\LibFsPath;
13
use Ktomk\Pipelines\LibTmp;
14
use Ktomk\Pipelines\Runner\Containers\NameBuilder;
15
use Ktomk\Pipelines\Runner\Containers\StepContainer;
16
use Ktomk\Pipelines\Runner\Docker\ArtifactSource;
17
use Ktomk\Pipelines\Runner\Docker\Binary\Repository;
18
use Ktomk\Pipelines\Runner\Docker\ImageLogin;
19
20
/**
21
 * Runner for a single step of a pipeline
22
 */
23
class StepRunner
24
{
25
    /**
26
     * list of temporary directory destructible markers
27
     *
28
     * @var array
29
     */
30
    private $temporaryDirectories = array();
31
32
    /**
33
     * @var Runner
34
     */
35
    private $runner;
36
37
    /**
38
     * DockerSession constructor.
39
     *
40
     * @param Runner $runner
41
     */
42 27
    public function __construct(Runner $runner)
43
    {
44 27
        $this->runner = $runner;
45 27
    }
46
47
    /**
48
     * @param Step $step
49
     *
50
     * @return null|int exist status of step script or null if the run operation failed
51
     */
52 25
    public function runStep(Step $step)
53
    {
54 25
        $dir = $this->runner->getDirectories()->getProjectDirectory();
55 25
        $env = $this->runner->getEnv();
56 25
        $exec = $this->runner->getExec();
57 25
        $streams = $this->runner->getStreams();
58
59 25
        $containers = new Containers($this->runner);
60
61 25
        $env->setPipelinesProjectPath($dir);
62
63 25
        $container = $containers->createStepContainer($step);
64
65 25
        $env->setContainerName($container->getName());
66
67 25
        $image = $step->getImage();
68
69
        # launch container
70 25
        $streams->out(sprintf(
71 25
            "\x1D+++ step #%d\n\n    name...........: %s\n    effective-image: %s\n    container......: %s\n",
72 25
            $step->getIndex() + 1,
73 25
            $step->getName() ? '"' . $step->getName() . '"' : '(unnamed)',
74 25
            $image->getName(),
75 25
            $container->getName()
76
        ));
77
78 25
        $id = $container->keepOrKill();
79
80 25
        $deployCopy = $this->runner->getFlags()->deployCopy();
81
82 25
        if (null === $id) {
83 23
            list($id, $status) = $this->runNewContainer($container, $dir, $deployCopy, $step);
84 22
            if (null === $id) {
85 3
                return $status;
86
            }
87
        }
88
89 21
        $streams->out(sprintf("    container-id...: %s\n\n", substr($id, 0, 12)));
90
91
        # TODO: different deployments, mount (default), mount-ro, copy
92 21
        if (null !== $result = $this->deployCopy($deployCopy, $id, $dir)) {
93 2
            return $result;
94
        }
95
96 19
        $status = StepScriptRunner::createRunStepScript($step, $streams, $exec, $container->getName());
97
98 19
        $this->captureStepArtifacts($step, $deployCopy && 0 === $status, $id, $dir);
99
100 19
        $container->shutdown($status);
101
102 19
        return $status;
103
    }
104
105
    /**
106
     * method to wrap new to have a test-point
107
     *
108
     * @return Repository
109
     */
110 2
    public function getDockerBinaryRepository()
111
    {
112 2
        $repo = Repository::create($this->runner->getExec(), $this->runner->getDirectories());
113 2
        $repo->resolve($this->runner->getRunOpts()->getBinaryPackage());
114
115 1
        return $repo;
116
    }
117
118
    /**
119
     * @param Step $step
120
     * @param bool $copy
121
     * @param string $id container id
122
     * @param string $dir to put artifacts in (project directory)
123
     *
124
     * @throws \RuntimeException
125
     *
126
     * @return void
127
     */
128 19
    private function captureStepArtifacts(Step $step, $copy, $id, $dir)
129
    {
130
        # capturing artifacts is only supported for deploy copy
131 19
        if (!$copy) {
132 14
            return;
133
        }
134
135 5
        $artifacts = $step->getArtifacts();
136
137 5
        if (null === $artifacts) {
138 2
            return;
139
        }
140
141 3
        $exec = $this->runner->getExec();
142 3
        $streams = $this->runner->getStreams();
143
144 3
        $streams->out("\x1D+++ copying artifacts from container...\n");
145
146 3
        $source = new ArtifactSource($exec, $id, $dir);
147
148 3
        $patterns = $artifacts->getPatterns();
149 3
        foreach ($patterns as $pattern) {
150 3
            $this->captureArtifactPattern($source, $pattern, $dir);
151
        }
152
153 3
        $streams('');
154 3
    }
155
156
    /**
157
     * capture artifact pattern
158
     *
159
     * @param ArtifactSource $source
160
     * @param string $pattern
161
     * @param string $dir
162
     *
163
     * @throws \RuntimeException
164
     *
165
     * @return void
166
     *
167
     * @see Runner::captureStepArtifacts()
168
     *
169
     */
170 3
    private function captureArtifactPattern(ArtifactSource $source, $pattern, $dir)
171
    {
172 3
        $exec = $this->runner->getExec();
173 3
        $streams = $this->runner->getStreams();
174
175 3
        $id = $source->getId();
176 3
        $paths = $source->findByPattern($pattern);
177 3
        if (empty($paths)) {
178 1
            return;
179
        }
180
181 2
        $chunks = Lib::arrayChunkByStringLength($paths, 131072, 4);
182
183 2
        foreach ($chunks as $paths) {
184 2
            $docker = Lib::cmd('docker', array('exec', '-w', '/app', $id));
185 2
            $tar = Lib::cmd('tar', array('c', '-f', '-', $paths));
186 2
            $unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir));
187
188 2
            $command = $docker . ' ' . $tar . ' | ' . $unTar;
189 2
            $status = $exec->pass($command, array());
190
191 2
            if (0 !== $status) {
192 1
                $streams->err(sprintf(
193 1
                    "pipelines: Artifact failure: '%s' (%d, %d paths, %d bytes)\n",
194
                    $pattern,
195
                    $status,
196 1
                    count($paths),
197 1
                    strlen($command)
198
                ));
199
            }
200
        }
201 2
    }
202
203
    /**
204
     * @param bool $copy
205
     * @param string $id container id
206
     * @param string $dir directory to copy contents into container
207
     *
208
     * @throws \RuntimeException
209
     *
210
     * @return null|int null if all clear, integer for exit status
211
     */
212 21
    private function deployCopy($copy, $id, $dir)
213
    {
214 21
        if (!$copy) {
215 14
            return null;
216
        }
217
218 7
        $streams = $this->runner->getStreams();
219 7
        $exec = $this->runner->getExec();
220
221 7
        $streams->out("\x1D+++ copying files into container...\n");
222
223 7
        $tmpDir = LibTmp::tmpDir('pipelines-cp.');
224 7
        $this->temporaryDirectories[] = DestructibleString::rmDir($tmpDir);
225 7
        LibFs::symlink($dir, $tmpDir . '/app');
226 7
        $cd = Lib::cmd('cd', array($tmpDir . '/.'));
227 7
        $tar = Lib::cmd('tar', array('c', '-h', '-f', '-', '--no-recursion', 'app'));
228 7
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/.'));
229
        $status = $exec->pass("${cd} && echo 'app' | ${tar} | ${dockerCp}", array());
230
        LibFs::unlink($tmpDir . '/app');
231 7
        if (0 !== $status) {
232 1
            $streams->err('pipelines: deploy copy failure\n');
233
234 1
            return $status;
235
        }
236
237 6
        $cd = Lib::cmd('cd', array($dir . '/.'));
238 6
        $tar = Lib::cmd('tar', array('c', '-f', '-', '.'));
239 6
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/app'));
240 6
        $status = $exec->pass("${cd} && ${tar} | ${dockerCp}", array());
241 6
        if (0 !== $status) {
242 1
            $streams->err('pipelines: deploy copy failure\n');
243
244 1
            return $status;
245
        }
246
247 5
        $streams('');
248
249 5
        return null;
250
    }
251
252
    /**
253
     * @param StepContainer $container
254
     * @param string $dir
255
     * @param bool $copy
256
     * @param Step $step
257
     *
258
     * @return array array(string|null $id, int $status)
259
     */
260
    private function runNewContainer(StepContainer $container, $dir, $copy, Step $step)
261
    {
262 23
        $env = $this->runner->getEnv();
263 23
        $streams = $this->runner->getStreams();
264
265 23
        $mountDockerSock = $this->obtainDockerSocketMount();
266
267 23
        $mountDockerClient = $this->obtainDockerClientMount($step);
268
269 22
        $mountWorkingDirectory = $this->obtainWorkingDirMount($copy, $dir, $mountDockerSock);
270 22
        if ($mountWorkingDirectory && is_int($mountWorkingDirectory[1])) {
271 2
            return $mountWorkingDirectory;
272
        }
273
274 20
        $network = $container->getServiceContainers()->obtainNetwork();
275
276
        # process docker login if image demands so, but continue on failure
277 20
        $image = $step->getImage();
278 20
        ImageLogin::loginImage($this->runner->getExec(), $this->runner->getEnv()->getResolver(), null, $image);
279
280 20
        list($status, $out, $err) = $container->run(
281
            array(
282 20
                $network,
283 20
                '-i', '--name', $container->getName(),
284 20
                $env->getArgs('-e'),
285 20
                $env::createArgVarDefinitions('-e', $step->getEnv()),
286 20
                $mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app',
287 20
                $mountDockerSock,
288 20
                $mountDockerClient,
289 20
                '--workdir', '/app', '--detach', '--entrypoint=/bin/sh', $image->getName(),
290
            )
291
        );
292 20
        if (0 !== $status) {
293 1
            $streams->out("    container-id...: *failure*\n\n");
294 1
            $streams->err("pipelines: setting up the container failed\n");
295 1
            $streams->err("${err}\n");
296 1
            $streams->out("${out}\n");
297 1
            $streams->out(sprintf("exit status: %d\n", $status));
298
299 1
            return array(null, $status);
300
        }
301 19
        $id = $container->getDisplayId();
302
303 19
        return array($id, $status);
304
    }
305
306
    /**
307
     * @param Step $step
308
     *
309
     * @return string[]
310
     */
311
    private function obtainDockerClientMount(Step $step)
312
    {
313 23
        if (!$step->getServices()->has('docker')) {
314 20
            return array();
315
        }
316
317 3
        $path = $this->runner->getRunOpts()->getOption('docker.client.path');
318
319
        // prefer pip mount over package
320 3
        $hostPath = $this->pipHostConfigBind($path);
321 3
        if (null !== $hostPath) {
322 1
            return array('-v', sprintf('%s:%s:ro', $hostPath, $path));
323
        }
324
325 2
        $local = $this->getDockerBinaryRepository()->getBinaryPath();
326 1
        chmod($local, 0755);
327
328 1
        return array('-v', sprintf('%s:%s:ro', $local, $path));
329
    }
330
331
    /**
332
     * enable docker client inside docker by mounting docker socket
333
     *
334
     * @return array docker socket volume args for docker run, empty if not mounting
335
     */
336
    private function obtainDockerSocketMount()
337
    {
338 23
        $args = array();
339
340
        // FIXME give more controlling options, this is serious /!\
341 23
        if (!$this->runner->getFlags()->useDockerSocket()) {
342 1
            return $args;
343
        }
344
345 22
        $defaultSocketPath = $this->runner->getRunOpts()->getOption('docker.socket.path');
346 22
        $hostPathDockerSocket = $defaultSocketPath;
347
348
        // pipelines inside pipelines
349 22
        $hostPath = $this->pipHostConfigBind($defaultSocketPath);
350 22
        if (null !== $hostPath) {
351
            return array(
352 1
                '-v', sprintf('%s:%s', $hostPath, $defaultSocketPath),
353
            );
354
        }
355
356 21
        $dockerHost = $this->runner->getEnv()->getInheritValue('DOCKER_HOST');
357 21
        if (null !== $dockerHost && 0 === strpos($dockerHost, 'unix://')) {
358 1
            $hostPathDockerSocket = LibFsPath::normalize(substr($dockerHost, 7));
359
        }
360
361 21
        $pathDockerSock = $defaultSocketPath;
362
363 21
        if (file_exists($hostPathDockerSocket)) {
364
            $args = array(
365 18
                '-v', sprintf('%s:%s', $hostPathDockerSocket, $pathDockerSock),
366
            );
367
        }
368
369 21
        return $args;
370
    }
371
372
    /**
373
     * @param bool $copy
374
     * @param string $dir
375
     * @param array $mountDockerSock
376
     *
377
     * @return array mount options or array(null, int $status) for error handling
378
     */
379
    private function obtainWorkingDirMount($copy, $dir, array $mountDockerSock)
380
    {
381 22
        if ($copy) {
382 7
            return array();
383
        }
384
385 15
        $parentName = $this->runner->getEnv()->getValue('PIPELINES_PARENT_CONTAINER_NAME');
386 15
        $hostDeviceDir = $this->pipHostConfigBind($dir);
387 15
        $checkMount = $mountDockerSock && null !== $parentName;
388 15
        $deviceDir = $hostDeviceDir ?: $dir;
389 15
        if ($checkMount && '/app' === $dir && null === $hostDeviceDir) { // FIXME(tk): hard encoded /app
390 2
            $deviceDir = $this->runner->getEnv()->getValue('PIPELINES_PROJECT_PATH');
391 2
            if ($deviceDir === $dir || null === $deviceDir) {
392 2
                $this->runner->getStreams()->err("pipelines: fatal: can not detect ${dir} mount point. preventing new container.\n");
393
394 2
                return array(null, 1);
395
            }
396
        }
397
398
        // FIXME(tk): Never mount anything not matching /home/[a-zA-Z][a-zA-Z0-9]*/[^.].*/...
399
        //   + do realpath checking
400
        //   + prevent dot path injections (logical fix first)
401 13
        return array('-v', "${deviceDir}:/app"); // FIXME(tk): hard encoded /app
402
    }
403
404
    /**
405
     * get host path from mount point if in pip level 2+
406
     *
407
     * @param mixed $mountPoint
408
     *
409
     * @return null|string
410
     */
411
    private function pipHostConfigBind($mountPoint)
412
    {
413
        // if there is a parent name, this is level 2+
414 23
        if (null === $pipName = $this->runner->getEnv()->getValue('PIPELINES_PIP_CONTAINER_NAME')) {
415 20
            return null;
416
        }
417
418 3
        return Docker::create($this->runner->getExec())->hostConfigBind($pipName, $mountPoint);
419
    }
420
}
421