StepRunner::deployStepCaches() - Code Metrics - Inspection of "... caches: functional in runner (pull tests)" - ktomk/pipelines - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — test ( bc2363...ce9d4d )

by Tom

created 2020-07-19 19:51 UTC

StepRunner::deployStepCaches() A

↳ Parent: StepRunner

Complexity

Conditions	5
Paths	4

Size

Total Lines	35
Code Lines	21

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	20
CRAP Score	5

Importance

Changes

Metric	Value
cc	5
eloc	21
c	0
b	0
f	0
nc	4
nop	2
dl	0
loc	35
ccs	20
cts	20
cp	1
crap	5
rs	9.2728

<?php

/* this file is part of pipelines */

namespace Ktomk\Pipelines\Runner;

use Ktomk\Pipelines\Cli\Docker;
use Ktomk\Pipelines\DestructibleString;
use Ktomk\Pipelines\File\Pipeline\Step;
use Ktomk\Pipelines\Lib;
use Ktomk\Pipelines\LibFs;
use Ktomk\Pipelines\LibFsPath;
use Ktomk\Pipelines\LibTmp;
use Ktomk\Pipelines\Runner\Containers\StepContainer;
use Ktomk\Pipelines\Runner\Docker\ArgsBuilder;
use Ktomk\Pipelines\Runner\Docker\ArtifactSource;
use Ktomk\Pipelines\Runner\Docker\Binary\Repository;
use Ktomk\Pipelines\Runner\Docker\ImageLogin;

/**
 * Runner for a single step of a pipeline
 */
class StepRunner
{
    /**
     * list of temporary directory destructible markers
     *
     * @var array
     */
    private $temporaryDirectories = array();

    /**
     * @var Runner
     */
    private $runner;

    /**
     * DockerSession constructor.
     *
     * @param Runner $runner
     */
    public function __construct(Runner $runner)
    {
        $this->runner = $runner;
    }

    /**
     * @param Step $step
     *
     * @return null|int exist status of step script or null if the run operation failed
     */
    public function runStep(Step $step)
    {
        $dir = $this->runner->getDirectories()->getProjectDirectory();
        $env = $this->runner->getEnv();
        $streams = $this->runner->getStreams();

        $containers = new Containers($this->runner);

        $env->setPipelinesProjectPath($dir);

        $container = $containers->createStepContainer($step);

        $env->setContainerName($container->getName());

        $image = $step->getImage();

        # launch container
        $streams->out(sprintf(
            "\x1D+++ step #%d\n\n    name...........: %s\n    effective-image: %s\n    container......: %s\n",
            $step->getIndex() + 1,
            $step->getName() ? '"' . $step->getName() . '"' : '(unnamed)',
            $image->getName(),
            $container->getName()
        ));

        $id = $container->keepOrKill();

        $deployCopy = $this->runner->getFlags()->deployCopy();

        if (null === $id) {
            list($id, $status, $out, $err) = $this->runNewContainer($container, $dir, $deployCopy, $step);
            if (null === $id) {
                $streams->out("    container-id...: *failure*\n\n");
                $streams->err("pipelines: setting up the container failed\n");
                empty($err) || $streams->err("${err}\n");
                empty($out) || $streams->out("${out}\n");
                $streams->out(sprintf("exit status: %d\n", $status));

                return $status;
            }
        }

        $streams->out(sprintf("    container-id...: %s\n\n", substr($id, 0, 12)));

        # TODO: different deployments, mount (default), mount-ro, copy
        if (null !== $result = $this->deployCopy($deployCopy, $id, $dir)) {
            $streams->err('pipelines: deploy copy failure\n');

            return $result;
        }

        $deployCopy && $streams('');

        $this->deployStepCaches($step, $id);

        $status = StepScriptRunner::createRunStepScript($this->runner, $container->getName(), $step);

        $this->captureStepCaches($step, 0 === $status, $id);

        $this->captureStepArtifacts($step, $deployCopy && 0 === $status, $id, $dir);

        $container->shutdown($status);

        return $status;
    }

    /**
     * method to wrap new to have a test-point
     *
     * @return Repository
     */
    public function getDockerBinaryRepository()
    {
        $repo = Repository::create($this->runner->getExec(), $this->runner->getDirectories());
        $repo->resolve($this->runner->getRunOpts()->getBinaryPackage());

        return $repo;
    }

    /**
     * @param Step $step
     * @param string $id container to deploy caches into
     */
    private function deployStepCaches(Step $step, $id)
    {
        $cachesDirectory = $this->getCachesDirectory();

        // skip capturing caches if caches directory is empty
        if (empty($cachesDirectory)) {
            return;
        }

        $exec = $this->runner->getExec();
        $streams = $this->runner->getStreams();
        $streams->out("\x1D+++ populating caches...\n");

        foreach ($step->getCaches() as $name => $path) {
            $tarFile = sprintf('%s/%s.tar', $cachesDirectory, $name);
            $tarExists = LibFs::isReadableFile($tarFile);
            $streams->out(sprintf(" - %s %s (%s)\n", $name, $path, $tarExists ? 'hit' : 'miss'));

            if (!LibFs::isReadableFile($tarFile)) {
                continue;
            }

            $containerPath = $this->mapCachePath($id, $path);

            $exec->pass('docker', array('exec', $id, 'mkdir', '-p', $containerPath));

            $cmd = Lib::cmd(
                sprintf('<%s docker', Lib::quoteArg($tarFile)),
                array(
                    'cp',
                    '-',
                    sprintf('%s:%s', $id, $containerPath),
                )
            );
            $exec->pass($cmd, array());
        }
    }

    /**
     * Capture cache from container
     *
     * @param Step $step
     * @param $capture
     * @param string $id container to capture caches from
     */
    private function captureStepCaches(Step $step, $capture, $id)
    {
        // skip capturing if disabled
        if (!$capture) {
            return;
        }

        $cachesDirectory = $this->getCachesDirectory();

        // skip capturing caches if caches directory is empty
        if (empty($cachesDirectory)) {
            return;
        }

        $exec = $this->runner->getExec();
        $streams = $this->runner->getStreams();
        $streams->out("\x1D+++ updating caches from container...\n");

        $cachesDirectory = LibFs::mkDir($cachesDirectory);

        foreach ($step->getCaches() as $name => $path) {
            $tarFile = sprintf('%s/%s.tar', $cachesDirectory, $name);
            $tarExists = LibFs::isReadableFile($tarFile);
            $streams->out(sprintf(" - %s %s (%s)\n", $name, $path, $tarExists ? 'update' : 'create'));

            $containerPath = $this->mapCachePath($id, $path);

            $cmd = Lib::cmd(
                sprintf('>%s docker', Lib::quoteArg($tarFile)),
                array(
                    'cp',
                    sprintf('%s:%s/.', $id, $containerPath),
                    '-',
                )
            );
            $exec->pass($cmd, array());
        }
    }

    /**
     * @return string
     */
    private function getCachesDirectory()
    {
        return $this->runner->getDirectories()->getBaseDirectory(
            'XDG_CACHE_HOME',
            sprintf(
                'caches/%s',
                $this->runner->getProject()
            )
        );
    }

    /**
     * map cache path to container path
     *
     * @param string $id of container
     * @param string $path
     *
     * @return string
     */
    private function mapCachePath($id, $path)
    {
        $containerPath = $path;

        // let the container map the path expression
        $status = $this->runner->getExec()->capture(
            'docker',
            array('exec', $id, '/bin/sh', '-c', sprintf('echo %s', $containerPath)),
            $out
        );

        if (0 === $status && 0 < strlen($out) && ('/' === $out[0])) {
            $containerPath = trim($out);
        }

        // fallback to '~' -> /root assumption (can fail easy as root might not be the user)
        if ('~' === $containerPath[0]) {
            $containerPath = '/root/' . ltrim(substr($containerPath, 1), '/');
        }

        // handle relative paths
        if (!LibFsPath::isAbsolute($containerPath)) {
            $containerPath = LibFsPath::normalizeSegments(
                $this->runner->getRunOpts()->getOption('step.clone-path') . '/' . $containerPath
            );
        }

        return $containerPath;
    }

    /**
     * @param Step $step
     * @param bool $copy
     * @param string $id container id
     * @param string $dir to put artifacts in (project directory)
     *
     * @throws \RuntimeException
     *
     * @return void
     */
    private function captureStepArtifacts(Step $step, $copy, $id, $dir)
    {
        # capturing artifacts is only supported for deploy copy
        if (!$copy) {
            return;
        }

        $artifacts = $step->getArtifacts();

        if (null === $artifacts) {
            return;
        }

        $exec = $this->runner->getExec();
        $streams = $this->runner->getStreams();

        $streams->out("\x1D+++ copying artifacts from container...\n");

        $source = new ArtifactSource($exec, $id, $dir);

        $patterns = $artifacts->getPatterns();
        foreach ($patterns as $pattern) {
            $this->captureArtifactPattern($source, $pattern, $dir);
        }

        $streams('');
    }

    /**
     * capture artifact pattern
     *
     * @param ArtifactSource $source
     * @param string $pattern
     * @param string $dir
     *
     * @throws \RuntimeException
     *
     * @return void
     *
     * @see Runner::captureStepArtifacts()
     *
     */
    private function captureArtifactPattern(ArtifactSource $source, $pattern, $dir)
    {
        $exec = $this->runner->getExec();
        $streams = $this->runner->getStreams();

        $id = $source->getId();
        $paths = $source->findByPattern($pattern);
        if (empty($paths)) {
            return;
        }

        $chunks = Lib::arrayChunkByStringLength($paths, 131072, 4);

        $clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');

        foreach ($chunks as $paths) {
            $docker = Lib::cmd('docker', array('exec', '-w', $clonePath, $id));
            $tar = Lib::cmd('tar', array('c', '-f', '-', $paths));
            $unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir));

            $command = $docker . ' ' . $tar . ' | ' . $unTar;
            $status = $exec->pass($command, array());

            if (0 !== $status) {
                $streams->err(sprintf(
                    "pipelines: Artifact failure: '%s' (%d, %d paths, %d bytes)\n",
                    $pattern,
                    $status,
                    count($paths),
                    strlen($command)
                ));
            }
        }
    }

    /**
     * @param bool $copy
     * @param string $id container id
     * @param string $dir directory to copy contents into container
     *
     * @throws \RuntimeException
     *
     * @return null|int null if all clear, integer for exit status
     */
    private function deployCopy($copy, $id, $dir)
    {
        if (!$copy) {
            return null;
        }

        $streams = $this->runner->getStreams();
        $exec = $this->runner->getExec();

        $streams->out("\x1D+++ copying files into container...\n");

        $tmpDir = LibTmp::tmpDir('pipelines-cp.');
        $this->temporaryDirectories[] = DestructibleString::rmDir($tmpDir);

        $clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');

        LibFs::symlink($dir, $tmpDir . $clonePath);
        $cd = Lib::cmd('cd', array($tmpDir . '/.'));

        $tar = Lib::cmd('tar', array('c', '-h', '-f', '-', '--no-recursion', '.' . $clonePath));
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/.'));
        $status = $exec->pass("${cd} && echo 'app' | ${tar} | ${dockerCp}", array());
        LibFs::unlink($tmpDir . $clonePath);
        if (0 !== $status) {
            return $status;
        }

        $cd = Lib::cmd('cd', array($dir . '/.'));
        $tar = Lib::cmd('tar', array('c', '-f', '-', '.'));
        $dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':' . $clonePath));
        $status = $exec->pass("${cd} && ${tar} | ${dockerCp}", array());
        if (0 !== $status) {
            return $status;
        }

        return null;
    }

    /**
     * @param StepContainer $container
     * @param string $dir project directory (host)
     * @param bool $copy deployment
     * @param Step $step
     *
     * @return array array(string|null $id, int $status, string $out, string $err)
     */
    private function runNewContainer(StepContainer $container, $dir, $copy, Step $step)
    {
        $env = $this->runner->getEnv();

        $mountDockerSock = $this->obtainDockerSocketMount();

        $mountDockerClient = $this->obtainDockerClientMount($step);

        $mountWorkingDirectory = $this->obtainWorkingDirMount($copy, $dir, $mountDockerSock);
        if ($mountWorkingDirectory && is_int($mountWorkingDirectory[1])) {
            return $mountWorkingDirectory + array(2 => '', 3 => '');
        }

        $network = $container->getServiceContainers()->obtainNetwork();

        # process docker login if image demands so, but continue on failure
        $image = $step->getImage();
        ImageLogin::loginImage($this->runner->getExec(), $this->runner->getEnv()->getResolver(), null, $image);

        $clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');

        list($status, $out, $err) = $container->run(
            array(
                $network,
                '-i', '--name', $container->getName(),
                $container->obtainLabelOptions(),
                $env->getArgs('-e'),
                ArgsBuilder::optMap('-e', $step->getEnv(), true),
                $mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app',
                $mountDockerSock,
                $mountDockerClient,
                $container->obtainUserOptions(),
                $container->obtainSshOptions(),
                '--workdir', $clonePath, '--detach', '--entrypoint=/bin/sh',
                $image->getName(),
            )
        );
        $id = $status ? null : $container->getDisplayId();

        return array($id, $status, $out, $err);
    }

    /**
     * @param Step $step
     *
     * @return string[]
     */
    private function obtainDockerClientMount(Step $step)
    {
        if (!$step->getServices()->has('docker')) {
            return array();
        }

        $path = $this->runner->getRunOpts()->getOption('docker.client.path');

        // prefer pip mount over package
        $hostPath = $this->pipHostConfigBind($path);
        if (null !== $hostPath) {
            return array('-v', sprintf('%s:%s:ro', $hostPath, $path));
        }

        $local = $this->getDockerBinaryRepository()->getBinaryPath();
        chmod($local, 0755);

        return array('-v', sprintf('%s:%s:ro', $local, $path));
    }

    /**
     * enable docker client inside docker by mounting docker socket
     *
     * @return array docker socket volume args for docker run, empty if not mounting
     */
    private function obtainDockerSocketMount()
    {
        $args = array();

        // FIXME give more controlling options, this is serious /!\
        if (!$this->runner->getFlags()->useDockerSocket()) {
            return $args;
        }

        $defaultSocketPath = $this->runner->getRunOpts()->getOption('docker.socket.path');
        $hostPathDockerSocket = $defaultSocketPath;

        // pipelines inside pipelines
        $hostPath = $this->pipHostConfigBind($defaultSocketPath);
        if (null !== $hostPath) {
            return array(
                '-v', sprintf('%s:%s', $hostPath, $defaultSocketPath),
            );
        }

        $dockerHost = $this->runner->getEnv()->getInheritValue('DOCKER_HOST');
        if (null !== $dockerHost && 0 === strpos($dockerHost, 'unix://')) {
            $hostPathDockerSocket = LibFsPath::normalize(substr($dockerHost, 7));
        }

        $pathDockerSock = $defaultSocketPath;

        if (file_exists($hostPathDockerSocket)) {
            $args = array(
                '-v', sprintf('%s:%s', $hostPathDockerSocket, $pathDockerSock),
            );
        }

        return $args;
    }

    /**
     * @param bool $copy deployment
     * @param string $dir project directory
     * @param array $mountDockerSock docker socket volume args for docker run, empty if not mounting
     *
     * @return array mount options or array(null, int $status) for error handling
     */
    private function obtainWorkingDirMount($copy, $dir, array $mountDockerSock)
    {
        if ($copy) {
            return array();
        }

        $parentName = $this->runner->getEnv()->getValue('PIPELINES_PARENT_CONTAINER_NAME');
        $hostDeviceDir = $this->pipHostConfigBind($dir);
        $checkMount = $mountDockerSock && null !== $parentName;
        $deviceDir = $hostDeviceDir ?: $dir;
        $clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
        if ($checkMount && $clonePath === $dir && null === $hostDeviceDir) {
            $deviceDir = $this->runner->getEnv()->getValue('PIPELINES_PROJECT_PATH');
            if ($deviceDir === $dir || null === $deviceDir) {
                $this->runner->getStreams()->err("pipelines: fatal: can not detect ${dir} mount point\n");

                return array(null, 1);
            }
        }

        // FIXME(tk): Never mount anything not matching /home/[a-zA-Z][a-zA-Z0-9]*/[^.].*/...
        //   + do realpath checking
        //   + prevent dot path injections (logical fix first)
        return array('-v', "${deviceDir}:${clonePath}");
    }

    /**
     * get host path from mount point if in pip level 2+
     *
     * @param mixed $mountPoint
     *
     * @return null|string
     */
    private function pipHostConfigBind($mountPoint)
    {
        // if there is a parent name, this is level 2+
        if (null === $pipName = $this->runner->getEnv()->getValue('PIPELINES_PIP_CONTAINER_NAME')) {
            return null;
        }

        return Docker::create($this->runner->getExec())->hostConfigBind($pipName, $mountPoint);
    }
}


1		<?php
2
3		/* this file is part of pipelines */
4
5		namespace Ktomk\Pipelines\Runner;
6
7		use Ktomk\Pipelines\Cli\Docker;
8		use Ktomk\Pipelines\DestructibleString;
9		use Ktomk\Pipelines\File\Pipeline\Step;
10		use Ktomk\Pipelines\Lib;
11		use Ktomk\Pipelines\LibFs;
12		use Ktomk\Pipelines\LibFsPath;
13		use Ktomk\Pipelines\LibTmp;
14		use Ktomk\Pipelines\Runner\Containers\StepContainer;
15		use Ktomk\Pipelines\Runner\Docker\ArgsBuilder;
16		use Ktomk\Pipelines\Runner\Docker\ArtifactSource;
17		use Ktomk\Pipelines\Runner\Docker\Binary\Repository;
18		use Ktomk\Pipelines\Runner\Docker\ImageLogin;
19
20		/**
21		* Runner for a single step of a pipeline
22		*/
23		class StepRunner
24		{
25		/**
26		* list of temporary directory destructible markers
27		*
28		* @var array
29		*/
30		private $temporaryDirectories = array();
31
32		/**
33		* @var Runner
34		*/
35		private $runner;
36
37		/**
38		* DockerSession constructor.
39		*
40		* @param Runner $runner
41		*/
42	30	public function __construct(Runner $runner)
43		{
44	30	$this->runner = $runner;
45	30	}
46
47		/**
48		* @param Step $step
49		*
50		* @return null\|int exist status of step script or null if the run operation failed
51		*/
52	28	public function runStep(Step $step)
53		{
54	28	$dir = $this->runner->getDirectories()->getProjectDirectory();
55	28	$env = $this->runner->getEnv();
56	28	$streams = $this->runner->getStreams();
57
58	28	$containers = new Containers($this->runner);
59
60	28	$env->setPipelinesProjectPath($dir);
61
62	28	$container = $containers->createStepContainer($step);
63
64	28	$env->setContainerName($container->getName());
65
66	28	$image = $step->getImage();
67
68		# launch container
69	28	$streams->out(sprintf(
70	28	"\x1D+++ step #%d\n\n name...........: %s\n effective-image: %s\n container......: %s\n",
71	28	$step->getIndex() + 1,
72	28	$step->getName() ? '"' . $step->getName() . '"' : '(unnamed)',
73	28	$image->getName(),
74	28	$container->getName()
75		));
76
77	28	$id = $container->keepOrKill();
78
79	28	$deployCopy = $this->runner->getFlags()->deployCopy();
80
81	28	if (null === $id) {
82	26	list($id, $status, $out, $err) = $this->runNewContainer($container, $dir, $deployCopy, $step);
83	25	if (null === $id) {
84	3	$streams->out(" container-id...: failure\n\n");
85	3	$streams->err("pipelines: setting up the container failed\n");
86	3	empty($err) \|\| $streams->err("${err}\n");
87	3	empty($out) \|\| $streams->out("${out}\n");
88	3	$streams->out(sprintf("exit status: %d\n", $status));
89
90	3	return $status;
91		}
92		}
93
94	24	$streams->out(sprintf(" container-id...: %s\n\n", substr($id, 0, 12)));
95
96		# TODO: different deployments, mount (default), mount-ro, copy
97	24	if (null !== $result = $this->deployCopy($deployCopy, $id, $dir)) {
98	2	$streams->err('pipelines: deploy copy failure\n');
99
100	2	return $result;
101		}
102
103	22	$deployCopy && $streams('');
104
105	22	$this->deployStepCaches($step, $id);
106
107	22	$status = StepScriptRunner::createRunStepScript($this->runner, $container->getName(), $step);
108
109	22	$this->captureStepCaches($step, 0 === $status, $id);
110
111	22	$this->captureStepArtifacts($step, $deployCopy && 0 === $status, $id, $dir);
112
113	22	$container->shutdown($status);
114
115	22	return $status;
116		}
117
118		/**
119		* method to wrap new to have a test-point
120		*
121		* @return Repository
122		*/
123		public function getDockerBinaryRepository()
124		{
125	2	$repo = Repository::create($this->runner->getExec(), $this->runner->getDirectories());
126	2	$repo->resolve($this->runner->getRunOpts()->getBinaryPackage());
127
128	1	return $repo;
129		}
130
131		/**
132		* @param Step $step
133		* @param string $id container to deploy caches into
134		*/
135		private function deployStepCaches(Step $step, $id)
136		{
137	22	$cachesDirectory = $this->getCachesDirectory();
138
139		// skip capturing caches if caches directory is empty
140	22	if (empty($cachesDirectory)) {
141	1	return;
142		}
143
144	21	$exec = $this->runner->getExec();
145	21	$streams = $this->runner->getStreams();
146	21	$streams->out("\x1D+++ populating caches...\n");
147
148	21	foreach ($step->getCaches() as $name => $path) {
149	2	$tarFile = sprintf('%s/%s.tar', $cachesDirectory, $name);
150	2	$tarExists = LibFs::isReadableFile($tarFile);
151	2	$streams->out(sprintf(" - %s %s (%s)\n", $name, $path, $tarExists ? 'hit' : 'miss'));
152
153	2	if (!LibFs::isReadableFile($tarFile)) {
154	1	continue;
155		}
156
157	1	$containerPath = $this->mapCachePath($id, $path);
158
159	1	$exec->pass('docker', array('exec', $id, 'mkdir', '-p', $containerPath));
160
161	1	$cmd = Lib::cmd(
162	1	sprintf('<%s docker', Lib::quoteArg($tarFile)),
163		array(
164	1	'cp',
165	1	'-',
166	1	sprintf('%s:%s', $id, $containerPath),
167		)
168		);
169	1	$exec->pass($cmd, array());
170		}
171	21	}
172
173		/**
174		* Capture cache from container
175		*
176		* @param Step $step
177		* @param $capture
178		* @param string $id container to capture caches from
179		*/
180		private function captureStepCaches(Step $step, $capture, $id)
181		{
182		// skip capturing if disabled
183	22	if (!$capture) {
184	2	return;
185		}
186
187	20	$cachesDirectory = $this->getCachesDirectory();
188
189		// skip capturing caches if caches directory is empty
190	20	if (empty($cachesDirectory)) {
191	1	return;
192		}
193
194	19	$exec = $this->runner->getExec();
195	19	$streams = $this->runner->getStreams();
196	19	$streams->out("\x1D+++ updating caches from container...\n");
197
198	19	$cachesDirectory = LibFs::mkDir($cachesDirectory);
199
200	19	foreach ($step->getCaches() as $name => $path) {
201	2	$tarFile = sprintf('%s/%s.tar', $cachesDirectory, $name);
202	2	$tarExists = LibFs::isReadableFile($tarFile);
203	2	$streams->out(sprintf(" - %s %s (%s)\n", $name, $path, $tarExists ? 'update' : 'create'));
204
205	2	$containerPath = $this->mapCachePath($id, $path);
206
207	2	$cmd = Lib::cmd(
208	2	sprintf('>%s docker', Lib::quoteArg($tarFile)),
209		array(
210	2	'cp',
211	2	sprintf('%s:%s/.', $id, $containerPath),
212	2	'-',
213		)
214		);
215	2	$exec->pass($cmd, array());
216		}
217	19	}
218
219		/**
220		* @return string
221		*/
222		private function getCachesDirectory()
223		{
224	22	return $this->runner->getDirectories()->getBaseDirectory(
225	22	'XDG_CACHE_HOME',
226	22	sprintf(
227	22	'caches/%s',
228	22	$this->runner->getProject()
229		)
230		);
231		}
232
233		/**
234		* map cache path to container path
235		*
236		* @param string $id of container
237		* @param string $path
238		*
239		* @return string
240		*/
241		private function mapCachePath($id, $path)
242		{
243	2	$containerPath = $path;
244
245		// let the container map the path expression
246	2	$status = $this->runner->getExec()->capture(
247	2	'docker',
248	2	array('exec', $id, '/bin/sh', '-c', sprintf('echo %s', $containerPath)),
249		$out
250		);
251
252	2	if (0 === $status && 0 < strlen($out) && ('/' === $out[0])) {
253		$containerPath = trim($out);
254		}
255
256		// fallback to '~' -> /root assumption (can fail easy as root might not be the user)
257	2	if ('~' === $containerPath[0]) {
258	2	$containerPath = '/root/' . ltrim(substr($containerPath, 1), '/');
259		}
260
261		// handle relative paths
262	2	if (!LibFsPath::isAbsolute($containerPath)) {
263		$containerPath = LibFsPath::normalizeSegments(
264		$this->runner->getRunOpts()->getOption('step.clone-path') . '/' . $containerPath
265		);
266		}
267
268	2	return $containerPath;
269		}
270
271		/**
272		* @param Step $step
273		* @param bool $copy
274		* @param string $id container id
275		* @param string $dir to put artifacts in (project directory)
276		*
277		* @throws \RuntimeException
278		*
279		* @return void
280		*/
281		private function captureStepArtifacts(Step $step, $copy, $id, $dir)
282		{
283		# capturing artifacts is only supported for deploy copy
284	22	if (!$copy) {
285	17	return;
286		}
287
288	5	$artifacts = $step->getArtifacts();
289
290	5	if (null === $artifacts) {
291	2	return;
292		}
293
294	3	$exec = $this->runner->getExec();
295	3	$streams = $this->runner->getStreams();
296
297	3	$streams->out("\x1D+++ copying artifacts from container...\n");
298
299	3	$source = new ArtifactSource($exec, $id, $dir);
300
301	3	$patterns = $artifacts->getPatterns();
302	3	foreach ($patterns as $pattern) {
303	3	$this->captureArtifactPattern($source, $pattern, $dir);
304		}
305
306	3	$streams('');
307	3	}
308
309		/**
310		* capture artifact pattern
311		*
312		* @param ArtifactSource $source
313		* @param string $pattern
314		* @param string $dir
315		*
316		* @throws \RuntimeException
317		*
318		* @return void
319		*
320		* @see Runner::captureStepArtifacts()
321		*
322		*/
323		private function captureArtifactPattern(ArtifactSource $source, $pattern, $dir)
324		{
325	3	$exec = $this->runner->getExec();
326	3	$streams = $this->runner->getStreams();
327
328	3	$id = $source->getId();
329	3	$paths = $source->findByPattern($pattern);
330	3	if (empty($paths)) {
331	1	return;
332		}
333
334	2	$chunks = Lib::arrayChunkByStringLength($paths, 131072, 4);
335
336	2	$clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
337
338	2	foreach ($chunks as $paths) {
339	2	$docker = Lib::cmd('docker', array('exec', '-w', $clonePath, $id));
340	2	$tar = Lib::cmd('tar', array('c', '-f', '-', $paths));
341	2	$unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir));
342
343	2	$command = $docker . ' ' . $tar . ' \| ' . $unTar;
344	2	$status = $exec->pass($command, array());
345
346	2	if (0 !== $status) {
347	1	$streams->err(sprintf(
348	1	"pipelines: Artifact failure: '%s' (%d, %d paths, %d bytes)\n",
349		$pattern,
350		$status,
351	1	count($paths),
352	1	strlen($command)
353		));
354		}
355		}
356	2	}
357
358		/**
359		* @param bool $copy
360		* @param string $id container id
361		* @param string $dir directory to copy contents into container
362		*
363		* @throws \RuntimeException
364		*
365		* @return null\|int null if all clear, integer for exit status
366		*/
367		private function deployCopy($copy, $id, $dir)
368		{
369	24	if (!$copy) {
370	17	return null;
371		}
372
373	7	$streams = $this->runner->getStreams();
374	7	$exec = $this->runner->getExec();
375
376	7	$streams->out("\x1D+++ copying files into container...\n");
377
378	7	$tmpDir = LibTmp::tmpDir('pipelines-cp.');
379	7	$this->temporaryDirectories[] = DestructibleString::rmDir($tmpDir);
380
381	7	$clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
382
383	7	LibFs::symlink($dir, $tmpDir . $clonePath);
384	7	$cd = Lib::cmd('cd', array($tmpDir . '/.'));
385
386	7	$tar = Lib::cmd('tar', array('c', '-h', '-f', '-', '--no-recursion', '.' . $clonePath));
387	7	$dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':/.'));
388	7	$status = $exec->pass("${cd} && echo 'app' \| ${tar} \| ${dockerCp}", array());
389	7	LibFs::unlink($tmpDir . $clonePath);
390	7	if (0 !== $status) {
391	1	return $status;
392		}
393
394	6	$cd = Lib::cmd('cd', array($dir . '/.'));
395	6	$tar = Lib::cmd('tar', array('c', '-f', '-', '.'));
396	6	$dockerCp = Lib::cmd('docker ', array('cp', '-', $id . ':' . $clonePath));
397	6	$status = $exec->pass("${cd} && ${tar} \| ${dockerCp}", array());
398	6	if (0 !== $status) {
399	1	return $status;
400		}
401
402	5	return null;
403		}
404
405		/**
406		* @param StepContainer $container
407		* @param string $dir project directory (host)
408		* @param bool $copy deployment
409		* @param Step $step
410		*
411		* @return array array(string\|null $id, int $status, string $out, string $err)
412		*/
413		private function runNewContainer(StepContainer $container, $dir, $copy, Step $step)
414		{
415	26	$env = $this->runner->getEnv();
416
417	26	$mountDockerSock = $this->obtainDockerSocketMount();
418
419	26	$mountDockerClient = $this->obtainDockerClientMount($step);
420
421	25	$mountWorkingDirectory = $this->obtainWorkingDirMount($copy, $dir, $mountDockerSock);
422	25	if ($mountWorkingDirectory && is_int($mountWorkingDirectory[1])) {
423	2	return $mountWorkingDirectory + array(2 => '', 3 => '');
424		}
425
426	23	$network = $container->getServiceContainers()->obtainNetwork();
427
428		# process docker login if image demands so, but continue on failure
429	23	$image = $step->getImage();
430	23	ImageLogin::loginImage($this->runner->getExec(), $this->runner->getEnv()->getResolver(), null, $image);
431
432	23	$clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
433
434	23	list($status, $out, $err) = $container->run(
435		array(
436	23	$network,
437	23	'-i', '--name', $container->getName(),
438	23	$container->obtainLabelOptions(),
439	23	$env->getArgs('-e'),
440	23	ArgsBuilder::optMap('-e', $step->getEnv(), true),
441	23	$mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app',
442	23	$mountDockerSock,
443	23	$mountDockerClient,
444	23	$container->obtainUserOptions(),
445	23	$container->obtainSshOptions(),
446	23	'--workdir', $clonePath, '--detach', '--entrypoint=/bin/sh',
447	23	$image->getName(),
448		)
449		);
450	23	$id = $status ? null : $container->getDisplayId();
451
452	23	return array($id, $status, $out, $err);
453		}
454
455		/**
456		* @param Step $step
457		*
458		* @return string[]
459		*/
460		private function obtainDockerClientMount(Step $step)
461		{
462	26	if (!$step->getServices()->has('docker')) {
463	23	return array();
464		}
465
466	3	$path = $this->runner->getRunOpts()->getOption('docker.client.path');
467
468		// prefer pip mount over package
469	3	$hostPath = $this->pipHostConfigBind($path);
470	3	if (null !== $hostPath) {
471	1	return array('-v', sprintf('%s:%s:ro', $hostPath, $path));
472		}
473
474	2	$local = $this->getDockerBinaryRepository()->getBinaryPath();
475	1	chmod($local, 0755);
476
477	1	return array('-v', sprintf('%s:%s:ro', $local, $path));
478		}
479
480		/**
481		* enable docker client inside docker by mounting docker socket
482		*
483		* @return array docker socket volume args for docker run, empty if not mounting
484		*/
485		private function obtainDockerSocketMount()
486		{
487	26	$args = array();
488
489		// FIXME give more controlling options, this is serious /!\
490	26	if (!$this->runner->getFlags()->useDockerSocket()) {
491	1	return $args;
492		}
493
494	25	$defaultSocketPath = $this->runner->getRunOpts()->getOption('docker.socket.path');
495	25	$hostPathDockerSocket = $defaultSocketPath;
496
497		// pipelines inside pipelines
498	25	$hostPath = $this->pipHostConfigBind($defaultSocketPath);
499	25	if (null !== $hostPath) {
500		return array(
501	1	'-v', sprintf('%s:%s', $hostPath, $defaultSocketPath),
502		);
503		}
504
505	24	$dockerHost = $this->runner->getEnv()->getInheritValue('DOCKER_HOST');
506	24	if (null !== $dockerHost && 0 === strpos($dockerHost, 'unix://')) {
507	1	$hostPathDockerSocket = LibFsPath::normalize(substr($dockerHost, 7));
508		}
509
510	24	$pathDockerSock = $defaultSocketPath;
511
512	24	if (file_exists($hostPathDockerSocket)) {
513		$args = array(
514	21	'-v', sprintf('%s:%s', $hostPathDockerSocket, $pathDockerSock),
515		);
516		}
517
518	24	return $args;
519		}
520
521		/**
522		* @param bool $copy deployment
523		* @param string $dir project directory
524		* @param array $mountDockerSock docker socket volume args for docker run, empty if not mounting
525		*
526		* @return array mount options or array(null, int $status) for error handling
527		*/
528		private function obtainWorkingDirMount($copy, $dir, array $mountDockerSock)
529		{
530	25	if ($copy) {
531	7	return array();
532		}
533
534	18	$parentName = $this->runner->getEnv()->getValue('PIPELINES_PARENT_CONTAINER_NAME');
535	18	$hostDeviceDir = $this->pipHostConfigBind($dir);
536	18	$checkMount = $mountDockerSock && null !== $parentName;
537	18	$deviceDir = $hostDeviceDir ?: $dir;
538	18	$clonePath = $this->runner->getRunOpts()->getOption('step.clone-path');
539	18	if ($checkMount && $clonePath === $dir && null === $hostDeviceDir) {
540	2	$deviceDir = $this->runner->getEnv()->getValue('PIPELINES_PROJECT_PATH');
541	2	if ($deviceDir === $dir \|\| null === $deviceDir) {
542	2	$this->runner->getStreams()->err("pipelines: fatal: can not detect ${dir} mount point\n");
543
544	2	return array(null, 1);
545		}
546		}
547
548		// FIXME(tk): Never mount anything not matching /home/[a-zA-Z][a-zA-Z0-9]/[^.]./...
549		// + do realpath checking
550		// + prevent dot path injections (logical fix first)
551	16	return array('-v', "${deviceDir}:${clonePath}");
552		}
553
554		/**
555		* get host path from mount point if in pip level 2+
556		*
557		* @param mixed $mountPoint
558		*
559		* @return null\|string
560		*/
561		private function pipHostConfigBind($mountPoint)
562		{
563		// if there is a parent name, this is level 2+
564	26	if (null === $pipName = $this->runner->getEnv()->getValue('PIPELINES_PIP_CONTAINER_NAME')) {
565	23	return null;
566		}
567
568	3	return Docker::create($this->runner->getExec())->hostConfigBind($pipName, $mountPoint);
569		}
570		}
571

ktomk / pipelines

Push — test ( bc2363...ce9d4d )

StepRunner::deployStepCaches() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like