Runner::captureArtifactPattern() - Code Metrics - Inspection of "prepare release" - ktomk/pipelines - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 72fc34...cfd30c )

by Tom

created 2018-02-14 17:41 UTC

Runner::captureArtifactPattern() B

↳ Parent: Project

Complexity

Conditions	4
Paths	4

Size

Total Lines	23
Code Lines	15

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	16
CRAP Score	4

Importance

Changes

Metric	Value
cc	4
eloc	15
nc	4
nop	4
dl	0
loc	23
ccs	16
cts	16
cp	1
crap	4
rs	8.7972
c	0
b	0
f	0

<?php

/* this file is part of pipelines */

namespace Ktomk\Pipelines;

use Ktomk\Pipelines\Cli\Docker;
use Ktomk\Pipelines\Cli\Exec;
use Ktomk\Pipelines\Cli\Streams;
use Ktomk\Pipelines\File\Image;
use Ktomk\Pipelines\Runner\ArtifactSource;
use Ktomk\Pipelines\Runner\DockerLogin;
use Ktomk\Pipelines\Runner\Env;

/**
 * Pipeline runner with docker under the hood
 */
class Runner
{
    const FLAGS = 11;
    const FLAG_DOCKER_REMOVE = 1;
    const FLAG_DOCKER_KILL = 2;
    const FLAG_DEPLOY_COPY = 4; # copy working dir into container
    const FLAG_KEEP_ON_ERROR = 8;

    const STATUS_NO_STEPS = 1;
    const STATUS_RECURSION_DETECTED = 127;

    /**
     * @var string
     */
    private $prefix;

    /**
     * @var string
     */
    private $directory;

    /**
     * @var Exec
     */
    private $exec;

    /**
     * @var int
     */
    private $flags;

    /**
     * @var Env
     */
    private $env;
    /**
     * @var Streams
     */
    private $streams;

    /**
     * DockerSession constructor.
     *
     * @param string $prefix
     * @param string $directory source repository root
     * @param Exec $exec
     * @param int $flags [optional]
     * @param Env $env [optional]
     * @param Streams $streams [optional]
     */
    public function __construct(
        $prefix,
        $directory,
        Exec $exec,
        $flags = null,
        Env $env = null,
        Streams $streams = null
    )
    {
        $this->prefix = $prefix;

        $this->directory = $directory;
        $this->exec = $exec;
        $this->flags = $flags === null ? self::FLAGS : $flags;
        $this->env = null === $env ? Env::create() : $env;
        $this->streams = null === $streams ? Streams::create() : $streams;
    }

    public function run(Pipeline $pipeline)
    {
        $hasId = $this->env->setPipelinesId($pipeline->getId()); # TODO give Env an addPipeline() method (compare addReference)
        if ($hasId) {
            $this->streams->err(sprintf(
                "pipelines: won't start pipeline '%s'; pipeline inside pipelines recursion detected\n",
                $pipeline->getId()
            ));
            return self::STATUS_RECURSION_DETECTED;
        }

        $steps = $pipeline->getSteps();
        foreach ($steps as $index => $step) {
            $status = $this->runStep($step, $index);
            if ($status !== 0) {
                return $status;
            }
        }

        if (!isset($status)) {
            $this->streams->err("pipelines: pipeline with no step to execute\n");
            return self::STATUS_NO_STEPS;
        }

        return $status;
    }

    /**
     * @param Step $step
     * @param int $index
     * @return int exit status
     */
    public function runStep(Step $step, $index)
    {
        $prefix = $this->prefix;
        $dir = $this->directory;
        $env = $this->env;
        $exec = $this->exec;
        $streams = $this->streams;

        $docker = new Docker($exec);

        $name = $prefix . '-' . Lib::generateUuid();
        $image = $step->getImage();
        $env->setContainerName($name);

        # launch container
        $streams->out(sprintf(
            "\x1D+++ step #%d\n\n    name...........: %s\n    effective-image: %s\n    container......: %s\n",
            $index,
            $step->getName() ? '"' . $step->getName() . '"' : '(unnamed)',
            $image->getName(),
            $name
        ));

        # process docker login if image demands so, but continue on failure
        $this->imageLogin($image);

        $copy = (bool)($this->flags & self::FLAG_DEPLOY_COPY);

        // docker client inside docker
        // FIXME give controlling options, this is serious /!\
        $mountDockerSock = array();
        if (file_exists('/var/run/docker.sock')) {
            $mountDockerSock = array(
                '-v', '/var/run/docker.sock:/var/run/docker.sock',
            );
        }

        $parentName = $env->getValue('PIPELINES_PARENT_CONTAINER_NAME');
        $checkMount = $mountDockerSock && null !== $parentName;
        $deviceDir = $checkMount ? $docker->hostDevice($parentName, $dir) : $dir;

        $mountWorkingDirectory = $copy
            ? array()
            : array('--volume', "$deviceDir:/app"); // FIXME(tk): hard encoded /app

        $status = $exec->capture('docker', array(
            'run', '-i', '--name', $name,
            $env->getArgs('-e'),
            $mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app',
            $mountDockerSock,
            '--workdir', '/app', '--detach', $image->getName()
        ), $out, $err);
        if ($status !== 0) {
            $streams->out("    container-id...: *failure*\n\n");
            $streams->err("pipelines: setting up the container failed\n");
            $streams->err("$err\n");
            $streams->out("$out\n");
            $streams->out(sprintf("exit status: %d\n", $status));
            return $status;
        }
        $id = rtrim($out) ?: "*dry-run*"; # side-effect: internal exploit of no output with true exit status
        $streams->out(sprintf("    container-id...: %s\n\n", substr($id, 0, 12)));

        # TODO: different deployments, mount (default), mount-ro, copy
        if (null !== $result = $this->deployCopy($copy, $id, $dir)) {
            return $result;
        }

        $status = $this->runStepScript($step, $streams, $exec, $name);

        $this->captureStepArtifacts($step, $copy, $id, $dir);

        $this->shutdownStepContainer($status, $id, $exec, $name);

        return $status;
    }

    /**
     * @param Image $image
     */
    private function imageLogin(Image $image)
    {
        $login = new DockerLogin($this->exec, $this->env->getResolver());
        $login->byImage($image);
    }

    /**
     * @param bool $copy
     * @param string $id container id
     * @param string $dir directory to copy contents into container
     * @return int|null null if all clear, integer for exit status
     */
    private function deployCopy($copy, $id, $dir)
    {
        if (!$copy) {
            return null;
        }

        $streams = $this->streams;
        $exec = $this->exec;

        $streams->out("\x1D+++ copying files into container...\n");
        $cd = Lib::cmd('cd', array($dir . '/.'));
        $tar = Lib::cmd('tar', array(
                'c', '-f', '-', '.')
        );
        $dockerCp = Lib::cmd('docker ', array(
                'cp', '-', $id . ':/app')
        );
        $status = $exec->pass("$cd && $tar | $dockerCp", array());
        if ($status !== 0) {
            $streams->err('pipelines: deploy copy failure\n');
            return $status;
        }

        $streams("");

        return null;
    }

    /**
     * @param Step $step
     * @param Streams $streams
     * @param Exec $exec
     * @param string $name container name
     * @return int|null should never be null, status, non-zero if a command failed
     */
    private function runStepScript(Step $step, Streams $streams, Exec $exec, $name)
    {
        $script = $step->getScript();
        $status = null;

        foreach ($script as $line => $command) {
            $streams->out(sprintf("\x1D+ %s\n", $command));
            $status = $exec->pass('docker', array(
                'exec', '-i', $name, '/bin/sh', '-c', $command,
            ));
            $streams->out(sprintf("\n"));
            if ($status !== 0) {
                break;
            }
        }

        return $status;
    }

    /**
     * @param Step $step
     * @param bool $copy
     * @param string $id container id
     * @param string $dir to put artifacts in (project directory)
     */
    private function captureStepArtifacts(Step $step, $copy, $id, $dir)
    {
        # capturing artifacts is only supported for deploy copy
        if (!$copy) {
            return;
        }

        $artifacts = $step->getArtifacts();

        if (null === $artifacts) {
            return;
        }

        $exec = $this->exec;
        $streams = $this->streams;

        $streams->out("\x1D+++ copying artifacts from container...\n");

        $source = new ArtifactSource($exec, $id, $dir);

        $patterns = $artifacts->getPatterns();
        foreach ($patterns as $pattern) {
            $this->captureArtifactPattern($source, $pattern, $id, $dir);
        }

        $streams("");
    }

    /**
     * @see Runner::captureStepArtifacts()
     *
     * @param ArtifactSource $source
     * @param string $pattern
     * @param string $id
     * @param string $dir
     */
    private function captureArtifactPattern(ArtifactSource $source, $pattern, $id, $dir)
    {
        $exec = $this->exec;
        $streams = $this->streams;

        $paths = $source->findByPattern($pattern);
        if (empty($paths)) {
            return;
        }

        $chunkSize = 2048;
        $chunks = array_chunk($paths, $chunkSize, true);

        foreach ($chunks as $paths) {
            $tar = Lib::cmd('tar', array('c', '-f', '-', $paths));
            $docker = Lib::cmd('docker', array('exec', '-w', '/app', $id));
            $unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir));

            $status = $exec->pass($docker . ' ' . $tar . ' | ' . $unTar, array());

            if ($status !== 0) {
                $streams->err(
                    sprintf("pipelines: Artifact failure: '%s' (%d, %d paths)\n", $pattern, $status, count($paths))
                );
            }
        }
    }

    /**
     * @param int $status
     * @param string $id container id
     * @param Exec $exec
     * @param string $name container name
     */
    private function shutdownStepContainer($status, $id, Exec $exec, $name)
    {
        $flags = $this->flags;

        # keep container on error
        if (0 !== $status && $flags & self::FLAG_KEEP_ON_ERROR) {
            $this->streams->err(sprintf(
                "exit status %d, keeping container id %s\n",
                $status,
                substr($id, 0, 12)
            ));

            return;
        }

        # keep or remove container
        if ($flags & self::FLAG_DOCKER_KILL) {
            $exec->capture('docker', array('kill', $name));
        }

        if ($flags & self::FLAG_DOCKER_REMOVE) {
            $exec->capture('docker', array('rm', $name));
        }
    }
}


1		<?php
2
3		/* this file is part of pipelines */
4
5		namespace Ktomk\Pipelines;
6
7		use Ktomk\Pipelines\Cli\Docker;
8		use Ktomk\Pipelines\Cli\Exec;
9		use Ktomk\Pipelines\Cli\Streams;
10		use Ktomk\Pipelines\File\Image;
11		use Ktomk\Pipelines\Runner\ArtifactSource;
12		use Ktomk\Pipelines\Runner\DockerLogin;
13		use Ktomk\Pipelines\Runner\Env;
14
15		/**
16		* Pipeline runner with docker under the hood
17		*/
18		class Runner
19		{
20		const FLAGS = 11;
21		const FLAG_DOCKER_REMOVE = 1;
22		const FLAG_DOCKER_KILL = 2;
23		const FLAG_DEPLOY_COPY = 4; # copy working dir into container
24		const FLAG_KEEP_ON_ERROR = 8;
25
26		const STATUS_NO_STEPS = 1;
27		const STATUS_RECURSION_DETECTED = 127;
28
29		/**
30		* @var string
31		*/
32		private $prefix;
33
34		/**
35		* @var string
36		*/
37		private $directory;
38
39		/**
40		* @var Exec
41		*/
42		private $exec;
43
44		/**
45		* @var int
46		*/
47		private $flags;
48
49		/**
50		* @var Env
51		*/
52		private $env;
53		/**
54		* @var Streams
55		*/
56		private $streams;
57
58		/**
59		* DockerSession constructor.
60		*
61		* @param string $prefix
62		* @param string $directory source repository root
63		* @param Exec $exec
64		* @param int $flags [optional]
65		* @param Env $env [optional]
66		* @param Streams $streams [optional]
67		*/
68	11	public function __construct(
69		$prefix,
70		$directory,
71		Exec $exec,
72		$flags = null,
73		Env $env = null,
74		Streams $streams = null
75		)
76		{
77	11	$this->prefix = $prefix;
78
79	11	$this->directory = $directory;
80	11	$this->exec = $exec;
81	11	$this->flags = $flags === null ? self::FLAGS : $flags;
82	11	$this->env = null === $env ? Env::create() : $env;
83	11	$this->streams = null === $streams ? Streams::create() : $streams;
84	11	}
85
86	11	public function run(Pipeline $pipeline)
87		{
88	11	$hasId = $this->env->setPipelinesId($pipeline->getId()); # TODO give Env an addPipeline() method (compare addReference)
89	11	if ($hasId) {
90	1	$this->streams->err(sprintf(
91	1	"pipelines: won't start pipeline '%s'; pipeline inside pipelines recursion detected\n",
92	1	$pipeline->getId()
93		));
94	1	return self::STATUS_RECURSION_DETECTED;
95		}
96
97	10	$steps = $pipeline->getSteps();
98	10	foreach ($steps as $index => $step) {
99	9	$status = $this->runStep($step, $index);
100	9	if ($status !== 0) {
101	9	return $status;
102		}
103		}
104
105	7	if (!isset($status)) {
106	1	$this->streams->err("pipelines: pipeline with no step to execute\n");
107	1	return self::STATUS_NO_STEPS;
108		}
109
110	6	return $status;
111		}
112
113		/**
114		* @param Step $step
115		* @param int $index
116		* @return int exit status
117		*/
118	9	public function runStep(Step $step, $index)
119		{
120	9	$prefix = $this->prefix;
121	9	$dir = $this->directory;
122	9	$env = $this->env;
123	9	$exec = $this->exec;
124	9	$streams = $this->streams;
125
126	9	$docker = new Docker($exec);
127
128	9	$name = $prefix . '-' . Lib::generateUuid();
129	9	$image = $step->getImage();
130	9	$env->setContainerName($name);
131
132		# launch container
133	9	$streams->out(sprintf(
134	9	"\x1D+++ step #%d\n\n name...........: %s\n effective-image: %s\n container......: %s\n",
135	9	$index,
136	9	$step->getName() ? '"' . $step->getName() . '"' : '(unnamed)',
137	9	$image->getName(),
138	9	$name
139		));
140
141		# process docker login if image demands so, but continue on failure
142	9	$this->imageLogin($image);
143
144	9	$copy = (bool)($this->flags & self::FLAG_DEPLOY_COPY);
145
146		// docker client inside docker
147		// FIXME give controlling options, this is serious /!\
148	9	$mountDockerSock = array();
149	9	if (file_exists('/var/run/docker.sock')) {
150		$mountDockerSock = array(
151		'-v', '/var/run/docker.sock:/var/run/docker.sock',
152		);
153		}
154
155	9	$parentName = $env->getValue('PIPELINES_PARENT_CONTAINER_NAME');
156	9	$checkMount = $mountDockerSock && null !== $parentName;
157	9	$deviceDir = $checkMount ? $docker->hostDevice($parentName, $dir) : $dir;
158
159	9	$mountWorkingDirectory = $copy
160	5	? array()
161	9	: array('--volume', "$deviceDir:/app"); // FIXME(tk): hard encoded /app
162
163	9	$status = $exec->capture('docker', array(
164	9	'run', '-i', '--name', $name,
165	9	$env->getArgs('-e'),
166	9	$mountWorkingDirectory, '-e', 'BITBUCKET_CLONE_DIR=/app',
167	9	$mountDockerSock,
168	9	'--workdir', '/app', '--detach', $image->getName()
169	9	), $out, $err);
170	9	if ($status !== 0) {
171	1	$streams->out(" container-id...: failure\n\n");
172	1	$streams->err("pipelines: setting up the container failed\n");
173	1	$streams->err("$err\n");
174	1	$streams->out("$out\n");
175	1	$streams->out(sprintf("exit status: %d\n", $status));
176	1	return $status;
177		}
178	8	$id = rtrim($out) ?: "dry-run"; # side-effect: internal exploit of no output with true exit status
179	8	$streams->out(sprintf(" container-id...: %s\n\n", substr($id, 0, 12)));
180
181		# TODO: different deployments, mount (default), mount-ro, copy
182	8	if (null !== $result = $this->deployCopy($copy, $id, $dir)) {
183	1	return $result;
184		}
185
186	7	$status = $this->runStepScript($step, $streams, $exec, $name);
187
188	7	$this->captureStepArtifacts($step, $copy, $id, $dir);
189
190	7	$this->shutdownStepContainer($status, $id, $exec, $name);
191
192	7	return $status;
193		}
194
195		/**
196		* @param Image $image
197		*/
198	9	private function imageLogin(Image $image)
199		{
200	9	$login = new DockerLogin($this->exec, $this->env->getResolver());
201	9	$login->byImage($image);
202	9	}
203
204		/**
205		* @param bool $copy
206		* @param string $id container id
207		* @param string $dir directory to copy contents into container
208		* @return int\|null null if all clear, integer for exit status
209		*/
210	8	private function deployCopy($copy, $id, $dir)
211		{
212	8	if (!$copy) {
213	3	return null;
214		}
215
216	5	$streams = $this->streams;
217	5	$exec = $this->exec;
218
219	5	$streams->out("\x1D+++ copying files into container...\n");
220	5	$cd = Lib::cmd('cd', array($dir . '/.'));
221	5	$tar = Lib::cmd('tar', array(
222	5	'c', '-f', '-', '.')
223		);
224	5	$dockerCp = Lib::cmd('docker ', array(
225	5	'cp', '-', $id . ':/app')
226		);
227	5	$status = $exec->pass("$cd && $tar \| $dockerCp", array());
228	5	if ($status !== 0) {
229	1	$streams->err('pipelines: deploy copy failure\n');
230	1	return $status;
231		}
232
233	4	$streams("");
234
235	4	return null;
236		}
237
238		/**
239		* @param Step $step
240		* @param Streams $streams
241		* @param Exec $exec
242		* @param string $name container name
243		* @return int\|null should never be null, status, non-zero if a command failed
244		*/
245	7	private function runStepScript(Step $step, Streams $streams, Exec $exec, $name)
246		{
247	7	$script = $step->getScript();
248	7	$status = null;
249
250	7	foreach ($script as $line => $command) {
251	7	$streams->out(sprintf("\x1D+ %s\n", $command));
252	7	$status = $exec->pass('docker', array(
253	7	'exec', '-i', $name, '/bin/sh', '-c', $command,
254		));
255	7	$streams->out(sprintf("\n"));
256	7	if ($status !== 0) {
257	7	break;
258		}
259		}
260
261	7	return $status;
262		}
263
264		/**
265		* @param Step $step
266		* @param bool $copy
267		* @param string $id container id
268		* @param string $dir to put artifacts in (project directory)
269		*/
270	7	private function captureStepArtifacts(Step $step, $copy, $id, $dir)
271		{
272		# capturing artifacts is only supported for deploy copy
273	7	if (!$copy) {
274	3	return;
275		}
276
277	4	$artifacts = $step->getArtifacts();
278
279	4	if (null === $artifacts) {
280	1	return;
281		}
282
283	3	$exec = $this->exec;
284	3	$streams = $this->streams;
285
286	3	$streams->out("\x1D+++ copying artifacts from container...\n");
287
288	3	$source = new ArtifactSource($exec, $id, $dir);
289
290	3	$patterns = $artifacts->getPatterns();
291	3	foreach ($patterns as $pattern) {
292	3	$this->captureArtifactPattern($source, $pattern, $id, $dir);
293		}
294
295	3	$streams("");
296	3	}
297
298		/**
299		* @see Runner::captureStepArtifacts()
300		*
301		* @param ArtifactSource $source
302		* @param string $pattern
303		* @param string $id
304		* @param string $dir
305		*/
306	3	private function captureArtifactPattern(ArtifactSource $source, $pattern, $id, $dir)
307		{
308	3	$exec = $this->exec;
309	3	$streams = $this->streams;
310
311	3	$paths = $source->findByPattern($pattern);
312	3	if (empty($paths)) {
313	1	return;
314		}
315
316	2	$chunkSize = 2048;
317	2	$chunks = array_chunk($paths, $chunkSize, true);
318
319	2	foreach ($chunks as $paths) {
320	2	$tar = Lib::cmd('tar', array('c', '-f', '-', $paths));
321	2	$docker = Lib::cmd('docker', array('exec', '-w', '/app', $id));
322	2	$unTar = Lib::cmd('tar', array('x', '-f', '-', '-C', $dir));
323
324	2	$status = $exec->pass($docker . ' ' . $tar . ' \| ' . $unTar, array());
325
326	2	if ($status !== 0) {
327	1	$streams->err(
328	2	sprintf("pipelines: Artifact failure: '%s' (%d, %d paths)\n", $pattern, $status, count($paths))
329		);
330		}
331		}
332	2	}
333
334		/**
335		* @param int $status
336		* @param string $id container id
337		* @param Exec $exec
338		* @param string $name container name
339		*/
340	7	private function shutdownStepContainer($status, $id, Exec $exec, $name)
341		{
342	7	$flags = $this->flags;
343
344		# keep container on error
345	7	if (0 !== $status && $flags & self::FLAG_KEEP_ON_ERROR) {
346	1	$this->streams->err(sprintf(
347	1	"exit status %d, keeping container id %s\n",
348	1	$status,
349	1	substr($id, 0, 12)
350		));
351
352	1	return;
353		}
354
355		# keep or remove container
356	6	if ($flags & self::FLAG_DOCKER_KILL) {
357	2	$exec->capture('docker', array('kill', $name));
358		}
359
360	6	if ($flags & self::FLAG_DOCKER_REMOVE) {
361	2	$exec->capture('docker', array('rm', $name));
362		}
363	6	}
364		}
365

ktomk / pipelines

Push — master ( 72fc34...cfd30c )

Runner::captureArtifactPattern() B

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like