CacheIo::deployStepCaches() - Code Metrics - Inspection of ".. integrate caches w/ travis" - ktomk/pipelines - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — test ( 9bb2d0...b603bb )

by Tom

created 2020-07-20 07:29 UTC

CacheIo::deployStepCaches() A

↳ Parent: CacheIo

Complexity

Conditions	5
Paths	4

Size

Total Lines	37
Code Lines	22

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	22
CRAP Score	5

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	5
eloc	22
c	1
b	0
f	0
nc	4
nop	1
dl	0
loc	37
ccs	22
cts	22
cp	1
crap	5
rs	9.2568

<?php

/* this file is part of pipelines */

namespace Ktomk\Pipelines\Runner\Docker;

use Ktomk\Pipelines\Cli\Exec;
use Ktomk\Pipelines\Cli\Streams;
use Ktomk\Pipelines\File\Pipeline\Step;
use Ktomk\Pipelines\Lib;
use Ktomk\Pipelines\LibFs;
use Ktomk\Pipelines\LibFsPath;
use Ktomk\Pipelines\Runner\Runner;

/**
 * Class CacheHandler
 *
 * Directory path related cache operations for docker containers when
 * running pipelines.
 *
 * Delegates between local cache storage (directory based, one tar
 * archive per cache) and (running) step-runner docker container.
 *
 * @package Ktomk\Pipelines\Runner\Docker
 */
class CacheIo
{
    /**
     * @var string
     */
    private $id;

    /**
     * @var string
     */
    private $cachesDirectory;

    /**
     * @var Streams
     */
    private $streams;

    /**
     * @var Exec
     */
    private $exec;

    /**
     * @var string
     */
    private $clonePath;

    /**
     * @param Runner $runner
     * @param $id
     *
     * @return CacheIo
     */
    public static function createByRunner(Runner $runner, $id)
    {
        return new self(
            self::runnerCachesDirectory($runner),
            $id,
            $runner->getRunOpts()->getOption('step.clone-path'),
            $runner->getStreams(),
            $runner->getExec()
        );
    }

    /**
     * @param Runner $runner
     *
     * @return string path of base-directory in which the piplines project tar files are located
     */
    public static function runnerCachesDirectory(Runner $runner)
    {
        return (string)$runner->getDirectories()->getBaseDirectory(
            'XDG_CACHE_HOME',
            sprintf(
                'caches/%s',
                $runner->getProject()
            )
        );
    }

    /**
     * CacheIo constructor.
     *
     * @param string $caches path to directory where tar files are stored
     * @param string $id container
     * @param string $clonePath
     * @param Streams $streams
     * @param Exec $exec
     */
    public function __construct($caches, $id, $clonePath, Streams $streams, Exec $exec)
    {
        $this->id = $id;
        $this->setCachesDirectory($caches);
        $this->clonePath = $clonePath;
        $this->streams = $streams;
        $this->exec = $exec;
    }

    /**
     * Deploy step caches into container
     *
     * @param Step $step
     */
    public function deployStepCaches(Step $step)
    {
        $cachesDirectory = $this->cachesDirectory;

        // skip deploying caches if there are no caches
        if (!$this->skip($cachesDirectory, $step)) {
            return;
        }

        $id = $this->id;
        $streams = $this->streams;
        $exec = $this->exec;

        $streams->out("\x1D+++ populating caches...\n");

        foreach ($step->getCaches() as $name => $path) {
            $tarFile = sprintf('%s/%s.tar', $cachesDirectory, $name);
            $tarExists = LibFs::isReadableFile($tarFile);
            $streams->out(sprintf(" - %s %s (%s)\n", $name, $path, $tarExists ? 'hit' : 'miss'));

            if (!LibFs::isReadableFile($tarFile)) {
                continue;
            }

            $containerPath = $this->mapCachePath($path);

            $exec->pass('docker', array('exec', $id, 'mkdir', '-p', $containerPath));

            $cmd = Lib::cmd(
                sprintf('<%s docker', Lib::quoteArg($tarFile)),
                array(
                    'cp',
                    '-',
                    sprintf('%s:%s', $id, $containerPath),
                )
            );
            $exec->pass($cmd, array());
        }
    }

    /**
     * Capture cache from container
     *
     * @param Step $step
     * @param bool $capture
     */
    public function captureStepCaches(Step $step, $capture)
    {
        // skip capturing if disabled
        if (!$capture) {
            return;
        }

        $cachesDirectory = $this->cachesDirectory;

        // skip capturing caches if there are no caches
        if (!$this->skip($cachesDirectory, $step)) {
            return;
        }

        $id = $this->id;
        $exec = $this->exec;
        $streams = $this->streams;

        $streams->out("\x1D+++ updating caches from container...\n");

        $cachesDirectory = LibFs::mkDir($cachesDirectory);

        foreach ($step->getCaches() as $name => $path) {
            $tarFile = sprintf('%s/%s.tar', $cachesDirectory, $name);
            $tarExists = LibFs::isReadableFile($tarFile);
            $streams->out(sprintf(" - %s %s (%s)\n", $name, $path, $tarExists ? 'update' : 'create'));

            $containerPath = $this->mapCachePath($path);

            $cmd = Lib::cmd(
                sprintf('>%s docker', Lib::quoteArg($tarFile)),
                array(
                    'cp',
                    sprintf('%s:%s/.', $id, $containerPath),
                    '-',
                )
            );
            $exec->pass($cmd, array());
        }
    }

    /**
     * Map cache path to container path
     *
     * A cache path in a cache definition can be with information for
     * container context, e.g. $HOME, ~ or just being relative to the
     * clone directory.
     *
     * This requires mapping for the "real" path
     *
     * @param string $path component of cache definition
     *
     * @return string
     */
    public function mapCachePath($path)
    {
        $id = $this->id;

        $containerPath = $path;

        // let the container map the path expression
        $status = $this->exec->capture(
            'docker',
            array('exec', $id, '/bin/sh', '-c', sprintf('echo %s', $containerPath)),
            $out
        );

        if (0 === $status && 0 < strlen($out) && ('/' === $out[0])) {
            $containerPath = trim($out);
        }

        // fallback to '~' -> /root assumption (can fail easy as root might not be the user)
        if ('~' === $containerPath[0]) {
            $containerPath = '/root/' . ltrim(substr($containerPath, 1), '/');
        }

        // handle relative paths
        if (!LibFsPath::isAbsolute($containerPath)) {
            $containerPath = LibFsPath::normalizeSegments(
                $this->clonePath . '/' . $containerPath
            );
        }

        return $containerPath;
    }

    /**
     * @param string $cachesDirectory
     * @param Step $step
     *
     * @return bool|void
     */
    private function skip($cachesDirectory, Step $step)
    {
        // caches directory is empty?
        if (empty($cachesDirectory)) {
            return false;
        }

        // step has caches?
        $caches = $step->getCaches()->getIterator();
        if (!count($caches)) {
            return false;
        }

        return true;
    }

    /**
     * @param string $path
     */
    private function setCachesDirectory($path)
    {
        if (!empty($path) && !LibFsPath::isAbsolute($path)) {
            throw new \InvalidArgumentException(sprintf('Caches directory: Not an absolute path: %s', $path));
        }

        $this->cachesDirectory = (string)$path;
    }
}


1		<?php
2
3		/* this file is part of pipelines */
4
5		namespace Ktomk\Pipelines\Runner\Docker;
6
7		use Ktomk\Pipelines\Cli\Exec;
8		use Ktomk\Pipelines\Cli\Streams;
9		use Ktomk\Pipelines\File\Pipeline\Step;
10		use Ktomk\Pipelines\Lib;
11		use Ktomk\Pipelines\LibFs;
12		use Ktomk\Pipelines\LibFsPath;
13		use Ktomk\Pipelines\Runner\Runner;
14
15		/**
16		* Class CacheHandler
17		*
18		* Directory path related cache operations for docker containers when
19		* running pipelines.
20		*
21		* Delegates between local cache storage (directory based, one tar
22		* archive per cache) and (running) step-runner docker container.
23		*
24		* @package Ktomk\Pipelines\Runner\Docker
25		*/
26		class CacheIo
27		{
28		/**
29		* @var string
30		*/
31		private $id;
32
33		/**
34		* @var string
35		*/
36		private $cachesDirectory;
37
38		/**
39		* @var Streams
40		*/
41		private $streams;
42
43		/**
44		* @var Exec
45		*/
46		private $exec;
47
48		/**
49		* @var string
50		*/
51		private $clonePath;
52
53		/**
54		* @param Runner $runner
55		* @param $id
56		*
57		* @return CacheIo
58		*/
59	3	public static function createByRunner(Runner $runner, $id)
60		{
61	3	return new self(
62	3	self::runnerCachesDirectory($runner),
63		$id,
64	3	$runner->getRunOpts()->getOption('step.clone-path'),
65	3	$runner->getStreams(),
66	3	$runner->getExec()
67		);
68		}
69
70		/**
71		* @param Runner $runner
72		*
73		* @return string path of base-directory in which the piplines project tar files are located
74		*/
75	4	public static function runnerCachesDirectory(Runner $runner)
76		{
77	4	return (string)$runner->getDirectories()->getBaseDirectory(
78	4	'XDG_CACHE_HOME',
79	4	sprintf(
80	4	'caches/%s',
81	4	$runner->getProject()
82		)
83		);
84		}
85
86		/**
87		* CacheIo constructor.
88		*
89		* @param string $caches path to directory where tar files are stored
90		* @param string $id container
91		* @param string $clonePath
92		* @param Streams $streams
93		* @param Exec $exec
94		*/
95	9	public function __construct($caches, $id, $clonePath, Streams $streams, Exec $exec)
96		{
97	9	$this->id = $id;
98	9	$this->setCachesDirectory($caches);
99	8	$this->clonePath = $clonePath;
100	8	$this->streams = $streams;
101	8	$this->exec = $exec;
102	8	}
103
104		/**
105		* Deploy step caches into container
106		*
107		* @param Step $step
108		*/
109	4	public function deployStepCaches(Step $step)
110		{
111	4	$cachesDirectory = $this->cachesDirectory;
112
113		// skip deploying caches if there are no caches
114	4	if (!$this->skip($cachesDirectory, $step)) {
115	2	return;
116		}
117
118	2	$id = $this->id;
119	2	$streams = $this->streams;
120	2	$exec = $this->exec;
121
122	2	$streams->out("\x1D+++ populating caches...\n");
123
124	2	foreach ($step->getCaches() as $name => $path) {
125	2	$tarFile = sprintf('%s/%s.tar', $cachesDirectory, $name);
126	2	$tarExists = LibFs::isReadableFile($tarFile);
127	2	$streams->out(sprintf(" - %s %s (%s)\n", $name, $path, $tarExists ? 'hit' : 'miss'));
128
129	2	if (!LibFs::isReadableFile($tarFile)) {
130	1	continue;
131		}
132
133	1	$containerPath = $this->mapCachePath($path);
134
135	1	$exec->pass('docker', array('exec', $id, 'mkdir', '-p', $containerPath));
136
137	1	$cmd = Lib::cmd(
138	1	sprintf('<%s docker', Lib::quoteArg($tarFile)),
139		array(
140	1	'cp',
141	1	'-',
142	1	sprintf('%s:%s', $id, $containerPath),
143		)
144		);
145	1	$exec->pass($cmd, array());
146		}
147	2	}
148
149		/**
150		* Capture cache from container
151		*
152		* @param Step $step
153		* @param bool $capture
154		*/
155	5	public function captureStepCaches(Step $step, $capture)
156		{
157		// skip capturing if disabled
158	5	if (!$capture) {
159	1	return;
160		}
161
162	4	$cachesDirectory = $this->cachesDirectory;
163
164		// skip capturing caches if there are no caches
165	4	if (!$this->skip($cachesDirectory, $step)) {
166	2	return;
167		}
168
169	2	$id = $this->id;
170	2	$exec = $this->exec;
171	2	$streams = $this->streams;
172
173	2	$streams->out("\x1D+++ updating caches from container...\n");
174
175	2	$cachesDirectory = LibFs::mkDir($cachesDirectory);
176
177	2	foreach ($step->getCaches() as $name => $path) {
178	2	$tarFile = sprintf('%s/%s.tar', $cachesDirectory, $name);
179	2	$tarExists = LibFs::isReadableFile($tarFile);
180	2	$streams->out(sprintf(" - %s %s (%s)\n", $name, $path, $tarExists ? 'update' : 'create'));
181
182	2	$containerPath = $this->mapCachePath($path);
183
184	2	$cmd = Lib::cmd(
185	2	sprintf('>%s docker', Lib::quoteArg($tarFile)),
186		array(
187	2	'cp',
188	2	sprintf('%s:%s/.', $id, $containerPath),
189	2	'-',
190		)
191		);
192	2	$exec->pass($cmd, array());
193		}
194	2	}
195
196		/**
197		* Map cache path to container path
198		*
199		* A cache path in a cache definition can be with information for
200		* container context, e.g. $HOME, ~ or just being relative to the
201		* clone directory.
202		*
203		* This requires mapping for the "real" path
204		*
205		* @param string $path component of cache definition
206		*
207		* @return string
208		*/
209	4	public function mapCachePath($path)
210		{
211	4	$id = $this->id;
212
213	4	$containerPath = $path;
214
215		// let the container map the path expression
216	4	$status = $this->exec->capture(
217	4	'docker',
218	4	array('exec', $id, '/bin/sh', '-c', sprintf('echo %s', $containerPath)),
219		$out
220		);
221
222	4	if (0 === $status && 0 < strlen($out) && ('/' === $out[0])) {
223	1	$containerPath = trim($out);
224		}
225
226		// fallback to '~' -> /root assumption (can fail easy as root might not be the user)
227	4	if ('~' === $containerPath[0]) {
228	2	$containerPath = '/root/' . ltrim(substr($containerPath, 1), '/');
229		}
230
231		// handle relative paths
232	4	if (!LibFsPath::isAbsolute($containerPath)) {
233	1	$containerPath = LibFsPath::normalizeSegments(
234	1	$this->clonePath . '/' . $containerPath
235		);
236		}
237
238	4	return $containerPath;
239		}
240
241		/**
242		* @param string $cachesDirectory
243		* @param Step $step
244		*
245		* @return bool\|void
246		*/
247	4	private function skip($cachesDirectory, Step $step)
248		{
249		// caches directory is empty?
250	4	if (empty($cachesDirectory)) {
251	1	return false;
252		}
253
254		// step has caches?
255	3	$caches = $step->getCaches()->getIterator();
256	3	if (!count($caches)) {
257	1	return false;
258		}
259
260	2	return true;
261		}
262
263		/**
264		* @param string $path
265		*/
266	9	private function setCachesDirectory($path)
267		{
268	9	if (!empty($path) && !LibFsPath::isAbsolute($path)) {
269	1	throw new \InvalidArgumentException(sprintf('Caches directory: Not an absolute path: %s', $path));
270		}
271
272	8	$this->cachesDirectory = (string)$path;
273	8	}
274		}
275

ktomk / pipelines

Push — test ( 9bb2d0...b603bb )

CacheIo::deployStepCaches() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like