aiscalator.jupyter.docker_image._include_apt_repo() - Code Metrics - Inspection of "Bump version: 0.1.4 → 0.1.5" - Aiscalate/aiscalator - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( e787a9...1a4fb3 )

by Christophe

created 2019-02-08 15:29 UTC

_include_apt_repo() A

↳ Parent: aiscalator.jupyter.docker_image

Complexity

Conditions

Size

Total Lines	31
Code Lines	15

Duplication

Lines	31
Ratio	100 %

Importance

Changes

Metric	Value
eloc	15
dl	31
loc	31
rs	9.65
c	0
b	0
f	0
cc	3
nop	3

# -*- coding: utf-8 -*-
# Apache Software License 2.0
#
# Copyright (c) 2018, Christophe Duong
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Module responsible to build docker images for jupyter subcommands
"""
import logging
from os import chdir
from os import getcwd
from os import listdir
from os.path import isdir
from os.path import isfile
from os.path import join
from shutil import copy
from tempfile import TemporaryDirectory

from aiscalator.core import utils
from aiscalator.core.config import AiscalatorConfig
from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer


def build(conf: AiscalatorConfig):
    """
    Builds the docker image following the parameters specified in the
    focused step's configuration file

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step

    Returns
    -------
    string
        the docker artifact name of the image built
    """
    input_docker_src = conf.step_field("docker_image.input_docker_src")
    # TODO check if input_docker_src refers to an existing docker image
    # in which case, if no customization is needed, no need to build
    cwd = getcwd()
    result = None
    try:
        # Prepare a temp folder to build docker image
        with TemporaryDirectory(prefix="aiscalator_") as tmp:
            _prepare_build_dir(conf, tmp, input_docker_src)
            chdir(tmp)
            result = _run_build(conf)
    finally:
        chdir(cwd)
    return result


def _prepare_build_dir(conf, dst, input_docker_src):
    """
    Copies all necessary files for building docker images in a tmp folder,
    substituting some specific macros accordingly to handle customized
    images such as:
    - add-apt-repository
    - apt-install packages
    - pip install packages
    - jupyter lab extensions

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dst : str
        temporary folder where to prepare the files
    input_docker_src : str
        name of the dockerfile package to use

    """
    input_docker_dir = utils.data_file("../config/docker/" + input_docker_src)

    if conf.app_config_has("jupyter.dockerfile_src"):
        # dockerfile is redefined in application configuration
        dockerfile_src = conf.app_config()["jupyter.dockerfile_src"]
        input_docker_dir = _find_docker_src(input_docker_src, dockerfile_src)

    if isdir(input_docker_dir):
        dockerfile = input_docker_dir + "/Dockerfile"
        with TemporaryDirectory(prefix="aiscalator_") as tmp:
            stg = "jupyter.docker_image"
            allow = (conf.app_config_has(stg + ".allow_apt_repository") and
                     conf.app_config()[stg + ".allow_apt_repository"])
            if allow:
                dockerfile = _include_apt_repo(conf, dockerfile,
                                               join(tmp, "apt_repository"))
            allow = (conf.app_config_has(stg + ".allow_apt_packages") and
                     conf.app_config()[stg + ".allow_apt_packages"])
            if allow:
                dockerfile = _include_apt_package(conf, dockerfile,
                                                  join(tmp, "apt_package"))
            allow = (conf.app_config_has(stg + ".allow_requirements") and
                     conf.app_config()[stg + ".allow_requirements"])
            if allow:
                dockerfile = _include_requirements(conf, dockerfile,
                                                   join(tmp, "requirements"),
                                                   dst)
            allow = (conf.app_config_has(stg +
                                         ".allow_lab_extensions") and
                     conf.app_config()[stg + ".allow_lab_extensions"])
            if allow:
                dockerfile = _include_lab_extensions(conf, dockerfile,
                                                     join(tmp,
                                                          "lab_extension"))
            copy(dockerfile, dst + '/Dockerfile')
        # copy the other files other than Dockerfile
        for file in listdir(input_docker_dir):
            if file != "Dockerfile":
                copy(join(input_docker_dir, file), join(dst, file))


def _find_docker_src(input_docker_src, dirs):
    """
    Finds a pre-configured dockerfile package or return the default one.

    Parameters
    ----------
    input_docker_src : str
        name of the dockerfile package to use
    dirs : list
        list of directories to check

    Returns
    -------
    str
        path to the corresponding dockerfile package
    """
    for src in dirs:
        if isfile(join(src, input_docker_src, "Dockerfile")):
            return src
    return utils.data_file("../config/docker/" + input_docker_src)


def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp):

    """
    Include add-apt-repository packages into the dockerfile

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dockerfile : str
        path to the dockerfile to modify
    tmp : str
        path to the temporary dockerfile output

    Returns
    -------
        path to the resulting dockerfile
    """
    if conf.has_step_field("docker_image.apt_repository_path"):
        content = conf.step_file_path("docker_image.apt_repository_path")
        value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
        if value:
            value = ("RUN apt-get update \\\n" +
                     " && apt-get install -yqq \\\n" +
                     "      software-properties-common \\\n" +
                     " && apt-add-repository \\\n" + value +
                     " && apt-get update")
            utils.copy_replace(dockerfile, tmp,
                               pattern="# apt_repository.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):

    """
    Include apt-install packages into the dockerfile

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dockerfile : str
        path to the dockerfile to modify
    tmp : str
        path to the temporary dockerfile output

    Returns
    -------
        path to the resulting dockerfile
    """
    if conf.has_step_field("docker_image.apt_package_path"):
        content = conf.step_file_path("docker_image.apt_package_path")
        value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
        if value:
            value = ("RUN apt-get install -yqq \\\n" + value +
                     """    && apt-get purge --auto-remove -yqq $buildDeps \\
    && apt-get autoremove -yqq --purge \\
    && apt-get clean \\
    && rm -rf \\
    /var/lib/apt/lists/* \\
    /tmp/* \\
    /var/tmp/* \\
    /usr/share/man \\
    /usr/share/doc \\
    /usr/share/doc-base
""")
            utils.copy_replace(dockerfile, tmp,
                               pattern="# apt_packages.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
    """
        Include pip install packages into the dockerfile

        Parameters
        ----------
        conf : AiscalatorConfig
            Configuration object for this step
        dockerfile : str
            path to the dockerfile to modify
        tmp : str
            path to the temporary dockerfile output
        dst : str
            path to the final temporary directory

        Returns
        -------
            path to the resulting dockerfile
        """
    if conf.has_step_field("docker_image.requirements_path"):
        content = conf.step_file_path("docker_image.requirements_path")
        copy(content, join(dst, 'requirements.txt'))
        utils.copy_replace(dockerfile, tmp,
                           pattern="# requirements.txt #",
                           replace_value="""
    COPY requirements.txt requirements.txt
    RUN pip install -r requirements.txt
    RUN rm requirements.txt""")
        return tmp
    return dockerfile


def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):

    """
        Include jupyter lab extensions packages into the dockerfile

        Parameters
        ----------
        conf : AiscalatorConfig
            Configuration object for this step
        dockerfile : str
            path to the dockerfile to modify
        tmp : str
            path to the temporary dockerfile output

        Returns
        -------
            path to the resulting dockerfile
        """
    if conf.has_step_field("docker_image.lab_extension_path"):
        content = conf.step_file_path("docker_image.lab_extension_path")
        prefix = "&& jupyter labextension install "
        value = utils.format_file_content(content,
                                          prefix=prefix, suffix=" \\\n")
        if value:
            value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
            utils.copy_replace(dockerfile, tmp,
                               pattern="# lab_extensions.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _run_build(conf: AiscalatorConfig):
    """
    Run the docker build command to produce the image and tag it.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step

    Returns
    -------
    str
        the docker image ID that was built
    """
    logger = logging.getLogger(__name__)
    commands = ["docker", "build", "--rm"]
    output_docker_name = None
    if conf.has_step_field("docker_image.output_docker_name"):
        output_docker_name = conf.step_field("docker_image.output_docker_name")
        commands += ["-t", output_docker_name + ":latest"]
    commands += ["."]
    log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
    logger.info("Running...: %s", " ".join(commands))
    utils.subprocess_run(commands, log_function=log.grep_logs)
    result = log.artifact()
    test = (
        result and
        output_docker_name is not None and
        conf.has_step_field("docker_image.output_docker_tag")
    )
    if test:
        commands = ["docker", "tag"]
        output_docker_tag = conf.step_field("docker_image.output_docker_tag")
        commands += [result, output_docker_name + ":" + output_docker_tag]
        # TODO implement docker tag output_docker_tag_commit_hash
        logger.info("Running...: %s", " ".join(commands))
        utils.subprocess_run(commands)
    return result


1		# -- coding: utf-8 --
2		# Apache Software License 2.0
3		#
4		# Copyright (c) 2018, Christophe Duong
5		#
6		# Licensed under the Apache License, Version 2.0 (the "License");
7		# you may not use this file except in compliance with the License.
8		# You may obtain a copy of the License at
9		#
10		# http://www.apache.org/licenses/LICENSE-2.0
11		#
12		# Unless required by applicable law or agreed to in writing, software
13		# distributed under the License is distributed on an "AS IS" BASIS,
14		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15		# See the License for the specific language governing permissions and
16		# limitations under the License.
17		"""
18		Module responsible to build docker images for jupyter subcommands
19		"""
20		import logging
21		from os import chdir
22		from os import getcwd
23		from os import listdir
24		from os.path import isdir
25		from os.path import isfile
26		from os.path import join
27		from shutil import copy
28		from tempfile import TemporaryDirectory
29
30		from aiscalator.core import utils
31		from aiscalator.core.config import AiscalatorConfig
32		from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
33
34
35		def build(conf: AiscalatorConfig):
36		"""
37		Builds the docker image following the parameters specified in the
38		focused step's configuration file
39
40		Parameters
41		----------
42		conf : AiscalatorConfig
43		Configuration object for this step
44
45		Returns
46		-------
47		string
48		the docker artifact name of the image built
49		"""
50		input_docker_src = conf.step_field("docker_image.input_docker_src")
51		# TODO check if input_docker_src refers to an existing docker image
52		# in which case, if no customization is needed, no need to build
53		cwd = getcwd()
54		result = None
55		try:
56		# Prepare a temp folder to build docker image
57		with TemporaryDirectory(prefix="aiscalator_") as tmp:
58		_prepare_build_dir(conf, tmp, input_docker_src)
59		chdir(tmp)
60		result = _run_build(conf)
61		finally:
62		chdir(cwd)
63		return result
64
65
66		def _prepare_build_dir(conf, dst, input_docker_src):
67		"""
68		Copies all necessary files for building docker images in a tmp folder,
69		substituting some specific macros accordingly to handle customized
70		images such as:
71		- add-apt-repository
72		- apt-install packages
73		- pip install packages
74		- jupyter lab extensions
75
76		Parameters
77		----------
78		conf : AiscalatorConfig
79		Configuration object for this step
80		dst : str
81		temporary folder where to prepare the files
82		input_docker_src : str
83		name of the dockerfile package to use
84
85		"""
86		input_docker_dir = utils.data_file("../config/docker/" + input_docker_src)
87
88		if conf.app_config_has("jupyter.dockerfile_src"):
89		# dockerfile is redefined in application configuration
90		dockerfile_src = conf.app_config()["jupyter.dockerfile_src"]
91		input_docker_dir = _find_docker_src(input_docker_src, dockerfile_src)
92
93		if isdir(input_docker_dir):
94		dockerfile = input_docker_dir + "/Dockerfile"
95		with TemporaryDirectory(prefix="aiscalator_") as tmp:
96		stg = "jupyter.docker_image"
97		allow = (conf.app_config_has(stg + ".allow_apt_repository") and
98		conf.app_config()[stg + ".allow_apt_repository"])
99		if allow:
100		dockerfile = _include_apt_repo(conf, dockerfile,
101		join(tmp, "apt_repository"))
102		allow = (conf.app_config_has(stg + ".allow_apt_packages") and
103		conf.app_config()[stg + ".allow_apt_packages"])
104		if allow:
105		dockerfile = _include_apt_package(conf, dockerfile,
106		join(tmp, "apt_package"))
107		allow = (conf.app_config_has(stg + ".allow_requirements") and
108		conf.app_config()[stg + ".allow_requirements"])
109		if allow:
110		dockerfile = _include_requirements(conf, dockerfile,
111		join(tmp, "requirements"),
112		dst)
113		allow = (conf.app_config_has(stg +
114		".allow_lab_extensions") and
115		conf.app_config()[stg + ".allow_lab_extensions"])
116		if allow:
117		dockerfile = _include_lab_extensions(conf, dockerfile,
118		join(tmp,
119		"lab_extension"))
120		copy(dockerfile, dst + '/Dockerfile')
121		# copy the other files other than Dockerfile
122		for file in listdir(input_docker_dir):
123		if file != "Dockerfile":
124		copy(join(input_docker_dir, file), join(dst, file))
125
126
127		def _find_docker_src(input_docker_src, dirs):
128		"""
129		Finds a pre-configured dockerfile package or return the default one.
130
131		Parameters
132		----------
133		input_docker_src : str
134		name of the dockerfile package to use
135		dirs : list
136		list of directories to check
137
138		Returns
139		-------
140		str
141		path to the corresponding dockerfile package
142		"""
143		for src in dirs:
144		if isfile(join(src, input_docker_src, "Dockerfile")):
145		return src
146		return utils.data_file("../config/docker/" + input_docker_src)
147
148
149	View Code Duplication	def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp):
		0 ignored issues – show Duplication introduced 2019-02-08 15:31 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
150		"""
151		Include add-apt-repository packages into the dockerfile
152
153		Parameters
154		----------
155		conf : AiscalatorConfig
156		Configuration object for this step
157		dockerfile : str
158		path to the dockerfile to modify
159		tmp : str
160		path to the temporary dockerfile output
161
162		Returns
163		-------
164		path to the resulting dockerfile
165		"""
166		if conf.has_step_field("docker_image.apt_repository_path"):
167		content = conf.step_file_path("docker_image.apt_repository_path")
168		value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
169		if value:
170		value = ("RUN apt-get update \\\n" +
171		" && apt-get install -yqq \\\n" +
172		" software-properties-common \\\n" +
173		" && apt-add-repository \\\n" + value +
174		" && apt-get update")
175		utils.copy_replace(dockerfile, tmp,
176		pattern="# apt_repository.txt #",
177		replace_value=value)
178		return tmp
179		return dockerfile
180
181
182	View Code Duplication	def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):
		0 ignored issues – show Duplication introduced 2019-02-08 15:31 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
183		"""
184		Include apt-install packages into the dockerfile
185
186		Parameters
187		----------
188		conf : AiscalatorConfig
189		Configuration object for this step
190		dockerfile : str
191		path to the dockerfile to modify
192		tmp : str
193		path to the temporary dockerfile output
194
195		Returns
196		-------
197		path to the resulting dockerfile
198		"""
199		if conf.has_step_field("docker_image.apt_package_path"):
200		content = conf.step_file_path("docker_image.apt_package_path")
201		value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
202		if value:
203		value = ("RUN apt-get install -yqq \\\n" + value +
204		""" && apt-get purge --auto-remove -yqq $buildDeps \\
205		&& apt-get autoremove -yqq --purge \\
206		&& apt-get clean \\
207		&& rm -rf \\
208		/var/lib/apt/lists/* \\
209		/tmp/* \\
210		/var/tmp/* \\
211		/usr/share/man \\
212		/usr/share/doc \\
213		/usr/share/doc-base
214		""")
215		utils.copy_replace(dockerfile, tmp,
216		pattern="# apt_packages.txt #",
217		replace_value=value)
218		return tmp
219		return dockerfile
220
221
222		def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
223		"""
224		Include pip install packages into the dockerfile
225
226		Parameters
227		----------
228		conf : AiscalatorConfig
229		Configuration object for this step
230		dockerfile : str
231		path to the dockerfile to modify
232		tmp : str
233		path to the temporary dockerfile output
234		dst : str
235		path to the final temporary directory
236
237		Returns
238		-------
239		path to the resulting dockerfile
240		"""
241		if conf.has_step_field("docker_image.requirements_path"):
242		content = conf.step_file_path("docker_image.requirements_path")
243		copy(content, join(dst, 'requirements.txt'))
244		utils.copy_replace(dockerfile, tmp,
245		pattern="# requirements.txt #",
246		replace_value="""
247		COPY requirements.txt requirements.txt
248		RUN pip install -r requirements.txt
249		RUN rm requirements.txt""")
250		return tmp
251		return dockerfile
252
253
254	View Code Duplication	def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):
		0 ignored issues – show Duplication introduced 2019-02-08 15:31 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
255		"""
256		Include jupyter lab extensions packages into the dockerfile
257
258		Parameters
259		----------
260		conf : AiscalatorConfig
261		Configuration object for this step
262		dockerfile : str
263		path to the dockerfile to modify
264		tmp : str
265		path to the temporary dockerfile output
266
267		Returns
268		-------
269		path to the resulting dockerfile
270		"""
271		if conf.has_step_field("docker_image.lab_extension_path"):
272		content = conf.step_file_path("docker_image.lab_extension_path")
273		prefix = "&& jupyter labextension install "
274		value = utils.format_file_content(content,
275		prefix=prefix, suffix=" \\\n")
276		if value:
277		value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
278		utils.copy_replace(dockerfile, tmp,
279		pattern="# lab_extensions.txt #",
280		replace_value=value)
281		return tmp
282		return dockerfile
283
284
285		def _run_build(conf: AiscalatorConfig):
286		"""
287		Run the docker build command to produce the image and tag it.
288
289		Parameters
290		----------
291		conf : AiscalatorConfig
292		Configuration object for this step
293
294		Returns
295		-------
296		str
297		the docker image ID that was built
298		"""
299		logger = logging.getLogger(__name__)
300		commands = ["docker", "build", "--rm"]
301		output_docker_name = None
302		if conf.has_step_field("docker_image.output_docker_name"):
303		output_docker_name = conf.step_field("docker_image.output_docker_name")
304		commands += ["-t", output_docker_name + ":latest"]
305		commands += ["."]
306		log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
307		logger.info("Running...: %s", " ".join(commands))
308		utils.subprocess_run(commands, log_function=log.grep_logs)
309		result = log.artifact()
310		test = (
311		result and
312		output_docker_name is not None and
313		conf.has_step_field("docker_image.output_docker_tag")
314		)
315		if test:
316		commands = ["docker", "tag"]
317		output_docker_tag = conf.step_field("docker_image.output_docker_tag")
318		commands += [result, output_docker_name + ":" + output_docker_tag]
319		# TODO implement docker tag output_docker_tag_commit_hash
320		logger.info("Running...: %s", " ".join(commands))
321		utils.subprocess_run(commands)
322		return result
323

Aiscalate / aiscalator

Push — master ( e787a9...1a4fb3 )

_include_apt_repo() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like