aiscalator.jupyter.docker_image.build() - Code Metrics - Aiscalate/aiscalator - Measure and Improve Code Quality continuously with Scrutinizer

aiscalator.jupyter.docker_image.build() A
last analyzed 2020-07-24 19:29 UTC

↳ Parent: aiscalator.jupyter.docker_image

Complexity

Conditions

Size

Total Lines	29
Code Lines	11

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	11
dl	0
loc	29
rs	9.85
c	0
b	0
f	0
cc	2
nop	1

# -*- coding: utf-8 -*-
# Apache Software License 2.0
#
# Copyright (c) 2018, Christophe Duong
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Module responsible to build docker images for jupyter subcommands
"""
import logging
from os import chdir
from os import getcwd
from os import listdir
from os.path import isdir
from os.path import isfile
from os.path import join
from shutil import copy
from tempfile import TemporaryDirectory

from aiscalator.core import utils
from aiscalator.core.config import AiscalatorConfig
from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer


def build(conf: AiscalatorConfig):
    """
    Builds the docker image following the parameters specified in the
    focused step's configuration file

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step

    Returns
    -------
    string
        the docker artifact name of the image built
    """
    input_docker_src = conf.step_field("docker_image.input_docker_src")
    # TODO check if input_docker_src refers to an existing docker image
    # in which case, if no customization is needed, no need to build
    cwd = getcwd()
    result = None
    try:
        # Prepare a temp folder to build docker image
        with TemporaryDirectory(prefix="aiscalator_") as tmp:
            _prepare_build_dir(conf, tmp, input_docker_src)
            chdir(tmp)
            result = _run_build(conf)
    finally:
        chdir(cwd)
    return result


def _prepare_build_dir(conf, dst, input_docker_src):
    """
    Copies all necessary files for building docker images in a tmp folder,
    substituting some specific macros accordingly to handle customized
    images such as:
    - add-apt-repository
    - apt-install packages
    - pip install packages
    - jupyter lab extensions

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dst : str
        temporary folder where to prepare the files
    input_docker_src : str
        name of the dockerfile package to use

    """
    input_docker_dir = utils.data_file("../config/docker/" + input_docker_src)

    if conf.app_config_has("jupyter.dockerfile_src"):
        # dockerfile is redefined in application configuration
        dockerfile_src = conf.app_config()["jupyter.dockerfile_src"]
        input_docker_dir = _find_docker_src(input_docker_src, dockerfile_src)

    if isdir(input_docker_dir):
        dockerfile = input_docker_dir + "/Dockerfile"
        with TemporaryDirectory(prefix="aiscalator_") as tmp:
            stg = "jupyter.docker_image"
            allow = (conf.app_config_has(stg + ".allow_apt_repository") and
                     conf.app_config()[stg + ".allow_apt_repository"])
            if allow:
                dockerfile = _include_apt_repo(conf, dockerfile,
                                               join(tmp, "apt_repository"))
            allow = (conf.app_config_has(stg + ".allow_apt_packages") and
                     conf.app_config()[stg + ".allow_apt_packages"])
            if allow:
                dockerfile = _include_apt_package(conf, dockerfile,
                                                  join(tmp, "apt_package"))
            allow = (conf.app_config_has(stg + ".allow_requirements") and
                     conf.app_config()[stg + ".allow_requirements"])
            if allow:
                dockerfile = _include_requirements(conf, dockerfile,
                                                   join(tmp, "requirements"),
                                                   dst)
            allow = (conf.app_config_has(stg +
                                         ".allow_lab_extensions") and
                     conf.app_config()[stg + ".allow_lab_extensions"])
            if allow:
                dockerfile = _include_lab_extensions(conf, dockerfile,
                                                     join(tmp,
                                                          "lab_extension"))
            copy(dockerfile, dst + '/Dockerfile')
        # copy the other files other than Dockerfile
        for file in listdir(input_docker_dir):
            if file != "Dockerfile":
                copy(join(input_docker_dir, file), join(dst, file))


def _find_docker_src(input_docker_src, dirs):
    """
    Finds a pre-configured dockerfile package or return the default one.

    Parameters
    ----------
    input_docker_src : str
        name of the dockerfile package to use
    dirs : list
        list of directories to check

    Returns
    -------
    str
        path to the corresponding dockerfile package
    """
    for src in dirs:
        if isfile(join(src, input_docker_src, "Dockerfile")):
            return src
    return utils.data_file("../config/docker/" + input_docker_src)


def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp):

    """
    Include add-apt-repository packages into the dockerfile

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dockerfile : str
        path to the dockerfile to modify
    tmp : str
        path to the temporary dockerfile output

    Returns
    -------
        path to the resulting dockerfile
    """
    if conf.has_step_field("docker_image.apt_repository_path"):
        content = conf.step_file_path("docker_image.apt_repository_path")
        value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
        if value:
            value = ("RUN apt-get update \\\n" +
                     " && apt-get install -yqq \\\n" +
                     "      software-properties-common \\\n" +
                     " && apt-add-repository \\\n" + value +
                     " && apt-get update")
            utils.copy_replace(dockerfile, tmp,
                               pattern="# apt_repository.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):

    """
    Include apt-install packages into the dockerfile

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dockerfile : str
        path to the dockerfile to modify
    tmp : str
        path to the temporary dockerfile output

    Returns
    -------
        path to the resulting dockerfile
    """
    if conf.has_step_field("docker_image.apt_package_path"):
        content = conf.step_file_path("docker_image.apt_package_path")
        value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
        if value:
            value = ("RUN apt-get update && apt-get install -yqq \\\n" +
                     value +
                     """    && apt-get purge --auto-remove -yqq $buildDeps \\
    && apt-get autoremove -yqq --purge \\
    && apt-get clean \\
    && rm -rf \\
    /var/lib/apt/lists/* \\
    /tmp/* \\
    /var/tmp/* \\
    /usr/share/man \\
    /usr/share/doc \\
    /usr/share/doc-base
""")
            utils.copy_replace(dockerfile, tmp,
                               pattern="# apt_packages.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
    """
        Include pip install packages into the dockerfile

        Parameters
        ----------
        conf : AiscalatorConfig
            Configuration object for this step
        dockerfile : str
            path to the dockerfile to modify
        tmp : str
            path to the temporary dockerfile output
        dst : str
            path to the final temporary directory

        Returns
        -------
            path to the resulting dockerfile
        """
    if conf.has_step_field("docker_image.requirements_path"):
        content = conf.step_file_path("docker_image.requirements_path")
        copy(content, join(dst, 'requirements.txt'))
        utils.copy_replace(dockerfile, tmp,
                           pattern="# requirements.txt #",
                           replace_value="""
    COPY requirements.txt requirements.txt
    RUN pip install -r requirements.txt
    RUN rm requirements.txt""")
        return tmp
    return dockerfile


def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):

    """
        Include jupyter lab extensions packages into the dockerfile

        Parameters
        ----------
        conf : AiscalatorConfig
            Configuration object for this step
        dockerfile : str
            path to the dockerfile to modify
        tmp : str
            path to the temporary dockerfile output

        Returns
        -------
            path to the resulting dockerfile
        """
    if conf.has_step_field("docker_image.lab_extension_path"):
        content = conf.step_file_path("docker_image.lab_extension_path")
        prefix = "&& jupyter labextension install "
        value = utils.format_file_content(content,
                                          prefix=prefix, suffix=" \\\n")
        if value:
            value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
            utils.copy_replace(dockerfile, tmp,
                               pattern="# lab_extensions.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _run_build(conf: AiscalatorConfig):
    """
    Run the docker build command to produce the image and tag it.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step

    Returns
    -------
    str
        the docker image ID that was built
    """
    logger = logging.getLogger(__name__)
    commands = ["docker", "build", "--rm"]
    output_docker_name = None
    if conf.has_step_field("docker_image.output_docker_name"):
        output_docker_name = conf.step_field("docker_image.output_docker_name")
        commands += ["-t", output_docker_name + ":latest"]
    commands += ["."]
    log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
    logger.info("Running...: %s", " ".join(commands))
    utils.subprocess_run(commands, log_function=log.grep_logs)
    result = log.artifact()
    test = (
        result and
        output_docker_name is not None and
        conf.has_step_field("docker_image.output_docker_tag")
    )
    if test:
        commands = ["docker", "tag"]
        output_docker_tag = conf.step_field("docker_image.output_docker_tag")
        commands += [result, output_docker_name + ":" + output_docker_tag]
        # TODO implement docker tag output_docker_tag_commit_hash
        logger.info("Running...: %s", " ".join(commands))
        utils.subprocess_run(commands)
    return result


1		# -- coding: utf-8 --
2		# Apache Software License 2.0
3		#
4		# Copyright (c) 2018, Christophe Duong
5		#
6		# Licensed under the Apache License, Version 2.0 (the "License");
7		# you may not use this file except in compliance with the License.
8		# You may obtain a copy of the License at
9		#
10		# http://www.apache.org/licenses/LICENSE-2.0
11		#
12		# Unless required by applicable law or agreed to in writing, software
13		# distributed under the License is distributed on an "AS IS" BASIS,
14		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15		# See the License for the specific language governing permissions and
16		# limitations under the License.
17		"""
18		Module responsible to build docker images for jupyter subcommands
19		"""
20		import logging
21		from os import chdir
22		from os import getcwd
23		from os import listdir
24		from os.path import isdir
25		from os.path import isfile
26		from os.path import join
27		from shutil import copy
28		from tempfile import TemporaryDirectory
29
30		from aiscalator.core import utils
31		from aiscalator.core.config import AiscalatorConfig
32		from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
33
34
35		def build(conf: AiscalatorConfig):
36		"""
37		Builds the docker image following the parameters specified in the
38		focused step's configuration file
39
40		Parameters
41		----------
42		conf : AiscalatorConfig
43		Configuration object for this step
44
45		Returns
46		-------
47		string
48		the docker artifact name of the image built
49		"""
50		input_docker_src = conf.step_field("docker_image.input_docker_src")
51		# TODO check if input_docker_src refers to an existing docker image
52		# in which case, if no customization is needed, no need to build
53		cwd = getcwd()
54		result = None
55		try:
56		# Prepare a temp folder to build docker image
57		with TemporaryDirectory(prefix="aiscalator_") as tmp:
58		_prepare_build_dir(conf, tmp, input_docker_src)
59		chdir(tmp)
60		result = _run_build(conf)
61		finally:
62		chdir(cwd)
63		return result
64
65
66		def _prepare_build_dir(conf, dst, input_docker_src):
67		"""
68		Copies all necessary files for building docker images in a tmp folder,
69		substituting some specific macros accordingly to handle customized
70		images such as:
71		- add-apt-repository
72		- apt-install packages
73		- pip install packages
74		- jupyter lab extensions
75
76		Parameters
77		----------
78		conf : AiscalatorConfig
79		Configuration object for this step
80		dst : str
81		temporary folder where to prepare the files
82		input_docker_src : str
83		name of the dockerfile package to use
84
85		"""
86		input_docker_dir = utils.data_file("../config/docker/" + input_docker_src)
87
88		if conf.app_config_has("jupyter.dockerfile_src"):
89		# dockerfile is redefined in application configuration
90		dockerfile_src = conf.app_config()["jupyter.dockerfile_src"]
91		input_docker_dir = _find_docker_src(input_docker_src, dockerfile_src)
92
93		if isdir(input_docker_dir):
94		dockerfile = input_docker_dir + "/Dockerfile"
95		with TemporaryDirectory(prefix="aiscalator_") as tmp:
96		stg = "jupyter.docker_image"
97		allow = (conf.app_config_has(stg + ".allow_apt_repository") and
98		conf.app_config()[stg + ".allow_apt_repository"])
99		if allow:
100		dockerfile = _include_apt_repo(conf, dockerfile,
101		join(tmp, "apt_repository"))
102		allow = (conf.app_config_has(stg + ".allow_apt_packages") and
103		conf.app_config()[stg + ".allow_apt_packages"])
104		if allow:
105		dockerfile = _include_apt_package(conf, dockerfile,
106		join(tmp, "apt_package"))
107		allow = (conf.app_config_has(stg + ".allow_requirements") and
108		conf.app_config()[stg + ".allow_requirements"])
109		if allow:
110		dockerfile = _include_requirements(conf, dockerfile,
111		join(tmp, "requirements"),
112		dst)
113		allow = (conf.app_config_has(stg +
114		".allow_lab_extensions") and
115		conf.app_config()[stg + ".allow_lab_extensions"])
116		if allow:
117		dockerfile = _include_lab_extensions(conf, dockerfile,
118		join(tmp,
119		"lab_extension"))
120		copy(dockerfile, dst + '/Dockerfile')
121		# copy the other files other than Dockerfile
122		for file in listdir(input_docker_dir):
123		if file != "Dockerfile":
124		copy(join(input_docker_dir, file), join(dst, file))
125
126
127		def _find_docker_src(input_docker_src, dirs):
128		"""
129		Finds a pre-configured dockerfile package or return the default one.
130
131		Parameters
132		----------
133		input_docker_src : str
134		name of the dockerfile package to use
135		dirs : list
136		list of directories to check
137
138		Returns
139		-------
140		str
141		path to the corresponding dockerfile package
142		"""
143		for src in dirs:
144		if isfile(join(src, input_docker_src, "Dockerfile")):
145		return src
146		return utils.data_file("../config/docker/" + input_docker_src)
147
148
149	View Code Duplication	def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp):
		0 ignored issues – show Duplication introduced 2019-02-08 15:31 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
150		"""
151		Include add-apt-repository packages into the dockerfile
152
153		Parameters
154		----------
155		conf : AiscalatorConfig
156		Configuration object for this step
157		dockerfile : str
158		path to the dockerfile to modify
159		tmp : str
160		path to the temporary dockerfile output
161
162		Returns
163		-------
164		path to the resulting dockerfile
165		"""
166		if conf.has_step_field("docker_image.apt_repository_path"):
167		content = conf.step_file_path("docker_image.apt_repository_path")
168		value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
169		if value:
170		value = ("RUN apt-get update \\\n" +
171		" && apt-get install -yqq \\\n" +
172		" software-properties-common \\\n" +
173		" && apt-add-repository \\\n" + value +
174		" && apt-get update")
175		utils.copy_replace(dockerfile, tmp,
176		pattern="# apt_repository.txt #",
177		replace_value=value)
178		return tmp
179		return dockerfile
180
181
182	View Code Duplication	def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):
		0 ignored issues – show Duplication introduced 2019-02-08 15:31 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
183		"""
184		Include apt-install packages into the dockerfile
185
186		Parameters
187		----------
188		conf : AiscalatorConfig
189		Configuration object for this step
190		dockerfile : str
191		path to the dockerfile to modify
192		tmp : str
193		path to the temporary dockerfile output
194
195		Returns
196		-------
197		path to the resulting dockerfile
198		"""
199		if conf.has_step_field("docker_image.apt_package_path"):
200		content = conf.step_file_path("docker_image.apt_package_path")
201		value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
202		if value:
203		value = ("RUN apt-get update && apt-get install -yqq \\\n" +
204		value +
205		""" && apt-get purge --auto-remove -yqq $buildDeps \\
206		&& apt-get autoremove -yqq --purge \\
207		&& apt-get clean \\
208		&& rm -rf \\
209		/var/lib/apt/lists/* \\
210		/tmp/* \\
211		/var/tmp/* \\
212		/usr/share/man \\
213		/usr/share/doc \\
214		/usr/share/doc-base
215		""")
216		utils.copy_replace(dockerfile, tmp,
217		pattern="# apt_packages.txt #",
218		replace_value=value)
219		return tmp
220		return dockerfile
221
222
223		def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
224		"""
225		Include pip install packages into the dockerfile
226
227		Parameters
228		----------
229		conf : AiscalatorConfig
230		Configuration object for this step
231		dockerfile : str
232		path to the dockerfile to modify
233		tmp : str
234		path to the temporary dockerfile output
235		dst : str
236		path to the final temporary directory
237
238		Returns
239		-------
240		path to the resulting dockerfile
241		"""
242		if conf.has_step_field("docker_image.requirements_path"):
243		content = conf.step_file_path("docker_image.requirements_path")
244		copy(content, join(dst, 'requirements.txt'))
245		utils.copy_replace(dockerfile, tmp,
246		pattern="# requirements.txt #",
247		replace_value="""
248		COPY requirements.txt requirements.txt
249		RUN pip install -r requirements.txt
250		RUN rm requirements.txt""")
251		return tmp
252		return dockerfile
253
254
255	View Code Duplication	def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):
		0 ignored issues – show Duplication introduced 2019-02-08 15:31 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
256		"""
257		Include jupyter lab extensions packages into the dockerfile
258
259		Parameters
260		----------
261		conf : AiscalatorConfig
262		Configuration object for this step
263		dockerfile : str
264		path to the dockerfile to modify
265		tmp : str
266		path to the temporary dockerfile output
267
268		Returns
269		-------
270		path to the resulting dockerfile
271		"""
272		if conf.has_step_field("docker_image.lab_extension_path"):
273		content = conf.step_file_path("docker_image.lab_extension_path")
274		prefix = "&& jupyter labextension install "
275		value = utils.format_file_content(content,
276		prefix=prefix, suffix=" \\\n")
277		if value:
278		value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
279		utils.copy_replace(dockerfile, tmp,
280		pattern="# lab_extensions.txt #",
281		replace_value=value)
282		return tmp
283		return dockerfile
284
285
286		def _run_build(conf: AiscalatorConfig):
287		"""
288		Run the docker build command to produce the image and tag it.
289
290		Parameters
291		----------
292		conf : AiscalatorConfig
293		Configuration object for this step
294
295		Returns
296		-------
297		str
298		the docker image ID that was built
299		"""
300		logger = logging.getLogger(__name__)
301		commands = ["docker", "build", "--rm"]
302		output_docker_name = None
303		if conf.has_step_field("docker_image.output_docker_name"):
304		output_docker_name = conf.step_field("docker_image.output_docker_name")
305		commands += ["-t", output_docker_name + ":latest"]
306		commands += ["."]
307		log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
308		logger.info("Running...: %s", " ".join(commands))
309		utils.subprocess_run(commands, log_function=log.grep_logs)
310		result = log.artifact()
311		test = (
312		result and
313		output_docker_name is not None and
314		conf.has_step_field("docker_image.output_docker_tag")
315		)
316		if test:
317		commands = ["docker", "tag"]
318		output_docker_tag = conf.step_field("docker_image.output_docker_tag")
319		commands += [result, output_docker_name + ":" + output_docker_tag]
320		# TODO implement docker tag output_docker_tag_commit_hash
321		logger.info("Running...: %s", " ".join(commands))
322		utils.subprocess_run(commands)
323		return result
324

Aiscalate / aiscalator

aiscalator.jupyter.docker_image.build() A last analyzed 2020-07-24 19:29 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

aiscalator.jupyter.docker_image.build() A
last analyzed 2020-07-24 19:29 UTC