aiscalator.jupyter.docker_image._prepare_build_dir() - Code Metrics - Inspection of "Merge pull request #6 from Aiscalate/dev" - Aiscalate/aiscalator - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 722b81...0abd85 )

by Christophe

created 2018-11-14 21:12 UTC

_prepare_build_dir() C

↳ Parent: aiscalator.jupyter.docker_image

Complexity

Conditions

Size

Total Lines	53
Code Lines	31

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	31
dl	0
loc	53
rs	6.6666
c	0
b	0
f	0
cc	9
nop	3

How to fix Long Method

# -*- coding: utf-8 -*-
# Apache Software License 2.0
#
# Copyright (c) 2018, Christophe Duong
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Module responsible to build docker images for jupyter subcommands
"""
import logging
from os import chdir
from os import getcwd
from os import listdir
from os.path import isdir
from os.path import isfile
from os.path import join
from shutil import copy
from tempfile import TemporaryDirectory

from aiscalator.core import utils
from aiscalator.core.config import AiscalatorConfig
from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer


def build(conf: AiscalatorConfig):
    """
    Builds the docker image following the parameters specified in the
    focused step's configuration file

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step

    Returns
    -------
    string
        the docker artifact name of the image built
    """
    input_docker_src = conf.step_field("docker_image.input_docker_src")
    # TODO check if input_docker_src refers to an existing docker image
    # in which case, if no customization is needed, no need to build
    cwd = getcwd()
    result = None
    try:
        # Prepare a temp folder to build docker image
        with TemporaryDirectory(prefix="aiscalator_") as tmp:
            _prepare_build_dir(conf, tmp, input_docker_src)
            chdir(tmp)
            result = _run_build(conf)
    finally:
        chdir(cwd)
    return result


def _prepare_build_dir(conf, dst, input_docker_src):
    """
    Copies all necessary files for building docker images in a tmp folder,
    substituting some specific macros accordingly to handle customized
    images such as:
    - apt-install packages
    - pip install packages
    - jupyter lab extensions

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dst : str
        temporary folder where to prepare the files
    input_docker_src : str
        name of the dockerfile package to use

    """
    input_docker_dir = utils.data_file("../config/docker/" + input_docker_src)

    if conf.app_config_has("jupyter.dockerfile_src"):
        # dockerfile is redefined in application configuration
        dockerfile_src = conf.app_config()["jupyter.dockerfile_src"]
        input_docker_dir = _find_docker_src(input_docker_src, dockerfile_src)

    if isdir(input_docker_dir):
        dockerfile = input_docker_dir + "/Dockerfile"
        with TemporaryDirectory(prefix="aiscalator_") as tmp:
            settings = "jupyter.docker_image"
            allow = (conf.app_config_has(settings + ".allow_apt_packages") and
                     conf.app_config()[settings + ".allow_apt_packages"])
            if allow:
                dockerfile = _include_apt_package(conf, dockerfile,
                                                  join(tmp, "apt_package"))
            allow = (conf.app_config_has(settings + ".allow_requirements") and
                     conf.app_config()[settings + ".allow_requirements"])
            if allow:
                dockerfile = _include_requirements(conf, dockerfile,
                                                   join(tmp, "requirements"),
                                                   dst)
            allow = (conf.app_config_has(settings +
                                         ".allow_lab_extensions") and
                     conf.app_config()[settings + ".allow_lab_extensions"])
            if allow:
                dockerfile = _include_lab_extensions(conf, dockerfile,
                                                     join(tmp,
                                                          "lab_extension"))
            copy(dockerfile, dst + '/Dockerfile')
        # copy the other files other than Dockerfile
        for file in listdir(input_docker_dir):
            if file != "Dockerfile":
                copy(join(input_docker_dir, file), join(dst, file))


def _find_docker_src(input_docker_src, dirs):
    """
    Finds a pre-configured dockerfile package or return the default one.

    Parameters
    ----------
    input_docker_src : str
        name of the dockerfile package to use
    dirs : list
        list of directories to check

    Returns
    -------
    str
        path to the corresponding dockerfile package
    """
    for src in dirs:
        if isfile(join(src, input_docker_src, "Dockerfile")):
            return src
    return utils.data_file("../config/docker/" + input_docker_src)


def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):
    """
    Include apt-install packages into the dockerfile

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dockerfile : str
        path to the dockerfile to modify
    tmp : str
        path to the temporary dockerfile output

    Returns
    -------
        path to the resulting dockerfile
    """
    if conf.has_step_field("docker_image.apt_package_path"):

        content = conf.step_file_path("docker_image.apt_package_path")
        value = utils.format_file_content(content, suffix="\\\n")
        if value:
            value = ("RUN apt-get install -yqq \\\n" + value + """
    && apt-get purge --auto-remove -yqq $buildDeps \\
    && apt-get autoremove -yqq --purge \\
    && apt-get clean \\
    && rm -rf \\
        /var/lib/apt/lists/* \\
        /tmp/* \\
        /var/tmp/* \\
        /usr/share/man \\
        /usr/share/doc \\
        /usr/share/doc-base
        """)
            utils.copy_replace(dockerfile, tmp,
                               pattern="# apt_packages.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
    """
        Include pip install packages into the dockerfile

        Parameters
        ----------
        conf : AiscalatorConfig
            Configuration object for this step
        dockerfile : str
            path to the dockerfile to modify
        tmp : str
            path to the temporary dockerfile output
        dst : str
            path to the final temporary directory

        Returns
        -------
            path to the resulting dockerfile
        """
    if conf.has_step_field("docker_image.requirements_path"):
        content = conf.step_file_path("docker_image.requirements_path")
        copy(content, join(dst, 'requirements.txt'))
        utils.copy_replace(dockerfile, tmp,
                           pattern="# requirements.txt #",
                           replace_value="""
    COPY requirements.txt requirements.txt
    RUN pip install -r requirements.txt
    RUN rm requirements.txt""")
        return tmp
    return dockerfile


def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):
    """
        Include jupyter lab extensions packages into the dockerfile

        Parameters
        ----------
        conf : AiscalatorConfig
            Configuration object for this step
        dockerfile : str
            path to the dockerfile to modify
        tmp : str
            path to the temporary dockerfile output

        Returns
        -------
            path to the resulting dockerfile
        """
    if conf.has_step_field("docker_image.lab_extension_path"):

        content = conf.step_file_path("docker_image.lab_extension_path")
        prefix = "&& jupyter labextension install "
        value = utils.format_file_content(content,
                                          prefix=prefix, suffix=" \\\n")
        if value:
            value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
            utils.copy_replace(dockerfile, tmp,
                               pattern="# lab_extensions.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _run_build(conf: AiscalatorConfig):
    """
    Run the docker build command to produce the image and tag it.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step

    Returns
    -------
    str
        the docker image ID that was built
    """
    logger = logging.getLogger(__name__)
    commands = ["docker", "build", "--rm"]
    output_docker_name = None
    if conf.has_step_field("docker_image.output_docker_name"):
        output_docker_name = conf.step_field("docker_image.output_docker_name")
        commands += ["-t", output_docker_name + ":latest"]
    commands += ["."]
    log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
    logger.info("Running...: %s", " ".join(commands))
    utils.subprocess_run(commands, log_function=log.grep_logs)
    result = log.artifact()
    test = (
        result and
        output_docker_name is not None and
        conf.has_step_field("docker_image.output_docker_tag")
    )
    if test:
        commands = ["docker", "tag"]
        output_docker_tag = conf.step_field("docker_image.output_docker_tag")
        commands += [result, output_docker_name + ":" + output_docker_tag]
        # TODO implement docker tag output_docker_tag_commit_hash
        logger.info("Running...: %s", " ".join(commands))
        utils.subprocess_run(commands)
    return result


1		# -- coding: utf-8 --
2		# Apache Software License 2.0
3		#
4		# Copyright (c) 2018, Christophe Duong
5		#
6		# Licensed under the Apache License, Version 2.0 (the "License");
7		# you may not use this file except in compliance with the License.
8		# You may obtain a copy of the License at
9		#
10		# http://www.apache.org/licenses/LICENSE-2.0
11		#
12		# Unless required by applicable law or agreed to in writing, software
13		# distributed under the License is distributed on an "AS IS" BASIS,
14		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15		# See the License for the specific language governing permissions and
16		# limitations under the License.
17		"""
18		Module responsible to build docker images for jupyter subcommands
19		"""
20		import logging
21		from os import chdir
22		from os import getcwd
23		from os import listdir
24		from os.path import isdir
25		from os.path import isfile
26		from os.path import join
27		from shutil import copy
28		from tempfile import TemporaryDirectory
29
30		from aiscalator.core import utils
31		from aiscalator.core.config import AiscalatorConfig
32		from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
33
34
35		def build(conf: AiscalatorConfig):
36		"""
37		Builds the docker image following the parameters specified in the
38		focused step's configuration file
39
40		Parameters
41		----------
42		conf : AiscalatorConfig
43		Configuration object for this step
44
45		Returns
46		-------
47		string
48		the docker artifact name of the image built
49		"""
50		input_docker_src = conf.step_field("docker_image.input_docker_src")
51		# TODO check if input_docker_src refers to an existing docker image
52		# in which case, if no customization is needed, no need to build
53		cwd = getcwd()
54		result = None
55		try:
56		# Prepare a temp folder to build docker image
57		with TemporaryDirectory(prefix="aiscalator_") as tmp:
58		_prepare_build_dir(conf, tmp, input_docker_src)
59		chdir(tmp)
60		result = _run_build(conf)
61		finally:
62		chdir(cwd)
63		return result
64
65
66		def _prepare_build_dir(conf, dst, input_docker_src):
67		"""
68		Copies all necessary files for building docker images in a tmp folder,
69		substituting some specific macros accordingly to handle customized
70		images such as:
71		- apt-install packages
72		- pip install packages
73		- jupyter lab extensions
74
75		Parameters
76		----------
77		conf : AiscalatorConfig
78		Configuration object for this step
79		dst : str
80		temporary folder where to prepare the files
81		input_docker_src : str
82		name of the dockerfile package to use
83
84		"""
85		input_docker_dir = utils.data_file("../config/docker/" + input_docker_src)
86
87		if conf.app_config_has("jupyter.dockerfile_src"):
88		# dockerfile is redefined in application configuration
89		dockerfile_src = conf.app_config()["jupyter.dockerfile_src"]
90		input_docker_dir = _find_docker_src(input_docker_src, dockerfile_src)
91
92		if isdir(input_docker_dir):
93		dockerfile = input_docker_dir + "/Dockerfile"
94		with TemporaryDirectory(prefix="aiscalator_") as tmp:
95		settings = "jupyter.docker_image"
96		allow = (conf.app_config_has(settings + ".allow_apt_packages") and
97		conf.app_config()[settings + ".allow_apt_packages"])
98		if allow:
99		dockerfile = _include_apt_package(conf, dockerfile,
100		join(tmp, "apt_package"))
101		allow = (conf.app_config_has(settings + ".allow_requirements") and
102		conf.app_config()[settings + ".allow_requirements"])
103		if allow:
104		dockerfile = _include_requirements(conf, dockerfile,
105		join(tmp, "requirements"),
106		dst)
107		allow = (conf.app_config_has(settings +
108		".allow_lab_extensions") and
109		conf.app_config()[settings + ".allow_lab_extensions"])
110		if allow:
111		dockerfile = _include_lab_extensions(conf, dockerfile,
112		join(tmp,
113		"lab_extension"))
114		copy(dockerfile, dst + '/Dockerfile')
115		# copy the other files other than Dockerfile
116		for file in listdir(input_docker_dir):
117		if file != "Dockerfile":
118		copy(join(input_docker_dir, file), join(dst, file))
119
120
121		def _find_docker_src(input_docker_src, dirs):
122		"""
123		Finds a pre-configured dockerfile package or return the default one.
124
125		Parameters
126		----------
127		input_docker_src : str
128		name of the dockerfile package to use
129		dirs : list
130		list of directories to check
131
132		Returns
133		-------
134		str
135		path to the corresponding dockerfile package
136		"""
137		for src in dirs:
138		if isfile(join(src, input_docker_src, "Dockerfile")):
139		return src
140		return utils.data_file("../config/docker/" + input_docker_src)
141
142
143		def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):
144		"""
145		Include apt-install packages into the dockerfile
146
147		Parameters
148		----------
149		conf : AiscalatorConfig
150		Configuration object for this step
151		dockerfile : str
152		path to the dockerfile to modify
153		tmp : str
154		path to the temporary dockerfile output
155
156		Returns
157		-------
158		path to the resulting dockerfile
159		"""
160	View Code Duplication	if conf.has_step_field("docker_image.apt_package_path"):
		0 ignored issues – show Duplication introduced 2018-11-11 11:04 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
161		content = conf.step_file_path("docker_image.apt_package_path")
162		value = utils.format_file_content(content, suffix="\\\n")
163		if value:
164		value = ("RUN apt-get install -yqq \\\n" + value + """
165		&& apt-get purge --auto-remove -yqq $buildDeps \\
166		&& apt-get autoremove -yqq --purge \\
167		&& apt-get clean \\
168		&& rm -rf \\
169		/var/lib/apt/lists/* \\
170		/tmp/* \\
171		/var/tmp/* \\
172		/usr/share/man \\
173		/usr/share/doc \\
174		/usr/share/doc-base
175		""")
176		utils.copy_replace(dockerfile, tmp,
177		pattern="# apt_packages.txt #",
178		replace_value=value)
179		return tmp
180		return dockerfile
181
182
183		def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
184		"""
185		Include pip install packages into the dockerfile
186
187		Parameters
188		----------
189		conf : AiscalatorConfig
190		Configuration object for this step
191		dockerfile : str
192		path to the dockerfile to modify
193		tmp : str
194		path to the temporary dockerfile output
195		dst : str
196		path to the final temporary directory
197
198		Returns
199		-------
200		path to the resulting dockerfile
201		"""
202		if conf.has_step_field("docker_image.requirements_path"):
203		content = conf.step_file_path("docker_image.requirements_path")
204		copy(content, join(dst, 'requirements.txt'))
205		utils.copy_replace(dockerfile, tmp,
206		pattern="# requirements.txt #",
207		replace_value="""
208		COPY requirements.txt requirements.txt
209		RUN pip install -r requirements.txt
210		RUN rm requirements.txt""")
211		return tmp
212		return dockerfile
213
214
215		def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):
216		"""
217		Include jupyter lab extensions packages into the dockerfile
218
219		Parameters
220		----------
221		conf : AiscalatorConfig
222		Configuration object for this step
223		dockerfile : str
224		path to the dockerfile to modify
225		tmp : str
226		path to the temporary dockerfile output
227
228		Returns
229		-------
230		path to the resulting dockerfile
231		"""
232	View Code Duplication	if conf.has_step_field("docker_image.lab_extension_path"):
		0 ignored issues – show Duplication introduced 2018-11-11 11:04 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
233		content = conf.step_file_path("docker_image.lab_extension_path")
234		prefix = "&& jupyter labextension install "
235		value = utils.format_file_content(content,
236		prefix=prefix, suffix=" \\\n")
237		if value:
238		value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
239		utils.copy_replace(dockerfile, tmp,
240		pattern="# lab_extensions.txt #",
241		replace_value=value)
242		return tmp
243		return dockerfile
244
245
246		def _run_build(conf: AiscalatorConfig):
247		"""
248		Run the docker build command to produce the image and tag it.
249
250		Parameters
251		----------
252		conf : AiscalatorConfig
253		Configuration object for this step
254
255		Returns
256		-------
257		str
258		the docker image ID that was built
259		"""
260		logger = logging.getLogger(__name__)
261		commands = ["docker", "build", "--rm"]
262		output_docker_name = None
263		if conf.has_step_field("docker_image.output_docker_name"):
264		output_docker_name = conf.step_field("docker_image.output_docker_name")
265		commands += ["-t", output_docker_name + ":latest"]
266		commands += ["."]
267		log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
268		logger.info("Running...: %s", " ".join(commands))
269		utils.subprocess_run(commands, log_function=log.grep_logs)
270		result = log.artifact()
271		test = (
272		result and
273		output_docker_name is not None and
274		conf.has_step_field("docker_image.output_docker_tag")
275		)
276		if test:
277		commands = ["docker", "tag"]
278		output_docker_tag = conf.step_field("docker_image.output_docker_tag")
279		commands += [result, output_docker_name + ":" + output_docker_tag]
280		# TODO implement docker tag output_docker_tag_commit_hash
281		logger.info("Running...: %s", " ".join(commands))
282		utils.subprocess_run(commands)
283		return result
284

Aiscalate / aiscalator

Push — master ( 722b81...0abd85 )

_prepare_build_dir() C

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like