aiscalator.jupyter.docker_image

Complexity

Total Complexity

29

Size/Duplication

Total Lines	324
Duplicated Lines	30.25 %

Importance

Changes

0

8 Functions

Rating	Name	Duplication	Size	Complexity
A	build()	0	29	2
A	_find_docker_src()	0	20	3
A	_include_apt_repo()	31	31	3
C	_prepare_build_dir()	0	59	10
A	_run_build()	0	38	3
A	_include_requirements()	0	30	2
A	_include_apt_package()	38	39	3
A	_include_lab_extensions()	29	29	3

# -*- coding: utf-8 -*-
# Apache Software License 2.0
#
# Copyright (c) 2018, Christophe Duong
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Module responsible to build docker images for jupyter subcommands
"""
import logging
from os import chdir
from os import getcwd
from os import listdir
from os.path import isdir
from os.path import isfile
from os.path import join
from shutil import copy
from tempfile import TemporaryDirectory

from aiscalator.core import utils
from aiscalator.core.config import AiscalatorConfig
from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer


def build(conf: AiscalatorConfig):
    """
    Builds the docker image following the parameters specified in the
    focused step's configuration file

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step

    Returns
    -------
    string
        the docker artifact name of the image built
    """
    input_docker_src = conf.step_field("docker_image.input_docker_src")
    # TODO check if input_docker_src refers to an existing docker image
    # in which case, if no customization is needed, no need to build
    cwd = getcwd()
    result = None
    try:
        # Prepare a temp folder to build docker image
        with TemporaryDirectory(prefix="aiscalator_") as tmp:
            _prepare_build_dir(conf, tmp, input_docker_src)
            chdir(tmp)
            result = _run_build(conf)
    finally:
        chdir(cwd)
    return result


def _prepare_build_dir(conf, dst, input_docker_src):
    """
    Copies all necessary files for building docker images in a tmp folder,
    substituting some specific macros accordingly to handle customized
    images such as:
    - add-apt-repository
    - apt-install packages
    - pip install packages
    - jupyter lab extensions

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dst : str
        temporary folder where to prepare the files
    input_docker_src : str
        name of the dockerfile package to use

    """
    input_docker_dir = utils.data_file("../config/docker/" + input_docker_src)

    if conf.app_config_has("jupyter.dockerfile_src"):
        # dockerfile is redefined in application configuration
        dockerfile_src = conf.app_config()["jupyter.dockerfile_src"]
        input_docker_dir = _find_docker_src(input_docker_src, dockerfile_src)

    if isdir(input_docker_dir):
        dockerfile = input_docker_dir + "/Dockerfile"
        with TemporaryDirectory(prefix="aiscalator_") as tmp:
            stg = "jupyter.docker_image"
            allow = (conf.app_config_has(stg + ".allow_apt_repository") and
                     conf.app_config()[stg + ".allow_apt_repository"])
            if allow:
                dockerfile = _include_apt_repo(conf, dockerfile,
                                               join(tmp, "apt_repository"))
            allow = (conf.app_config_has(stg + ".allow_apt_packages") and
                     conf.app_config()[stg + ".allow_apt_packages"])
            if allow:
                dockerfile = _include_apt_package(conf, dockerfile,
                                                  join(tmp, "apt_package"))
            allow = (conf.app_config_has(stg + ".allow_requirements") and
                     conf.app_config()[stg + ".allow_requirements"])
            if allow:
                dockerfile = _include_requirements(conf, dockerfile,
                                                   join(tmp, "requirements"),
                                                   dst)
            allow = (conf.app_config_has(stg +
                                         ".allow_lab_extensions") and
                     conf.app_config()[stg + ".allow_lab_extensions"])
            if allow:
                dockerfile = _include_lab_extensions(conf, dockerfile,
                                                     join(tmp,
                                                          "lab_extension"))
            copy(dockerfile, dst + '/Dockerfile')
        # copy the other files other than Dockerfile
        for file in listdir(input_docker_dir):
            if file != "Dockerfile":
                copy(join(input_docker_dir, file), join(dst, file))


def _find_docker_src(input_docker_src, dirs):
    """
    Finds a pre-configured dockerfile package or return the default one.

    Parameters
    ----------
    input_docker_src : str
        name of the dockerfile package to use
    dirs : list
        list of directories to check

    Returns
    -------
    str
        path to the corresponding dockerfile package
    """
    for src in dirs:
        if isfile(join(src, input_docker_src, "Dockerfile")):
            return src
    return utils.data_file("../config/docker/" + input_docker_src)


def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp):

This code seems to be duplicated in your project.

    """
    Include add-apt-repository packages into the dockerfile

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dockerfile : str
        path to the dockerfile to modify
    tmp : str
        path to the temporary dockerfile output

    Returns
    -------
        path to the resulting dockerfile
    """
    if conf.has_step_field("docker_image.apt_repository_path"):
        content = conf.step_file_path("docker_image.apt_repository_path")
        value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
        if value:
            value = ("RUN apt-get update \\\n" +
                     " && apt-get install -yqq \\\n" +
                     "      software-properties-common \\\n" +
                     " && apt-add-repository \\\n" + value +
                     " && apt-get update")
            utils.copy_replace(dockerfile, tmp,
                               pattern="# apt_repository.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):

This code seems to be duplicated in your project.

    """
    Include apt-install packages into the dockerfile

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step
    dockerfile : str
        path to the dockerfile to modify
    tmp : str
        path to the temporary dockerfile output

    Returns
    -------
        path to the resulting dockerfile
    """
    if conf.has_step_field("docker_image.apt_package_path"):
        content = conf.step_file_path("docker_image.apt_package_path")
        value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
        if value:
            value = ("RUN apt-get update && apt-get install -yqq \\\n" +
                     value +
                     """    && apt-get purge --auto-remove -yqq $buildDeps \\
    && apt-get autoremove -yqq --purge \\
    && apt-get clean \\
    && rm -rf \\
    /var/lib/apt/lists/* \\
    /tmp/* \\
    /var/tmp/* \\
    /usr/share/man \\
    /usr/share/doc \\
    /usr/share/doc-base
""")
            utils.copy_replace(dockerfile, tmp,
                               pattern="# apt_packages.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
    """
        Include pip install packages into the dockerfile

        Parameters
        ----------
        conf : AiscalatorConfig
            Configuration object for this step
        dockerfile : str
            path to the dockerfile to modify
        tmp : str
            path to the temporary dockerfile output
        dst : str
            path to the final temporary directory

        Returns
        -------
            path to the resulting dockerfile
        """
    if conf.has_step_field("docker_image.requirements_path"):
        content = conf.step_file_path("docker_image.requirements_path")
        copy(content, join(dst, 'requirements.txt'))
        utils.copy_replace(dockerfile, tmp,
                           pattern="# requirements.txt #",
                           replace_value="""
    COPY requirements.txt requirements.txt
    RUN pip install -r requirements.txt
    RUN rm requirements.txt""")
        return tmp
    return dockerfile


def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):

This code seems to be duplicated in your project.

    """
        Include jupyter lab extensions packages into the dockerfile

        Parameters
        ----------
        conf : AiscalatorConfig
            Configuration object for this step
        dockerfile : str
            path to the dockerfile to modify
        tmp : str
            path to the temporary dockerfile output

        Returns
        -------
            path to the resulting dockerfile
        """
    if conf.has_step_field("docker_image.lab_extension_path"):
        content = conf.step_file_path("docker_image.lab_extension_path")
        prefix = "&& jupyter labextension install "
        value = utils.format_file_content(content,
                                          prefix=prefix, suffix=" \\\n")
        if value:
            value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
            utils.copy_replace(dockerfile, tmp,
                               pattern="# lab_extensions.txt #",
                               replace_value=value)
            return tmp
    return dockerfile


def _run_build(conf: AiscalatorConfig):
    """
    Run the docker build command to produce the image and tag it.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for this step

    Returns
    -------
    str
        the docker image ID that was built
    """
    logger = logging.getLogger(__name__)
    commands = ["docker", "build", "--rm"]
    output_docker_name = None
    if conf.has_step_field("docker_image.output_docker_name"):
        output_docker_name = conf.step_field("docker_image.output_docker_name")
        commands += ["-t", output_docker_name + ":latest"]
    commands += ["."]
    log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
    logger.info("Running...: %s", " ".join(commands))
    utils.subprocess_run(commands, log_function=log.grep_logs)
    result = log.artifact()
    test = (
        result and
        output_docker_name is not None and
        conf.has_step_field("docker_image.output_docker_tag")
    )
    if test:
        commands = ["docker", "tag"]
        output_docker_tag = conf.step_field("docker_image.output_docker_tag")
        commands += [result, output_docker_name + ":" + output_docker_tag]
        # TODO implement docker tag output_docker_tag_commit_hash
        logger.info("Running...: %s", " ".join(commands))
        utils.subprocess_run(commands)
    return result