aiscalator.jupyter.command._prepare_docker_image_env() - Code Metrics - Inspection of "Bump version: 0.1.4 → 0.1.5" - Aiscalate/aiscalator - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( e787a9...1a4fb3 )

by Christophe

created 2019-02-08 15:29 UTC

_prepare_docker_image_env() F

↳ Parent: aiscalator.jupyter.command

Complexity

Conditions

Size

Total Lines	52
Code Lines	33

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	33
dl	0
loc	52
rs	3.6
c	0
b	0
f	0
cc	14
nop	1

How to fix Long Method Complexity

# -*- coding: utf-8 -*-
# Apache Software License 2.0
#
# Copyright (c) 2018, Christophe Duong
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Implementations of commands for Jupyter
"""
import logging
import os.path
from os import makedirs

from aiscalator.core.config import AiscalatorConfig
from aiscalator.core.config import convert_to_format
from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
from aiscalator.core.utils import check_notebook_dir
from aiscalator.core.utils import copy_replace
from aiscalator.core.utils import data_file
from aiscalator.core.utils import notebook_file
from aiscalator.core.utils import subprocess_run
from aiscalator.core.utils import wait_for_jupyter_lab
from aiscalator.jupyter.docker_image import build


def _prepare_docker_env(conf: AiscalatorConfig, program, reason):
    """
    Assembles the list of commands to execute a docker run call

    When calling "docker run ...", this function also adds a set of
    additional parameters to mount the proper volumes and expose the
    correct environment for the call in the docker image mapped to the
    host directories. This is done so only some specific data and code
    folders are accessible within the docker image.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    program : List
        the rest of the commands to execute as part of
        the docker run call

    Returns
    -------
    List
        The full Array of Strings representing the commands to execute
        in the docker run call
    """
    # TODO: refactor using https://github.com/docker/docker-py ?
    commands = [
        "docker", "run", "--name", conf.step_container_name() + "_" + reason,
        "--rm",
        # TODO improve port publishing
        "-p", "10000:8888",
        "-p", "4040:4040"
    ]
    for env in conf.user_env_file(conf.step_field("task.env")):
        if os.path.isfile(env):
            commands += ["--env-file", env]
    commands += _prepare_docker_image_env(conf)
    code_path = conf.step_file_path('task.code_path')
    notebook, _ = notebook_file(code_path)
    check_notebook_dir(notebook)
    commands += [
        "--mount", "type=bind,source=" + os.path.dirname(notebook) +
        ",target=/home/jovyan/work/notebook/",
    ]
    commands += _prepare_task_env(conf)
    if conf.has_step_field("task.execution_dir_path"):
        execution_dir_path = conf.step_file_path('task.execution_dir_path')
        if execution_dir_path:
            makedirs(execution_dir_path, exist_ok=True)
        commands += [
            "--mount", "type=bind,source=" +
            execution_dir_path +
            ",target=/home/jovyan/work/notebook_run/"
        ]
    commands += program
    return commands


def _prepare_docker_image_env(conf: AiscalatorConfig):
    """
    Assemble the list of volumes to mount specific to
    building the docker image

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step

    Returns
    -------
    list
        list of commands to bind those volumes
    """
    commands = []
    if conf.config_path() is not None:
        commands += [
            "--mount",
            "type=bind,source=" + os.path.realpath(conf.config_path()) +
            ",target="
            "/home/jovyan/work/" + os.path.basename(conf.config_path()),
        ]
    if conf.has_step_field("docker_image.apt_repository_path"):
        apt_repo = conf.step_file_path('docker_image.apt_repository_path')
        if apt_repo and os.path.isfile(apt_repo):
            commands += [
                "--mount", "type=bind,source=" + apt_repo +
                ",target=/home/jovyan/work/apt_repository.txt",
            ]
    if conf.has_step_field("docker_image.apt_package_path"):
        apt_packages = conf.step_file_path('docker_image.apt_package_path')
        if apt_packages and os.path.isfile(apt_packages):
            commands += [
                "--mount", "type=bind,source=" + apt_packages +
                ",target=/home/jovyan/work/apt_packages.txt",
            ]
    if conf.has_step_field("docker_image.requirements_path"):
        requirements = conf.step_file_path('docker_image.requirements_path')
        if requirements and os.path.isfile(requirements):
            commands += [
                "--mount", "type=bind,source=" + requirements +
                ",target=/home/jovyan/work/requirements.txt",
            ]
    if conf.has_step_field("docker_image.lab_extension_path"):
        lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
        if lab_extensions and os.path.isfile(lab_extensions):
            commands += [
                "--mount", "type=bind,source=" + lab_extensions +
                ",target=/home/jovyan/work/lab_extensions.txt",
            ]
    return commands


def _prepare_task_env(conf: AiscalatorConfig):
    """
    Assemble the list of volumes to mount specific to
    the task execution

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step

    Returns
    -------
    list
        list of commands to bind those volumes
    """
    commands = []
    if conf.root_dir():
        commands += _mount_path(conf, "task.modules_src_path",
                                "/home/jovyan/work/modules/")
        commands += _mount_path(conf, "task.input_data_path",
                                "/home/jovyan/work/data/input/",
                                readonly=True)
        commands += _mount_path(conf, "task.output_data_path",
                                "/home/jovyan/work/data/output/",
                                make_dirs=True)
    return commands


def _mount_path(conf: AiscalatorConfig, field, target_path,
                readonly=False, make_dirs=False):
    """
    Returu commands to mount path from list field into the
    docker image when running.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    field : str
        the field in the configuration step that contains the path
    target_path : str
        where to mount them inside the container
    readonly : bool
        flag to mount the path as read-only
    make_dirs : bool
        flag to create the folder on the host before mounting if
        it doesn't exists.

    Returns
    -------
    list
        commands to mount all the paths from the field

    """
    commands = []
    if conf.has_step_field(field):
        for value in conf.step_field(field):
            # TODO handle URL
            for i in value:
                if make_dirs:
                    makedirs(os.path.realpath(conf.root_dir() + value[i]),
                             exist_ok=True)
                if os.path.exists(conf.root_dir() + value[i]):
                    commands += [
                        "--mount",
                        "type=bind,source=" +
                        os.path.realpath(conf.root_dir() + value[i]) +
                        ",target=" + os.path.join(target_path, i) +
                        (",readonly" if readonly else "")
                    ]
    return commands


def jupyter_run(conf: AiscalatorConfig, prepare_only=False,
                param=None, param_raw=None):
    """
    Executes the step in browserless mode using papermill

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    prepare_only : bool
        Indicates if papermill should replace the parameters of the
        notebook only or it should execute all the cells too

    Returns
    -------
    string
        the path to the output notebook resulting from the execution
        of this step
    """
    logger = logging.getLogger(__name__)
    conf.validate_config()
    docker_image = build(conf)
    if not docker_image:
        raise Exception("Failed to build docker image")
    notebook, _ = notebook_file(conf.step_file_path('task.code_path'))
    notebook = os.path.join("/home/jovyan/work/notebook/",
                            os.path.basename(notebook))
    notebook_output = conf.step_notebook_output_path(notebook)
    commands = _prepare_docker_env(conf, [
        docker_image, "bash", "start-papermill.sh",
        "papermill",
        notebook, notebook_output
    ], "run")
    if prepare_only:
        commands.append("--prepare-only")
    parameters = conf.step_extract_parameters()
    if parameters:
        commands += parameters
    if param:
        for parameter in param:
            commands += ["-p", parameter[0], parameter[1]]
    if param_raw:
        for raw_parameter in param_raw:
            commands += ["-r", raw_parameter[0], raw_parameter[1]]
    log = LogRegexAnalyzer()
    logger.info("Running...: %s", " ".join(commands))
    subprocess_run(commands, log_function=log.grep_logs)
    return os.path.join(conf.step_file_path('task.execution_dir_path'),
                        os.path.basename(notebook_output))


def jupyter_edit(conf: AiscalatorConfig, param=None, param_raw=None):
    """
    Starts a Jupyter Lab environment configured to edit the focused step

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    param : list
        list of tuples of parameters
    param_raw : list
        list of tuples of raw parameters
    Returns
    -------
    string
        Url of the running jupyter lab
    """
    logger = logging.getLogger(__name__)
    conf.validate_config()
    docker_image = build(conf)
    if docker_image:
        # TODO: shutdown other jupyter lab still running
        notebook, _ = notebook_file(conf.step_field('task.code_path'))
        notebook = os.path.basename(notebook)
        if conf.step_extract_parameters():
            jupyter_run(conf, prepare_only=True,
                        param=param,
                        param_raw=param_raw)
        commands = _prepare_docker_env(conf, [
            docker_image, "start.sh",
            'jupyter', 'lab'
        ], "edit")
        return wait_for_jupyter_lab(commands, logger, notebook,
                                    10000, "work/notebook")
    raise Exception("Failed to build docker image")


def jupyter_new(name, path, output_format="hocon"):
    """
    Starts a Jupyter Lab environment configured to edit a brand new step

    Parameters
    ----------
    name : str
        name of the new step
    path : str
        path to where the new step files should be created
    output_format : str
        the format of the new configuration file to produce
    Returns
    -------
    string
        Url of the running jupyter lab
    """
    step_file = os.path.join(path, name, name) + '.conf'
    if os.path.dirname(step_file):
        makedirs(os.path.dirname(step_file), exist_ok=True)
    copy_replace(data_file("../config/template/step.conf"), step_file,
                 pattern="Untitled", replace_value=name)
    if output_format != 'hocon':
        file = os.path.join(path, name, name) + '.' + output_format
        step_file = convert_to_format(step_file, output=file,
                                      output_format=output_format)

    notebook = os.path.join(path, name, 'notebook', name) + '.ipynb'
    if os.path.dirname(notebook):
        makedirs(os.path.dirname(notebook), exist_ok=True)
    copy_replace(data_file("../config/template/notebook.json"), notebook)

    open(os.path.join(path, name, "apt_repository.txt"), 'a').close()
    open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
    open(os.path.join(path, name, "requirements.txt"), 'a').close()
    open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
    jupyter_edit(AiscalatorConfig(config=step_file,
                                  step_selection=name))


1			# -- coding: utf-8 --
2			# Apache Software License 2.0
3			#
4			# Copyright (c) 2018, Christophe Duong
5			#
6			# Licensed under the Apache License, Version 2.0 (the "License");
7			# you may not use this file except in compliance with the License.
8			# You may obtain a copy of the License at
9			#
10			# http://www.apache.org/licenses/LICENSE-2.0
11			#
12			# Unless required by applicable law or agreed to in writing, software
13			# distributed under the License is distributed on an "AS IS" BASIS,
14			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15			# See the License for the specific language governing permissions and
16			# limitations under the License.
17			"""
18			Implementations of commands for Jupyter
19			"""
20			import logging
21			import os.path
22			from os import makedirs
23
24			from aiscalator.core.config import AiscalatorConfig
25			from aiscalator.core.config import convert_to_format
26			from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
27			from aiscalator.core.utils import check_notebook_dir
28			from aiscalator.core.utils import copy_replace
29			from aiscalator.core.utils import data_file
30			from aiscalator.core.utils import notebook_file
31			from aiscalator.core.utils import subprocess_run
32			from aiscalator.core.utils import wait_for_jupyter_lab
33			from aiscalator.jupyter.docker_image import build
34
35
36			def _prepare_docker_env(conf: AiscalatorConfig, program, reason):
37			"""
38			Assembles the list of commands to execute a docker run call
39
40			When calling "docker run ...", this function also adds a set of
41			additional parameters to mount the proper volumes and expose the
42			correct environment for the call in the docker image mapped to the
43			host directories. This is done so only some specific data and code
44			folders are accessible within the docker image.
45
46			Parameters
47			----------
48			conf : AiscalatorConfig
49			Configuration object for the step
50			program : List
51			the rest of the commands to execute as part of
52			the docker run call
53
54			Returns
55			-------
56			List
57			The full Array of Strings representing the commands to execute
58			in the docker run call
59			"""
60			# TODO: refactor using https://github.com/docker/docker-py ?
61			commands = [
62			"docker", "run", "--name", conf.step_container_name() + "_" + reason,
63			"--rm",
64			# TODO improve port publishing
65			"-p", "10000:8888",
66			"-p", "4040:4040"
67			]
68			for env in conf.user_env_file(conf.step_field("task.env")):
69			if os.path.isfile(env):
70			commands += ["--env-file", env]
71			commands += _prepare_docker_image_env(conf)
72			code_path = conf.step_file_path('task.code_path')
73			notebook, _ = notebook_file(code_path)
74			check_notebook_dir(notebook)
75			commands += [
76			"--mount", "type=bind,source=" + os.path.dirname(notebook) +
77			",target=/home/jovyan/work/notebook/",
78			]
79			commands += _prepare_task_env(conf)
80			if conf.has_step_field("task.execution_dir_path"):
81			execution_dir_path = conf.step_file_path('task.execution_dir_path')
82			if execution_dir_path:
83			makedirs(execution_dir_path, exist_ok=True)
84			commands += [
85			"--mount", "type=bind,source=" +
86			execution_dir_path +
87			",target=/home/jovyan/work/notebook_run/"
88			]
89			commands += program
90			return commands
91
92
93			def _prepare_docker_image_env(conf: AiscalatorConfig):
94			"""
95			Assemble the list of volumes to mount specific to
96			building the docker image
97
98			Parameters
99			----------
100			conf : AiscalatorConfig
101			Configuration object for the step
102
103			Returns
104			-------
105			list
106			list of commands to bind those volumes
107			"""
108			commands = []
109			if conf.config_path() is not None:
110			commands += [
111			"--mount",
112			"type=bind,source=" + os.path.realpath(conf.config_path()) +
113			",target="
114			"/home/jovyan/work/" + os.path.basename(conf.config_path()),
115			]
116			if conf.has_step_field("docker_image.apt_repository_path"):
117			apt_repo = conf.step_file_path('docker_image.apt_repository_path')
118			if apt_repo and os.path.isfile(apt_repo):
119			commands += [
120			"--mount", "type=bind,source=" + apt_repo +
121			",target=/home/jovyan/work/apt_repository.txt",
122			]
123			if conf.has_step_field("docker_image.apt_package_path"):
124			apt_packages = conf.step_file_path('docker_image.apt_package_path')
125			if apt_packages and os.path.isfile(apt_packages):
126			commands += [
127			"--mount", "type=bind,source=" + apt_packages +
128			",target=/home/jovyan/work/apt_packages.txt",
129			]
130			if conf.has_step_field("docker_image.requirements_path"):
131			requirements = conf.step_file_path('docker_image.requirements_path')
132			if requirements and os.path.isfile(requirements):
133			commands += [
134			"--mount", "type=bind,source=" + requirements +
135			",target=/home/jovyan/work/requirements.txt",
136			]
137			if conf.has_step_field("docker_image.lab_extension_path"):
138			lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
139			if lab_extensions and os.path.isfile(lab_extensions):
140			commands += [
141			"--mount", "type=bind,source=" + lab_extensions +
142			",target=/home/jovyan/work/lab_extensions.txt",
143			]
144			return commands
145
146
147			def _prepare_task_env(conf: AiscalatorConfig):
148			"""
149			Assemble the list of volumes to mount specific to
150			the task execution
151
152			Parameters
153			----------
154			conf : AiscalatorConfig
155			Configuration object for the step
156
157			Returns
158			-------
159			list
160			list of commands to bind those volumes
161			"""
162			commands = []
163			if conf.root_dir():
164			commands += _mount_path(conf, "task.modules_src_path",
165			"/home/jovyan/work/modules/")
166			commands += _mount_path(conf, "task.input_data_path",
167			"/home/jovyan/work/data/input/",
168			readonly=True)
169			commands += _mount_path(conf, "task.output_data_path",
170			"/home/jovyan/work/data/output/",
171			make_dirs=True)
172			return commands
173
174
175			def _mount_path(conf: AiscalatorConfig, field, target_path,
176			readonly=False, make_dirs=False):
177			"""
178			Returu commands to mount path from list field into the
179			docker image when running.
180
181			Parameters
182			----------
183			conf : AiscalatorConfig
184			Configuration object for the step
185			field : str
186			the field in the configuration step that contains the path
187			target_path : str
188			where to mount them inside the container
189			readonly : bool
190			flag to mount the path as read-only
191			make_dirs : bool
192			flag to create the folder on the host before mounting if
193			it doesn't exists.
194
195			Returns
196			-------
197			list
198			commands to mount all the paths from the field
199
200			"""
201			commands = []
202			if conf.has_step_field(field):
203			for value in conf.step_field(field):
204			# TODO handle URL
205			for i in value:
206			if make_dirs:
207			makedirs(os.path.realpath(conf.root_dir() + value[i]),
208			exist_ok=True)
209			if os.path.exists(conf.root_dir() + value[i]):
210			commands += [
211			"--mount",
212			"type=bind,source=" +
213			os.path.realpath(conf.root_dir() + value[i]) +
214			",target=" + os.path.join(target_path, i) +
215			(",readonly" if readonly else "")
216			]
217			return commands
218
219
220			def jupyter_run(conf: AiscalatorConfig, prepare_only=False,
221			param=None, param_raw=None):
222			"""
223			Executes the step in browserless mode using papermill
224
225			Parameters
226			----------
227			conf : AiscalatorConfig
228			Configuration object for the step
229			prepare_only : bool
230			Indicates if papermill should replace the parameters of the
231			notebook only or it should execute all the cells too
232
233			Returns
234			-------
235			string
236			the path to the output notebook resulting from the execution
237			of this step
238			"""
239			logger = logging.getLogger(__name__)
240			conf.validate_config()
241			docker_image = build(conf)
242			if not docker_image:
243			raise Exception("Failed to build docker image")
244			notebook, _ = notebook_file(conf.step_file_path('task.code_path'))
245			notebook = os.path.join("/home/jovyan/work/notebook/",
246			os.path.basename(notebook))
247			notebook_output = conf.step_notebook_output_path(notebook)
248			commands = _prepare_docker_env(conf, [
249			docker_image, "bash", "start-papermill.sh",
250			"papermill",
251			notebook, notebook_output
252			], "run")
253			if prepare_only:
254			commands.append("--prepare-only")
255			parameters = conf.step_extract_parameters()
256			if parameters:
257			commands += parameters
258			if param:
259			for parameter in param:
260			commands += ["-p", parameter[0], parameter[1]]
261			if param_raw:
262			for raw_parameter in param_raw:
263			commands += ["-r", raw_parameter[0], raw_parameter[1]]
264			log = LogRegexAnalyzer()
265			logger.info("Running...: %s", " ".join(commands))
266			subprocess_run(commands, log_function=log.grep_logs)
267			return os.path.join(conf.step_file_path('task.execution_dir_path'),
268			os.path.basename(notebook_output))
269
270
271			def jupyter_edit(conf: AiscalatorConfig, param=None, param_raw=None):
272			"""
273			Starts a Jupyter Lab environment configured to edit the focused step
274
275			Parameters
276			----------
277			conf : AiscalatorConfig
278			Configuration object for the step
279			param : list
280			list of tuples of parameters
281			param_raw : list
282			list of tuples of raw parameters
283			Returns
284			-------
285			string
286			Url of the running jupyter lab
287			"""
288			logger = logging.getLogger(__name__)
289			conf.validate_config()
290			docker_image = build(conf)
291			if docker_image:
292			# TODO: shutdown other jupyter lab still running
293			notebook, _ = notebook_file(conf.step_field('task.code_path'))
294			notebook = os.path.basename(notebook)
295			if conf.step_extract_parameters():
296			jupyter_run(conf, prepare_only=True,
297			param=param,
298			param_raw=param_raw)
299			commands = _prepare_docker_env(conf, [
300			docker_image, "start.sh",
301			'jupyter', 'lab'
302			], "edit")
303			return wait_for_jupyter_lab(commands, logger, notebook,
304			10000, "work/notebook")
305			raise Exception("Failed to build docker image")
306
307
308			def jupyter_new(name, path, output_format="hocon"):
309			"""
310			Starts a Jupyter Lab environment configured to edit a brand new step
311
312			Parameters
313			----------
314			name : str
315			name of the new step
316			path : str
317			path to where the new step files should be created
318			output_format : str
319			the format of the new configuration file to produce
320			Returns
321			-------
322			string
323			Url of the running jupyter lab
324			"""
325			step_file = os.path.join(path, name, name) + '.conf'
326			if os.path.dirname(step_file):
327			makedirs(os.path.dirname(step_file), exist_ok=True)
328			copy_replace(data_file("../config/template/step.conf"), step_file,
329			pattern="Untitled", replace_value=name)
330			if output_format != 'hocon':
331			file = os.path.join(path, name, name) + '.' + output_format
332			step_file = convert_to_format(step_file, output=file,
333			output_format=output_format)
334
335			notebook = os.path.join(path, name, 'notebook', name) + '.ipynb'
336			if os.path.dirname(notebook):
337			makedirs(os.path.dirname(notebook), exist_ok=True)
338			copy_replace(data_file("../config/template/notebook.json"), notebook)
339
340			open(os.path.join(path, name, "apt_repository.txt"), 'a').close()
341			open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
342			open(os.path.join(path, name, "requirements.txt"), 'a').close()
343			open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
344			jupyter_edit(AiscalatorConfig(config=step_file,
345			step_selection=name))
346

Aiscalate / aiscalator

Push — master ( e787a9...1a4fb3 )

_prepare_docker_image_env() F

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like