aiscalator.jupyter.command._prepare_docker_env() - Code Metrics - Inspection of "Merge pull request #10 from Aiscalate/dev" - Aiscalate/aiscalator - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 5927b1...ccc1cd )

by Christophe

created 2018-11-18 19:25 UTC

aiscalator.jupyter.command._prepare_docker_env() B

↳ Parent: aiscalator.jupyter.command

Complexity

Conditions

Size

Total Lines	55
Code Lines	27

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	27
dl	0
loc	55
rs	8.7653
c	0
b	0
f	0
cc	5
nop	3

How to fix Long Method

# -*- coding: utf-8 -*-
# Apache Software License 2.0
#
# Copyright (c) 2018, Christophe Duong
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Implementations of commands for Jupyter
"""
import logging
import os.path
from os import makedirs

from aiscalator.core.config import AiscalatorConfig
from aiscalator.core.config import convert_to_format
from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
from aiscalator.core.utils import check_notebook_dir
from aiscalator.core.utils import copy_replace
from aiscalator.core.utils import data_file
from aiscalator.core.utils import notebook_file
from aiscalator.core.utils import subprocess_run
from aiscalator.core.utils import wait_for_jupyter_lab
from aiscalator.jupyter.docker_image import build


def _prepare_docker_env(conf: AiscalatorConfig, program, reason):
    """
    Assembles the list of commands to execute a docker run call

    When calling "docker run ...", this function also adds a set of
    additional parameters to mount the proper volumes and expose the
    correct environment for the call in the docker image mapped to the
    host directories. This is done so only some specific data and code
    folders are accessible within the docker image.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    program : List
        the rest of the commands to execute as part of
        the docker run call

    Returns
    -------
    List
        The full Array of Strings representing the commands to execute
        in the docker run call
    """
    # TODO: refactor using https://github.com/docker/docker-py ?
    commands = [
        "docker", "run", "--name", conf.step_container_name() + "_" + reason,
        "--rm",
        # TODO improve port publishing
        "-p", "10000:8888",
        "-p", "4040:4040"
    ]
    for env in conf.user_env_file():
        if os.path.isfile(env):
            commands += ["--env-file", env]
    commands += _prepare_docker_image_env(conf)
    code_path = conf.step_file_path('task.code_path')
    notebook, _ = notebook_file(code_path)
    check_notebook_dir(notebook)
    commands += [
        "--mount", "type=bind,source=" + os.path.dirname(notebook) +
        ",target=/home/jovyan/work/notebook/",
    ]
    commands += _prepare_task_env(conf)
    if conf.has_step_field("task.execution_dir_path"):
        execution_dir_path = conf.step_file_path('task.execution_dir_path')
        if execution_dir_path:
            makedirs(execution_dir_path, exist_ok=True)
        commands += [
            "--mount", "type=bind,source=" +
            execution_dir_path +
            ",target=/home/jovyan/work/notebook_run/"
        ]
    commands += program
    return commands


def _prepare_docker_image_env(conf: AiscalatorConfig):
    """
    Assemble the list of volumes to mount specific to
    building the docker image

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step

    Returns
    -------
    list
        list of commands to bind those volumes
    """
    commands = []
    if conf.config_path() is not None:
        commands += [
            "--mount",
            "type=bind,source=" + os.path.realpath(conf.config_path()) +
            ",target="
            "/home/jovyan/work/" + os.path.basename(conf.config_path()),
        ]
    if conf.has_step_field("docker_image.apt_package_path"):
        apt_packages = conf.step_file_path('docker_image.apt_package_path')
        if apt_packages and os.path.isfile(apt_packages):
            commands += [
                "--mount", "type=bind,source=" + apt_packages +
                ",target=/home/jovyan/work/apt_packages.txt",
            ]
    if conf.has_step_field("docker_image.requirements_path"):
        requirements = conf.step_file_path('docker_image.requirements_path')
        if requirements and os.path.isfile(requirements):
            commands += [
                "--mount", "type=bind,source=" + requirements +
                ",target=/home/jovyan/work/requirements.txt",
            ]
    if conf.has_step_field("docker_image.lab_extension_path"):
        lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
        if lab_extensions and os.path.isfile(lab_extensions):
            commands += [
                "--mount", "type=bind,source=" + lab_extensions +
                ",target=/home/jovyan/work/lab_extensions.txt",
            ]
    return commands


def _prepare_task_env(conf: AiscalatorConfig):
    """
    Assemble the list of volumes to mount specific to
    the task execution

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step

    Returns
    -------
    list
        list of commands to bind those volumes
    """
    commands = []
    if conf.root_dir():
        commands += _mount_path(conf, "task.modules_src_path",
                                "/home/jovyan/work/modules/")
        commands += _mount_path(conf, "task.input_data_path",
                                "/home/jovyan/work/data/input/",
                                readonly=True)
        commands += _mount_path(conf, "task.output_data_path",
                                "/home/jovyan/work/data/output/",
                                make_dirs=True)
    return commands


def _mount_path(conf: AiscalatorConfig, field, target_path,
                readonly=False, make_dirs=False):
    """
    Returu commands to mount path from list field into the
    docker image when running.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    field : str
        the field in the configuration step that contains the path
    target_path : str
        where to mount them inside the container
    readonly : bool
        flag to mount the path as read-only
    make_dirs : bool
        flag to create the folder on the host before mounting if
        it doesn't exists.

    Returns
    -------
    list
        commands to mount all the paths from the field

    """
    commands = []
    if conf.has_step_field(field):
        for value in conf.step_field(field):
            # TODO handle URL
            for i in value:
                if make_dirs:
                    makedirs(os.path.realpath(conf.root_dir() + value[i]),
                             exist_ok=True)
                if os.path.exists(conf.root_dir() + value[i]):
                    commands += [
                        "--mount",
                        "type=bind,source=" +
                        os.path.realpath(conf.root_dir() + value[i]) +
                        ",target=" + target_path + i +
                        (",readonly" if readonly else "")
                    ]
    return commands


def jupyter_run(conf: AiscalatorConfig, prepare_only=False,
                param=None, param_raw=None):
    """
    Executes the step in browserless mode using papermill

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    prepare_only : bool
        Indicates if papermill should replace the parameters of the
        notebook only or it should execute all the cells too

    Returns
    -------
    string
        the path to the output notebook resulting from the execution
        of this step
    """
    logger = logging.getLogger(__name__)
    conf.validate_config()
    docker_image = build(conf)
    if not docker_image:
        raise Exception("Failed to build docker image")
    notebook, _ = notebook_file(conf.step_file_path('task.code_path'))
    notebook = os.path.join("/home/jovyan/work/notebook/",
                            os.path.basename(notebook))
    notebook_output = conf.step_notebook_output_path(notebook)
    commands = _prepare_docker_env(conf, [
        docker_image, "bash", "start-papermill.sh",
        "papermill",
        notebook, notebook_output
    ], "run")
    if prepare_only:
        commands.append("--prepare-only")
    parameters = conf.step_extract_parameters()
    if parameters:
        commands += parameters
    if param:
        for parameter in param:
            commands += ["-p", parameter[0], parameter[1]]
    if param_raw:
        for raw_parameter in param_raw:
            commands += ["-r", raw_parameter[0], raw_parameter[1]]
    log = LogRegexAnalyzer()
    logger.info("Running...: %s", " ".join(commands))
    subprocess_run(commands, log_function=log.grep_logs)
    return os.path.join(conf.step_file_path('task.execution_dir_path'),
                        os.path.basename(notebook_output))


def jupyter_edit(conf: AiscalatorConfig, param=None, param_raw=None):
    """
    Starts a Jupyter Lab environment configured to edit the focused step

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    param : list
        list of tuples of parameters
    param_raw : list
        list of tuples of raw parameters
    Returns
    -------
    string
        Url of the running jupyter lab
    """
    logger = logging.getLogger(__name__)
    conf.validate_config()
    docker_image = build(conf)
    if docker_image:
        # TODO: shutdown other jupyter lab still running
        notebook, _ = notebook_file(conf.step_field('task.code_path'))
        notebook = os.path.basename(notebook)
        if conf.step_extract_parameters():
            jupyter_run(conf, prepare_only=True,
                        param=param,
                        param_raw=param_raw)
        commands = _prepare_docker_env(conf, [
            docker_image, "start.sh",
            'jupyter', 'lab'
        ], "edit")
        return wait_for_jupyter_lab(commands, logger, notebook,
                                    10000, "work/notebook")
    raise Exception("Failed to build docker image")


def jupyter_new(name, path, output_format="hocon"):
    """
    Starts a Jupyter Lab environment configured to edit a brand new step

    Parameters
    ----------
    name : str
        name of the new step
    path : str
        path to where the new step files should be created
    output_format : str
        the format of the new configuration file to produce
    Returns
    -------
    string
        Url of the running jupyter lab
    """
    step_file = os.path.join(path, name, name) + '.conf'
    if os.path.dirname(step_file):
        makedirs(os.path.dirname(step_file), exist_ok=True)
    copy_replace(data_file("../config/template/step.conf"), step_file,
                 pattern="Untitled", replace_value=name)
    if output_format != 'hocon':
        file = os.path.join(path, name, name) + '.' + output_format
        step_file = convert_to_format(step_file, output=file,
                                      output_format=output_format)

    notebook = os.path.join(path, name, 'notebook', name) + '.ipynb'
    if os.path.dirname(notebook):
        makedirs(os.path.dirname(notebook), exist_ok=True)
    copy_replace(data_file("../config/template/notebook.json"), notebook)

    open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
    open(os.path.join(path, name, "requirements.txt"), 'a').close()
    open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
    jupyter_edit(AiscalatorConfig(config=step_file,
                                  step_selection=name))


1			# -- coding: utf-8 --
2			# Apache Software License 2.0
3			#
4			# Copyright (c) 2018, Christophe Duong
5			#
6			# Licensed under the Apache License, Version 2.0 (the "License");
7			# you may not use this file except in compliance with the License.
8			# You may obtain a copy of the License at
9			#
10			# http://www.apache.org/licenses/LICENSE-2.0
11			#
12			# Unless required by applicable law or agreed to in writing, software
13			# distributed under the License is distributed on an "AS IS" BASIS,
14			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15			# See the License for the specific language governing permissions and
16			# limitations under the License.
17			"""
18			Implementations of commands for Jupyter
19			"""
20			import logging
21			import os.path
22			from os import makedirs
23
24			from aiscalator.core.config import AiscalatorConfig
25			from aiscalator.core.config import convert_to_format
26			from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
27			from aiscalator.core.utils import check_notebook_dir
28			from aiscalator.core.utils import copy_replace
29			from aiscalator.core.utils import data_file
30			from aiscalator.core.utils import notebook_file
31			from aiscalator.core.utils import subprocess_run
32			from aiscalator.core.utils import wait_for_jupyter_lab
33			from aiscalator.jupyter.docker_image import build
34
35
36			def _prepare_docker_env(conf: AiscalatorConfig, program, reason):
37			"""
38			Assembles the list of commands to execute a docker run call
39
40			When calling "docker run ...", this function also adds a set of
41			additional parameters to mount the proper volumes and expose the
42			correct environment for the call in the docker image mapped to the
43			host directories. This is done so only some specific data and code
44			folders are accessible within the docker image.
45
46			Parameters
47			----------
48			conf : AiscalatorConfig
49			Configuration object for the step
50			program : List
51			the rest of the commands to execute as part of
52			the docker run call
53
54			Returns
55			-------
56			List
57			The full Array of Strings representing the commands to execute
58			in the docker run call
59			"""
60			# TODO: refactor using https://github.com/docker/docker-py ?
61			commands = [
62			"docker", "run", "--name", conf.step_container_name() + "_" + reason,
63			"--rm",
64			# TODO improve port publishing
65			"-p", "10000:8888",
66			"-p", "4040:4040"
67			]
68			for env in conf.user_env_file():
69			if os.path.isfile(env):
70			commands += ["--env-file", env]
71			commands += _prepare_docker_image_env(conf)
72			code_path = conf.step_file_path('task.code_path')
73			notebook, _ = notebook_file(code_path)
74			check_notebook_dir(notebook)
75			commands += [
76			"--mount", "type=bind,source=" + os.path.dirname(notebook) +
77			",target=/home/jovyan/work/notebook/",
78			]
79			commands += _prepare_task_env(conf)
80			if conf.has_step_field("task.execution_dir_path"):
81			execution_dir_path = conf.step_file_path('task.execution_dir_path')
82			if execution_dir_path:
83			makedirs(execution_dir_path, exist_ok=True)
84			commands += [
85			"--mount", "type=bind,source=" +
86			execution_dir_path +
87			",target=/home/jovyan/work/notebook_run/"
88			]
89			commands += program
90			return commands
91
92
93			def _prepare_docker_image_env(conf: AiscalatorConfig):
94			"""
95			Assemble the list of volumes to mount specific to
96			building the docker image
97
98			Parameters
99			----------
100			conf : AiscalatorConfig
101			Configuration object for the step
102
103			Returns
104			-------
105			list
106			list of commands to bind those volumes
107			"""
108			commands = []
109			if conf.config_path() is not None:
110			commands += [
111			"--mount",
112			"type=bind,source=" + os.path.realpath(conf.config_path()) +
113			",target="
114			"/home/jovyan/work/" + os.path.basename(conf.config_path()),
115			]
116			if conf.has_step_field("docker_image.apt_package_path"):
117			apt_packages = conf.step_file_path('docker_image.apt_package_path')
118			if apt_packages and os.path.isfile(apt_packages):
119			commands += [
120			"--mount", "type=bind,source=" + apt_packages +
121			",target=/home/jovyan/work/apt_packages.txt",
122			]
123			if conf.has_step_field("docker_image.requirements_path"):
124			requirements = conf.step_file_path('docker_image.requirements_path')
125			if requirements and os.path.isfile(requirements):
126			commands += [
127			"--mount", "type=bind,source=" + requirements +
128			",target=/home/jovyan/work/requirements.txt",
129			]
130			if conf.has_step_field("docker_image.lab_extension_path"):
131			lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
132			if lab_extensions and os.path.isfile(lab_extensions):
133			commands += [
134			"--mount", "type=bind,source=" + lab_extensions +
135			",target=/home/jovyan/work/lab_extensions.txt",
136			]
137			return commands
138
139
140			def _prepare_task_env(conf: AiscalatorConfig):
141			"""
142			Assemble the list of volumes to mount specific to
143			the task execution
144
145			Parameters
146			----------
147			conf : AiscalatorConfig
148			Configuration object for the step
149
150			Returns
151			-------
152			list
153			list of commands to bind those volumes
154			"""
155			commands = []
156			if conf.root_dir():
157			commands += _mount_path(conf, "task.modules_src_path",
158			"/home/jovyan/work/modules/")
159			commands += _mount_path(conf, "task.input_data_path",
160			"/home/jovyan/work/data/input/",
161			readonly=True)
162			commands += _mount_path(conf, "task.output_data_path",
163			"/home/jovyan/work/data/output/",
164			make_dirs=True)
165			return commands
166
167
168			def _mount_path(conf: AiscalatorConfig, field, target_path,
169			readonly=False, make_dirs=False):
170			"""
171			Returu commands to mount path from list field into the
172			docker image when running.
173
174			Parameters
175			----------
176			conf : AiscalatorConfig
177			Configuration object for the step
178			field : str
179			the field in the configuration step that contains the path
180			target_path : str
181			where to mount them inside the container
182			readonly : bool
183			flag to mount the path as read-only
184			make_dirs : bool
185			flag to create the folder on the host before mounting if
186			it doesn't exists.
187
188			Returns
189			-------
190			list
191			commands to mount all the paths from the field
192
193			"""
194			commands = []
195			if conf.has_step_field(field):
196			for value in conf.step_field(field):
197			# TODO handle URL
198			for i in value:
199			if make_dirs:
200			makedirs(os.path.realpath(conf.root_dir() + value[i]),
201			exist_ok=True)
202			if os.path.exists(conf.root_dir() + value[i]):
203			commands += [
204			"--mount",
205			"type=bind,source=" +
206			os.path.realpath(conf.root_dir() + value[i]) +
207			",target=" + target_path + i +
208			(",readonly" if readonly else "")
209			]
210			return commands
211
212
213			def jupyter_run(conf: AiscalatorConfig, prepare_only=False,
214			param=None, param_raw=None):
215			"""
216			Executes the step in browserless mode using papermill
217
218			Parameters
219			----------
220			conf : AiscalatorConfig
221			Configuration object for the step
222			prepare_only : bool
223			Indicates if papermill should replace the parameters of the
224			notebook only or it should execute all the cells too
225
226			Returns
227			-------
228			string
229			the path to the output notebook resulting from the execution
230			of this step
231			"""
232			logger = logging.getLogger(__name__)
233			conf.validate_config()
234			docker_image = build(conf)
235			if not docker_image:
236			raise Exception("Failed to build docker image")
237			notebook, _ = notebook_file(conf.step_file_path('task.code_path'))
238			notebook = os.path.join("/home/jovyan/work/notebook/",
239			os.path.basename(notebook))
240			notebook_output = conf.step_notebook_output_path(notebook)
241			commands = _prepare_docker_env(conf, [
242			docker_image, "bash", "start-papermill.sh",
243			"papermill",
244			notebook, notebook_output
245			], "run")
246			if prepare_only:
247			commands.append("--prepare-only")
248			parameters = conf.step_extract_parameters()
249			if parameters:
250			commands += parameters
251			if param:
252			for parameter in param:
253			commands += ["-p", parameter[0], parameter[1]]
254			if param_raw:
255			for raw_parameter in param_raw:
256			commands += ["-r", raw_parameter[0], raw_parameter[1]]
257			log = LogRegexAnalyzer()
258			logger.info("Running...: %s", " ".join(commands))
259			subprocess_run(commands, log_function=log.grep_logs)
260			return os.path.join(conf.step_file_path('task.execution_dir_path'),
261			os.path.basename(notebook_output))
262
263
264			def jupyter_edit(conf: AiscalatorConfig, param=None, param_raw=None):
265			"""
266			Starts a Jupyter Lab environment configured to edit the focused step
267
268			Parameters
269			----------
270			conf : AiscalatorConfig
271			Configuration object for the step
272			param : list
273			list of tuples of parameters
274			param_raw : list
275			list of tuples of raw parameters
276			Returns
277			-------
278			string
279			Url of the running jupyter lab
280			"""
281			logger = logging.getLogger(__name__)
282			conf.validate_config()
283			docker_image = build(conf)
284			if docker_image:
285			# TODO: shutdown other jupyter lab still running
286			notebook, _ = notebook_file(conf.step_field('task.code_path'))
287			notebook = os.path.basename(notebook)
288			if conf.step_extract_parameters():
289			jupyter_run(conf, prepare_only=True,
290			param=param,
291			param_raw=param_raw)
292			commands = _prepare_docker_env(conf, [
293			docker_image, "start.sh",
294			'jupyter', 'lab'
295			], "edit")
296			return wait_for_jupyter_lab(commands, logger, notebook,
297			10000, "work/notebook")
298			raise Exception("Failed to build docker image")
299
300
301			def jupyter_new(name, path, output_format="hocon"):
302			"""
303			Starts a Jupyter Lab environment configured to edit a brand new step
304
305			Parameters
306			----------
307			name : str
308			name of the new step
309			path : str
310			path to where the new step files should be created
311			output_format : str
312			the format of the new configuration file to produce
313			Returns
314			-------
315			string
316			Url of the running jupyter lab
317			"""
318			step_file = os.path.join(path, name, name) + '.conf'
319			if os.path.dirname(step_file):
320			makedirs(os.path.dirname(step_file), exist_ok=True)
321			copy_replace(data_file("../config/template/step.conf"), step_file,
322			pattern="Untitled", replace_value=name)
323			if output_format != 'hocon':
324			file = os.path.join(path, name, name) + '.' + output_format
325			step_file = convert_to_format(step_file, output=file,
326			output_format=output_format)
327
328			notebook = os.path.join(path, name, 'notebook', name) + '.ipynb'
329			if os.path.dirname(notebook):
330			makedirs(os.path.dirname(notebook), exist_ok=True)
331			copy_replace(data_file("../config/template/notebook.json"), notebook)
332
333			open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
334			open(os.path.join(path, name, "requirements.txt"), 'a').close()
335			open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
336			jupyter_edit(AiscalatorConfig(config=step_file,
337			step_selection=name))
338

Aiscalate / aiscalator

Push — master ( 5927b1...ccc1cd )

aiscalator.jupyter.command._prepare_docker_env() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like