aiscalator.jupyter.command._mount_path() - Code Metrics - Inspection of "Merge pull request #6 from Aiscalate/dev" - Aiscalate/aiscalator - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 722b81...0abd85 )

by Christophe

created 2018-11-14 21:12 UTC

aiscalator.jupyter.command._mount_path() B

↳ Parent: aiscalator.jupyter.command

Complexity

Conditions

Size

Total Lines	43
Code Lines	17

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	17
dl	0
loc	43
rs	8
c	0
b	0
f	0
cc	7
nop	5

# -*- coding: utf-8 -*-
# Apache Software License 2.0
#
# Copyright (c) 2018, Christophe Duong
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Implementations of commands for Jupyter
"""
import logging
import os.path
from os import makedirs

from aiscalator.core.config import AiscalatorConfig
from aiscalator.core.config import convert_to_format
from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
from aiscalator.core.utils import copy_replace
from aiscalator.core.utils import data_file
from aiscalator.core.utils import subprocess_run
from aiscalator.core.utils import wait_for_jupyter_lab
from aiscalator.jupyter.docker_image import build


def _prepare_docker_env(conf: AiscalatorConfig, program):
    """
    Assembles the list of commands to execute a docker run call

    When calling "docker run ...", this function also adds a set of
    additional parameters to mount the proper volumes and expose the
    correct environment for the call in the docker image mapped to the
    host directories. This is done so only some specific data and code
    folders are accessible within the docker image.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    program : List
        the rest of the commands to execute as part of
        the docker run call

    Returns
    -------
    List
        The full Array of Strings representing the commands to execute
        in the docker run call
    """
    # TODO: refactor using https://github.com/docker/docker-py ?
    commands = [
        "docker", "run", "--name", conf.step_container_name(), "--rm",
        # TODO improve port publishing
        "-p", "10000:8888",
        "-p", "4040:4040"
    ]
    for env in conf.user_env_file():
        if os.path.isfile(env):
            commands += ["--env-file", env]
    commands += _prepare_docker_image_env(conf)
    code_path = conf.step_file_path('task.code_path')
    commands += [
        "--mount", "type=bind,source=" + os.path.dirname(code_path) +
        ",target=/home/jovyan/work/notebook/",
    ]
    commands += _prepare_task_env(conf)
    if conf.has_step_field("task.execution_dir_path"):
        makedirs(conf.step_file_path('task.execution_dir_path'), exist_ok=True)
        commands += [
            "--mount", "type=bind,source=" +
            conf.step_file_path('task.execution_dir_path') +
            ",target=/home/jovyan/work/notebook_run/"
        ]
    commands += program
    return commands


def _prepare_docker_image_env(conf: AiscalatorConfig):
    """
    Assemble the list of volumes to mount specific to
    building the docker image

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step

    Returns
    -------
    list
        list of commands to bind those volumes
    """
    commands = []
    if conf.config_path() is not None:
        commands += [
            "--mount",
            "type=bind,source=" + os.path.abspath(conf.config_path()) +
            ",target="
            "/home/jovyan/work/" + os.path.basename(conf.config_path()),
        ]
    if conf.has_step_field("docker_image.apt_package_path"):
        apt_packages = conf.step_file_path('docker_image.apt_package_path')
        if apt_packages and os.path.isfile(apt_packages):
            commands += [
                "--mount", "type=bind,source=" + apt_packages +
                ",target=/home/jovyan/work/apt_packages.txt",
            ]
    if conf.has_step_field("docker_image.requirements_path"):
        requirements = conf.step_file_path('docker_image.requirements_path')
        if requirements and os.path.isfile(requirements):
            commands += [
                "--mount", "type=bind,source=" + requirements +
                ",target=/home/jovyan/work/requirements.txt",
            ]
    if conf.has_step_field("docker_image.lab_extension_path"):
        lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
        if lab_extensions and os.path.isfile(lab_extensions):
            commands += [
                "--mount", "type=bind,source=" + lab_extensions +
                ",target=/home/jovyan/work/lab_extensions.txt",
            ]
    return commands


def _prepare_task_env(conf: AiscalatorConfig):
    """
    Assemble the list of volumes to mount specific to
    the task execution

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step

    Returns
    -------
    list
        list of commands to bind those volumes
    """
    commands = []
    if conf.root_dir():
        commands += _mount_path(conf, "task.modules_src_path",
                                "/home/jovyan/work/modules/")
        commands += _mount_path(conf, "task.input_data_path",
                                "/home/jovyan/work/data/input/",
                                readonly=True)
        commands += _mount_path(conf, "task.output_data_path",
                                "/home/jovyan/work/data/output/",
                                make_dirs=True)
    return commands


def _mount_path(conf: AiscalatorConfig, field, target_path,
                readonly=False, make_dirs=False):
    """
    Returu commands to mount path from list field into the
    docker image when running.

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    field : str
        the field in the configuration step that contains the path
    target_path : str
        where to mount them inside the container
    readonly : bool
        flag to mount the path as read-only
    make_dirs : bool
        flag to create the folder on the host before mounting if
        it doesn't exists.

    Returns
    -------
    list
        commands to mount all the paths from the field

    """
    commands = []
    if conf.has_step_field(field):
        for value in conf.step_field(field):
            # TODO handle URL
            for i in value:
                if make_dirs:
                    makedirs(os.path.abspath(conf.root_dir() + i),
                             exist_ok=True)
                if os.path.exists(conf.root_dir() + i):
                    commands += [
                        "--mount",
                        "type=bind,source=" +
                        os.path.abspath(conf.root_dir() + i) +
                        ",target=" + target_path + value[i] +
                        (",readonly" if readonly else "")
                    ]
    return commands


def jupyter_run(conf: AiscalatorConfig, prepare_only=False):
    """
    Executes the step in browserless mode using papermill

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step
    prepare_only : bool
        Indicates if papermill should replace the parameters of the
        notebook only or it should execute all the cells too

    Returns
    -------
    string
        the path to the output notebook resulting from the execution
        of this step
    """
    logger = logging.getLogger(__name__)
    conf.validate_config()
    docker_image = build(conf)
    if not docker_image:
        raise Exception("Failed to build docker image")
    notebook = ("/home/jovyan/work/notebook/" +
                os.path.basename(conf.step_file_path('task.code_path')))
    notebook_output = conf.step_notebook_output_path(notebook)
    parameters = conf.step_extract_parameters()
    commands = _prepare_docker_env(conf, [
        docker_image, "start.sh",
        # TODO: check step type, if jupyter then papermill
        "papermill",
        notebook, notebook_output
    ])
    if prepare_only:
        commands.append("--prepare-only")
    commands += parameters
    log = LogRegexAnalyzer()
    logger.info("Running...: %s", " ".join(commands))
    subprocess_run(commands, log_function=log.grep_logs)
    # TODO handle notebook_output execution history and latest successful run
    return notebook_output


def jupyter_edit(conf: AiscalatorConfig):
    """
    Starts a Jupyter Lab environment configured to edit the focused step

    Parameters
    ----------
    conf : AiscalatorConfig
        Configuration object for the step

    Returns
    -------
    string
        Url of the running jupyter lab
    """
    logger = logging.getLogger(__name__)
    conf.validate_config()
    docker_image = build(conf)
    if docker_image:
        # TODO: shutdown other jupyter lab still running
        notebook = os.path.basename(conf.step_field('task.code_path'))
        if conf.step_extract_parameters():
            jupyter_run(conf, prepare_only=True)
        commands = _prepare_docker_env(conf, [
            docker_image, "start.sh",
            'jupyter', 'lab'
        ])
        return wait_for_jupyter_lab(commands, logger, notebook,
                                    10000, "notebook")
    raise Exception("Failed to build docker image")


def jupyter_new(name, path, output_format="hocon"):
    """
    Starts a Jupyter Lab environment configured to edit a brand new step

    Parameters
    ----------
    name : str
        name of the new step
    path : str
        path to where the new step files should be created
    output_format : str
        the format of the new configuration file to produce
    Returns
    -------
    string
        Url of the running jupyter lab
    """
    step_file = os.path.join(path, name, name) + '.conf'
    makedirs(os.path.dirname(step_file), exist_ok=True)
    copy_replace(data_file("../config/template/step.conf"), step_file,
                 pattern="Untitled", replace_value=name)
    if output_format != 'hocon':
        file = os.path.join(path, name, name) + '.' + output_format
        step_file = convert_to_format(step_file, output=file,
                                      output_format=output_format)

    notebook_file = os.path.join(path, name, 'notebook', name) + '.ipynb'
    makedirs(os.path.dirname(notebook_file), exist_ok=True)
    copy_replace(data_file("../config/template/notebook.json"), notebook_file)

    open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
    open(os.path.join(path, name, "requirements.txt"), 'a').close()
    open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
    jupyter_edit(AiscalatorConfig(config=step_file,
                                  step_selection=name))


1			# -- coding: utf-8 --
2			# Apache Software License 2.0
3			#
4			# Copyright (c) 2018, Christophe Duong
5			#
6			# Licensed under the Apache License, Version 2.0 (the "License");
7			# you may not use this file except in compliance with the License.
8			# You may obtain a copy of the License at
9			#
10			# http://www.apache.org/licenses/LICENSE-2.0
11			#
12			# Unless required by applicable law or agreed to in writing, software
13			# distributed under the License is distributed on an "AS IS" BASIS,
14			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15			# See the License for the specific language governing permissions and
16			# limitations under the License.
17			"""
18			Implementations of commands for Jupyter
19			"""
20			import logging
21			import os.path
22			from os import makedirs
23
24			from aiscalator.core.config import AiscalatorConfig
25			from aiscalator.core.config import convert_to_format
26			from aiscalator.core.log_regex_analyzer import LogRegexAnalyzer
27			from aiscalator.core.utils import copy_replace
28			from aiscalator.core.utils import data_file
29			from aiscalator.core.utils import subprocess_run
30			from aiscalator.core.utils import wait_for_jupyter_lab
31			from aiscalator.jupyter.docker_image import build
32
33
34			def _prepare_docker_env(conf: AiscalatorConfig, program):
35			"""
36			Assembles the list of commands to execute a docker run call
37
38			When calling "docker run ...", this function also adds a set of
39			additional parameters to mount the proper volumes and expose the
40			correct environment for the call in the docker image mapped to the
41			host directories. This is done so only some specific data and code
42			folders are accessible within the docker image.
43
44			Parameters
45			----------
46			conf : AiscalatorConfig
47			Configuration object for the step
48			program : List
49			the rest of the commands to execute as part of
50			the docker run call
51
52			Returns
53			-------
54			List
55			The full Array of Strings representing the commands to execute
56			in the docker run call
57			"""
58			# TODO: refactor using https://github.com/docker/docker-py ?
59			commands = [
60			"docker", "run", "--name", conf.step_container_name(), "--rm",
61			# TODO improve port publishing
62			"-p", "10000:8888",
63			"-p", "4040:4040"
64			]
65			for env in conf.user_env_file():
66			if os.path.isfile(env):
67			commands += ["--env-file", env]
68			commands += _prepare_docker_image_env(conf)
69			code_path = conf.step_file_path('task.code_path')
70			commands += [
71			"--mount", "type=bind,source=" + os.path.dirname(code_path) +
72			",target=/home/jovyan/work/notebook/",
73			]
74			commands += _prepare_task_env(conf)
75			if conf.has_step_field("task.execution_dir_path"):
76			makedirs(conf.step_file_path('task.execution_dir_path'), exist_ok=True)
77			commands += [
78			"--mount", "type=bind,source=" +
79			conf.step_file_path('task.execution_dir_path') +
80			",target=/home/jovyan/work/notebook_run/"
81			]
82			commands += program
83			return commands
84
85
86			def _prepare_docker_image_env(conf: AiscalatorConfig):
87			"""
88			Assemble the list of volumes to mount specific to
89			building the docker image
90
91			Parameters
92			----------
93			conf : AiscalatorConfig
94			Configuration object for the step
95
96			Returns
97			-------
98			list
99			list of commands to bind those volumes
100			"""
101			commands = []
102			if conf.config_path() is not None:
103			commands += [
104			"--mount",
105			"type=bind,source=" + os.path.abspath(conf.config_path()) +
106			",target="
107			"/home/jovyan/work/" + os.path.basename(conf.config_path()),
108			]
109			if conf.has_step_field("docker_image.apt_package_path"):
110			apt_packages = conf.step_file_path('docker_image.apt_package_path')
111			if apt_packages and os.path.isfile(apt_packages):
112			commands += [
113			"--mount", "type=bind,source=" + apt_packages +
114			",target=/home/jovyan/work/apt_packages.txt",
115			]
116			if conf.has_step_field("docker_image.requirements_path"):
117			requirements = conf.step_file_path('docker_image.requirements_path')
118			if requirements and os.path.isfile(requirements):
119			commands += [
120			"--mount", "type=bind,source=" + requirements +
121			",target=/home/jovyan/work/requirements.txt",
122			]
123			if conf.has_step_field("docker_image.lab_extension_path"):
124			lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
125			if lab_extensions and os.path.isfile(lab_extensions):
126			commands += [
127			"--mount", "type=bind,source=" + lab_extensions +
128			",target=/home/jovyan/work/lab_extensions.txt",
129			]
130			return commands
131
132
133			def _prepare_task_env(conf: AiscalatorConfig):
134			"""
135			Assemble the list of volumes to mount specific to
136			the task execution
137
138			Parameters
139			----------
140			conf : AiscalatorConfig
141			Configuration object for the step
142
143			Returns
144			-------
145			list
146			list of commands to bind those volumes
147			"""
148			commands = []
149			if conf.root_dir():
150			commands += _mount_path(conf, "task.modules_src_path",
151			"/home/jovyan/work/modules/")
152			commands += _mount_path(conf, "task.input_data_path",
153			"/home/jovyan/work/data/input/",
154			readonly=True)
155			commands += _mount_path(conf, "task.output_data_path",
156			"/home/jovyan/work/data/output/",
157			make_dirs=True)
158			return commands
159
160
161			def _mount_path(conf: AiscalatorConfig, field, target_path,
162			readonly=False, make_dirs=False):
163			"""
164			Returu commands to mount path from list field into the
165			docker image when running.
166
167			Parameters
168			----------
169			conf : AiscalatorConfig
170			Configuration object for the step
171			field : str
172			the field in the configuration step that contains the path
173			target_path : str
174			where to mount them inside the container
175			readonly : bool
176			flag to mount the path as read-only
177			make_dirs : bool
178			flag to create the folder on the host before mounting if
179			it doesn't exists.
180
181			Returns
182			-------
183			list
184			commands to mount all the paths from the field
185
186			"""
187			commands = []
188			if conf.has_step_field(field):
189			for value in conf.step_field(field):
190			# TODO handle URL
191			for i in value:
192			if make_dirs:
193			makedirs(os.path.abspath(conf.root_dir() + i),
194			exist_ok=True)
195			if os.path.exists(conf.root_dir() + i):
196			commands += [
197			"--mount",
198			"type=bind,source=" +
199			os.path.abspath(conf.root_dir() + i) +
200			",target=" + target_path + value[i] +
201			(",readonly" if readonly else "")
202			]
203			return commands
204
205
206			def jupyter_run(conf: AiscalatorConfig, prepare_only=False):
207			"""
208			Executes the step in browserless mode using papermill
209
210			Parameters
211			----------
212			conf : AiscalatorConfig
213			Configuration object for the step
214			prepare_only : bool
215			Indicates if papermill should replace the parameters of the
216			notebook only or it should execute all the cells too
217
218			Returns
219			-------
220			string
221			the path to the output notebook resulting from the execution
222			of this step
223			"""
224			logger = logging.getLogger(__name__)
225			conf.validate_config()
226			docker_image = build(conf)
227			if not docker_image:
228			raise Exception("Failed to build docker image")
229			notebook = ("/home/jovyan/work/notebook/" +
230			os.path.basename(conf.step_file_path('task.code_path')))
231			notebook_output = conf.step_notebook_output_path(notebook)
232			parameters = conf.step_extract_parameters()
233			commands = _prepare_docker_env(conf, [
234			docker_image, "start.sh",
235			# TODO: check step type, if jupyter then papermill
236			"papermill",
237			notebook, notebook_output
238			])
239			if prepare_only:
240			commands.append("--prepare-only")
241			commands += parameters
242			log = LogRegexAnalyzer()
243			logger.info("Running...: %s", " ".join(commands))
244			subprocess_run(commands, log_function=log.grep_logs)
245			# TODO handle notebook_output execution history and latest successful run
246			return notebook_output
247
248
249			def jupyter_edit(conf: AiscalatorConfig):
250			"""
251			Starts a Jupyter Lab environment configured to edit the focused step
252
253			Parameters
254			----------
255			conf : AiscalatorConfig
256			Configuration object for the step
257
258			Returns
259			-------
260			string
261			Url of the running jupyter lab
262			"""
263			logger = logging.getLogger(__name__)
264			conf.validate_config()
265			docker_image = build(conf)
266			if docker_image:
267			# TODO: shutdown other jupyter lab still running
268			notebook = os.path.basename(conf.step_field('task.code_path'))
269			if conf.step_extract_parameters():
270			jupyter_run(conf, prepare_only=True)
271			commands = _prepare_docker_env(conf, [
272			docker_image, "start.sh",
273			'jupyter', 'lab'
274			])
275			return wait_for_jupyter_lab(commands, logger, notebook,
276			10000, "notebook")
277			raise Exception("Failed to build docker image")
278
279
280			def jupyter_new(name, path, output_format="hocon"):
281			"""
282			Starts a Jupyter Lab environment configured to edit a brand new step
283
284			Parameters
285			----------
286			name : str
287			name of the new step
288			path : str
289			path to where the new step files should be created
290			output_format : str
291			the format of the new configuration file to produce
292			Returns
293			-------
294			string
295			Url of the running jupyter lab
296			"""
297			step_file = os.path.join(path, name, name) + '.conf'
298			makedirs(os.path.dirname(step_file), exist_ok=True)
299			copy_replace(data_file("../config/template/step.conf"), step_file,
300			pattern="Untitled", replace_value=name)
301			if output_format != 'hocon':
302			file = os.path.join(path, name, name) + '.' + output_format
303			step_file = convert_to_format(step_file, output=file,
304			output_format=output_format)
305
306			notebook_file = os.path.join(path, name, 'notebook', name) + '.ipynb'
307			makedirs(os.path.dirname(notebook_file), exist_ok=True)
308			copy_replace(data_file("../config/template/notebook.json"), notebook_file)
309
310			open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
311			open(os.path.join(path, name, "requirements.txt"), 'a').close()
312			open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
313			jupyter_edit(AiscalatorConfig(config=step_file,
314			step_selection=name))
315

Aiscalate / aiscalator

Push — master ( 722b81...0abd85 )

aiscalator.jupyter.command._mount_path() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like