start_pipeline() - Code Metrics - Inspection of "Removed unused tasks + Used log dir from conf" - usnistgov/ocr-pipeline - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( f3119e...d9d5a0 )

by Philippe

created 2017-07-19 21:46 UTC

start_pipeline() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	1

Metric	Value
cc	1
c	1
b	0
f	1
dl	0
loc	7
rs	9.4285

"""Manage the OCR pipeline, both remotely and locally

.. Authors:
    Philippe Dessauw
    [email protected]

.. Sponsor:
    Alden Dima
    [email protected]
    Information Systems Group
    Software and Systems Division
    Information Technology Laboratory
    National Institute of Standards and Technology
    http://www.nist.gov/itl/ssd/is
"""
from __future__ import division
import logging
from os import listdir
import os
from apputils.config import load_config
from apputils.fileop import create_directories
from denoiser import Denoiser
from os.path import join, isdir, exists, abspath
from fabric.contrib.console import confirm
from fabric.contrib.project import upload_project
from fabric.contrib.files import exists as fab_exists
from fabric.decorators import task, roles, runs_once
from fabric.api import env, run, local, cd
from fabric.operations import sudo
from fabric.tasks import Task, execute
from fabric.state import output
from shutil import copytree, rmtree

# Initialize the app_config variable
load_config("conf/app.yaml", os.environ['ROOT'])
from apputils.config import app_config

logger = logging.getLogger('app')

# Hosts configurations
env.roledefs = app_config["machines"]
env.hosts = [ip for ips in env.roledefs.values() for ip in ips]

# Extra default configuration
env.warn_only = True
output.update({
    "warnings": False,
    "running": False
})

# Global variable to know if the execution is done locally or remotely
local_exec = False


# @task
# def install():
#     """Install the pipeline on the specified cluster
#     """
#     logger.debug("Installing pipeline...")
#
#     local_root = os.environ['ROOT']
#     remote_root = app_config['root']
#
#     if local_exec:
#         if abspath(local_root) == abspath(remote_root):
#             logger.error("Source and destination folder are the same")
#             exit(1)
#
#         if exists(remote_root):
#             if confirm("Existing data will be deleted. Do you want to proceed anyway?", default=False):
#                 rmtree(remote_root)
#             else:
#                 logger.error("Pipeline destination folder already exists")
#                 exit(2)
#
#         copytree(local_root, remote_root)
#         local(remote_root+'/utils/install.sh')
#     else:
#         if app_config["use_sudo"]:
#             run_fn = sudo
#         else:
#             run_fn = run
#
#         if not fab_exists(remote_root):
#             logging.debug("Building remote directory...")
#             run_fn("mkdir -p "+remote_root)
#         else:
#             if not confirm("Existing data will be deleted. Do you want to proceed anyway?", default=False):
#                 logger.error("Pipeline destination folder already exists")
#                 exit(2)
#
#         logging.debug("Uploading project...")
#         upload_project(
#             local_dir=local_root,
#             remote_dir=remote_root,
#             use_sudo=app_config["use_sudo"]
#         )
#
#         if run_fn(remote_root+"/utils/auth.sh").failed:
#             logger.error("An error occured with modifying the right for the pipeline")
#             exit(3)
#
#         if run(remote_root+"/utils/install.sh").failed:
#             logger.error("An error occured with the install script")
#             exit(4)
#
#     logger.info("Pipeline successfully installed")


@task
def init():
    """Initialize the app (available for localhost only)
    """
    logger.debug("Initializing app...")

    if not local_exec:
        logger.error("Pipeline can only be initialized locally")
        exit(1)

    # Create project tree
    create_directories(app_config["dirs"])
    logger.info("App initialized")


@task
def create_models(dataset_dir):
    """Initialize the app (available for localhost only)

    Parameters:
        dataset_dir (:func:`str`): Path to the training set
    """
    logger.debug("Creating models...")

    if not local_exec:
        logger.error("Models can only be generated locally")
        exit(1)

    # Modify the configuration for local execution
    app_config['root'] = os.environ['ROOT']

    # Generate inline models and train classifier
    denoiser = Denoiser(app_config)

    if not exists(dataset_dir) or not isdir(dataset_dir):
        logger.error(dataset_dir+" is not a valid directory")
        exit(2)

    dataset = [join(dataset_dir, f) for f in listdir(dataset_dir)]

    denoiser.generate_models(dataset)
    logger.info("Inline models generated")

    denoiser.train(dataset)
    logger.info("Classifier trained")


@task
def check():
    """Check that the 3rd party software are all installed
    """
    base_dir = "utils/check"
    scripts = [join(base_dir, sc) for sc in listdir(base_dir) if sc.endswith(".sh")]

    for script in scripts:
        launch_script(script)


# @task
# @runs_once
# def start_pipeline():
#     """Start the conversion process across the platform.
#     """
#     execute(start_master)
#     execute(start_slave)


@task
@roles("master")
def start_master():
    """Start the master server on the local machine
    """
    launch_script("utils/run-wrapper.sh", ["--master", ("> %s/master.log", app_config["dirs"]["logs"])], True)


@task
@roles("slaves")
def start_slave():
    """Start a slave on the local machine
    """
    launch_script("utils/run-wrapper.sh", ["--slave", ("> %s/slave.log", app_config["dirs"]["logs"])], True)


def launch_script(script_name, script_opts=list(), background=False):
    """Launch any script you specify

    Parameters:
        script_name (:func:`str`): Path of the script to run
        script_opts (:func:`str`): Options to pass to the script
    """
    if local_exec:
        root_dir = os.environ['ROOT']
    else:
        root_dir = app_config['root']

    command = join(root_dir, script_name)+" "+" ".join(script_opts)

    logger.debug("Execute: "+command)

    with cd(root_dir):
        if local_exec:
            if background:
                local(command)
                return

            if local(command).failed:
                logger.error("Command failed "+command)
                exit(1)
        else:
            if background:
                run(command)
                return

            if run(command).failed:
                logger.error("Command failed "+command)
                exit(1)


def print_help(light=False):
    """Print the help message

    Parameter
        light (bool): If true, only print the usage
    """
    print "Usage: python ui.py [-h] command [args [args ...]]"

    if light:
        return

    print "Commands: "

    default_func_help = "No description available"
    for func in functions.keys():
        help_str = "* "+func+" "*(20-len(func))

        if func in functions_help.keys():
            help_str += functions_help[func]
        else:
            help_str += default_func_help

        print help_str

    print ""
    print "Options:"
    print_opts = ', '.join(help_opts)

    print print_opts+" "*(22-len(print_opts))+"Print this help message"


def setup_local_exec():
    """Set the local_exec variable to true
    """
    global local_exec
    local_exec = True


if __name__ == "__main__":
    import sys
    args = sys.argv[1:]

    setup_local_exec()

    help_opts = ["--help", "-h"]
    functions = {k: v for k, v in locals().items() if isinstance(v, Task) and v.__module__ == "__main__"
                 and k != "print_help"}
    functions_help = {k: v.__doc__.split("\n")[0] for k, v in functions.items() if v.__doc__ is not None}

    if len(args) == 0:
        logger.error("No function specified")
        print_help(True)
        exit(1)

    if len(args) == 1 and args[0] in help_opts:
        print_help()
        exit(0)

    if args[0] not in functions.keys():
        logger.fatal("Command "+args[0]+" unknown")
        print_help(True)
        exit(1)

    if len(args) > 1:
        functions[args[0]](*args[1:])
    else:
        functions[args[0]]()




1			"""Manage the OCR pipeline, both remotely and locally
2
3			.. Authors:
4			Philippe Dessauw
5			[email protected]
6
7			.. Sponsor:
8			Alden Dima
9			[email protected]
10			Information Systems Group
11			Software and Systems Division
12			Information Technology Laboratory
13			National Institute of Standards and Technology
14			http://www.nist.gov/itl/ssd/is
15			"""
16			from __future__ import division
17			import logging
18			from os import listdir
19			import os
20			from apputils.config import load_config
21			from apputils.fileop import create_directories
22			from denoiser import Denoiser
23			from os.path import join, isdir, exists, abspath
24			from fabric.contrib.console import confirm
25			from fabric.contrib.project import upload_project
26			from fabric.contrib.files import exists as fab_exists
27			from fabric.decorators import task, roles, runs_once
28			from fabric.api import env, run, local, cd
29			from fabric.operations import sudo
30			from fabric.tasks import Task, execute
31			from fabric.state import output
32			from shutil import copytree, rmtree
33
34			# Initialize the app_config variable
35			load_config("conf/app.yaml", os.environ['ROOT'])
36			from apputils.config import app_config
37
38			logger = logging.getLogger('app')
39
40			# Hosts configurations
41			env.roledefs = app_config["machines"]
42			env.hosts = [ip for ips in env.roledefs.values() for ip in ips]
43
44			# Extra default configuration
45			env.warn_only = True
46			output.update({
47			"warnings": False,
48			"running": False
49			})
50
51			# Global variable to know if the execution is done locally or remotely
52			local_exec = False
53
54
55			# @task
56			# def install():
57			# """Install the pipeline on the specified cluster
58			# """
59			# logger.debug("Installing pipeline...")
60			#
61			# local_root = os.environ['ROOT']
62			# remote_root = app_config['root']
63			#
64			# if local_exec:
65			# if abspath(local_root) == abspath(remote_root):
66			# logger.error("Source and destination folder are the same")
67			# exit(1)
68			#
69			# if exists(remote_root):
70			# if confirm("Existing data will be deleted. Do you want to proceed anyway?", default=False):
71			# rmtree(remote_root)
72			# else:
73			# logger.error("Pipeline destination folder already exists")
74			# exit(2)
75			#
76			# copytree(local_root, remote_root)
77			# local(remote_root+'/utils/install.sh')
78			# else:
79			# if app_config["use_sudo"]:
80			# run_fn = sudo
81			# else:
82			# run_fn = run
83			#
84			# if not fab_exists(remote_root):
85			# logging.debug("Building remote directory...")
86			# run_fn("mkdir -p "+remote_root)
87			# else:
88			# if not confirm("Existing data will be deleted. Do you want to proceed anyway?", default=False):
89			# logger.error("Pipeline destination folder already exists")
90			# exit(2)
91			#
92			# logging.debug("Uploading project...")
93			# upload_project(
94			# local_dir=local_root,
95			# remote_dir=remote_root,
96			# use_sudo=app_config["use_sudo"]
97			# )
98			#
99			# if run_fn(remote_root+"/utils/auth.sh").failed:
100			# logger.error("An error occured with modifying the right for the pipeline")
101			# exit(3)
102			#
103			# if run(remote_root+"/utils/install.sh").failed:
104			# logger.error("An error occured with the install script")
105			# exit(4)
106			#
107			# logger.info("Pipeline successfully installed")
108
109
110			@task
111			def init():
112			"""Initialize the app (available for localhost only)
113			"""
114			logger.debug("Initializing app...")
115
116			if not local_exec:
117			logger.error("Pipeline can only be initialized locally")
118			exit(1)
119
120			# Create project tree
121			create_directories(app_config["dirs"])
122			logger.info("App initialized")
123
124
125			@task
126			def create_models(dataset_dir):
127			"""Initialize the app (available for localhost only)
128
129			Parameters:
130			dataset_dir (:func:`str`): Path to the training set
131			"""
132			logger.debug("Creating models...")
133
134			if not local_exec:
135			logger.error("Models can only be generated locally")
136			exit(1)
137
138			# Modify the configuration for local execution
139			app_config['root'] = os.environ['ROOT']
140
141			# Generate inline models and train classifier
142			denoiser = Denoiser(app_config)
143
144			if not exists(dataset_dir) or not isdir(dataset_dir):
145			logger.error(dataset_dir+" is not a valid directory")
146			exit(2)
147
148			dataset = [join(dataset_dir, f) for f in listdir(dataset_dir)]
149
150			denoiser.generate_models(dataset)
151			logger.info("Inline models generated")
152
153			denoiser.train(dataset)
154			logger.info("Classifier trained")
155
156
157			@task
158			def check():
159			"""Check that the 3rd party software are all installed
160			"""
161			base_dir = "utils/check"
162			scripts = [join(base_dir, sc) for sc in listdir(base_dir) if sc.endswith(".sh")]
163
164			for script in scripts:
165			launch_script(script)
166
167
168			# @task
169			# @runs_once
170			# def start_pipeline():
171			# """Start the conversion process across the platform.
172			# """
173			# execute(start_master)
174			# execute(start_slave)
175
176
177			@task
178			@roles("master")
179			def start_master():
180			"""Start the master server on the local machine
181			"""
182			launch_script("utils/run-wrapper.sh", ["--master", ("> %s/master.log", app_config["dirs"]["logs"])], True)
183
184
185			@task
186			@roles("slaves")
187			def start_slave():
188			"""Start a slave on the local machine
189			"""
190			launch_script("utils/run-wrapper.sh", ["--slave", ("> %s/slave.log", app_config["dirs"]["logs"])], True)
191
192
193			def launch_script(script_name, script_opts=list(), background=False):
194			"""Launch any script you specify
195
196			Parameters:
197			script_name (:func:`str`): Path of the script to run
198			script_opts (:func:`str`): Options to pass to the script
199			"""
200			if local_exec:
201			root_dir = os.environ['ROOT']
202			else:
203			root_dir = app_config['root']
204
205			command = join(root_dir, script_name)+" "+" ".join(script_opts)
206
207			logger.debug("Execute: "+command)
208
209			with cd(root_dir):
210			if local_exec:
211			if background:
212			local(command)
213			return
214
215			if local(command).failed:
216			logger.error("Command failed "+command)
217			exit(1)
218			else:
219			if background:
220			run(command)
221			return
222
223			if run(command).failed:
224			logger.error("Command failed "+command)
225			exit(1)
226
227
228			def print_help(light=False):
229			"""Print the help message
230
231			Parameter
232			light (bool): If true, only print the usage
233			"""
234			print "Usage: python ui.py [-h] command [args [args ...]]"
235
236			if light:
237			return
238
239			print "Commands: "
240
241			default_func_help = "No description available"
242			for func in functions.keys():
243			help_str = "* "+func+" "*(20-len(func))
244
245			if func in functions_help.keys():
246			help_str += functions_help[func]
247			else:
248			help_str += default_func_help
249
250			print help_str
251
252			print ""
253			print "Options:"
254			print_opts = ', '.join(help_opts)
255
256			print print_opts+" "*(22-len(print_opts))+"Print this help message"
257
258
259			def setup_local_exec():
260			"""Set the local_exec variable to true
261			"""
262			global local_exec
263			local_exec = True
264
265
266			if __name__ == "__main__":
267			import sys
268			args = sys.argv[1:]
269
270			setup_local_exec()
271
272			help_opts = ["--help", "-h"]
273			functions = {k: v for k, v in locals().items() if isinstance(v, Task) and v.__module__ == "__main__"
274			and k != "print_help"}
275			functions_help = {k: v.__doc__.split("\n")[0] for k, v in functions.items() if v.__doc__ is not None}
276
277			if len(args) == 0:
278			logger.error("No function specified")
279			print_help(True)
280			exit(1)
281
282			if len(args) == 1 and args[0] in help_opts:
283			print_help()
284			exit(0)
285
286			if args[0] not in functions.keys():
287			logger.fatal("Command "+args[0]+" unknown")
288			print_help(True)
289			exit(1)
290
291			if len(args) > 1:
292			functions[args[0]](*args[1:])
293			else:
294			functions[args[0]]()
295
296
297

usnistgov / ocr-pipeline

Push — master ( f3119e...d9d5a0 )

start_pipeline() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like