savu.tomo_recon.__create_folder_name() - Code Metrics - DiamondLightSource/Savu - Measure and Improve Code Quality continuously with Scrutinizer

savu.tomo_recon.__create_folder_name() A
last analyzed 2022-08-17 15:44 UTC

↳ Parent: savu.tomo_recon

Complexity

Conditions

Size

Total Lines	10
Code Lines	10

Duplication

Lines	10
Ratio	100 %

Importance

Changes

Metric	Value
cc	3
eloc	10
nop	1
dl	10
loc	10
rs	9.9
c	0
b	0
f	0

# Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: tomo_recon
   :platform: Unix
   :synopsis: Runner for the Savu framework

.. moduleauthor:: Mark Basham <[email protected]>

"""

import tempfile  # this import is required for pyFAI - DO NOT REMOVE!
import argparse
import traceback
import sys
import os
from mpi4py import MPI
from savu.version import __version__

import savu.core.utils as cu
from scripts.citation_extractor import citation_extractor
from savu.core.basic_plugin_runner import BasicPluginRunner
from savu.core.plugin_runner import PluginRunner
import pre_run as pr

def __option_parser(doc=True):
    """ Option parser for command line arguments.
    """
    version = "%(prog)s " + __version__
    parser = argparse.ArgumentParser(prog='savu')
    hide = argparse.SUPPRESS

    parser.add_argument('in_file', help='Input data file.')
    process_str = 'Process list, created with the savu configurator.'
    parser.add_argument('process_list', help=process_str)
    parser.add_argument('out_folder', help='Output folder.')
    parser.add_argument('--version', action='version', version=version)
    parser.add_argument("-f", "--folder", help="Override output folder name")

    parser.add_argument("--pre_run", help="Pre-run of savu to gather stats and cropping information.",
                        action="store_true", default=False)

    tmp_help = "Store intermediate files in a temp directory."
    parser.add_argument("-d", "--tmp", help=tmp_help)

    template_help = "Pass a template file of plugin input parameters."
    parser.add_argument("-t", "--template", help=template_help, default=None)

    log_help = "Store full log file in a separate location"
    parser.add_argument("-l", "--log", help=log_help)

    v_help = "Display all debug log messages"
    parser.add_argument("-v", "--verbose", help=v_help, action="store_true",
                        default=False)
    parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
                        help="Display only Errors and Info.", default=False)
    # temporary flag to fix lustre issue
    parser.add_argument("--lustre_workaround", action="store_true",
                        dest="lustre", help="Avoid lustre segmentation fault",
                        default=False)
    sys_params_help = "Override default path to Savu system parameters file."
    parser.add_argument("--system_params", help=sys_params_help, default=None)

    # Set stats off
    parser.add_argument("--stats", help="Turn stats 'on' or 'off'.", default="on", choices=["on", "off"])

    # Hidden arguments
    # process names
    parser.add_argument("-n", "--names", help=hide, default="CPU0")
    # transport mechanism
    parser.add_argument("--transport", help=hide, default="hdf5")
    # Set Savu mode
    parser.add_argument("-m", "--mode", help=hide, default="full",
                        choices=['basic', 'full'])
    # Set logging to cluster mode
    parser.add_argument("-c", "--cluster", action="store_true", help=hide,
                        default=False)
    # Send an email on completion
    parser.add_argument("-e", "--email", dest="email", help=hide, default=None)
    # Facility email for errors
    parser.add_argument("--facility_email", dest="femail", help=hide,
                        default=None)
    # Set beamline log file (for online processing)
    parser.add_argument("--bllog", dest="bllog", help=hide, default=None)
    # Location of syslog server
    parser.add_argument("-s", "--syslog", dest="syslog", help=hide,
                        default='localhost')
    # Port to connect to syslog server on
    parser.add_argument("-p", "--syslog_port", dest="syslog_port",
                        help=hide, default=514, type=int)
    parser.add_argument("--test_state", dest="test_state", default='False',
                        action='store_true', help=hide)

    # DosNa related parameters
    parser.add_argument("--dosna_backend", dest="dosna_backend", help=hide,
                        default=None)
    parser.add_argument("--dosna_engine", dest="dosna_engine", help=hide,
                        default=None)
    parser.add_argument("--dosna_connection", dest="dosna_connection",
                        help=hide, default=None)
    parser.add_argument("--dosna_connection_options",
                        dest="dosna_connection_options", help=hide,
                        nargs='+', default=[])

    check_help = "Continue Savu processing from a checkpoint."
    choices = ['plugin', 'subplugin']
    parser.add_argument("--checkpoint", nargs="?", choices=choices,
                        const='plugin', help=check_help, default=None)
    if doc==False:
        args = parser.parse_args()
        __check_conditions(parser, args)
        return args
    else:
        return parser


def __check_conditions(parser, args):
    if args.checkpoint and not args.folder:
        msg = "--checkpoint flag requires '-f folder_name', where folder_name"\
              " contains the partially completed Savu job.  The out_folder"\
              " should be the path to this folder."
        parser.error(msg)


def _set_options(args):

    """ Set run specific information in options dictionary.

    :params dict opt: input optional arguments (or defaults)
    :params args: input required arguments
    :returns options: optional and required arguments
    :rtype: dict
    """
    options = {}
    options['data_file'] = args.in_file
    options['process_file'] = args.process_list
    options['mode'] = args.mode
    options['template'] = args.template
    options['transport'] = 'basic' if args.mode == 'basic' else args.transport
    options['process_names'] = args.names
    options['verbose'] = args.verbose
    options['quiet'] = args.quiet
    options['cluster'] = args.cluster
    options['syslog_server'] = args.syslog
    options['syslog_port'] = args.syslog_port
    options['test_state'] = args.test_state
    options['lustre'] = args.lustre
    options['bllog'] = args.bllog
    options['email'] = args.email
    options['femail'] = args.femail
    options['system_params'] = args.system_params
    options['stats'] = args.stats
    options['pre_run'] = args.pre_run

    if args.folder:
        out_folder_name = os.path.basename(args.folder)
    else:
        out_folder_name = __create_folder_name(options['data_file'])

    out_folder_path = __create_output_folder(args.out_folder, out_folder_name)

    options['out_folder'] = out_folder_name
    options['out_path'] = out_folder_path

    basename = os.path.basename(args.in_file)
    options['datafile_name'] = os.path.splitext(basename)[0] if basename \
        else args.in_file.split(os.sep)[-2]

    inter_folder_path = __create_output_folder(args.tmp, out_folder_name)\
        if args.tmp else out_folder_path

    options['inter_path'] = inter_folder_path
    options['log_path'] = args.log if args.log else options['inter_path']
    options['nProcesses'] = len(options["process_names"].split(','))
    # DosNa related options
    options["dosna_backend"] = args.dosna_backend
    options["dosna_engine"] = args.dosna_engine
    options["dosna_connection"] = args.dosna_connection
    options["dosna_connection_options"] = args.dosna_connection_options
    options['checkpoint'] = args.checkpoint

    command_str = " ".join([str(i) for i in sys.argv[1:]])
    command_full = f"savu {command_str}"
    options["command"] = command_full

    return options


def __create_folder_name(dpath):

    if os.path.isfile(dpath):
        dpath = os.path.splitext(dpath)[0]
    elif os.path.isdir(dpath):
        dpath = os.path.dirname(dpath)
    import time
    MPI.COMM_WORLD.barrier()
    timestamp = time.strftime("%Y%m%d%H%M%S")
    MPI.COMM_WORLD.barrier()
    return "_".join([timestamp, os.path.basename(dpath)])


def __create_output_folder(path, folder_name):
    folder = os.path.join(path, folder_name)
    if MPI.COMM_WORLD.rank == 0:
        if not os.path.exists(folder):
            os.makedirs(folder)
    return folder


def main(input_args=None):
    args = __option_parser(doc=False)

    if input_args:
        args = input_args

    options = _set_options(args)


    pRunner = PluginRunner if options['mode'] == 'full' else BasicPluginRunner
    try:
        options["post_pre_run"] = False
        answer = "Y"
        if options["pre_run"]:
            pre_run_options = pr._set_options(args)
            pre_plugin_runner = pRunner(pre_run_options)
            pre_plugin_runner._run_plugin_list()
            pre_plugin_runner.exp._save_pre_run_log()
            #options["data_file"] = pre_plugin_runner.exp.meta_data.get("pre_run_file")
            folder = options['out_path']
            fname = options['datafile_name'] + '_pre_run.nxs'
            filename = os.path.join(folder, fname)
            options["data_file"] = filename
            options["pre_run"] = False
            options["post_pre_run"] = True
            if MPI.COMM_WORLD.rank == 0:
                while answer not in ("y", "N"):
                    cu.user_message("Pre-run complete. See run_log/pre_run_log.txt for details. Do you want to continue? [y/N]")
                    answer = input()
        if answer in ("y", "Y"):
            plugin_runner = pRunner(options)
            plugin_runner._run_plugin_list()
            if options['process'] == 0:
                in_file = plugin_runner.exp.meta_data['nxs_filename']
                citation_extractor.main(in_file=in_file, quiet=True)
    except Exception:
        # raise the error in the user log
        trace = traceback.format_exc()
        cu.user_message(trace)
        if options['nProcesses'] == 1:
            sys.exit(1)
        else:
            # Kill all MPI processes
            MPI.COMM_WORLD.Abort(1)


if __name__ == '__main__':
    main()


1		# Copyright 2014 Diamond Light Source Ltd.
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		"""
16		.. module:: tomo_recon
17		:platform: Unix
18		:synopsis: Runner for the Savu framework
19
20		.. moduleauthor:: Mark Basham <[email protected]>
21
22		"""
23
24		import tempfile # this import is required for pyFAI - DO NOT REMOVE!
25		import argparse
26		import traceback
27		import sys
28		import os
29		from mpi4py import MPI
30		from savu.version import __version__
31
32		import savu.core.utils as cu
33		from scripts.citation_extractor import citation_extractor
34		from savu.core.basic_plugin_runner import BasicPluginRunner
35		from savu.core.plugin_runner import PluginRunner
36		import pre_run as pr
37
38		def __option_parser(doc=True):
39		""" Option parser for command line arguments.
40		"""
41		version = "%(prog)s " + __version__
42		parser = argparse.ArgumentParser(prog='savu')
43		hide = argparse.SUPPRESS
44
45		parser.add_argument('in_file', help='Input data file.')
46		process_str = 'Process list, created with the savu configurator.'
47		parser.add_argument('process_list', help=process_str)
48		parser.add_argument('out_folder', help='Output folder.')
49		parser.add_argument('--version', action='version', version=version)
50		parser.add_argument("-f", "--folder", help="Override output folder name")
51
52		parser.add_argument("--pre_run", help="Pre-run of savu to gather stats and cropping information.",
53		action="store_true", default=False)
54
55		tmp_help = "Store intermediate files in a temp directory."
56		parser.add_argument("-d", "--tmp", help=tmp_help)
57
58		template_help = "Pass a template file of plugin input parameters."
59		parser.add_argument("-t", "--template", help=template_help, default=None)
60
61		log_help = "Store full log file in a separate location"
62		parser.add_argument("-l", "--log", help=log_help)
63
64		v_help = "Display all debug log messages"
65		parser.add_argument("-v", "--verbose", help=v_help, action="store_true",
66		default=False)
67		parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
68		help="Display only Errors and Info.", default=False)
69		# temporary flag to fix lustre issue
70		parser.add_argument("--lustre_workaround", action="store_true",
71		dest="lustre", help="Avoid lustre segmentation fault",
72		default=False)
73		sys_params_help = "Override default path to Savu system parameters file."
74		parser.add_argument("--system_params", help=sys_params_help, default=None)
75
76		# Set stats off
77		parser.add_argument("--stats", help="Turn stats 'on' or 'off'.", default="on", choices=["on", "off"])
78
79		# Hidden arguments
80		# process names
81		parser.add_argument("-n", "--names", help=hide, default="CPU0")
82		# transport mechanism
83		parser.add_argument("--transport", help=hide, default="hdf5")
84		# Set Savu mode
85		parser.add_argument("-m", "--mode", help=hide, default="full",
86		choices=['basic', 'full'])
87		# Set logging to cluster mode
88		parser.add_argument("-c", "--cluster", action="store_true", help=hide,
89		default=False)
90		# Send an email on completion
91		parser.add_argument("-e", "--email", dest="email", help=hide, default=None)
92		# Facility email for errors
93		parser.add_argument("--facility_email", dest="femail", help=hide,
94		default=None)
95		# Set beamline log file (for online processing)
96		parser.add_argument("--bllog", dest="bllog", help=hide, default=None)
97		# Location of syslog server
98		parser.add_argument("-s", "--syslog", dest="syslog", help=hide,
99		default='localhost')
100		# Port to connect to syslog server on
101		parser.add_argument("-p", "--syslog_port", dest="syslog_port",
102		help=hide, default=514, type=int)
103		parser.add_argument("--test_state", dest="test_state", default='False',
104		action='store_true', help=hide)
105
106		# DosNa related parameters
107		parser.add_argument("--dosna_backend", dest="dosna_backend", help=hide,
108		default=None)
109		parser.add_argument("--dosna_engine", dest="dosna_engine", help=hide,
110		default=None)
111		parser.add_argument("--dosna_connection", dest="dosna_connection",
112		help=hide, default=None)
113		parser.add_argument("--dosna_connection_options",
114		dest="dosna_connection_options", help=hide,
115		nargs='+', default=[])
116
117		check_help = "Continue Savu processing from a checkpoint."
118		choices = ['plugin', 'subplugin']
119		parser.add_argument("--checkpoint", nargs="?", choices=choices,
120		const='plugin', help=check_help, default=None)
121		if doc==False:
122		args = parser.parse_args()
123		__check_conditions(parser, args)
124		return args
125		else:
126		return parser
127
128
129		def __check_conditions(parser, args):
130		if args.checkpoint and not args.folder:
131		msg = "--checkpoint flag requires '-f folder_name', where folder_name"\
132		" contains the partially completed Savu job. The out_folder"\
133		" should be the path to this folder."
134		parser.error(msg)
135
136
137	View Code Duplication	def _set_options(args):
		0 ignored issues – show Duplication introduced 2022-07-26 15:17 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
138		""" Set run specific information in options dictionary.
139
140		:params dict opt: input optional arguments (or defaults)
141		:params args: input required arguments
142		:returns options: optional and required arguments
143		:rtype: dict
144		"""
145		options = {}
146		options['data_file'] = args.in_file
147		options['process_file'] = args.process_list
148		options['mode'] = args.mode
149		options['template'] = args.template
150		options['transport'] = 'basic' if args.mode == 'basic' else args.transport
151		options['process_names'] = args.names
152		options['verbose'] = args.verbose
153		options['quiet'] = args.quiet
154		options['cluster'] = args.cluster
155		options['syslog_server'] = args.syslog
156		options['syslog_port'] = args.syslog_port
157		options['test_state'] = args.test_state
158		options['lustre'] = args.lustre
159		options['bllog'] = args.bllog
160		options['email'] = args.email
161		options['femail'] = args.femail
162		options['system_params'] = args.system_params
163		options['stats'] = args.stats
164		options['pre_run'] = args.pre_run
165
166		if args.folder:
167		out_folder_name = os.path.basename(args.folder)
168		else:
169		out_folder_name = __create_folder_name(options['data_file'])
170
171		out_folder_path = __create_output_folder(args.out_folder, out_folder_name)
172
173		options['out_folder'] = out_folder_name
174		options['out_path'] = out_folder_path
175
176		basename = os.path.basename(args.in_file)
177		options['datafile_name'] = os.path.splitext(basename)[0] if basename \
178		else args.in_file.split(os.sep)[-2]
179
180		inter_folder_path = __create_output_folder(args.tmp, out_folder_name)\
181		if args.tmp else out_folder_path
182
183		options['inter_path'] = inter_folder_path
184		options['log_path'] = args.log if args.log else options['inter_path']
185		options['nProcesses'] = len(options["process_names"].split(','))
186		# DosNa related options
187		options["dosna_backend"] = args.dosna_backend
188		options["dosna_engine"] = args.dosna_engine
189		options["dosna_connection"] = args.dosna_connection
190		options["dosna_connection_options"] = args.dosna_connection_options
191		options['checkpoint'] = args.checkpoint
192
193		command_str = " ".join([str(i) for i in sys.argv[1:]])
194		command_full = f"savu {command_str}"
195		options["command"] = command_full
196
197		return options
198
199
200	View Code Duplication	def __create_folder_name(dpath):
		0 ignored issues – show Duplication introduced 2022-07-26 15:17 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
201		if os.path.isfile(dpath):
202		dpath = os.path.splitext(dpath)[0]
203		elif os.path.isdir(dpath):
204		dpath = os.path.dirname(dpath)
205		import time
206		MPI.COMM_WORLD.barrier()
207		timestamp = time.strftime("%Y%m%d%H%M%S")
208		MPI.COMM_WORLD.barrier()
209		return "_".join([timestamp, os.path.basename(dpath)])
210
211
212		def __create_output_folder(path, folder_name):
213		folder = os.path.join(path, folder_name)
214		if MPI.COMM_WORLD.rank == 0:
215		if not os.path.exists(folder):
216		os.makedirs(folder)
217		return folder
218
219
220		def main(input_args=None):
221		args = __option_parser(doc=False)
222
223		if input_args:
224		args = input_args
225
226		options = _set_options(args)
227
228
229		pRunner = PluginRunner if options['mode'] == 'full' else BasicPluginRunner
230		try:
231		options["post_pre_run"] = False
232		answer = "Y"
233		if options["pre_run"]:
234		pre_run_options = pr._set_options(args)
235		pre_plugin_runner = pRunner(pre_run_options)
236		pre_plugin_runner._run_plugin_list()
237		pre_plugin_runner.exp._save_pre_run_log()
238		#options["data_file"] = pre_plugin_runner.exp.meta_data.get("pre_run_file")
239		folder = options['out_path']
240		fname = options['datafile_name'] + '_pre_run.nxs'
241		filename = os.path.join(folder, fname)
242		options["data_file"] = filename
243		options["pre_run"] = False
244		options["post_pre_run"] = True
245		if MPI.COMM_WORLD.rank == 0:
246		while answer not in ("y", "N"):
247		cu.user_message("Pre-run complete. See run_log/pre_run_log.txt for details. Do you want to continue? [y/N]")
248		answer = input()
249		if answer in ("y", "Y"):
250		plugin_runner = pRunner(options)
251		plugin_runner._run_plugin_list()
252		if options['process'] == 0:
253		in_file = plugin_runner.exp.meta_data['nxs_filename']
254		citation_extractor.main(in_file=in_file, quiet=True)
255		except Exception:
256		# raise the error in the user log
257		trace = traceback.format_exc()
258		cu.user_message(trace)
259		if options['nProcesses'] == 1:
260		sys.exit(1)
261		else:
262		# Kill all MPI processes
263		MPI.COMM_WORLD.Abort(1)
264
265
266		if __name__ == '__main__':
267		main()
268

DiamondLightSource / Savu

savu.tomo_recon.__create_folder_name() A last analyzed 2022-08-17 15:44 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

savu.tomo_recon.__create_folder_name() A
last analyzed 2022-08-17 15:44 UTC