savu.pre_run._choose_process_list() - Code Metrics - DiamondLightSource/Savu - Measure and Improve Code Quality continuously with Scrutinizer

savu.pre_run._choose_process_list() A
last analyzed 2022-08-17 15:44 UTC

↳ Parent: savu.pre_run

Complexity

Conditions

Size

Total Lines	5
Code Lines	4

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	2
eloc	4
nop	2
dl	0
loc	5
rs	10
c	0
b	0
f	0

# Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: pre-run
   :platform: Unix
   :synopsis: Runner for the Savu framework during a pre-run

.. moduleauthor:: Jacob Williamson <[email protected]>

"""

import tempfile  # this import is required for pyFAI - DO NOT REMOVE!
import argparse
import traceback
import sys
import os

import h5py as h5
from mpi4py import MPI
from savu.version import __version__

import savu.core.utils as cu
from scripts.citation_extractor import citation_extractor
from savu.core.basic_plugin_runner import BasicPluginRunner
from savu.core.plugin_runner import PluginRunner
#from scripts.config_generator.savu_config import internal_config


def __option_parser(doc=True):
    """ Option parser for command line arguments.
    """
    version = "%(prog)s " + __version__
    parser = argparse.ArgumentParser(prog='savu')
    hide = argparse.SUPPRESS

    parser.add_argument('in_file', help='Input data file.')
    parser.add_argument('out_folder', help='Output folder.')
    parser.add_argument('--version', action='version', version=version)
    parser.add_argument("-f", "--folder", help="Override output folder name")

    tmp_help = "Store intermediate files in a temp directory."
    parser.add_argument("-d", "--tmp", help=tmp_help)

    template_help = "Pass a template file of plugin input parameters."
    parser.add_argument("-t", "--template", help=template_help, default=None)

    log_help = "Store full log file in a separate location"
    parser.add_argument("-l", "--log", help=log_help)

    v_help = "Display all debug log messages"
    parser.add_argument("-v", "--verbose", help=v_help, action="store_true",
                        default=False)
    parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
                        help="Display only Errors and Info.", default=False)
    # temporary flag to fix lustre issue
    parser.add_argument("--lustre_workaround", action="store_true",
                        dest="lustre", help="Avoid lustre segmentation fault",
                        default=False)
    sys_params_help = "Override default path to Savu system parameters file."
    parser.add_argument("--system_params", help=sys_params_help, default=None)

    # Set stats off
    parser.add_argument("--stats", help="Turn stats 'on' or 'off'.", default="on", choices=["on", "off"])

    # Hidden arguments
    # process names
    parser.add_argument("-n", "--names", help=hide, default="CPU0")
    # transport mechanism
    parser.add_argument("--transport", help=hide, default="hdf5")

    # Set logging to cluster mode
    parser.add_argument("-c", "--cluster", action="store_true", help=hide,
                        default=False)
    # Facility email for errors
    parser.add_argument("--facility_email", dest="femail", help=hide,
                        default=None)
    # Send an email on completion
    parser.add_argument("-e", "--email", dest="email", help=hide, default=None)

    parser.add_argument("--bllog", dest="bllog", help=hide, default=None)
    # Location of syslog server

    # Location of syslog server
    parser.add_argument("-s", "--syslog", dest="syslog", help=hide,
                        default='localhost')
    # Port to connect to syslog server on
    parser.add_argument("-p", "--syslog_port", dest="syslog_port",
                        help=hide, default=514, type=int)
    parser.add_argument("--test_state", dest="test_state", default='False',
                        action='store_true', help=hide)

    if doc==False:
        args = parser.parse_args()
        return args
    else:
        return parser

def _set_options(args):

    """ Set run specific information in options dictionary.

    :params dict opt: input optional arguments (or defaults)
    :params args: input required arguments
    :returns options: optional and required arguments
    :rtype: dict
    """
    options = {}
    options['data_file'] = args.in_file
    options['process_file'] = 'savu/data/stats/pre_run.nxs'
    options["process_file_name"] = options["process_file"].split("/")[-1]
    options["post_pre_run"] = False
    options['mode'] = 'full'
    options['template'] = args.template
    options['transport'] = args.transport
    options['process_names'] = args.names
    options['verbose'] = args.verbose
    options['quiet'] = args.quiet
    options['cluster'] = args.cluster
    options['syslog_server'] = args.syslog
    options['syslog_port'] = args.syslog_port
    options['test_state'] = args.test_state
    options['lustre'] = args.lustre
    options['bllog'] = args.bllog
    options['email'] = args.email
    options['femail'] = args.femail
    options['system_params'] = args.system_params
    options['stats'] = 'on'
    options['pre_run'] = True
    options['checkpoint'] = None

    if args.folder:
        out_folder_name = os.path.basename(args.folder)
    else:
        out_folder_name = __create_folder_name(options['data_file'])

    out_folder_path = __create_output_folder(args.out_folder, out_folder_name)

    options['out_folder'] = out_folder_name
    options['out_path'] = out_folder_path

    basename = os.path.basename(args.in_file)
    options['datafile_name'] = os.path.splitext(basename)[0] if basename \
        else args.in_file.split(os.sep)[-2]

    inter_folder_path = __create_output_folder(args.tmp, out_folder_name) \
        if args.tmp else out_folder_path

    options['inter_path'] = inter_folder_path
    options['log_path'] = args.log if args.log else options['inter_path']
    options['nProcesses'] = len(options["process_names"].split(','))

    command_str = " ".join([str(i) for i in sys.argv[1:]])
    command_full = f"savu {command_str}"
    options["command"] = command_full

    return options

def __create_folder_name(dpath):

    if os.path.isfile(dpath):
        dpath = os.path.splitext(dpath)[0]
    elif os.path.isdir(dpath):
        dpath = os.path.dirname(dpath)
    import time
    MPI.COMM_WORLD.barrier()
    timestamp = time.strftime("%Y%m%d%H%M%S")
    MPI.COMM_WORLD.barrier()
    return "_".join([timestamp, os.path.basename(dpath)])


def __create_output_folder(path, folder_name):
    folder = os.path.join(path, folder_name)
    if MPI.COMM_WORLD.rank == 0:
        if not os.path.exists(folder):
            os.makedirs(folder)
    return folder

def __get_beamline(options):
    try:
        with h5.File(options["data_file"], "r") as h5file:
            beamline = h5file['entry1/instrument/name'][()]
        return beamline
    except KeyError:
        return None

#def _edit_process_list(options, beamline):
#    #  Only 'open', 'set', 'mod' and 'save' should be used.
#    if beamline == "i23":
#        commands = [f"open {options['process_file']}",
#                    f"set 2 on",
#                    f"save {options['process_file']}"]
#    else:
#        commands = [f"open {options['process_file']}",
#                   f"set 2 off",
#                    f"save {options['process_file']}"]
#    plugin_list = internal_config(*commands)
#    active_plugins = []
#    for plugin in plugin_list:
#        if plugin["active"]:
#            active_plugins.append(plugin["name"])
#    return active_plugins

def _choose_process_list(options, beamline):
    #  Only 'open', 'set', 'mod' and 'save' should be used.
    if beamline == "i23":
        options['process_file'] = 'savu/data/stats/pre_run_i23.nxs'
        options["process_file_name"] = options["process_file"].split("/")[-1]


def main(input_args=None):
    args = __option_parser(doc=False)

    if input_args:
        args = input_args

    options = _set_options(args)

    pRunner = PluginRunner if options['mode'] == 'full' else BasicPluginRunner

    try:
        beamline = __get_beamline(options)
        _choose_process_list(options, beamline)
        plugin_runner = pRunner(options)
        plugin_runner._run_plugin_list()
        plugin_runner.exp._save_pre_run_log()
        if options['process'] == 0:
            in_file = plugin_runner.exp.meta_data['nxs_filename']
            citation_extractor.main(in_file=in_file, quiet=True)
    except Exception:
        # raise the error in the user log
        trace = traceback.format_exc()
        cu.user_message(trace)
        if options['nProcesses'] == 1:
            sys.exit(1)
        else:
            # Kill all MPI processes
            MPI.COMM_WORLD.Abort(1)

if __name__ == '__main__':
    main()


1		# Copyright 2014 Diamond Light Source Ltd.
2		#
3		# Licensed under the Apache License, Version 2.0 (the "License");
4		# you may not use this file except in compliance with the License.
5		# You may obtain a copy of the License at
6		#
7		# http://www.apache.org/licenses/LICENSE-2.0
8		#
9		# Unless required by applicable law or agreed to in writing, software
10		# distributed under the License is distributed on an "AS IS" BASIS,
11		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12		# See the License for the specific language governing permissions and
13		# limitations under the License.
14
15		"""
16		.. module:: pre-run
17		:platform: Unix
18		:synopsis: Runner for the Savu framework during a pre-run
19
20		.. moduleauthor:: Jacob Williamson <[email protected]>
21
22		"""
23
24		import tempfile # this import is required for pyFAI - DO NOT REMOVE!
25		import argparse
26		import traceback
27		import sys
28		import os
29
30		import h5py as h5
31		from mpi4py import MPI
32		from savu.version import __version__
33
34		import savu.core.utils as cu
35		from scripts.citation_extractor import citation_extractor
36		from savu.core.basic_plugin_runner import BasicPluginRunner
37		from savu.core.plugin_runner import PluginRunner
38		#from scripts.config_generator.savu_config import internal_config
39
40
41		def __option_parser(doc=True):
42		""" Option parser for command line arguments.
43		"""
44		version = "%(prog)s " + __version__
45		parser = argparse.ArgumentParser(prog='savu')
46		hide = argparse.SUPPRESS
47
48		parser.add_argument('in_file', help='Input data file.')
49		parser.add_argument('out_folder', help='Output folder.')
50		parser.add_argument('--version', action='version', version=version)
51		parser.add_argument("-f", "--folder", help="Override output folder name")
52
53		tmp_help = "Store intermediate files in a temp directory."
54		parser.add_argument("-d", "--tmp", help=tmp_help)
55
56		template_help = "Pass a template file of plugin input parameters."
57		parser.add_argument("-t", "--template", help=template_help, default=None)
58
59		log_help = "Store full log file in a separate location"
60		parser.add_argument("-l", "--log", help=log_help)
61
62		v_help = "Display all debug log messages"
63		parser.add_argument("-v", "--verbose", help=v_help, action="store_true",
64		default=False)
65		parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
66		help="Display only Errors and Info.", default=False)
67		# temporary flag to fix lustre issue
68		parser.add_argument("--lustre_workaround", action="store_true",
69		dest="lustre", help="Avoid lustre segmentation fault",
70		default=False)
71		sys_params_help = "Override default path to Savu system parameters file."
72		parser.add_argument("--system_params", help=sys_params_help, default=None)
73
74		# Set stats off
75		parser.add_argument("--stats", help="Turn stats 'on' or 'off'.", default="on", choices=["on", "off"])
76
77		# Hidden arguments
78		# process names
79		parser.add_argument("-n", "--names", help=hide, default="CPU0")
80		# transport mechanism
81		parser.add_argument("--transport", help=hide, default="hdf5")
82
83		# Set logging to cluster mode
84		parser.add_argument("-c", "--cluster", action="store_true", help=hide,
85		default=False)
86		# Facility email for errors
87		parser.add_argument("--facility_email", dest="femail", help=hide,
88		default=None)
89		# Send an email on completion
90		parser.add_argument("-e", "--email", dest="email", help=hide, default=None)
91
92		parser.add_argument("--bllog", dest="bllog", help=hide, default=None)
93		# Location of syslog server
94
95		# Location of syslog server
96		parser.add_argument("-s", "--syslog", dest="syslog", help=hide,
97		default='localhost')
98		# Port to connect to syslog server on
99		parser.add_argument("-p", "--syslog_port", dest="syslog_port",
100		help=hide, default=514, type=int)
101		parser.add_argument("--test_state", dest="test_state", default='False',
102		action='store_true', help=hide)
103
104		if doc==False:
105		args = parser.parse_args()
106		return args
107		else:
108		return parser
109
110	View Code Duplication	def _set_options(args):
		0 ignored issues – show Duplication introduced 2022-07-26 15:17 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
111		""" Set run specific information in options dictionary.
112
113		:params dict opt: input optional arguments (or defaults)
114		:params args: input required arguments
115		:returns options: optional and required arguments
116		:rtype: dict
117		"""
118		options = {}
119		options['data_file'] = args.in_file
120		options['process_file'] = 'savu/data/stats/pre_run.nxs'
121		options["process_file_name"] = options["process_file"].split("/")[-1]
122		options["post_pre_run"] = False
123		options['mode'] = 'full'
124		options['template'] = args.template
125		options['transport'] = args.transport
126		options['process_names'] = args.names
127		options['verbose'] = args.verbose
128		options['quiet'] = args.quiet
129		options['cluster'] = args.cluster
130		options['syslog_server'] = args.syslog
131		options['syslog_port'] = args.syslog_port
132		options['test_state'] = args.test_state
133		options['lustre'] = args.lustre
134		options['bllog'] = args.bllog
135		options['email'] = args.email
136		options['femail'] = args.femail
137		options['system_params'] = args.system_params
138		options['stats'] = 'on'
139		options['pre_run'] = True
140		options['checkpoint'] = None
141
142		if args.folder:
143		out_folder_name = os.path.basename(args.folder)
144		else:
145		out_folder_name = __create_folder_name(options['data_file'])
146
147		out_folder_path = __create_output_folder(args.out_folder, out_folder_name)
148
149		options['out_folder'] = out_folder_name
150		options['out_path'] = out_folder_path
151
152		basename = os.path.basename(args.in_file)
153		options['datafile_name'] = os.path.splitext(basename)[0] if basename \
154		else args.in_file.split(os.sep)[-2]
155
156		inter_folder_path = __create_output_folder(args.tmp, out_folder_name) \
157		if args.tmp else out_folder_path
158
159		options['inter_path'] = inter_folder_path
160		options['log_path'] = args.log if args.log else options['inter_path']
161		options['nProcesses'] = len(options["process_names"].split(','))
162
163		command_str = " ".join([str(i) for i in sys.argv[1:]])
164		command_full = f"savu {command_str}"
165		options["command"] = command_full
166
167		return options
168
169	View Code Duplication	def __create_folder_name(dpath):
		0 ignored issues – show Duplication introduced 2022-07-26 15:17 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
170		if os.path.isfile(dpath):
171		dpath = os.path.splitext(dpath)[0]
172		elif os.path.isdir(dpath):
173		dpath = os.path.dirname(dpath)
174		import time
175		MPI.COMM_WORLD.barrier()
176		timestamp = time.strftime("%Y%m%d%H%M%S")
177		MPI.COMM_WORLD.barrier()
178		return "_".join([timestamp, os.path.basename(dpath)])
179
180
181		def __create_output_folder(path, folder_name):
182		folder = os.path.join(path, folder_name)
183		if MPI.COMM_WORLD.rank == 0:
184		if not os.path.exists(folder):
185		os.makedirs(folder)
186		return folder
187
188		def __get_beamline(options):
189		try:
190		with h5.File(options["data_file"], "r") as h5file:
191		beamline = h5file['entry1/instrument/name'][()]
192		return beamline
193		except KeyError:
194		return None
195
196		#def _edit_process_list(options, beamline):
197		# # Only 'open', 'set', 'mod' and 'save' should be used.
198		# if beamline == "i23":
199		# commands = [f"open {options['process_file']}",
200		# f"set 2 on",
201		# f"save {options['process_file']}"]
202		# else:
203		# commands = [f"open {options['process_file']}",
204		# f"set 2 off",
205		# f"save {options['process_file']}"]
206		# plugin_list = internal_config(*commands)
207		# active_plugins = []
208		# for plugin in plugin_list:
209		# if plugin["active"]:
210		# active_plugins.append(plugin["name"])
211		# return active_plugins
212
213		def _choose_process_list(options, beamline):
214		# Only 'open', 'set', 'mod' and 'save' should be used.
215		if beamline == "i23":
216		options['process_file'] = 'savu/data/stats/pre_run_i23.nxs'
217		options["process_file_name"] = options["process_file"].split("/")[-1]
218
219
220		def main(input_args=None):
221		args = __option_parser(doc=False)
222
223		if input_args:
224		args = input_args
225
226		options = _set_options(args)
227
228		pRunner = PluginRunner if options['mode'] == 'full' else BasicPluginRunner
229
230		try:
231		beamline = __get_beamline(options)
232		_choose_process_list(options, beamline)
233		plugin_runner = pRunner(options)
234		plugin_runner._run_plugin_list()
235		plugin_runner.exp._save_pre_run_log()
236		if options['process'] == 0:
237		in_file = plugin_runner.exp.meta_data['nxs_filename']
238		citation_extractor.main(in_file=in_file, quiet=True)
239		except Exception:
240		# raise the error in the user log
241		trace = traceback.format_exc()
242		cu.user_message(trace)
243		if options['nProcesses'] == 1:
244		sys.exit(1)
245		else:
246		# Kill all MPI processes
247		MPI.COMM_WORLD.Abort(1)
248
249		if __name__ == '__main__':
250		main()
251

DiamondLightSource / Savu

savu.pre_run._choose_process_list() A last analyzed 2022-08-17 15:44 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

savu.pre_run._choose_process_list() A
last analyzed 2022-08-17 15:44 UTC