scripts.dawn_runner.run_savu.process_init() - Code Metrics - Inspection of "scrutinizer config" - DiamondLightSource/Savu - Measure and Improve Code Quality continuously with Scrutinizer

Test Failed

Pull Request — master (#708)

by Daniil

created 2021-02-25 17:10 UTC

scripts.dawn_runner.run_savu.process_init() A

↳ Parent: scripts.dawn_runner.run_savu

Complexity

Conditions

Size

Total Lines	10
Code Lines	10

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	2
eloc	10
nop	1
dl	0
loc	10
rs	9.9
c	0
b	0
f	0

'''
run_savu
This is a refactor of the code that used to be contained in dawn.
It's used to mock up a runner for individual savu plugins from a python shell.
It is currently very early in development and will be subject to massive refactor in the future.
'''
from savu.data.experiment_collection import Experiment
from savu.data.meta_data import MetaData
from savu.plugins.utils import get_plugin
import savu.plugins.loaders.utils.yaml_utils as yaml
import os, sys
import numpy as np
from copy import deepcopy as copy
import time
from collections import OrderedDict


def get_output_rank(path2plugin, inputs, params, persistence):
    sys_path_0_lock = persistence['sys_path_0_lock']
    sys_path_0_lock.acquire()
    try:
        parameters = {}
        # slight repack here
        for key in list(params.keys()):
            val = params[key]["value"]
            if type(val)==type(''):
                val = val.replace('\n','').strip()
                parameters[key] = val
        plugin = _savu_setup(path2plugin, inputs, parameters)
        persistence['plugin_object'] = plugin
    finally:
        sys_path_0_lock.release()

    return len(plugin.get_plugin_out_datasets()[0].get_core_dimensions())


def runSavu(path2plugin, params, metaOnly, inputs, persistence):
    '''
    path2plugin  - is the path to the user script that should be run
    params - are the savu parameters
    metaOnly - a boolean for whether the data is kept in metadata or is passed as data
    inputs      - is a dictionary of input objects 
    '''
    t1 = time.time()
    sys_path_0_lock = persistence['sys_path_0_lock']
    sys_path_0_set = persistence['sys_path_0_set']
    plugin_object = persistence['plugin_object']
    axis_labels = persistence['axis_labels']
    axis_values = persistence['axis_values']
    string_key = persistence['string_key']
    parameters = persistence['parameters']
    aux = persistence['aux']
    sys_path_0_lock.acquire()
    try:
        result = copy(inputs)

        scriptDir = os.path.dirname(path2plugin)
        sys_path_0 = sys.path[0]
        if sys_path_0_set and scriptDir != sys_path_0:
            raise Exception("runSavu attempted to change sys.path[0] in a way that "
                            "could cause a race condition. Current sys.path[0] is {!r}, "
                            "trying to set to {!r}".format(sys_path_0, scriptDir))
        else:
            sys.path[0] = scriptDir
            sys_path_0_set = True
        
        if not plugin_object:
            parameters = {}
                # slight repack here
            for key in list(params.keys()):
#                 print "here"
                val = params[key]["value"]
                if type(val)==type(''):
                    val = val.replace('\n','').strip()
#                 print val
                parameters[key] = val
                print(("val: {}".format(val)))
#             print "initialising the object"
            plugin_object = _savu_setup(path2plugin, inputs, parameters)
            persistence['plugin_object'] = plugin_object
            axis_labels, axis_values = process_init(plugin_object)
#             print "I did the initialisation"
#             print "axis labels",axis_labels
#             print "axis_values", axis_values
#             print plugin_object
            chkstring =  [any(isinstance(ix, str) for ix in axis_values[label]) for label in axis_labels]
            if any(chkstring): # are any axis values strings we instead make this an aux out
                metaOnly = True
#                 print "AXIS LABELS"+str(axis_values)
                string_key = axis_labels[chkstring.index(True)]
                aux = OrderedDict.fromkeys(axis_values[string_key])
#                 print aux.keys()
            else:
                string_key = axis_labels[0]# will it always be the first one?
            if not metaOnly:
                if len(axis_labels) == 1:
                    result['xaxis']=axis_values[axis_labels[0]]
                    result['xaxis_title']=axis_labels[0]
                if len(axis_labels) == 2:
#                     print "set the output axes"
                    x = axis_labels[0]
                    result['xaxis_title']=x
                    y = axis_labels[1]
                    result['yaxis_title']=y
                    result['yaxis']=axis_values[y]
                    result['xaxis']=axis_values[x]
        else:
            pass
    finally:
        sys_path_0_lock.release()

    if plugin_object.get_max_frames()>1: # we need to get round this since we are frame independant
        data = np.expand_dims(inputs['data'], 0)
    else:
        data = inputs['data']

    print(("metaOnly: {}".format(metaOnly)))

    if not metaOnly: 

        out = plugin_object.process_frames([data])
#         print "ran the plugin"

        result['data'] = out
    elif metaOnly:
        result['data'] = inputs['data']
#         print type(result['data'])
        out_array = plugin_object.process_frames([inputs['data']])

#         print aux.keys()

        for k,key in enumerate(aux.keys()):
            aux[key]=np.array([out_array[k]])# wow really

        result['auxiliary'] = aux
    t2 = time.time()
    print("time to runSavu = "+str((t2-t1)))
    return result


def _savu_setup(path2plugin, inputs, parameters):
    print("running _savu_setup")
    parameters['in_datasets'] = [inputs['dataset_name']]
    parameters['out_datasets'] = [inputs['dataset_name']]
    plugin = get_plugin(path2plugin.split('.py')[0]+'.py')
    plugin.exp = setup_exp_and_data(inputs, inputs['data'], plugin)
    plugin._set_parameters(parameters)
    plugin._set_plugin_datasets()
    plugin.setup()
    return plugin


def process_init(plugin):
    axis_labels = plugin.get_out_datasets()[0].get_axis_label_keys()
    axis_labels.remove('idx')  # get the labels
    axis_values = {}
    plugin._clean_up()  # this copies the metadata!
    for label in axis_labels:
        axis_values[label] = plugin.get_out_datasets()[0].meta_data.get(label)
    plugin.base_pre_process()
    plugin.pre_process()
    return axis_labels, axis_values


def setup_exp_and_data(inputs, data, plugin):
    exp = DawnExperiment(get_options())
    data_obj = exp.create_data_object('in_data', inputs['dataset_name'])
    data_obj.data = None
    if len(inputs['data'].shape)==1:
#         print data.shape
        if inputs['xaxis_title'] is None or inputs['xaxis_title'].isspace():
            inputs['xaxis_title']='x'
            inputs['xaxis'] = np.arange(inputs['data'].shape[0])
        data_obj.set_axis_labels('idx.units', inputs['xaxis_title'] + '.units')
        data_obj.meta_data.set('idx', np.array([1]))
        data_obj.meta_data.set(str(inputs['xaxis_title']), inputs['xaxis'])
        data_obj.add_pattern(plugin.get_plugin_pattern(), core_dims=(1,), slice_dims=(0, ))
        data_obj.add_pattern('SINOGRAM', core_dims=(1,), slice_dims=(0, )) # good to add these two on too
        data_obj.add_pattern('PROJECTION', core_dims=(1,), slice_dims=(0, ))
    if len(inputs['data'].shape)==2:
        if inputs['xaxis_title'] is None  or inputs['xaxis_title'].isspace():
            print("set x")
            inputs['xaxis_title']='x'
            inputs['xaxis'] = np.arange(inputs['data'].shape[0])
        if inputs['yaxis_title'] is None or inputs['yaxis_title'].isspace():
            print("set y")
            inputs['yaxis_title']='y'
            size_y_axis = inputs['data'].shape[1]
            inputs['yaxis'] = np.arange(size_y_axis)
        
        data_obj.set_axis_labels('idx.units', inputs['xaxis_title'] + '.units', inputs['yaxis_title'] + '.units')
        data_obj.meta_data.set('idx', np.array([1]))
        data_obj.meta_data.set(str(inputs['xaxis_title']), inputs['xaxis'])
        data_obj.meta_data.set(str(inputs['yaxis_title']), inputs['yaxis'])
        data_obj.add_pattern(plugin.get_plugin_pattern(), core_dims=(1,2,), slice_dims=(0, ))
        data_obj.add_pattern('SINOGRAM', core_dims=(1,2,), slice_dims=(0, )) # good to add these two on too
        data_obj.add_pattern('PROJECTION', core_dims=(1,2,), slice_dims=(0, ))
   
    data_obj.set_shape((1, ) + data.shape) # need to add for now for slicing...
    data_obj.get_preview().set_preview([])
    return exp

class DawnExperiment(Experiment):
    def __init__(self, options):
        self.index={"in_data": {}, "out_data": {}, "mapping": {}}
        self.meta_data = MetaData(get_options())
        self.nxs_file = None

def get_options():
    options = {}
    options['dawn_runner'] = True
    options['transport'] = 'hdf5'
    options['process_names'] = 'CPU0'
    options['processes'] = 'CPU0'
    options['data_file'] = ''
    options['process_file'] = ''
    options['out_path'] = ''
    options['inter_path'] = ''
    options['log_path'] = ''
    options['run_type'] = ''
    options['verbose'] = 'True'
    options['system_params'] = _set_system_params()
    return options

def _set_system_params():
    # look in conda environment to see which version is being used
    savu_path = sys.modules['savu'].__path__[0]
    sys_files = os.path.join(
            os.path.dirname(savu_path), 'system_files')
    subdirs = os.listdir(sys_files)
    sys_folder = 'dls' if len(subdirs) > 1 else subdirs[0]
    fname = 'system_parameters.yml'
    sys_file = os.path.join(sys_files, sys_folder, fname)
    return yaml.read_yaml(sys_file)


1			'''
2			run_savu
3			This is a refactor of the code that used to be contained in dawn.
4			It's used to mock up a runner for individual savu plugins from a python shell.
5			It is currently very early in development and will be subject to massive refactor in the future.
6			'''
7			from savu.data.experiment_collection import Experiment
8			from savu.data.meta_data import MetaData
9			from savu.plugins.utils import get_plugin
10			import savu.plugins.loaders.utils.yaml_utils as yaml
11			import os, sys
12			import numpy as np
13			from copy import deepcopy as copy
14			import time
15			from collections import OrderedDict
16
17
18			def get_output_rank(path2plugin, inputs, params, persistence):
19			sys_path_0_lock = persistence['sys_path_0_lock']
20			sys_path_0_lock.acquire()
21			try:
22			parameters = {}
23			# slight repack here
24			for key in list(params.keys()):
25			val = params[key]["value"]
26			if type(val)==type(''):
27			val = val.replace('\n','').strip()
28			parameters[key] = val
29			plugin = _savu_setup(path2plugin, inputs, parameters)
30			persistence['plugin_object'] = plugin
31			finally:
32			sys_path_0_lock.release()
33
34			return len(plugin.get_plugin_out_datasets()[0].get_core_dimensions())
35
36
37			def runSavu(path2plugin, params, metaOnly, inputs, persistence):
38			'''
39			path2plugin - is the path to the user script that should be run
40			params - are the savu parameters
41			metaOnly - a boolean for whether the data is kept in metadata or is passed as data
42			inputs - is a dictionary of input objects
43			'''
44			t1 = time.time()
45			sys_path_0_lock = persistence['sys_path_0_lock']
46			sys_path_0_set = persistence['sys_path_0_set']
47			plugin_object = persistence['plugin_object']
48			axis_labels = persistence['axis_labels']
49			axis_values = persistence['axis_values']
50			string_key = persistence['string_key']
51			parameters = persistence['parameters']
52			aux = persistence['aux']
53			sys_path_0_lock.acquire()
54			try:
55			result = copy(inputs)
56
57			scriptDir = os.path.dirname(path2plugin)
58			sys_path_0 = sys.path[0]
59			if sys_path_0_set and scriptDir != sys_path_0:
60			raise Exception("runSavu attempted to change sys.path[0] in a way that "
61			"could cause a race condition. Current sys.path[0] is {!r}, "
62			"trying to set to {!r}".format(sys_path_0, scriptDir))
63			else:
64			sys.path[0] = scriptDir
65			sys_path_0_set = True
66
67			if not plugin_object:
68			parameters = {}
69			# slight repack here
70			for key in list(params.keys()):
71			# print "here"
72			val = params[key]["value"]
73			if type(val)==type(''):
74			val = val.replace('\n','').strip()
75			# print val
76			parameters[key] = val
77			print(("val: {}".format(val)))
78			# print "initialising the object"
79			plugin_object = _savu_setup(path2plugin, inputs, parameters)
80			persistence['plugin_object'] = plugin_object
81			axis_labels, axis_values = process_init(plugin_object)
82			# print "I did the initialisation"
83			# print "axis labels",axis_labels
84			# print "axis_values", axis_values
85			# print plugin_object
86			chkstring = [any(isinstance(ix, str) for ix in axis_values[label]) for label in axis_labels]
87			if any(chkstring): # are any axis values strings we instead make this an aux out
88			metaOnly = True
89			# print "AXIS LABELS"+str(axis_values)
90			string_key = axis_labels[chkstring.index(True)]
91			aux = OrderedDict.fromkeys(axis_values[string_key])
92			# print aux.keys()
93			else:
94			string_key = axis_labels[0]# will it always be the first one?
95			if not metaOnly:
96			if len(axis_labels) == 1:
97			result['xaxis']=axis_values[axis_labels[0]]
98			result['xaxis_title']=axis_labels[0]
99			if len(axis_labels) == 2:
100			# print "set the output axes"
101			x = axis_labels[0]
102			result['xaxis_title']=x
103			y = axis_labels[1]
104			result['yaxis_title']=y
105			result['yaxis']=axis_values[y]
106			result['xaxis']=axis_values[x]
107			else:
108			pass
109			finally:
110			sys_path_0_lock.release()
111
112			if plugin_object.get_max_frames()>1: # we need to get round this since we are frame independant
113			data = np.expand_dims(inputs['data'], 0)
114			else:
115			data = inputs['data']
116
117			print(("metaOnly: {}".format(metaOnly)))
118
119			if not metaOnly:
120
121			out = plugin_object.process_frames([data])
122			# print "ran the plugin"
123
124			result['data'] = out
125			elif metaOnly:
126			result['data'] = inputs['data']
127			# print type(result['data'])
128			out_array = plugin_object.process_frames([inputs['data']])
129
130			# print aux.keys()
131
132			for k,key in enumerate(aux.keys()):
133			aux[key]=np.array([out_array[k]])# wow really
134
135			result['auxiliary'] = aux
136			t2 = time.time()
137			print("time to runSavu = "+str((t2-t1)))
138			return result
139
140
141			def _savu_setup(path2plugin, inputs, parameters):
142			print("running _savu_setup")
143			parameters['in_datasets'] = [inputs['dataset_name']]
144			parameters['out_datasets'] = [inputs['dataset_name']]
145			plugin = get_plugin(path2plugin.split('.py')[0]+'.py')
146			plugin.exp = setup_exp_and_data(inputs, inputs['data'], plugin)
147			plugin._set_parameters(parameters)
148			plugin._set_plugin_datasets()
149			plugin.setup()
150			return plugin
151
152
153			def process_init(plugin):
154			axis_labels = plugin.get_out_datasets()[0].get_axis_label_keys()
155			axis_labels.remove('idx') # get the labels
156			axis_values = {}
157			plugin._clean_up() # this copies the metadata!
158			for label in axis_labels:
159			axis_values[label] = plugin.get_out_datasets()[0].meta_data.get(label)
160			plugin.base_pre_process()
161			plugin.pre_process()
162			return axis_labels, axis_values
163
164
165			def setup_exp_and_data(inputs, data, plugin):
166			exp = DawnExperiment(get_options())
167			data_obj = exp.create_data_object('in_data', inputs['dataset_name'])
168			data_obj.data = None
169			if len(inputs['data'].shape)==1:
170			# print data.shape
171			if inputs['xaxis_title'] is None or inputs['xaxis_title'].isspace():
172			inputs['xaxis_title']='x'
173			inputs['xaxis'] = np.arange(inputs['data'].shape[0])
174			data_obj.set_axis_labels('idx.units', inputs['xaxis_title'] + '.units')
175			data_obj.meta_data.set('idx', np.array([1]))
176			data_obj.meta_data.set(str(inputs['xaxis_title']), inputs['xaxis'])
177			data_obj.add_pattern(plugin.get_plugin_pattern(), core_dims=(1,), slice_dims=(0, ))
178			data_obj.add_pattern('SINOGRAM', core_dims=(1,), slice_dims=(0, )) # good to add these two on too
179			data_obj.add_pattern('PROJECTION', core_dims=(1,), slice_dims=(0, ))
180			if len(inputs['data'].shape)==2:
181			if inputs['xaxis_title'] is None or inputs['xaxis_title'].isspace():
182			print("set x")
183			inputs['xaxis_title']='x'
184			inputs['xaxis'] = np.arange(inputs['data'].shape[0])
185			if inputs['yaxis_title'] is None or inputs['yaxis_title'].isspace():
186			print("set y")
187			inputs['yaxis_title']='y'
188			size_y_axis = inputs['data'].shape[1]
189			inputs['yaxis'] = np.arange(size_y_axis)
190
191			data_obj.set_axis_labels('idx.units', inputs['xaxis_title'] + '.units', inputs['yaxis_title'] + '.units')
192			data_obj.meta_data.set('idx', np.array([1]))
193			data_obj.meta_data.set(str(inputs['xaxis_title']), inputs['xaxis'])
194			data_obj.meta_data.set(str(inputs['yaxis_title']), inputs['yaxis'])
195			data_obj.add_pattern(plugin.get_plugin_pattern(), core_dims=(1,2,), slice_dims=(0, ))
196			data_obj.add_pattern('SINOGRAM', core_dims=(1,2,), slice_dims=(0, )) # good to add these two on too
197			data_obj.add_pattern('PROJECTION', core_dims=(1,2,), slice_dims=(0, ))
198
199			data_obj.set_shape((1, ) + data.shape) # need to add for now for slicing...
200			data_obj.get_preview().set_preview([])
201			return exp
202
203			class DawnExperiment(Experiment):
204			def __init__(self, options):
205			self.index={"in_data": {}, "out_data": {}, "mapping": {}}
206			self.meta_data = MetaData(get_options())
207			self.nxs_file = None
208
209			def get_options():
210			options = {}
211			options['dawn_runner'] = True
212			options['transport'] = 'hdf5'
213			options['process_names'] = 'CPU0'
214			options['processes'] = 'CPU0'
215			options['data_file'] = ''
216			options['process_file'] = ''
217			options['out_path'] = ''
218			options['inter_path'] = ''
219			options['log_path'] = ''
220			options['run_type'] = ''
221			options['verbose'] = 'True'
222			options['system_params'] = _set_system_params()
223			return options
224
225			def _set_system_params():
226			# look in conda environment to see which version is being used
227			savu_path = sys.modules['savu'].__path__[0]
228			sys_files = os.path.join(
229			os.path.dirname(savu_path), 'system_files')
230			subdirs = os.listdir(sys_files)
231			sys_folder = 'dls' if len(subdirs) > 1 else subdirs[0]
232			fname = 'system_parameters.yml'
233			sys_file = os.path.join(sys_files, sys_folder, fname)
234			return yaml.read_yaml(sys_file)
235

DiamondLightSource / Savu

Pull Request — master (#708)

scripts.dawn_runner.run_savu.process_init() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like