savu.plugins.loaders.full_field_loaders.nxtomo_loader.NxtomoLoader._get_data_file() - Code Metrics - Inspection of "Merge pull request #875 from DiamondLightSource/al..." - DiamondLightSource/Savu - Measure and Improve Code Quality continuously with Scrutinizer

Test Failed

Push — master ( f1d4ff...d83625 )

by Daniil

created 2021-12-07 16:09 UTC

NxtomoLoader._get_data_file() A

↳ Parent: savu.plugins.loaders.full_field_loaders.nxtomo_loader

Complexity

Conditions

Size

Total Lines	3
Code Lines	3

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
eloc	3
nop	1
dl	0
loc	3
rs	10
c	0
b	0
f	0

# Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: nxtomo_loader
   :platform: Unix
   :synopsis: A class for loading standard tomography data in Nexus format.

.. moduleauthor:: Nicola Wadeson <[email protected]>

"""

import h5py
import logging
import numpy as np

from savu.plugins.loaders.base_loader import BaseLoader
from savu.plugins.utils import register_plugin

from savu.data.data_structures.data_types.data_plus_darks_and_flats \
    import ImageKey, NoImageKey


@register_plugin
class NxtomoLoader(BaseLoader):
    """
    """
    def __init__(self, name='NxtomoLoader'):
        super(NxtomoLoader, self).__init__(name)
        self.warnings = []

    def log_warning(self, msg):
        logging.warning(msg)
        self.warnings.append(msg)

    def setup(self):
        exp = self.get_experiment()
        data_obj = exp.create_data_object('in_data', self.parameters['name'])

        data_obj.backing_file = self._get_data_file()

        data_obj.data = self._get_h5_entry(
            data_obj.backing_file, self.parameters['data_path'])

        self._set_dark_and_flat(data_obj)

        self.nFrames = self.__get_nFrames(data_obj)
        if self.nFrames > 1:
            self.__setup_4d(data_obj)
            self.__setup_3d_to_4d(data_obj, self.nFrames)
        else:
            if len(data_obj.data.shape) == 3:
                self._setup_3d(data_obj)
            else:
                self.__setup_4d(data_obj)
            data_obj.set_original_shape(data_obj.data.shape)
        self._set_rotation_angles(data_obj)
        self._set_projection_shifts(data_obj)

        try:
            control = self._get_h5_path(
                data_obj.backing_file, 'entry1/tomo_entry/control/data')
            data_obj.meta_data.set("control", control[...])
        except Exception:
            self.log_warning("No Control information available")

        nAngles = len(data_obj.meta_data.get('rotation_angle'))
        self.__check_angles(data_obj, nAngles)

        self.set_data_reduction_params(data_obj)
        data_obj.data._set_dark_and_flat()

    def _get_h5_entry(self, filename, path):
        if path in filename:
            return filename[path]
        else:
            raise Exception("Path %s not found in %s" % (path, filename))

    def __get_nFrames(self, dObj):
        if self.parameters['3d_to_4d'] is False:
            return 0
        if self.parameters['3d_to_4d'] is True:
            try:
                # for backwards compatibility
                n_frames = eval(self.parameters["angles"], {"builtins": None, "np": np})
                return np.array(n_frames).shape[0]
            except Exception:
                raise Exception("Please specify the angles, or the number of "
                                "frames per scan (via 3d_to_4d param) in the loader.")
        if isinstance(self.parameters['3d_to_4d'], int):
            return self.parameters['3d_to_4d']
        else:
            raise Exception("Unknown value for loader parameter '3d_to_4d', "
                            "please specify an integer value")

    def _setup_3d(self, data_obj):
        logging.debug("Setting up 3d tomography data.")
        rot = 0
        detY = 1
        detX = 2
        data_obj.set_axis_labels('rotation_angle.degrees',
                                 'detector_y.pixel',
                                 'detector_x.pixel')

        data_obj.add_pattern('PROJECTION', core_dims=(detX, detY),
                             slice_dims=(rot,))
        data_obj.add_pattern('SINOGRAM', core_dims=(detX, rot),
                             slice_dims=(detY,))
        data_obj.add_pattern('TANGENTOGRAM', core_dims=(rot, detY),
                             slice_dims=(detX,))

    def __setup_3d_to_4d(self, data_obj, n_frames):
        logging.debug("setting up 4d tomography data from 3d input.")
        from savu.data.data_structures.data_types.map_3dto4d_h5 \
            import Map3dto4dh5

        all_angles = data_obj.data.shape[0]
        if all_angles % n_frames != 0:
            self.log_warning("There are not a complete set of scans in this file, "
                             "loading complete ones only")
        data_obj.data = Map3dto4dh5(data_obj, n_frames)
        data_obj.set_original_shape(data_obj.data.get_shape())

    def __setup_4d(self, data_obj):
        logging.debug("setting up 4d tomography data.")
        rot = 0
        detY = 1
        detX = 2
        scan = 3

        data_obj.set_axis_labels('rotation_angle.degrees', 'detector_y.pixel',
                                 'detector_x.pixel', 'scan.number')

        data_obj.add_pattern('PROJECTION', core_dims=(detX, detY),
                             slice_dims=(rot, scan))
        data_obj.add_pattern('SINOGRAM', core_dims=(detX, rot),
                             slice_dims=(detY, scan))
        data_obj.add_pattern('TANGENTOGRAM', core_dims=(rot, detY),
                             slice_dims=(detX, scan))
        data_obj.add_pattern('SINOMOVIE', core_dims=(detX, rot, scan),
                             slice_dims=(detY,))

    def _set_dark_and_flat(self, data_obj):
        flat = self.parameters['flat'][0]
        dark = self.parameters['dark'][0]

        if not flat or not dark:
            self.__find_dark_and_flat(data_obj, flat=flat, dark=dark)
        else:
            self.__set_separate_dark_and_flat(data_obj)

    def __find_dark_and_flat(self, data_obj, flat=None, dark=None):
        ignore = self.parameters['ignore_flats'] if \
            self.parameters['ignore_flats'] else None
        try:
            image_key = data_obj.backing_file[
                self.parameters['image_key_path']][...]
            data_obj.data = \
                ImageKey(data_obj, image_key, 0, ignore=ignore)
        except KeyError:
            self.log_warning("An image key was not found.")
            try:
                data_obj.data = NoImageKey(data_obj, None, 0)
                entry = 'entry1/tomo_entry/instrument/detector/'
                data_obj.data._set_flat_path(entry + 'flatfield')
                data_obj.data._set_dark_path(entry + 'darkfield')
            except KeyError:
                self.log_warning("Dark/flat data was not found in input file.")
        data_obj.data._set_dark_and_flat()
        if dark:
            data_obj.data.update_dark(dark)
        if flat:
            data_obj.data.update_flat(flat)

    def __set_separate_dark_and_flat(self, data_obj):
        try:
            image_key = data_obj.backing_file[
                'entry1/tomo_entry/instrument/detector/image_key'][...]
        except:
            image_key = None
        data_obj.data = NoImageKey(data_obj, image_key, 0)
        self.__set_data(data_obj, 'flat', data_obj.data._set_flat_path)
        self.__set_data(data_obj, 'dark', data_obj.data._set_dark_path)

    def __set_data(self, data_obj, name, func):
        path, entry, scale = self.parameters[name]
            
        if path.split('/')[0] == 'Savu':
            import os
            savu_base_path = os.path.join(os.path.dirname(
                os.path.realpath(__file__)), '..', '..', '..', '..')
            path = os.path.join(savu_base_path, path.split('Savu')[1][1:])

        ffile = h5py.File(path, 'r')
        try:
            image_key = \
                ffile['entry1/tomo_entry/instrument/detector/image_key'][...]
            func(ffile[entry], imagekey=image_key)
        except:
            func(ffile[entry])

        data_obj.data._set_scale(name, scale)

    def _set_rotation_angles(self, data_obj):
        angles = self.parameters['angles']
        warn_ms = "No angles found so evenly distributing them between 0 and" \
                  " 180 degrees"
        if angles is None:
            angle_key = 'entry1/tomo_entry/data/rotation_angle'
            nxs_angles = self.__get_angles_from_nxs_file(data_obj, angle_key)
            if nxs_angles is None:
                self.log_warning(warn_ms)
                angles = np.linspace(0, 180, data_obj.get_shape()[0])
            else:
                angles = nxs_angles
        else:
            try:
                angles = eval(angles)
            except Exception as e:
                logging.warning(e)
                try:
                    angles = np.loadtxt(angles)
                except Exception as e:
                    logging.debug(e)
                    self.log_warning(warn_ms)
                    angles = np.linspace(0, 180, data_obj.get_shape()[0])
        data_obj.meta_data.set("rotation_angle", angles)
        return len(angles)

    def _set_projection_shifts(self, data_obj):
        proj_shifts = np.zeros((data_obj.get_shape()[0], 2)) # initialise a 2d array of projection shifts
        self.exp.meta_data.set('projection_shifts', proj_shifts)
        data_obj.meta_data.set("projection_shifts", proj_shifts)
        return len(proj_shifts)

    def __get_angles_from_nxs_file(self, data_obj, path):
        if path in data_obj.backing_file:
            idx = data_obj.data.get_image_key() == 0 if \
                isinstance(data_obj.data, ImageKey) else slice(None)
            return data_obj.backing_file[path][idx]
        else:
            self.log_warning("No rotation angle entry found in input file.")
            return None

    def _get_data_file(self):
        data = self.exp.meta_data.get("data_file")
        return h5py.File(data, 'r')

    def __check_angles(self, data_obj, n_angles):
        rot_dim = data_obj.get_data_dimension_by_axis_label("rotation_angle")
        data_angles = data_obj.data.get_shape()[rot_dim]
        if data_angles != n_angles:
            if self.nFrames > 1:
                rot_angles = data_obj.meta_data.get("rotation_angle")
                try:
                    full_rotations = n_angles // data_angles
                    cleaned_size = full_rotations * data_angles
                    if cleaned_size != n_angles:
                        rot_angles = rot_angles[0:cleaned_size]
                        self.log_warning(
                            "the angle list has more values than expected in it")
                    rot_angles = np.reshape(
                        rot_angles, [full_rotations, data_angles])
                    data_obj.meta_data.set("rotation_angle",
                                           np.transpose(rot_angles))
                    return
                except:
                    pass
            raise Exception("The number of angles %s does not match the data "
                            "dimension length %s" % (n_angles, data_angles))

    def executive_summary(self):
        """ Provide a summary to the user for the result of the plugin.

        e.g.
         - Warning, the sample may have shifted during data collection
         - Filter operated normally

        :returns:  A list of string summaries
        """
        if len(self.warnings) == 0:
            return ["Nothing to Report"]
        else:
            return self.warnings


1			# Copyright 2014 Diamond Light Source Ltd.
2			#
3			# Licensed under the Apache License, Version 2.0 (the "License");
4			# you may not use this file except in compliance with the License.
5			# You may obtain a copy of the License at
6			#
7			# http://www.apache.org/licenses/LICENSE-2.0
8			#
9			# Unless required by applicable law or agreed to in writing, software
10			# distributed under the License is distributed on an "AS IS" BASIS,
11			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12			# See the License for the specific language governing permissions and
13			# limitations under the License.
14
15			"""
16			.. module:: nxtomo_loader
17			:platform: Unix
18			:synopsis: A class for loading standard tomography data in Nexus format.
19
20			.. moduleauthor:: Nicola Wadeson <[email protected]>
21
22			"""
23
24			import h5py
25			import logging
26			import numpy as np
27
28			from savu.plugins.loaders.base_loader import BaseLoader
29			from savu.plugins.utils import register_plugin
30
31			from savu.data.data_structures.data_types.data_plus_darks_and_flats \
32			import ImageKey, NoImageKey
33
34
35			@register_plugin
36			class NxtomoLoader(BaseLoader):
37			"""
38			"""
39			def __init__(self, name='NxtomoLoader'):
40			super(NxtomoLoader, self).__init__(name)
41			self.warnings = []
42
43			def log_warning(self, msg):
44			logging.warning(msg)
45			self.warnings.append(msg)
46
47			def setup(self):
48			exp = self.get_experiment()
49			data_obj = exp.create_data_object('in_data', self.parameters['name'])
50
51			data_obj.backing_file = self._get_data_file()
52
53			data_obj.data = self._get_h5_entry(
54			data_obj.backing_file, self.parameters['data_path'])
55
56			self._set_dark_and_flat(data_obj)
57
58			self.nFrames = self.__get_nFrames(data_obj)
59			if self.nFrames > 1:
60			self.__setup_4d(data_obj)
61			self.__setup_3d_to_4d(data_obj, self.nFrames)
62			else:
63			if len(data_obj.data.shape) == 3:
64			self._setup_3d(data_obj)
65			else:
66			self.__setup_4d(data_obj)
67			data_obj.set_original_shape(data_obj.data.shape)
68			self._set_rotation_angles(data_obj)
69			self._set_projection_shifts(data_obj)
70
71			try:
72			control = self._get_h5_path(
73			data_obj.backing_file, 'entry1/tomo_entry/control/data')
74			data_obj.meta_data.set("control", control[...])
75			except Exception:
76			self.log_warning("No Control information available")
77
78			nAngles = len(data_obj.meta_data.get('rotation_angle'))
79			self.__check_angles(data_obj, nAngles)
80
81			self.set_data_reduction_params(data_obj)
82			data_obj.data._set_dark_and_flat()
83
84			def _get_h5_entry(self, filename, path):
85			if path in filename:
86			return filename[path]
87			else:
88			raise Exception("Path %s not found in %s" % (path, filename))
89
90			def __get_nFrames(self, dObj):
91			if self.parameters['3d_to_4d'] is False:
92			return 0
93			if self.parameters['3d_to_4d'] is True:
94			try:
95			# for backwards compatibility
96			n_frames = eval(self.parameters["angles"], {"builtins": None, "np": np})
97			return np.array(n_frames).shape[0]
98			except Exception:
99			raise Exception("Please specify the angles, or the number of "
100			"frames per scan (via 3d_to_4d param) in the loader.")
101			if isinstance(self.parameters['3d_to_4d'], int):
102			return self.parameters['3d_to_4d']
103			else:
104			raise Exception("Unknown value for loader parameter '3d_to_4d', "
105			"please specify an integer value")
106
107			def _setup_3d(self, data_obj):
108			logging.debug("Setting up 3d tomography data.")
109			rot = 0
110			detY = 1
111			detX = 2
112			data_obj.set_axis_labels('rotation_angle.degrees',
113			'detector_y.pixel',
114			'detector_x.pixel')
115
116			data_obj.add_pattern('PROJECTION', core_dims=(detX, detY),
117			slice_dims=(rot,))
118			data_obj.add_pattern('SINOGRAM', core_dims=(detX, rot),
119			slice_dims=(detY,))
120			data_obj.add_pattern('TANGENTOGRAM', core_dims=(rot, detY),
121			slice_dims=(detX,))
122
123			def __setup_3d_to_4d(self, data_obj, n_frames):
124			logging.debug("setting up 4d tomography data from 3d input.")
125			from savu.data.data_structures.data_types.map_3dto4d_h5 \
126			import Map3dto4dh5
127
128			all_angles = data_obj.data.shape[0]
129			if all_angles % n_frames != 0:
130			self.log_warning("There are not a complete set of scans in this file, "
131			"loading complete ones only")
132			data_obj.data = Map3dto4dh5(data_obj, n_frames)
133			data_obj.set_original_shape(data_obj.data.get_shape())
134
135			def __setup_4d(self, data_obj):
136			logging.debug("setting up 4d tomography data.")
137			rot = 0
138			detY = 1
139			detX = 2
140			scan = 3
141
142			data_obj.set_axis_labels('rotation_angle.degrees', 'detector_y.pixel',
143			'detector_x.pixel', 'scan.number')
144
145			data_obj.add_pattern('PROJECTION', core_dims=(detX, detY),
146			slice_dims=(rot, scan))
147			data_obj.add_pattern('SINOGRAM', core_dims=(detX, rot),
148			slice_dims=(detY, scan))
149			data_obj.add_pattern('TANGENTOGRAM', core_dims=(rot, detY),
150			slice_dims=(detX, scan))
151			data_obj.add_pattern('SINOMOVIE', core_dims=(detX, rot, scan),
152			slice_dims=(detY,))
153
154			def _set_dark_and_flat(self, data_obj):
155			flat = self.parameters['flat'][0]
156			dark = self.parameters['dark'][0]
157
158			if not flat or not dark:
159			self.__find_dark_and_flat(data_obj, flat=flat, dark=dark)
160			else:
161			self.__set_separate_dark_and_flat(data_obj)
162
163			def __find_dark_and_flat(self, data_obj, flat=None, dark=None):
164			ignore = self.parameters['ignore_flats'] if \
165			self.parameters['ignore_flats'] else None
166			try:
167			image_key = data_obj.backing_file[
168			self.parameters['image_key_path']][...]
169			data_obj.data = \
170			ImageKey(data_obj, image_key, 0, ignore=ignore)
171			except KeyError:
172			self.log_warning("An image key was not found.")
173			try:
174			data_obj.data = NoImageKey(data_obj, None, 0)
175			entry = 'entry1/tomo_entry/instrument/detector/'
176			data_obj.data._set_flat_path(entry + 'flatfield')
177			data_obj.data._set_dark_path(entry + 'darkfield')
178			except KeyError:
179			self.log_warning("Dark/flat data was not found in input file.")
180			data_obj.data._set_dark_and_flat()
181			if dark:
182			data_obj.data.update_dark(dark)
183			if flat:
184			data_obj.data.update_flat(flat)
185
186			def __set_separate_dark_and_flat(self, data_obj):
187			try:
188			image_key = data_obj.backing_file[
189			'entry1/tomo_entry/instrument/detector/image_key'][...]
190			except:
191			image_key = None
192			data_obj.data = NoImageKey(data_obj, image_key, 0)
193			self.__set_data(data_obj, 'flat', data_obj.data._set_flat_path)
194			self.__set_data(data_obj, 'dark', data_obj.data._set_dark_path)
195
196			def __set_data(self, data_obj, name, func):
197			path, entry, scale = self.parameters[name]
198
199			if path.split('/')[0] == 'Savu':
200			import os
201			savu_base_path = os.path.join(os.path.dirname(
202			os.path.realpath(__file__)), '..', '..', '..', '..')
203			path = os.path.join(savu_base_path, path.split('Savu')[1][1:])
204
205			ffile = h5py.File(path, 'r')
206			try:
207			image_key = \
208			ffile['entry1/tomo_entry/instrument/detector/image_key'][...]
209			func(ffile[entry], imagekey=image_key)
210			except:
211			func(ffile[entry])
212
213			data_obj.data._set_scale(name, scale)
214
215			def _set_rotation_angles(self, data_obj):
216			angles = self.parameters['angles']
217			warn_ms = "No angles found so evenly distributing them between 0 and" \
218			" 180 degrees"
219			if angles is None:
220			angle_key = 'entry1/tomo_entry/data/rotation_angle'
221			nxs_angles = self.__get_angles_from_nxs_file(data_obj, angle_key)
222			if nxs_angles is None:
223			self.log_warning(warn_ms)
224			angles = np.linspace(0, 180, data_obj.get_shape()[0])
225			else:
226			angles = nxs_angles
227			else:
228			try:
229			angles = eval(angles)
230			except Exception as e:
231			logging.warning(e)
232			try:
233			angles = np.loadtxt(angles)
234			except Exception as e:
235			logging.debug(e)
236			self.log_warning(warn_ms)
237			angles = np.linspace(0, 180, data_obj.get_shape()[0])
238			data_obj.meta_data.set("rotation_angle", angles)
239			return len(angles)
240
241			def _set_projection_shifts(self, data_obj):
242			proj_shifts = np.zeros((data_obj.get_shape()[0], 2)) # initialise a 2d array of projection shifts
243			self.exp.meta_data.set('projection_shifts', proj_shifts)
244			data_obj.meta_data.set("projection_shifts", proj_shifts)
245			return len(proj_shifts)
246
247			def __get_angles_from_nxs_file(self, data_obj, path):
248			if path in data_obj.backing_file:
249			idx = data_obj.data.get_image_key() == 0 if \
250			isinstance(data_obj.data, ImageKey) else slice(None)
251			return data_obj.backing_file[path][idx]
252			else:
253			self.log_warning("No rotation angle entry found in input file.")
254			return None
255
256			def _get_data_file(self):
257			data = self.exp.meta_data.get("data_file")
258			return h5py.File(data, 'r')
259
260			def __check_angles(self, data_obj, n_angles):
261			rot_dim = data_obj.get_data_dimension_by_axis_label("rotation_angle")
262			data_angles = data_obj.data.get_shape()[rot_dim]
263			if data_angles != n_angles:
264			if self.nFrames > 1:
265			rot_angles = data_obj.meta_data.get("rotation_angle")
266			try:
267			full_rotations = n_angles // data_angles
268			cleaned_size = full_rotations * data_angles
269			if cleaned_size != n_angles:
270			rot_angles = rot_angles[0:cleaned_size]
271			self.log_warning(
272			"the angle list has more values than expected in it")
273			rot_angles = np.reshape(
274			rot_angles, [full_rotations, data_angles])
275			data_obj.meta_data.set("rotation_angle",
276			np.transpose(rot_angles))
277			return
278			except:
279			pass
280			raise Exception("The number of angles %s does not match the data "
281			"dimension length %s" % (n_angles, data_angles))
282
283			def executive_summary(self):
284			""" Provide a summary to the user for the result of the plugin.
285
286			e.g.
287			- Warning, the sample may have shifted during data collection
288			- Filter operated normally
289
290			:returns: A list of string summaries
291			"""
292			if len(self.warnings) == 0:
293			return ["Nothing to Report"]
294			else:
295			return self.warnings
296

DiamondLightSource / Savu

Push — master ( f1d4ff...d83625 )

NxtomoLoader._get_data_file() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like