Test Failed
Pull Request — master (#785)
by
unknown
03:24
created

PluginParameters.check_dependencies()   B

Complexity

Conditions 8

Size

Total Lines 35
Code Lines 20

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
cc 8
eloc 20
nop 2
dl 0
loc 35
rs 7.3333
c 0
b 0
f 0
1
# Copyright 2014 Diamond Light Source Ltd.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14
15
"""
16
.. module:: plugin_tools
17
   :platform: Unix
18
   :synopsis: Plugin tools
19
20
.. moduleauthor:: Jessica Verschoyle <[email protected]>
21
22
"""
23
import os
24
import copy
25
import json
26
import logging
27
28
from colorama import Fore
29
from collections import OrderedDict
30
31
import savu.plugins.utils as pu
32
from savu.data.meta_data import MetaData
33
import savu.plugins.docstring_parser as doc
34
import scripts.config_generator.parameter_utils as param_u
35
from savu.data.plugin_list import CitationInformation
36
37
logger = logging.getLogger("documentationLog")
38
39
40
class PluginParameters(object):
41
    """Save the parameters for the plugin and base classes to a
42
    dictionary. The parameters are in yaml format inside the
43
    define_parameter function. These are read and checked for problems.
44
    """
45
46
    def __init__(self):
47
        super(PluginParameters, self).__init__()
48
        self.param = MetaData(ordered=True)
49
        self.docstring_info = {}
50
        self.parameters = {}
51
        self.multi_params_dict = {}
52
        self.extra_dims = []        
53
54
    def populate_parameters(self, tools_list):
55
        """ Set parameter definitions and default parameter values """
56
        # set the parameter definitions
57
        # populates the dictionary returned by self.get_param_definitions()
58
        list(map(lambda tool_class: 
59
                 self._set_parameter_definitions(tool_class), tools_list))
60
        # set the default parameter values
61
        # populates the dictionary returned by self.get_param_values()
62
        self._populate_default_parameters()
63
64
    def initialise(self, params):
65
        # Override default parameter values with plugin list entries
66
        self.set_plugin_list_parameters(copy.deepcopy(params))
67
        self._get_plugin().set_parameters(self.parameters)
68
69
    def _populate_default_parameters(self):
70
        """
71
        This method should populate all the required parameters with
72
        default values. It is used for checking to see if parameter
73
        values are appropriate
74
        """
75
        p_defs = self.get_param_definitions()
76
        self.set_docstring(self.get_doc())
77
        self.parameters = \
78
            OrderedDict([(k, v['default']) for k, v in p_defs.items()])
79
        # parameters holds current values, this is edited outside of the
80
        # tools class so default and dependency display values are updated here
81
        self.update_dependent_defaults()
82
        self.check_dependencies(self.parameters)
83
        self._get_plugin().set_parameters(self.parameters)
84
85
    def set_docstring(self, doc_str):
86
        self.docstring_info['info'] = doc_str.get('verbose')
87
        self.docstring_info['warn'] = doc_str.get('warn')
88
        self.docstring_info['documentation_link'] = doc_str.get('documentation_link')
89
        self.docstring_info['synopsis'] = doc.find_synopsis(self._get_plugin())
90
91
    def _set_parameters_this_instance(self, indices):
92
        """ Determines the parameters for this instance of the plugin, in the
93
        case of parameter tuning.
94
95
        param np.ndarray indices: the index of the current value in the
96
            parameter tuning list.
97
        """
98
        dims = set(self.multi_params_dict.keys())
99
        count = 0
100
        for dim in dims:
101
            info = self.multi_params_dict[dim]
102
            name = info['label'].split('_param')[0]
103
            self.parameters[name] = info['values'][indices[count]]
104
            count += 1
105
106
    def _set_parameter_definitions(self, tool_class):
107
        """Load the parameters for each base class, c, check the
108
        dataset visibility, check data types, set dictionary values.
109
        """
110
        param_info_dict = self._load_param_from_doc(tool_class)
111
        if param_info_dict:
112
            for p_name, p_value in param_info_dict.items():
113
                if p_name in self.param.get_dictionary():
114
                    for k,v in p_value.items():
115
                        self.param[p_name][k] = v
116
                else:
117
                    self.param.set(p_name, p_value)
118
        self._check_param_defs(tool_class)
119
120
    def _check_param_defs(self, tool_class):
121
        """Check the parameter definitions for errors
122
123
        :param tool_class: tool_class to use for error message
124
        """
125
        pdefs = self.param.get_dictionary()
126
        # Remove ignored parameters
127
        self._remove_ignored_params(pdefs)
128
        # Check that unknown keys aren't present
129
        self._check_all_keys(pdefs, tool_class)
130
        # Check if the required keys are included
131
        self._check_required_keys(pdefs, tool_class)
132
        # Check that option values are valid
133
        self._check_options(pdefs, tool_class)
134
        # Check that the visibility is valid
135
        self._check_visibility(pdefs, tool_class)
136
        # Check that the dtype is valid
137
        self._check_dtype(pdefs, tool_class)
138
        # Use a display option to apply to dependent parameters later.
139
        self._set_display(pdefs)
140
        for k,v in pdefs.items():
141
            # Change empty OrderedDict to dict (due to yaml loader)
142
            if isinstance(v['default'], OrderedDict):
143
                v['default'] = json.loads(json.dumps(v['default']))
144
            # Change the string to an integer, float, list, str, dict
145
            if not self.default_dependency_dict_exists(v):
146
                v['default'] = pu._dumps(v['default'])
147
148
    def _load_param_from_doc(self, tool_class):
149
        """Find the parameter information from the method docstring.
150
        This is provided in a yaml format.
151
        """
152
        # *** TO DO turn the dtype entry into a string
153
154
        param_info_dict = None
155
        if hasattr(tool_class, "define_parameters"):
156
            yaml_text = tool_class.define_parameters.__doc__
157
            if yaml_text and yaml_text.strip():
158
                # If yaml_text is not None and not empty or consisting of spaces
159
                param_info_dict = doc.load_yaml_doc(yaml_text)
160
                if param_info_dict:
161
                    if not isinstance(param_info_dict, OrderedDict):
162
                        error_msg = (
163
                            f"The parameters have not been read "
164
                            f"in correctly for {tool_class.__name__}"
165
                        )
166
                        raise Exception(error_msg)
167
168
        return param_info_dict
169
170
    def get_default_value(self, mod_param):
171
        """If the '--default' argument is passed, then get the original
172
        default value. If the default contains a dictionary, then search
173
        for the correct value
174
        """
175
        param_info_dict = self.param.get_dictionary()
176
        if self.default_dependency_dict_exists(param_info_dict[mod_param]):
177
            mod_value = self.get_dependent_default(param_info_dict[mod_param])
178
        else:
179
            mod_value = param_info_dict[mod_param]["default"]
180
        return mod_value
181
182
    def _check_required_keys(self, param_info_dict, tool_class):
183
        """Check the four keys ['dtype', 'description', 'visibility',
184
        'default'] are included inside the dictionary given for each
185
        parameter.
186
        """
187
        required_keys = ["dtype", "description", "visibility", "default"]
188
        missing_keys = False
189
        missing_key_dict = {}
190
191
        for p_key, p in param_info_dict.items():
192
            all_keys = p.keys()
193
            if p.get("visibility"):
194
                if p.get("visibility") == "hidden":
195
                    # For hidden keys, only require a default value key
196
                    required_keys = ["default"]
197
            else:
198
                required_keys = ["visibility"]
199
200
            if not all(d in all_keys for d in required_keys):
201
                missing_key_dict[p_key] = set(required_keys) - set(all_keys)
202
                missing_keys = True
203
204
        if missing_keys:
205
            self._print_incorrect_key_str(missing_key_dict, "missing required", tool_class)
206
207
    def _check_all_keys(self, param_info_dict, tool_class):
208
        allowed_keys = ["dtype", "description", "visibility", "default",
209
                        "options", "display", "example", "dependency",
210
                        "warning"]
211
        unknown_keys = False
212
        unknown_keys_dict = {}
213
214
        for p_key, p in param_info_dict.items():
215
            all_keys = p.keys()
216
            if any(d not in allowed_keys for d in all_keys):
217
                unknown_keys_dict[p_key] = set(all_keys) - set(allowed_keys)
218
                unknown_keys = True
219
220
        if unknown_keys:
221
            self._print_incorrect_key_str(unknown_keys_dict, "unknown",
222
                                        tool_class)
223
224
    def _print_incorrect_key_str(self, incorrect_key_dict, key_str, tool_class):
225
        """Create and print an error message and log message when certain keys are
226
        invalid or missing
227
228
        :param missing_key_dict: Dictionary of parameter option keys which are invalid/missing
229
        :param key_str: Error message string for this
230
        :param tool_class:
231
        :return:
232
        """
233
        print(
234
            f"{tool_class.__name__} has {key_str} keys."
235
        )
236
        for param, incorrect_values in incorrect_key_dict.items():
237
            print(f"The {key_str} keys for '{param}' are:")
238
            print(*incorrect_values, sep=", ")
239
        logger.error(f"ERROR: {key_str} keys inside {tool_class.__name__}")
240
        raise Exception(f"Please edit {tool_class.__name__}")
241
242
    def _check_dtype(self, param_info_dict, tool_class):
243
        """
244
        Make sure that the dtype input is valid and that the default value is
245
        compatible
246
        """
247
        plugin_error_str = f"There was an error with {tool_class.__name__}"
248
        for p_key, p_dict in param_info_dict.items():
249
            dtype = p_dict.get("dtype")
250
            if dtype:
251
                dtype = dtype.replace(" ", "")
252
                try:
253
                    pvalid, error_str = param_u.is_valid_dtype(dtype)
254
                    if not pvalid:
255
                        raise Exception("Invalid parameter definition %s:\n %s"
256
                                        % (p_key, error_str))
257
                except IndexError:
258
                    print(plugin_error_str)
259
                if not self.default_dependency_dict_exists(p_dict):
260
                    default_value = pu._dumps(p_dict["default"])
261
                    pvalid, error_str = param_u.is_valid(p_key, default_value,
262
                                                 p_dict, check=True)
263
                    if not pvalid:
264
                        raise Exception(f"{plugin_error_str}: {error_str}")
265
266
    def _check_visibility(self, param_info_dict, tool_class):
267
        """Make sure that the visibility choice is valid"""
268
        visibility_levels = [
269
            "basic",
270
            "intermediate",
271
            "advanced",
272
            "datasets",
273
            "hidden",
274
        ]
275
        visibility_valid = True
276
        for p_key, p in param_info_dict.items():
277
            # Check dataset visibility level is correct
278
            self._check_data_keys(p_key, p)
279
            # Check that the data types are valid choices
280
            if p["visibility"] not in visibility_levels:
281
                print(
282
                    f"Inside {tool_class.__name__} the {p_key}"
283
                    f" parameter is assigned an invalid visibility "
284
                    f"level '{p['visibility']}'"
285
                )
286
                print("Valid choices are:")
287
                print(*visibility_levels, sep=", ")
288
                visibility_valid = False
289
290
        if not visibility_valid:
291
            raise Exception(
292
                f"Please change the file for {tool_class.__name__}"
293
            )
294
295
    def _check_data_keys(self, p_key, p):
296
        """Make sure that the visibility of dataset parameters is 'datasets'
297
        so that the display order is unchanged.
298
        """
299
        datasets = ["in_datasets", "out_datasets"]
300
        exceptions = ["hidden"]
301
        if p_key in datasets:
302
            if p["visibility"] != "datasets" \
303
                    and p["visibility"] not in exceptions:
304
                p["visibility"] = "datasets"
305
306
    def _check_options(self, param_info_dict, tool_class):
307
        """Make sure that option verbose descriptions match the actual
308
        options
309
        """
310
        options_valid = True
311
        for p_key, p in param_info_dict.items():
312
            desc = param_info_dict[p_key].get("description")
313
            # desc not present for hidden keys
314
            if desc and isinstance(desc, dict):
315
                options = param_info_dict[p_key].get("options")
316
                option_desc = desc.get("options")
317
                if options and option_desc:
318
                    # Check that there is not an invalid option description
319
                    # inside the option list.
320
                    invalid_option = [
321
                        opt for opt in option_desc if opt not in options
322
                    ]
323
                    if invalid_option:
324
                        options_valid = False
325
                        break
326
327
        if options_valid is False:
328
            raise Exception(
329
                f"Please check the parameter options for {tool_class.__name__}"
330
            )
331
332
    def _remove_ignored_params(self, param_info_dict):
333
        """Remove any parameters with visibility = ignore"""
334
        p_dict_copy = param_info_dict.copy()
335
        for p_key, p in p_dict_copy.items():
336
            visibility = param_info_dict[p_key].get("visibility")
337
            if visibility == "ignore":
338
                del param_info_dict[p_key]
339
340
    def _set_display(self, param_info_dict):
341
        """Initially, set all of the parameters to display 'on'
342
        This is later altered when dependent parameters need to be shown
343
        or hidden
344
        """
345
        for k, v in param_info_dict.items():
346
            v["display"] = "on"
347
348
    def update_dependent_defaults(self):
349
        """
350
        Fix default values for parameters that have a dependency on the value 
351
        of another parameter, and are in dictionary form.
352
        """
353
        for name, pdict in self.get_param_definitions().items():
354
            if self.default_dependency_dict_exists(pdict):
355
                self.parameters[name] = self.get_dependent_default(pdict)
356
357
    def default_dependency_dict_exists(self, pdict):
358
        """ Check that the parameter default value is in a format with
359
        the parent parameter string and the dependent value
360
        e.g. default:
361
                algorithm: FGP
362
        and not an actual default value to be set
363
        e.g. default: {'2':5}
364
365
        :param pdict: The parameter definition dictionary
366
        :return: True if the default dictionary contains the
367
                correct format
368
        """
369
        if pdict["default"] and isinstance(pdict["default"], dict):
370
            if "dict" not in pdict["dtype"]:
371
                return True
372
            else:
373
                parent_name = list(pdict['default'].keys())[0]
374
                if parent_name in self.get_param_definitions():
375
                    return True
376
        return False
377
378
    def does_exist(self, key, ddict):
379
        if not key in ddict:
380
            raise Exception("The dependency %s does not exist" % key)
381
        return ddict[key]
382
383
    def get_dependent_default(self, child):
384
        """
385
        Recursive function to replace a dictionary of default parameters with
386
        a single value.
387
388
        Parameters
389
        ----------
390
        child : dict
391
            The parameter definition dictionary of the dependent parameter.
392
393
        Returns1
394
        -------
395
        value
396
            The correct default value based on the current value of the
397
            dependency, or parent, parameter.
398
399
        """
400
        pdefs = self.get_param_definitions()
401
        parent_name = list(child['default'].keys())[0]
402
        parent = self.does_exist(parent_name, pdefs)
403
404
        # if the parent default is a dictionary then apply the function
405
        # recursively
406
        if isinstance(parent['default'], dict):
407
            self.parameters[parent_name] = \
408
                self.get_dependent_default(parent['default'])
409
        return child['default'][parent_name][self.parameters[parent_name]]
410
411
    def warn_dependents(self, mod_param, mod_value): 
412
        """
413
        Find dependents of a modified parameter # complete the docstring
414
        """
415
        # find dependents
416
        for name, pdict in self.get_param_definitions().items():
417
            if self.default_dependency_dict_exists(pdict):
418
                default = pdict['default']
419
                parent_name = list(default.keys())[0]
420
                if parent_name == mod_param:
421
                    if mod_value in default[parent_name].keys():
422
                        value = default[parent_name][mod_value]
423
                        desc = pdict['description']
424
                        self.make_recommendation(
425
                            name, desc, parent_name, value)
426
427
    def make_recommendation(self, child_name, desc, parent_name, value): 
428
        if isinstance(desc, dict):
429
            desc["range"] = f"The recommended value with the chosen " \
430
                            f"{parent_name} would be {value}"
431
        recommendation = f"It's recommended that you update {child_name}"\
432
                         f" to {value}"
433
        print(Fore.RED + recommendation + Fore.RESET)
434
435
        
436
    def check_dependencies(self, parameters):
437
        """Determine which parameter values are dependent on a parent
438
        value and whether they should be hidden or shown
439
        """
440
        param_info_dict = self.param.get_dictionary()
441
        dep_list = {
442
            k: v["dependency"]
443
            for k, v in param_info_dict.items()
444
            if "dependency" in v
445
        }
446
        for p_name, dependency in dep_list.items():
447
            if isinstance(dependency, OrderedDict):
448
                # There is a dictionary of dependency values
449
                parent_param_name = list(dependency.keys())[0]
450
                # The choices which must be in the parent value
451
                parent_choice_list = dependency[parent_param_name]
452
453
                if parent_param_name in parameters:
454
                    """Check that the parameter is in the current plug in
455
                    This is relevant for base classes which have several
456
                    dependent classes
457
                    """
458
                    parent_value = parameters[parent_param_name]
459
460
                    if str(parent_value) in parent_choice_list:
461
                        param_info_dict[p_name]["display"] = "on"
462
                    else:
463
                        param_info_dict[p_name]["display"] = "off"
464
            else:
465
                if dependency in parameters:
466
                    parent_value = parameters[dependency]
467
                    if parent_value is None or str(parent_value) == "None":
468
                        param_info_dict[p_name]["display"] = "off"
469
                    else:
470
                        param_info_dict[p_name]["display"] = "on"
471
472
473
    def set_plugin_list_parameters(self, input_parameters):
474
        """
475
        This method is called after the plugin has been created by the
476
        pipeline framework.  It replaces ``self.parameters``
477
        default values with those given in the input process list. It
478
        checks for multi parameter strings, eg. 57;68;56;
479
480
        :param dict input_parameters: A dictionary of the input parameters
481
        for this plugin, or None if no customisation is required.
482
        """        
483
        for key in input_parameters.keys():
484
            if key in self.parameters.keys():
485
                new_value = input_parameters[key]
486
                self.__check_multi_params(
487
                    self.parameters, new_value, key
488
                )
489
            else:
490
                error = (
491
                    f"Parameter '{key}' is not valid for plugin "
492
                    f"{self.plugin_class.name}. \nTry opening and re-saving "
493
                    f"the process list in the configurator to auto remove "
494
                    f"\nobsolete parameters."
495
                )
496
                raise ValueError(error)
497
498
    def __check_multi_params(self, parameters, value, key):
499
        """
500
        Convert parameter value to a list if it uses parameter tuning
501
        and set associated parameters, so the framework knows the new size
502
        of the data and which plugins to re-run.
503
504
        :param parameters: Dictionary of parameters and current values
505
        :param value: Value to set parameter to
506
        :param key: Parameter name
507
        :return:
508
        """
509
        if param_u.is_multi_param(key, value):
510
            value, error_str = pu.convert_multi_params(key, value)
511
            if not error_str:
512
                parameters[key] = value
513
                label = key + "_params." + type(value[0]).__name__
514
                self.alter_multi_params_dict(
515
                    len(self.get_multi_params_dict()),
516
                    {"label": label, "values": value},
517
                )
518
                self.append_extra_dims(len(value))
519
        else:
520
            parameters[key] = value
521
522
    def get_expand_dict(self, preview, expand_dim):
523
        """Create dict for expand syntax
524
525
        :param preview: Preview parameter value
526
        :param expand_dim: Number of dimensions to return dict for
527
        :return: dict
528
        """
529
        expand_dict = {}
530
        preview_val = pu._dumps(preview)
531
        if not preview_val:
532
            # In the case that there is an empty dict, display the default
533
            preview_val = []
534
        if isinstance( preview_val, dict):
535
            for key, prev_list in preview_val.items():
536
                expand_dict[key] = self.get_expand_dict(prev_list, expand_dim)
537
            return expand_dict
538
        elif isinstance(preview_val, list):
539
            if expand_dim == "all":
540
                expand_dict = \
541
                    self._output_all_dimensions(preview_val,
542
                         self._get_dimensions(preview_val))
543
            else:
544
                pu.check_valid_dimension(expand_dim, preview_val)
545
                dim_key = f"dim{expand_dim}"
546
                expand_dict[dim_key] = \
547
                    self._dim_slice_output(preview_val, expand_dim)
548
            return expand_dict
549
        else:
550
            raise ValueError("This preview value was not a recognised list "
551
                             "or dictionary. This expand command currenty "
552
                             "only works with those two data types.")
553
554
    def _get_dimensions(self, preview_list):
555
        """
556
        :param preview_list: The preview parameter list
557
        :return: Dimensions to display
558
        """
559
        return 1 if not preview_list else len(preview_list)
560
561
    def _output_all_dimensions(self, preview_list, dims):
562
        """Compile output string lines for all dimensions
563
564
        :param preview_list: The preview parameter list
565
        :param dims: Number of dimensions to display
566
        :return: dict
567
        """
568
        prev_dict = {}
569
        for dim in range(1, dims + 1):
570
            dim_key = f"dim{dim}"
571
            prev_dict[dim_key] = self._dim_slice_output(preview_list, dim)
572
        return prev_dict
573
574
    def _dim_slice_output(self, preview_list, dim):
575
        """If there are multiple values in list format
576
        Only save the values for the dimensions chosen
577
578
        :param preview_list: The preview parameter list
579
        :param dim: dimension to return the slice notation dictionary for
580
        :return slice notation dictionary
581
        """
582
        if not preview_list:
583
            # If empty
584
            preview_display_value = ":"
585
        else:
586
            preview_display_value = preview_list[dim - 1]
587
        prev_val = self._set_all_syntax(preview_display_value)
588
        return self._get_slice_notation_dict(prev_val)
589
590
    def _get_slice_notation_dict(self, val):
591
        """Create a dict for slice notation information,
592
        start:stop:step (and chunk if provided)
593
594
        :param val: The list value in slice notation
595
        :return: dictionary of slice notation
596
        """
597
        import itertools
598
599
        basic_slice_keys = ["start", "stop", "step"]
600
        all_slice_keys = [*basic_slice_keys, "chunk"]
601
        slice_dict = {}
602
603
        if pu.is_slice_notation(val):
604
            val_list = val.split(":")
605
            if len(val_list) < 3:
606
                # Make sure the start stop step slice keys are always shown,
607
                # even when blank
608
                val_list.append("")
609
            for slice_name, v in zip(all_slice_keys, val_list):
610
                # Only print up to the shortest list.
611
                # (Only show the chunk value if it is in val_list)
612
                slice_dict[slice_name] = v
613
        else:
614
            val_list = [val]
615
            for slice_name, v in itertools.zip_longest(
616
                    basic_slice_keys, val_list, fillvalue=""
617
            ):
618
                slice_dict[slice_name] = v
619
        return slice_dict
620
621
    def _set_all_syntax(self, val, replacement_str=""):
622
        """Remove additional spaces from val, replace colon when 'all'
623
        data is selected
624
625
        :param val: Slice notation value
626
        :param replacement_str: String to replace ':' with
627
        :return:
628
        """
629
        if isinstance(val, str):
630
            if pu.is_slice_notation(val):
631
                if val == ":":
632
                    val = replacement_str
633
                else:
634
                    val = val.strip()
635
            else:
636
                val = val.strip()
637
        return val
638
639
    def get_multi_params_dict(self):
640
        """ Get the multi parameter dictionary. """
641
        return self.multi_params_dict
642
643
    def alter_multi_params_dict(self, key, value):
644
        self.multi_params_dict[key] = value
645
646
    def get_extra_dims(self):
647
        """ Get the extra dimensions. """
648
        return self.extra_dims
649
650
    def set_extra_dims(self, value):
651
        self.extra_dims = value
652
653
    def append_extra_dims(self, value):
654
        self.extra_dims.append(value)
655
656
    def define_parameters(self):
657
        pass
658
659
    """
660
    @dataclass
661
    class Parameter:
662
        ''' Descriptor of Parameter Information for plugins
663
        '''
664
        visibility: int
665
        datatype: specific_type
666
        description: str
667
        default: int
668
        Options: Optional[[str]]
669
        dependency: Optional[]
670
671
        def _get_param(self):
672
            param_dict = {}
673
            param_dict['visibility'] = self.visibility
674
            param_dict['type'] = self.dtype
675
            param_dict['description'] = self.description
676
            # and the remaining keys
677
            return param_dict
678
    """
679
680
681
class PluginCitations(object):
682
    """Get this citation dictionary so get_dictionary of the metadata type
683
    should return a dictionary of all the citation info as taken from
684
    docstring
685
    """
686
687
    def __init__(self):
688
        super(PluginCitations, self).__init__()
689
        self.cite = MetaData(ordered=True)
690
691
    def set_cite(self, tools_list):
692
        """Set the citations for each of the tools classes
693
        :param tools_list: List containing tool classes of parent plugins
694
        """
695
        list(
696
            map(
697
                lambda tool_class: self._set_plugin_citations(tool_class),
698
                tools_list
699
            )
700
        )
701
702
    def _set_plugin_citations(self, tool_class):
703
        """ Load the parameters for each base class and set values"""
704
        citations = self._load_cite_from_doc(tool_class)
705
        if citations:
706
            for citation in citations.values():
707
                if self._citation_keys_valid(citation, tool_class):
708
                    new_citation = CitationInformation(**citation)
709
                    self.cite.set(new_citation.name, new_citation)
710
                else:
711
                    print(f"The citation for {tool_class.__name__} "
712
                          f"was not saved.")
713
714
    def _citation_keys_valid(self, new_citation, tool_class):
715
        """Check that required citation keys are present. Return false if
716
        required keys are missing
717
        """
718
        required_keys = ["description"]
719
        # Inside the fresnel filter there is only a description
720
        citation_keys = [k for k in new_citation.keys()]
721
        # Check that all of the required keys are contained inside the
722
        # citation definition
723
        check_keys = all(item in citation_keys for item in required_keys)
724
        citation_keys_valid = False if check_keys is False else True
725
726
        all_keys = [
727
            "short_name_article",
728
            "description",
729
            "bibtex",
730
            "endnote",
731
            "doi",
732
            "dependency",
733
        ]
734
        # Keys which are not used
735
        additional_keys = [k for k in citation_keys if k not in all_keys]
736
        if additional_keys:
737
            print(f"Please only use the following keys inside the citation"
738
                  f" definition for {tool_class.__name__}:")
739
            print(*all_keys, sep=", ")
740
            print("The incorrect keys used:", additional_keys)
741
742
        return citation_keys_valid
743
744
    def _load_cite_from_doc(self, tool_class):
745
        """Find the citation information from the method docstring.
746
        This is provided in a yaml format.
747
748
        :param tool_class: Tool to retrieve citation docstring from
749
        :return: All citations from this tool class
750
        """
751
        all_c = OrderedDict()
752
        # Seperate the citation methods. __dict__ returns instance attributes.
753
        citation_methods = {key: value
754
                            for key, value in tool_class.__dict__.items()
755
                            if key.startswith('citation')}
756
        for c_method_name, c_method in citation_methods.items():
757
            yaml_text = c_method.__doc__
758
            if yaml_text is not None:
759
                yaml_text = self.seperate_description(yaml_text)
760
                current_citation = doc.load_yaml_doc(yaml_text)
761
                if not isinstance(current_citation, OrderedDict):
762
                    print(f"The citation information has not been read in "
763
                          f"correctly for {tool_class.__name__}.")
764
                else:
765
                    all_c[c_method_name] = current_citation
766
        return all_c
767
768
    def seperate_description(self, yaml_text):
769
        """Change the format of the docstring to retain new lines for the
770
        endnote and bibtex and create a key for the description so that
771
        it be read as a yaml file
772
773
        :param yaml_text:
774
        :return: Reformatted yaml text
775
        """
776
        description = doc.remove_new_lines(yaml_text.partition("bibtex:")[0])
777
        desc_str = "        description:" + description
778
779
        bibtex_text = \
780
            yaml_text.partition("bibtex:")[2].partition("endnote:")[0]
781
        end_text = \
782
            yaml_text.partition("bibtex:")[2].partition("endnote:")[2]
783
784
        if bibtex_text and end_text:
785
            final_str = desc_str + '\n        bibtex: |' + bibtex_text \
786
                      + 'endnote: |' + end_text
787
        elif end_text:
788
            final_str = desc_str + '\n        endnote: |' + end_text
789
        elif bibtex_text:
790
            final_str = desc_str + '\n        bibtex: |' + bibtex_text
791
        else:
792
            final_str = desc_str
793
794
        return final_str
795
796
797
class PluginDocumentation(object):
798
    """Get this documentation dictionary so get_dictionary of
799
    the metadata type should return a dictionary of all the
800
    documentation details taken from docstring
801
    """
802
803
    def __init__(self):
804
        super(PluginDocumentation, self).__init__()
805
        self.doc = MetaData()
806
807
    def set_doc(self, tools_list):
808
        # Use the tools class at the 'top'
809
        doc_lines = tools_list[-1].__doc__.splitlines()
810
        doc_lines = [line.strip() for line in doc_lines if line]
811
        docstring = " ".join(doc_lines)
812
        self.doc.set("verbose", docstring)
813
        self.doc.set("warn", self.set_warn(tools_list))
814
        self.set_doc_link()
815
816
    def set_warn(self, tools_list):
817
        """Remove new lines and save config warnings for the child tools
818
        class only.
819
        """
820
        config_str = tools_list[-1].config_warn.__doc__
821
        if config_str and "\n\n" in config_str:
822
            # Separate multiple warnings with two new lines \n\n
823
            config_warn_list = [doc.remove_new_lines(l)
824
                                for l in config_str.split("\n\n")]
825
            config_str = '\n'.join(config_warn_list)
826
        return config_str
827
828
    def set_doc_link(self):
829
        """If there is a restructured text documentation file inside the
830
        doc/source/documentation folder, then save the link to the page.
831
832
        """
833
        # determine Savu base path
834
        savu_base_path = \
835
            os.path.dirname(os.path.realpath(__file__)).split("savu")[0]
836
837
        # Locate documentation file
838
        doc_folder = savu_base_path + "doc/source/documentation"
839
        module_path = \
840
            self.plugin_class.__module__.replace(".", "/").replace("savu", "")
841
        file_ = module_path + "_doc"
842
        file_name = file_ + ".rst"
843
        file_path = doc_folder + file_name
844
        sphinx_link = 'https://savu.readthedocs.io/en/latest/' \
845
                      'documentation' + file_
846
        if os.path.isfile(file_path):
847
            self.doc.set("documentation_link", sphinx_link)
848
849
    def config_warn(self):
850
        pass
851
852
853
class PluginTools(PluginParameters, PluginCitations, PluginDocumentation):
854
    """Holds all of the parameter, citation and documentation information
855
    for one plugin class - cls"""
856
857
    def __init__(self, cls):
858
        super(PluginTools, self).__init__()
859
        self.plugin_class = cls
860
        self.tools_list = self._find_tools()
861
        self._set_tools_data()
862
863
    def _get_plugin(self):
864
        return self.plugin_class
865
866
    def _find_tools(self):
867
        """Using the method resolution order, find base class tools"""
868
        tool_list = []
869
        for tool_class in self.plugin_class.__class__.__mro__[::-1]:
870
            plugin_tools_id = tool_class.__module__ + "_tools"
871
            p_tools = pu.get_tools_class(plugin_tools_id)
872
            if p_tools:
873
                tool_list.append(p_tools)
874
        return tool_list
875
876
    def _set_tools_data(self):
877
        """Populate the parameters, citations and documentation
878
        with information from all of the tools classes
879
        """
880
        self.populate_parameters(self.tools_list)
881
        self.set_cite(self.tools_list)
882
        self.set_doc(self.tools_list)
883
884
    def get_param_definitions(self):
885
        """
886
        Returns
887
        -------
888
        dict
889
            Original parameter definitions read from tools file.
890
        """
891
        return self.param.get_dictionary()
892
893
    def get_param_values(self):
894
        """
895
        Returns
896
        -------
897
        dict
898
            Plugin parameter values for this instance.
899
900
        """
901
        return self.parameters
902
903
    def get_citations(self):
904
        return self.cite.get_dictionary()
905
906
    def get_doc(self):
907
        return self.doc.get_dictionary()
908