sacred.Ingredient.traverse_ingredients() - Code Metrics - Inspection of "minor refactorings" - IDSIA/sacred - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 5b2103...27f92f )

by Klaus

created 2016-01-14 11:37 UTC

sacred.Ingredient.traverse_ingredients() A

↳ Parent: sacred.Ingredient

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	4
dl	0
loc	10
rs	9.2

#!/usr/bin/env python
# coding=utf-8
from __future__ import division, print_function, unicode_literals

import inspect
import os.path

from collections import OrderedDict
from six import string_types

from sacred.config import (ConfigDict, ConfigScope, create_captured_function,
                           load_config_file)
from sacred.dependencies import (PEP440_VERSION_PATTERN, PackageDependency,
                                 Source, gather_sources_and_dependencies)
from sacred.initialize import create_run
from sacred.utils import CircularDependencyError, optional_kwargs_decorator

__sacred__ = True  # marks files that should be filtered from stack traces

__all__ = ('Ingredient',)


class Ingredient(object):
    """
    Ingredients are reusable parts of experiments.

    Each Ingredient can have its own configuration (visible as an entry in the
    parents configuration), named configurations, captured functions and
    commands.

    Ingredients can themselves use ingredients.
    """

    def __init__(self, path, ingredients=(), _caller_globals=None):
        self.path = path
        self.config_hooks = []
        self.configurations = []
        self.named_configs = dict()
        self.ingredients = list(ingredients)
        self.logger = None
        self.captured_functions = []
        self.post_run_hooks = []
        self.pre_run_hooks = []
        self._is_traversing = False
        self.commands = OrderedDict()
        # capture some context information
        _caller_globals = _caller_globals or inspect.stack()[1][0].f_globals
        self.doc = _caller_globals.get('__doc__', "")
        self.sources, self.dependencies = \
            gather_sources_and_dependencies(_caller_globals)

    # =========================== Decorators ==================================
    @optional_kwargs_decorator
    def capture(self, function=None, prefix=None):
        """
        Decorator to turn a function into a captured function.

        The missing arguments of captured functions are automatically filled
        from the configuration if possible.
        See :ref:`captured_functions` for more information.

        If a ``prefix`` is specified, the search for suitable
        entries is performed in the corresponding subtree of the configuration.
        """
        if function in self.captured_functions:
            return function
        captured_function = create_captured_function(function, prefix=prefix)
        self.captured_functions.append(captured_function)
        return captured_function

    @optional_kwargs_decorator
    def pre_run_hook(self, func, prefix=None):
        """
        Decorator to add a pre-run hook to this ingredient.

        Pre-run hooks are captured functions that are run, just before the
        main function is executed.
        """
        cf = self.capture(func, prefix=prefix)
        self.pre_run_hooks.append(cf)
        return cf

    @optional_kwargs_decorator
    def post_run_hook(self, func, prefix=None):
        """
        Decorator to add a post-run hook to this ingredient.

        Post-run hooks are captured functions that are run, just after the
        main function is executed.
        """
        cf = self.capture(func, prefix=prefix)
        self.post_run_hooks.append(cf)
        return cf

    @optional_kwargs_decorator
    def command(self, function=None, prefix=None, unobserved=False):
        """
        Decorator to define a new command for this Ingredient or Experiment.

        The name of the command will be the name of the function. It can be
        called from the command-line or by using the run_command function.

        Commands are automatically also captured functions.

        The command can be given a prefix, to restrict its configuration space
        to a subtree. (see ``capture`` for more information)

        A command can be made unobserved (i.e. ignoring all observers) by
        passing the unobserved=True keyword argument.
        """
        captured_f = self.capture(function, prefix=prefix)
        captured_f.unobserved = unobserved
        self.commands[function.__name__] = captured_f
        return captured_f

    def config(self, function):
        """
        Decorator to add a function to the configuration of the Experiment.

        The decorated function is turned into a
        :class:`~sacred.config_scope.ConfigScope` and added to the
        Ingredient/Experiment.

        When the experiment is run, this function will also be executed and
        all json-serializable local variables inside it will end up as entries
        in the configuration of the experiment.
        """
        self.configurations.append(ConfigScope(function))
        return self.configurations[-1]

    def named_config(self, func):
        """
        Decorator to turn a function into a named configuration.

        See :ref:`named_configurations`.
        """
        config_scope = ConfigScope(func)
        self._add_named_config(func.__name__, config_scope)
        return config_scope

    def config_hook(self, func):
        """
        Decorator to add a config hook to this ingredient.

        Config hooks need to be a function that takes 3 parameters and returns
        a dictionary:
        (config, command_name, logger) --> dict

        Config hooks are run after the configuration of this Ingredient, but
        before any further ingredient-configurations are run.
        The dictionary returned by a config hook is used to update the
        config updates.
        Note that they are not restricted to the local namespace of the
        ingredient.
        """
        argspec = inspect.getargspec(func)
        args = ['config', 'command_name', 'logger']
        if not (argspec.args == args and argspec.varargs is None and
                argspec.keywords is None and argspec.defaults is None):
            raise ValueError('Wrong signature for config_hook. Expected: '
                             '(config, command_name, logger)')
        self.config_hooks.append(func)
        return self.config_hooks[-1]

    # =========================== Public Interface ============================

    def add_config(self, cfg_or_file=None, **kw_conf):
        """
        Add a configuration entry to this ingredient/experiment.

        Can be called with a filename, a dictionary xor with keyword arguments.
        Supported formats for the config-file so far are: ``json``, ``pickle``
        and ``yaml``.

        The resulting dictionary will be converted into a
         :class:`~sacred.config_scope.ConfigDict`.

        :param cfg_or_file: Configuration dictionary of filename of config file
                            to add to this ingredient/experiment.
        :type cfg_or_file: dict or str
        :param kw_conf: Configuration entries to be added to this
                        ingredient/experiment.
        """
        self.configurations.append(self._create_config_dict(cfg_or_file,
                                                            kw_conf))

    def _add_named_config(self, name, conf):
        if name in self.named_configs:
            raise KeyError('Configuration name "{}" already in use!')
        self.named_configs[name] = conf

    @staticmethod
    def _create_config_dict(cfg_or_file, kw_conf):
        if cfg_or_file is not None and kw_conf:
            raise ValueError("cannot combine keyword config with "
                             "positional argument")
        if cfg_or_file is None:
            if not kw_conf:
                raise ValueError("attempted to add empty config")
            return ConfigDict(kw_conf)
        elif isinstance(cfg_or_file, dict):
            return ConfigDict(cfg_or_file)
        elif isinstance(cfg_or_file, string_types):
            if not os.path.exists(cfg_or_file):
                raise IOError('File not found {}'.format(cfg_or_file))
            abspath = os.path.abspath(cfg_or_file)
            return ConfigDict(load_config_file(abspath))
        else:
            raise TypeError("Invalid argument type {}"
                            .format(type(cfg_or_file)))

    def add_named_config(self, name, cfg_or_file=None, **kw_conf):
        """
        Add a **named** configuration entry to this ingredient/experiment.

        Can be called with a filename, a dictionary xor with keyword arguments.
        Supported formats for the config-file so far are: ``json``, ``pickle``
        and ``yaml``.

        The resulting dictionary will be converted into a
         :class:`~sacred.config_scope.ConfigDict`.

        See :ref:`named_configurations`

        :param name: name of the configuration
        :type name: str
        :param cfg_or_file: Configuration dictionary of filename of config file
                            to add to this ingredient/experiment.
        :type cfg_or_file: dict or str
        :param kw_conf: Configuration entries to be added to this
                        ingredient/experiment.
        """
        self._add_named_config(name, self._create_config_dict(cfg_or_file,
                                                              kw_conf))

    def add_source_file(self, filename):
        """
        Add a file as source dependency to this experiment/ingredient.

        :param filename: filename of the source to be added as dependency
        :type filename: str
        """
        self.sources.add(Source.create(filename))

    def add_package_dependency(self, package_name, version):
        """
        Add a package to the list of dependencies.

        :param package_name: The name of the package dependency
        :type package_name: str
        :param version: The (minimum) version of the package
        :type version: str
        """
        if not PEP440_VERSION_PATTERN.match(version):
            raise ValueError('Invalid Version: "{}"'.format(version))
        self.dependencies.add(PackageDependency(package_name, version))

    def gather_commands(self):
        for cmd_name, cmd in self.commands.items():
            yield self.path + '.' + cmd_name, cmd

        for ingred in self.ingredients:
            for cmd_name, cmd in ingred.gather_commands():
                yield cmd_name, cmd

    def get_experiment_info(self):
        """Get a dictionary with information about this experiment.

        Contains:
          * *name*: the name
          * *sources*: a list of sources (filename, md5)
          * *dependencies*: a list of package dependencies (name, version)
          * *doc*: the docstring

        :return: experiment information
        :rtype: dict
        """
        dependencies = set()
        sources = set()
        for ing, _ in self.traverse_ingredients():
            dependencies |= ing.dependencies
            sources |= ing.sources

        for dep in dependencies:
            dep.fill_missing_version()

        return dict(
            name=self.path,
            sources=[s.to_tuple() for s in sorted(sources)],
            dependencies=[d.to_tuple() for d in sorted(dependencies)],
            doc=self.doc)

    def traverse_ingredients(self):
        if self._is_traversing:
            raise CircularDependencyError()
        else:
            self._is_traversing = True
        yield self, 0
        for ingredient in self.ingredients:
            for ingred, depth in ingredient.traverse_ingredients():
                yield ingred, depth + 1
        self._is_traversing = False

    # ======================== Private Helpers ================================

    def _create_run_for_command(self, command_name, config_updates=None,
                                named_configs=()):
        run = create_run(self, command_name, config_updates,
                         named_configs=named_configs)
        return run


1			#!/usr/bin/env python
2			# coding=utf-8
3			from __future__ import division, print_function, unicode_literals
4
5			import inspect
6			import os.path
7
8			from collections import OrderedDict
9			from six import string_types
10
11			from sacred.config import (ConfigDict, ConfigScope, create_captured_function,
12			load_config_file)
13			from sacred.dependencies import (PEP440_VERSION_PATTERN, PackageDependency,
14			Source, gather_sources_and_dependencies)
15			from sacred.initialize import create_run
16			from sacred.utils import CircularDependencyError, optional_kwargs_decorator
17
18			__sacred__ = True # marks files that should be filtered from stack traces
19
20			__all__ = ('Ingredient',)
21
22
23			class Ingredient(object):
24			"""
25			Ingredients are reusable parts of experiments.
26
27			Each Ingredient can have its own configuration (visible as an entry in the
28			parents configuration), named configurations, captured functions and
29			commands.
30
31			Ingredients can themselves use ingredients.
32			"""
33
34			def __init__(self, path, ingredients=(), _caller_globals=None):
35			self.path = path
36			self.config_hooks = []
37			self.configurations = []
38			self.named_configs = dict()
39			self.ingredients = list(ingredients)
40			self.logger = None
41			self.captured_functions = []
42			self.post_run_hooks = []
43			self.pre_run_hooks = []
44			self._is_traversing = False
45			self.commands = OrderedDict()
46			# capture some context information
47			_caller_globals = _caller_globals or inspect.stack()[1][0].f_globals
48			self.doc = _caller_globals.get('__doc__', "")
49			self.sources, self.dependencies = \
50			gather_sources_and_dependencies(_caller_globals)
51
52			# =========================== Decorators ==================================
53			@optional_kwargs_decorator
54			def capture(self, function=None, prefix=None):
55			"""
56			Decorator to turn a function into a captured function.
57
58			The missing arguments of captured functions are automatically filled
59			from the configuration if possible.
60			See :ref:`captured_functions` for more information.
61
62			If a ``prefix`` is specified, the search for suitable
63			entries is performed in the corresponding subtree of the configuration.
64			"""
65			if function in self.captured_functions:
66			return function
67			captured_function = create_captured_function(function, prefix=prefix)
68			self.captured_functions.append(captured_function)
69			return captured_function
70
71			@optional_kwargs_decorator
72			def pre_run_hook(self, func, prefix=None):
73			"""
74			Decorator to add a pre-run hook to this ingredient.
75
76			Pre-run hooks are captured functions that are run, just before the
77			main function is executed.
78			"""
79			cf = self.capture(func, prefix=prefix)
80			self.pre_run_hooks.append(cf)
81			return cf
82
83			@optional_kwargs_decorator
84			def post_run_hook(self, func, prefix=None):
85			"""
86			Decorator to add a post-run hook to this ingredient.
87
88			Post-run hooks are captured functions that are run, just after the
89			main function is executed.
90			"""
91			cf = self.capture(func, prefix=prefix)
92			self.post_run_hooks.append(cf)
93			return cf
94
95			@optional_kwargs_decorator
96			def command(self, function=None, prefix=None, unobserved=False):
97			"""
98			Decorator to define a new command for this Ingredient or Experiment.
99
100			The name of the command will be the name of the function. It can be
101			called from the command-line or by using the run_command function.
102
103			Commands are automatically also captured functions.
104
105			The command can be given a prefix, to restrict its configuration space
106			to a subtree. (see ``capture`` for more information)
107
108			A command can be made unobserved (i.e. ignoring all observers) by
109			passing the unobserved=True keyword argument.
110			"""
111			captured_f = self.capture(function, prefix=prefix)
112			captured_f.unobserved = unobserved
113			self.commands[function.__name__] = captured_f
114			return captured_f
115
116			def config(self, function):
117			"""
118			Decorator to add a function to the configuration of the Experiment.
119
120			The decorated function is turned into a
121			:class:`~sacred.config_scope.ConfigScope` and added to the
122			Ingredient/Experiment.
123
124			When the experiment is run, this function will also be executed and
125			all json-serializable local variables inside it will end up as entries
126			in the configuration of the experiment.
127			"""
128			self.configurations.append(ConfigScope(function))
129			return self.configurations[-1]
130
131			def named_config(self, func):
132			"""
133			Decorator to turn a function into a named configuration.
134
135			See :ref:`named_configurations`.
136			"""
137			config_scope = ConfigScope(func)
138			self._add_named_config(func.__name__, config_scope)
139			return config_scope
140
141			def config_hook(self, func):
142			"""
143			Decorator to add a config hook to this ingredient.
144
145			Config hooks need to be a function that takes 3 parameters and returns
146			a dictionary:
147			(config, command_name, logger) --> dict
148
149			Config hooks are run after the configuration of this Ingredient, but
150			before any further ingredient-configurations are run.
151			The dictionary returned by a config hook is used to update the
152			config updates.
153			Note that they are not restricted to the local namespace of the
154			ingredient.
155			"""
156			argspec = inspect.getargspec(func)
157			args = ['config', 'command_name', 'logger']
158			if not (argspec.args == args and argspec.varargs is None and
159			argspec.keywords is None and argspec.defaults is None):
160			raise ValueError('Wrong signature for config_hook. Expected: '
161			'(config, command_name, logger)')
162			self.config_hooks.append(func)
163			return self.config_hooks[-1]
164
165			# =========================== Public Interface ============================
166
167			def add_config(self, cfg_or_file=None, **kw_conf):
168			"""
169			Add a configuration entry to this ingredient/experiment.
170
171			Can be called with a filename, a dictionary xor with keyword arguments.
172			Supported formats for the config-file so far are: ``json``, ``pickle``
173			and ``yaml``.
174
175			The resulting dictionary will be converted into a
176			:class:`~sacred.config_scope.ConfigDict`.
177
178			:param cfg_or_file: Configuration dictionary of filename of config file
179			to add to this ingredient/experiment.
180			:type cfg_or_file: dict or str
181			:param kw_conf: Configuration entries to be added to this
182			ingredient/experiment.
183			"""
184			self.configurations.append(self._create_config_dict(cfg_or_file,
185			kw_conf))
186
187			def _add_named_config(self, name, conf):
188			if name in self.named_configs:
189			raise KeyError('Configuration name "{}" already in use!')
190			self.named_configs[name] = conf
191
192			@staticmethod
193			def _create_config_dict(cfg_or_file, kw_conf):
194			if cfg_or_file is not None and kw_conf:
195			raise ValueError("cannot combine keyword config with "
196			"positional argument")
197			if cfg_or_file is None:
198			if not kw_conf:
199			raise ValueError("attempted to add empty config")
200			return ConfigDict(kw_conf)
201			elif isinstance(cfg_or_file, dict):
202			return ConfigDict(cfg_or_file)
203			elif isinstance(cfg_or_file, string_types):
204			if not os.path.exists(cfg_or_file):
205			raise IOError('File not found {}'.format(cfg_or_file))
206			abspath = os.path.abspath(cfg_or_file)
207			return ConfigDict(load_config_file(abspath))
208			else:
209			raise TypeError("Invalid argument type {}"
210			.format(type(cfg_or_file)))
211
212			def add_named_config(self, name, cfg_or_file=None, **kw_conf):
213			"""
214			Add a named configuration entry to this ingredient/experiment.
215
216			Can be called with a filename, a dictionary xor with keyword arguments.
217			Supported formats for the config-file so far are: ``json``, ``pickle``
218			and ``yaml``.
219
220			The resulting dictionary will be converted into a
221			:class:`~sacred.config_scope.ConfigDict`.
222
223			See :ref:`named_configurations`
224
225			:param name: name of the configuration
226			:type name: str
227			:param cfg_or_file: Configuration dictionary of filename of config file
228			to add to this ingredient/experiment.
229			:type cfg_or_file: dict or str
230			:param kw_conf: Configuration entries to be added to this
231			ingredient/experiment.
232			"""
233			self._add_named_config(name, self._create_config_dict(cfg_or_file,
234			kw_conf))
235
236			def add_source_file(self, filename):
237			"""
238			Add a file as source dependency to this experiment/ingredient.
239
240			:param filename: filename of the source to be added as dependency
241			:type filename: str
242			"""
243			self.sources.add(Source.create(filename))
244
245			def add_package_dependency(self, package_name, version):
246			"""
247			Add a package to the list of dependencies.
248
249			:param package_name: The name of the package dependency
250			:type package_name: str
251			:param version: The (minimum) version of the package
252			:type version: str
253			"""
254			if not PEP440_VERSION_PATTERN.match(version):
255			raise ValueError('Invalid Version: "{}"'.format(version))
256			self.dependencies.add(PackageDependency(package_name, version))
257
258			def gather_commands(self):
259			for cmd_name, cmd in self.commands.items():
260			yield self.path + '.' + cmd_name, cmd
261
262			for ingred in self.ingredients:
263			for cmd_name, cmd in ingred.gather_commands():
264			yield cmd_name, cmd
265
266			def get_experiment_info(self):
267			"""Get a dictionary with information about this experiment.
268
269			Contains:
270			* name: the name
271			* sources: a list of sources (filename, md5)
272			* dependencies: a list of package dependencies (name, version)
273			* doc: the docstring
274
275			:return: experiment information
276			:rtype: dict
277			"""
278			dependencies = set()
279			sources = set()
280			for ing, _ in self.traverse_ingredients():
281			dependencies \|= ing.dependencies
282			sources \|= ing.sources
283
284			for dep in dependencies:
285			dep.fill_missing_version()
286
287			return dict(
288			name=self.path,
289			sources=[s.to_tuple() for s in sorted(sources)],
290			dependencies=[d.to_tuple() for d in sorted(dependencies)],
291			doc=self.doc)
292
293			def traverse_ingredients(self):
294			if self._is_traversing:
295			raise CircularDependencyError()
296			else:
297			self._is_traversing = True
298			yield self, 0
299			for ingredient in self.ingredients:
300			for ingred, depth in ingredient.traverse_ingredients():
301			yield ingred, depth + 1
302			self._is_traversing = False
303
304			# ======================== Private Helpers ================================
305
306			def _create_run_for_command(self, command_name, config_updates=None,
307			named_configs=()):
308			run = create_run(self, command_name, config_updates,
309			named_configs=named_configs)
310			return run
311

IDSIA / sacred

Push — master ( 5b2103...27f92f )

sacred.Ingredient.traverse_ingredients() A

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like