sacred.Ingredient.add_config() - Code Metrics - Inspection of "Merge branch 'master' of github.com:IDSIA/sacred" - IDSIA/sacred - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( f83703...e975b1 )

by Klaus

created 2015-12-22 09:19 UTC

sacred.Ingredient.add_config() A

↳ Parent: sacred.Ingredient

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	1
dl	0
loc	19
rs	9.4286

#!/usr/bin/env python
# coding=utf-8
from __future__ import division, print_function, unicode_literals

import inspect
import os.path

from collections import OrderedDict
from six import string_types

from sacred.config import (ConfigDict, ConfigScope, create_captured_function,
                           load_config_file)
from sacred.dependencies import (PEP440_VERSION_PATTERN, PackageDependency,
                                 Source, gather_sources_and_dependencies)
from sacred.initialize import create_run
from sacred.utils import CircularDependencyError, optional_kwargs_decorator

__sacred__ = True  # marks files that should be filtered from stack traces

__all__ = ('Ingredient',)


class Ingredient(object):

    """
    Ingredients are reusable parts of experiments.

    Each Ingredient can have its own configuration (visible as an entry in the
    parents configuration), named configurations, captured functions and
    commands.

    Ingredients can themselves use ingredients.
    """

    def __init__(self, path, ingredients=(), _caller_globals=None):
        self.path = path
        self.config_hooks = []
        self.configurations = []
        self.named_configs = dict()
        self.ingredients = list(ingredients)
        self.logger = None
        self.captured_functions = []
        self.post_run_hooks = []
        self.pre_run_hooks = []
        self._is_traversing = False
        self.commands = OrderedDict()
        # capture some context information
        _caller_globals = _caller_globals or inspect.stack()[1][0].f_globals
        self.doc = _caller_globals.get('__doc__', "")
        self.sources, self.dependencies = \
            gather_sources_and_dependencies(_caller_globals)

    # =========================== Decorators ==================================
    @optional_kwargs_decorator
    def capture(self, function=None, prefix=None):
        """
        Decorator to turn a function into a captured function.

        The missing arguments of captured functions are automatically filled
        from the configuration if possible.
        See :ref:`captured_functions` for more information.

        If a ``prefix`` is specified, the search for suitable
        entries is performed in the corresponding subtree of the configuration.
        """
        if function in self.captured_functions:
            return function
        captured_function = create_captured_function(function, prefix=prefix)
        self.captured_functions.append(captured_function)
        return captured_function

    @optional_kwargs_decorator
    def pre_run_hook(self, func, prefix=None):
        """
        Decorator to add a pre-run hook to this ingredient.

        Pre-run hooks are captured functions that are run, just before the
        main function is executed.
        """
        cf = self.capture(func, prefix=prefix)
        self.pre_run_hooks.append(cf)
        return cf

    @optional_kwargs_decorator
    def post_run_hook(self, func, prefix=None):
        """
        Decorator to add a post-run hook to this ingredient.

        Post-run hooks are captured functions that are run, just after the
        main function is executed.
        """
        cf = self.capture(func, prefix=prefix)
        self.post_run_hooks.append(cf)
        return cf

    @optional_kwargs_decorator
    def command(self, function=None, prefix=None, unobserved=False):
        """
        Decorator to define a new command for this Ingredient or Experiment.

        The name of the command will be the name of the function. It can be
        called from the command-line or by using the run_command function.

        Commands are automatically also captured functions.

        The command can be given a prefix, to restrict its configuration space
        to a subtree. (see ``capture`` for more information)

        A command can be made unobserved (i.e. ignoring all observers) by
        passing the unobserved=True keyword argument.
        """
        captured_f = self.capture(function, prefix=prefix)
        captured_f.unobserved = unobserved
        self.commands[function.__name__] = captured_f
        return captured_f

    def config(self, function):
        """
        Decorator to add a function to the configuration of the Experiment.

        The decorated function is turned into a
        :class:`~sacred.config_scope.ConfigScope` and added to the
        Ingredient/Experiment.

        When the experiment is run, this function will also be executed and
        all json-serializable local variables inside it will end up as entries
        in the configuration of the experiment.
        """
        self.configurations.append(ConfigScope(function))
        return self.configurations[-1]

    def named_config(self, func):
        """
        Decorator to turn a function into a named configuration.

        See :ref:`named_configurations`.
        """
        config_scope = ConfigScope(func)
        self._add_named_config(func.__name__, config_scope)
        return config_scope

    def config_hook(self, func):
        """
        Decorator to add a config hook to this ingredient.

        Config hooks need to be a function that takes 3 parameters and returns
        a dictionary:
        (config, command_name, logger) --> dict

        Config hooks are run after the configuration of this Ingredient, but
        before any further ingredient-configurations are run.
        The dictionary returned by a config hook is used to update the
        config updates.
        Note that they are not restricted to the local namespace of the
        ingredient.
        """
        argspec = inspect.getargspec(func)
        args = ['config', 'command_name', 'logger']
        if not (argspec.args == args and argspec.varargs is None and
                argspec.keywords is None and argspec.defaults is None):
            raise ValueError('Wrong signature for config_hook. Expected: '
                             '(config, command_name, logger)')
        self.config_hooks.append(func)
        return self.config_hooks[-1]

    # =========================== Public Interface ============================

    def add_config(self, cfg_or_file=None, **kw_conf):
        """
        Add a configuration entry to this ingredient/experiment.

        Can be called with a filename, a dictionary xor with keyword arguments.
        Supported formats for the config-file so far are: ``json``, ``pickle``
        and ``yaml``.

        The resulting dictionary will be converted into a
         :class:`~sacred.config_scope.ConfigDict`.

        :param cfg_or_file: Configuration dictionary of filename of config file
                            to add to this ingredient/experiment.
        :type cfg_or_file: dict or str
        :param kw_conf: Configuration entries to be added to this
                        ingredient/experiment.
        """
        self.configurations.append(self._create_config_dict(cfg_or_file,
                                                            kw_conf))

    def _add_named_config(self, name, conf):
        if name in self.named_configs:
            raise KeyError('Configuration name "{}" already in use!')
        self.named_configs[name] = conf

    @staticmethod
    def _create_config_dict(cfg_or_file, kw_conf):
        if cfg_or_file is not None and kw_conf:
            raise ValueError("cannot combine keyword config with "
                             "positional argument")
        if cfg_or_file is None:
            if not kw_conf:
                raise ValueError("attempted to add empty config")
            return ConfigDict(kw_conf)
        elif isinstance(cfg_or_file, dict):
            return ConfigDict(cfg_or_file)
        elif isinstance(cfg_or_file, string_types):
            if not os.path.exists(cfg_or_file):
                raise IOError('File not found {}'.format(cfg_or_file))
            abspath = os.path.abspath(cfg_or_file)
            return ConfigDict(load_config_file(abspath))
        else:
            raise TypeError("Invalid argument type {}"
                            .format(type(cfg_or_file)))

    def add_named_config(self, name, cfg_or_file=None, **kw_conf):
        """
        Add a **named** configuration entry to this ingredient/experiment.
        (See :ref:`named_configurations`)

        Can be called with a filename, a dictionary xor with keyword arguments.
        Supported formats for the config-file so far are: ``json``, ``pickle``
        and ``yaml``.

        The resulting dictionary will be converted into a
         :class:`~sacred.config_scope.ConfigDict`.

        :param name: name of the configuration
        :type name: str
        :param cfg_or_file: Configuration dictionary of filename of config file
                            to add to this ingredient/experiment.
        :type cfg_or_file: dict or str
        :param kw_conf: Configuration entries to be added to this
                        ingredient/experiment.
        """
        self._add_named_config(name, self._create_config_dict(cfg_or_file,
                                                              kw_conf))

    def add_source_file(self, filename):
        """
        Add a file as source dependency to this experiment/ingredient.

        :param filename: filename of the source to be added as dependency
        :type filename: str
        """
        self.sources.add(Source.create(filename))

    def add_package_dependency(self, package_name, version):
        """
        Add a package to the list of dependencies.

        :param package_name: The name of the package dependency
        :type package_name: str
        :param version: The (minimum) version of the package
        :type version: str
        """
        if not PEP440_VERSION_PATTERN.match(version):
            raise ValueError('Invalid Version: "{}"'.format(version))
        self.dependencies.add(PackageDependency(package_name, version))

    def gather_commands(self):
        for cmd_name, cmd in self.commands.items():
            yield self.path + '.' + cmd_name, cmd

        for ingred in self.ingredients:
            for cmd_name, cmd in ingred.gather_commands():
                yield cmd_name, cmd

    # ======================== Private Helpers ================================

    def get_experiment_info(self):
        """Get a dictionary with information about this experiment.

        Contains:
          * *name*: the name
          * *sources*: a list of sources (filename, md5)
          * *dependencies*: a list of package dependencies (name, version)
          * *doc*: the docstring

        :return: experiment information
        :rtype: dict
        """
        dependencies = set()
        sources = set()
        for ing, _ in self._traverse_ingredients():
            dependencies |= ing.dependencies
            sources |= ing.sources

        for dep in dependencies:
            dep.fill_missing_version()

        return dict(
            name=self.path,
            sources=[s.to_tuple() for s in sorted(sources)],
            dependencies=[d.to_tuple() for d in sorted(dependencies)],
            doc=self.doc)

    def _traverse_ingredients(self):
        if self._is_traversing:
            raise CircularDependencyError()
        else:
            self._is_traversing = True
        yield self, 0
        for ingredient in self.ingredients:
            for ingred, depth in ingredient._traverse_ingredients():
                yield ingred, depth + 1
        self._is_traversing = False

    def _create_run_for_command(self, command_name, config_updates=None,
                                named_configs=()):
        run = create_run(self, command_name, config_updates,
                         named_configs=named_configs)
        return run


1			#!/usr/bin/env python
2			# coding=utf-8
3			from __future__ import division, print_function, unicode_literals
4
5			import inspect
6			import os.path
7
8			from collections import OrderedDict
9			from six import string_types
10
11			from sacred.config import (ConfigDict, ConfigScope, create_captured_function,
12			load_config_file)
13			from sacred.dependencies import (PEP440_VERSION_PATTERN, PackageDependency,
14			Source, gather_sources_and_dependencies)
15			from sacred.initialize import create_run
16			from sacred.utils import CircularDependencyError, optional_kwargs_decorator
17
18			__sacred__ = True # marks files that should be filtered from stack traces
19
20			__all__ = ('Ingredient',)
21
22
23			class Ingredient(object):
24
25			"""
26			Ingredients are reusable parts of experiments.
27
28			Each Ingredient can have its own configuration (visible as an entry in the
29			parents configuration), named configurations, captured functions and
30			commands.
31
32			Ingredients can themselves use ingredients.
33			"""
34
35			def __init__(self, path, ingredients=(), _caller_globals=None):
36			self.path = path
37			self.config_hooks = []
38			self.configurations = []
39			self.named_configs = dict()
40			self.ingredients = list(ingredients)
41			self.logger = None
42			self.captured_functions = []
43			self.post_run_hooks = []
44			self.pre_run_hooks = []
45			self._is_traversing = False
46			self.commands = OrderedDict()
47			# capture some context information
48			_caller_globals = _caller_globals or inspect.stack()[1][0].f_globals
49			self.doc = _caller_globals.get('__doc__', "")
50			self.sources, self.dependencies = \
51			gather_sources_and_dependencies(_caller_globals)
52
53			# =========================== Decorators ==================================
54			@optional_kwargs_decorator
55			def capture(self, function=None, prefix=None):
56			"""
57			Decorator to turn a function into a captured function.
58
59			The missing arguments of captured functions are automatically filled
60			from the configuration if possible.
61			See :ref:`captured_functions` for more information.
62
63			If a ``prefix`` is specified, the search for suitable
64			entries is performed in the corresponding subtree of the configuration.
65			"""
66			if function in self.captured_functions:
67			return function
68			captured_function = create_captured_function(function, prefix=prefix)
69			self.captured_functions.append(captured_function)
70			return captured_function
71
72			@optional_kwargs_decorator
73			def pre_run_hook(self, func, prefix=None):
74			"""
75			Decorator to add a pre-run hook to this ingredient.
76
77			Pre-run hooks are captured functions that are run, just before the
78			main function is executed.
79			"""
80			cf = self.capture(func, prefix=prefix)
81			self.pre_run_hooks.append(cf)
82			return cf
83
84			@optional_kwargs_decorator
85			def post_run_hook(self, func, prefix=None):
86			"""
87			Decorator to add a post-run hook to this ingredient.
88
89			Post-run hooks are captured functions that are run, just after the
90			main function is executed.
91			"""
92			cf = self.capture(func, prefix=prefix)
93			self.post_run_hooks.append(cf)
94			return cf
95
96			@optional_kwargs_decorator
97			def command(self, function=None, prefix=None, unobserved=False):
98			"""
99			Decorator to define a new command for this Ingredient or Experiment.
100
101			The name of the command will be the name of the function. It can be
102			called from the command-line or by using the run_command function.
103
104			Commands are automatically also captured functions.
105
106			The command can be given a prefix, to restrict its configuration space
107			to a subtree. (see ``capture`` for more information)
108
109			A command can be made unobserved (i.e. ignoring all observers) by
110			passing the unobserved=True keyword argument.
111			"""
112			captured_f = self.capture(function, prefix=prefix)
113			captured_f.unobserved = unobserved
114			self.commands[function.__name__] = captured_f
115			return captured_f
116
117			def config(self, function):
118			"""
119			Decorator to add a function to the configuration of the Experiment.
120
121			The decorated function is turned into a
122			:class:`~sacred.config_scope.ConfigScope` and added to the
123			Ingredient/Experiment.
124
125			When the experiment is run, this function will also be executed and
126			all json-serializable local variables inside it will end up as entries
127			in the configuration of the experiment.
128			"""
129			self.configurations.append(ConfigScope(function))
130			return self.configurations[-1]
131
132			def named_config(self, func):
133			"""
134			Decorator to turn a function into a named configuration.
135
136			See :ref:`named_configurations`.
137			"""
138			config_scope = ConfigScope(func)
139			self._add_named_config(func.__name__, config_scope)
140			return config_scope
141
142			def config_hook(self, func):
143			"""
144			Decorator to add a config hook to this ingredient.
145
146			Config hooks need to be a function that takes 3 parameters and returns
147			a dictionary:
148			(config, command_name, logger) --> dict
149
150			Config hooks are run after the configuration of this Ingredient, but
151			before any further ingredient-configurations are run.
152			The dictionary returned by a config hook is used to update the
153			config updates.
154			Note that they are not restricted to the local namespace of the
155			ingredient.
156			"""
157			argspec = inspect.getargspec(func)
158			args = ['config', 'command_name', 'logger']
159			if not (argspec.args == args and argspec.varargs is None and
160			argspec.keywords is None and argspec.defaults is None):
161			raise ValueError('Wrong signature for config_hook. Expected: '
162			'(config, command_name, logger)')
163			self.config_hooks.append(func)
164			return self.config_hooks[-1]
165
166			# =========================== Public Interface ============================
167
168			def add_config(self, cfg_or_file=None, **kw_conf):
169			"""
170			Add a configuration entry to this ingredient/experiment.
171
172			Can be called with a filename, a dictionary xor with keyword arguments.
173			Supported formats for the config-file so far are: ``json``, ``pickle``
174			and ``yaml``.
175
176			The resulting dictionary will be converted into a
177			:class:`~sacred.config_scope.ConfigDict`.
178
179			:param cfg_or_file: Configuration dictionary of filename of config file
180			to add to this ingredient/experiment.
181			:type cfg_or_file: dict or str
182			:param kw_conf: Configuration entries to be added to this
183			ingredient/experiment.
184			"""
185			self.configurations.append(self._create_config_dict(cfg_or_file,
186			kw_conf))
187
188			def _add_named_config(self, name, conf):
189			if name in self.named_configs:
190			raise KeyError('Configuration name "{}" already in use!')
191			self.named_configs[name] = conf
192
193			@staticmethod
194			def _create_config_dict(cfg_or_file, kw_conf):
195			if cfg_or_file is not None and kw_conf:
196			raise ValueError("cannot combine keyword config with "
197			"positional argument")
198			if cfg_or_file is None:
199			if not kw_conf:
200			raise ValueError("attempted to add empty config")
201			return ConfigDict(kw_conf)
202			elif isinstance(cfg_or_file, dict):
203			return ConfigDict(cfg_or_file)
204			elif isinstance(cfg_or_file, string_types):
205			if not os.path.exists(cfg_or_file):
206			raise IOError('File not found {}'.format(cfg_or_file))
207			abspath = os.path.abspath(cfg_or_file)
208			return ConfigDict(load_config_file(abspath))
209			else:
210			raise TypeError("Invalid argument type {}"
211			.format(type(cfg_or_file)))
212
213			def add_named_config(self, name, cfg_or_file=None, **kw_conf):
214			"""
215			Add a named configuration entry to this ingredient/experiment.
216			(See :ref:`named_configurations`)
217
218			Can be called with a filename, a dictionary xor with keyword arguments.
219			Supported formats for the config-file so far are: ``json``, ``pickle``
220			and ``yaml``.
221
222			The resulting dictionary will be converted into a
223			:class:`~sacred.config_scope.ConfigDict`.
224
225			:param name: name of the configuration
226			:type name: str
227			:param cfg_or_file: Configuration dictionary of filename of config file
228			to add to this ingredient/experiment.
229			:type cfg_or_file: dict or str
230			:param kw_conf: Configuration entries to be added to this
231			ingredient/experiment.
232			"""
233			self._add_named_config(name, self._create_config_dict(cfg_or_file,
234			kw_conf))
235
236			def add_source_file(self, filename):
237			"""
238			Add a file as source dependency to this experiment/ingredient.
239
240			:param filename: filename of the source to be added as dependency
241			:type filename: str
242			"""
243			self.sources.add(Source.create(filename))
244
245			def add_package_dependency(self, package_name, version):
246			"""
247			Add a package to the list of dependencies.
248
249			:param package_name: The name of the package dependency
250			:type package_name: str
251			:param version: The (minimum) version of the package
252			:type version: str
253			"""
254			if not PEP440_VERSION_PATTERN.match(version):
255			raise ValueError('Invalid Version: "{}"'.format(version))
256			self.dependencies.add(PackageDependency(package_name, version))
257
258			def gather_commands(self):
259			for cmd_name, cmd in self.commands.items():
260			yield self.path + '.' + cmd_name, cmd
261
262			for ingred in self.ingredients:
263			for cmd_name, cmd in ingred.gather_commands():
264			yield cmd_name, cmd
265
266			# ======================== Private Helpers ================================
267
268			def get_experiment_info(self):
269			"""Get a dictionary with information about this experiment.
270
271			Contains:
272			* name: the name
273			* sources: a list of sources (filename, md5)
274			* dependencies: a list of package dependencies (name, version)
275			* doc: the docstring
276
277			:return: experiment information
278			:rtype: dict
279			"""
280			dependencies = set()
281			sources = set()
282			for ing, _ in self._traverse_ingredients():
283			dependencies \|= ing.dependencies
284			sources \|= ing.sources
285
286			for dep in dependencies:
287			dep.fill_missing_version()
288
289			return dict(
290			name=self.path,
291			sources=[s.to_tuple() for s in sorted(sources)],
292			dependencies=[d.to_tuple() for d in sorted(dependencies)],
293			doc=self.doc)
294
295			def _traverse_ingredients(self):
296			if self._is_traversing:
297			raise CircularDependencyError()
298			else:
299			self._is_traversing = True
300			yield self, 0
301			for ingredient in self.ingredients:
302			for ingred, depth in ingredient._traverse_ingredients():
303			yield ingred, depth + 1
304			self._is_traversing = False
305
306			def _create_run_for_command(self, command_name, config_updates=None,
307			named_configs=()):
308			run = create_run(self, command_name, config_updates,
309			named_configs=named_configs)
310			return run
311

IDSIA / sacred

Push — master ( f83703...e975b1 )

sacred.Ingredient.add_config() A

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like