Ingredient._create_run_for_command() - Code Metrics - Inspection of "fixed file storage observer" - IDSIA/sacred - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( dbc38f...56accc )

by Klaus

created 2017-01-16 09:53 UTC

Ingredient._create_run_for_command() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	3
Bugs	1	Features	0

Metric	Value
cc	1
c	3
b	1
f	0
dl	0
loc	5
rs	9.4285

#!/usr/bin/env python
# coding=utf-8
from __future__ import division, print_function, unicode_literals

import inspect
import os.path

from collections import OrderedDict

from sacred.config import (ConfigDict, ConfigScope, create_captured_function,
                           load_config_file)
from sacred.dependencies import (PEP440_VERSION_PATTERN, PackageDependency,
                                 Source, gather_sources_and_dependencies)
from sacred.optional import basestring
from sacred.utils import CircularDependencyError, optional_kwargs_decorator

__sacred__ = True  # marks files that should be filtered from stack traces

__all__ = ('Ingredient',)


def collect_repositories(sources):
    return [{'url': s.repo, 'commit': s.commit, 'dirty': s.is_dirty}
            for s in sources if s.repo]


class Ingredient(object):
    """
    Ingredients are reusable parts of experiments.

    Each Ingredient can have its own configuration (visible as an entry in the
    parents configuration), named configurations, captured functions and
    commands.

    Ingredients can themselves use ingredients.
    """

    def __init__(self, path, ingredients=(), interactive=False,
                 _caller_globals=None):
        self.path = path
        self.config_hooks = []
        self.configurations = []
        self.named_configs = dict()
        self.ingredients = list(ingredients)
        self.logger = None
        self.captured_functions = []
        self.post_run_hooks = []
        self.pre_run_hooks = []
        self._is_traversing = False
        self.commands = OrderedDict()
        # capture some context information
        _caller_globals = _caller_globals or inspect.stack()[1][0].f_globals
        mainfile_name = _caller_globals.get('__file__', '.')
        self.base_dir = os.path.dirname(os.path.abspath(mainfile_name))
        self.doc = _caller_globals.get('__doc__', "")
        self.sources, self.dependencies = \
            gather_sources_and_dependencies(_caller_globals, interactive)

    # =========================== Decorators ==================================
    @optional_kwargs_decorator
    def capture(self, function=None, prefix=None):
        """
        Decorator to turn a function into a captured function.

        The missing arguments of captured functions are automatically filled
        from the configuration if possible.
        See :ref:`captured_functions` for more information.

        If a ``prefix`` is specified, the search for suitable
        entries is performed in the corresponding subtree of the configuration.
        """
        if function in self.captured_functions:
            return function
        captured_function = create_captured_function(function, prefix=prefix)
        self.captured_functions.append(captured_function)
        return captured_function

    @optional_kwargs_decorator
    def pre_run_hook(self, func, prefix=None):
        """
        Decorator to add a pre-run hook to this ingredient.

        Pre-run hooks are captured functions that are run, just before the
        main function is executed.
        """
        cf = self.capture(func, prefix=prefix)
        self.pre_run_hooks.append(cf)
        return cf

    @optional_kwargs_decorator
    def post_run_hook(self, func, prefix=None):
        """
        Decorator to add a post-run hook to this ingredient.

        Post-run hooks are captured functions that are run, just after the
        main function is executed.
        """
        cf = self.capture(func, prefix=prefix)
        self.post_run_hooks.append(cf)
        return cf

    @optional_kwargs_decorator
    def command(self, function=None, prefix=None, unobserved=False):
        """
        Decorator to define a new command for this Ingredient or Experiment.

        The name of the command will be the name of the function. It can be
        called from the command-line or by using the run_command function.

        Commands are automatically also captured functions.

        The command can be given a prefix, to restrict its configuration space
        to a subtree. (see ``capture`` for more information)

        A command can be made unobserved (i.e. ignoring all observers) by
        passing the unobserved=True keyword argument.
        """
        captured_f = self.capture(function, prefix=prefix)
        captured_f.unobserved = unobserved
        self.commands[function.__name__] = captured_f
        return captured_f

    def config(self, function):
        """
        Decorator to add a function to the configuration of the Experiment.

        The decorated function is turned into a
        :class:`~sacred.config_scope.ConfigScope` and added to the
        Ingredient/Experiment.

        When the experiment is run, this function will also be executed and
        all json-serializable local variables inside it will end up as entries
        in the configuration of the experiment.
        """
        self.configurations.append(ConfigScope(function))
        return self.configurations[-1]

    def named_config(self, func):
        """
        Decorator to turn a function into a named configuration.

        See :ref:`named_configurations`.
        """
        config_scope = ConfigScope(func)
        self._add_named_config(func.__name__, config_scope)
        return config_scope

    def config_hook(self, func):
        """
        Decorator to add a config hook to this ingredient.

        Config hooks need to be a function that takes 3 parameters and returns
        a dictionary:
        (config, command_name, logger) --> dict

        Config hooks are run after the configuration of this Ingredient, but
        before any further ingredient-configurations are run.
        The dictionary returned by a config hook is used to update the
        config updates.
        Note that they are not restricted to the local namespace of the
        ingredient.
        """
        argspec = inspect.getargspec(func)
        args = ['config', 'command_name', 'logger']
        if not (argspec.args == args and argspec.varargs is None and
                argspec.keywords is None and argspec.defaults is None):
            raise ValueError('Wrong signature for config_hook. Expected: '
                             '(config, command_name, logger)')
        self.config_hooks.append(func)
        return self.config_hooks[-1]

    # =========================== Public Interface ============================

    def add_config(self, cfg_or_file=None, **kw_conf):
        """
        Add a configuration entry to this ingredient/experiment.

        Can be called with a filename, a dictionary xor with keyword arguments.
        Supported formats for the config-file so far are: ``json``, ``pickle``
        and ``yaml``.

        The resulting dictionary will be converted into a
         :class:`~sacred.config_scope.ConfigDict`.

        :param cfg_or_file: Configuration dictionary of filename of config file
                            to add to this ingredient/experiment.
        :type cfg_or_file: dict or str
        :param kw_conf: Configuration entries to be added to this
                        ingredient/experiment.
        """
        self.configurations.append(self._create_config_dict(cfg_or_file,
                                                            kw_conf))

    def _add_named_config(self, name, conf):
        if name in self.named_configs:
            raise KeyError('Configuration name "{}" already in use!')
        self.named_configs[name] = conf

    @staticmethod
    def _create_config_dict(cfg_or_file, kw_conf):
        if cfg_or_file is not None and kw_conf:
            raise ValueError("cannot combine keyword config with "
                             "positional argument")
        if cfg_or_file is None:
            if not kw_conf:
                raise ValueError("attempted to add empty config")
            return ConfigDict(kw_conf)
        elif isinstance(cfg_or_file, dict):
            return ConfigDict(cfg_or_file)
        elif isinstance(cfg_or_file, basestring):
            if not os.path.exists(cfg_or_file):
                raise IOError('File not found {}'.format(cfg_or_file))
            abspath = os.path.abspath(cfg_or_file)
            return ConfigDict(load_config_file(abspath))
        else:
            raise TypeError("Invalid argument type {}"
                            .format(type(cfg_or_file)))

    def add_named_config(self, name, cfg_or_file=None, **kw_conf):
        """
        Add a **named** configuration entry to this ingredient/experiment.

        Can be called with a filename, a dictionary xor with keyword arguments.
        Supported formats for the config-file so far are: ``json``, ``pickle``
        and ``yaml``.

        The resulting dictionary will be converted into a
         :class:`~sacred.config_scope.ConfigDict`.

        See :ref:`named_configurations`

        :param name: name of the configuration
        :type name: str
        :param cfg_or_file: Configuration dictionary of filename of config file
                            to add to this ingredient/experiment.
        :type cfg_or_file: dict or str
        :param kw_conf: Configuration entries to be added to this
                        ingredient/experiment.
        """
        self._add_named_config(name, self._create_config_dict(cfg_or_file,
                                                              kw_conf))

    def add_source_file(self, filename):
        """
        Add a file as source dependency to this experiment/ingredient.

        :param filename: filename of the source to be added as dependency
        :type filename: str
        """
        self.sources.add(Source.create(filename))

    def add_package_dependency(self, package_name, version):
        """
        Add a package to the list of dependencies.

        :param package_name: The name of the package dependency
        :type package_name: str
        :param version: The (minimum) version of the package
        :type version: str
        """
        if not PEP440_VERSION_PATTERN.match(version):
            raise ValueError('Invalid Version: "{}"'.format(version))
        self.dependencies.add(PackageDependency(package_name, version))

    def gather_commands(self):
        for cmd_name, cmd in self.commands.items():
            yield self.path + '.' + cmd_name, cmd

        for ingred in self.ingredients:
            for cmd_name, cmd in ingred.gather_commands():
                yield cmd_name, cmd

    def get_experiment_info(self):
        """Get a dictionary with information about this experiment.

        Contains:
          * *name*: the name
          * *sources*: a list of sources (filename, md5)
          * *dependencies*: a list of package dependencies (name, version)

        :return: experiment information
        :rtype: dict
        """
        dependencies = set()
        sources = set()
        for ing, _ in self.traverse_ingredients():
            dependencies |= ing.dependencies
            sources |= ing.sources

        for dep in dependencies:
            dep.fill_missing_version()

        return dict(
            name=self.path,
            base_dir=self.base_dir,
            sources=[s.to_json(self.base_dir) for s in sorted(sources)],
            dependencies=[d.to_json() for d in sorted(dependencies)],
            repositories=collect_repositories(sources)
        )

    def traverse_ingredients(self):
        if self._is_traversing:
            raise CircularDependencyError()
        else:
            self._is_traversing = True
        yield self, 0
        for ingredient in self.ingredients:
            for ingred, depth in ingredient.traverse_ingredients():
                yield ingred, depth + 1
        self._is_traversing = False


1			#!/usr/bin/env python
2			# coding=utf-8
3			from __future__ import division, print_function, unicode_literals
4
5			import inspect
6			import os.path
7
8			from collections import OrderedDict
9
10			from sacred.config import (ConfigDict, ConfigScope, create_captured_function,
11			load_config_file)
12			from sacred.dependencies import (PEP440_VERSION_PATTERN, PackageDependency,
13			Source, gather_sources_and_dependencies)
14			from sacred.optional import basestring
15			from sacred.utils import CircularDependencyError, optional_kwargs_decorator
16
17			__sacred__ = True # marks files that should be filtered from stack traces
18
19			__all__ = ('Ingredient',)
20
21
22			def collect_repositories(sources):
23			return [{'url': s.repo, 'commit': s.commit, 'dirty': s.is_dirty}
24			for s in sources if s.repo]
25
26
27			class Ingredient(object):
28			"""
29			Ingredients are reusable parts of experiments.
30
31			Each Ingredient can have its own configuration (visible as an entry in the
32			parents configuration), named configurations, captured functions and
33			commands.
34
35			Ingredients can themselves use ingredients.
36			"""
37
38			def __init__(self, path, ingredients=(), interactive=False,
39			_caller_globals=None):
40			self.path = path
41			self.config_hooks = []
42			self.configurations = []
43			self.named_configs = dict()
44			self.ingredients = list(ingredients)
45			self.logger = None
46			self.captured_functions = []
47			self.post_run_hooks = []
48			self.pre_run_hooks = []
49			self._is_traversing = False
50			self.commands = OrderedDict()
51			# capture some context information
52			_caller_globals = _caller_globals or inspect.stack()[1][0].f_globals
53			mainfile_name = _caller_globals.get('__file__', '.')
54			self.base_dir = os.path.dirname(os.path.abspath(mainfile_name))
55			self.doc = _caller_globals.get('__doc__', "")
56			self.sources, self.dependencies = \
57			gather_sources_and_dependencies(_caller_globals, interactive)
58
59			# =========================== Decorators ==================================
60			@optional_kwargs_decorator
61			def capture(self, function=None, prefix=None):
62			"""
63			Decorator to turn a function into a captured function.
64
65			The missing arguments of captured functions are automatically filled
66			from the configuration if possible.
67			See :ref:`captured_functions` for more information.
68
69			If a ``prefix`` is specified, the search for suitable
70			entries is performed in the corresponding subtree of the configuration.
71			"""
72			if function in self.captured_functions:
73			return function
74			captured_function = create_captured_function(function, prefix=prefix)
75			self.captured_functions.append(captured_function)
76			return captured_function
77
78			@optional_kwargs_decorator
79			def pre_run_hook(self, func, prefix=None):
80			"""
81			Decorator to add a pre-run hook to this ingredient.
82
83			Pre-run hooks are captured functions that are run, just before the
84			main function is executed.
85			"""
86			cf = self.capture(func, prefix=prefix)
87			self.pre_run_hooks.append(cf)
88			return cf
89
90			@optional_kwargs_decorator
91			def post_run_hook(self, func, prefix=None):
92			"""
93			Decorator to add a post-run hook to this ingredient.
94
95			Post-run hooks are captured functions that are run, just after the
96			main function is executed.
97			"""
98			cf = self.capture(func, prefix=prefix)
99			self.post_run_hooks.append(cf)
100			return cf
101
102			@optional_kwargs_decorator
103			def command(self, function=None, prefix=None, unobserved=False):
104			"""
105			Decorator to define a new command for this Ingredient or Experiment.
106
107			The name of the command will be the name of the function. It can be
108			called from the command-line or by using the run_command function.
109
110			Commands are automatically also captured functions.
111
112			The command can be given a prefix, to restrict its configuration space
113			to a subtree. (see ``capture`` for more information)
114
115			A command can be made unobserved (i.e. ignoring all observers) by
116			passing the unobserved=True keyword argument.
117			"""
118			captured_f = self.capture(function, prefix=prefix)
119			captured_f.unobserved = unobserved
120			self.commands[function.__name__] = captured_f
121			return captured_f
122
123			def config(self, function):
124			"""
125			Decorator to add a function to the configuration of the Experiment.
126
127			The decorated function is turned into a
128			:class:`~sacred.config_scope.ConfigScope` and added to the
129			Ingredient/Experiment.
130
131			When the experiment is run, this function will also be executed and
132			all json-serializable local variables inside it will end up as entries
133			in the configuration of the experiment.
134			"""
135			self.configurations.append(ConfigScope(function))
136			return self.configurations[-1]
137
138			def named_config(self, func):
139			"""
140			Decorator to turn a function into a named configuration.
141
142			See :ref:`named_configurations`.
143			"""
144			config_scope = ConfigScope(func)
145			self._add_named_config(func.__name__, config_scope)
146			return config_scope
147
148			def config_hook(self, func):
149			"""
150			Decorator to add a config hook to this ingredient.
151
152			Config hooks need to be a function that takes 3 parameters and returns
153			a dictionary:
154			(config, command_name, logger) --> dict
155
156			Config hooks are run after the configuration of this Ingredient, but
157			before any further ingredient-configurations are run.
158			The dictionary returned by a config hook is used to update the
159			config updates.
160			Note that they are not restricted to the local namespace of the
161			ingredient.
162			"""
163			argspec = inspect.getargspec(func)
164			args = ['config', 'command_name', 'logger']
165			if not (argspec.args == args and argspec.varargs is None and
166			argspec.keywords is None and argspec.defaults is None):
167			raise ValueError('Wrong signature for config_hook. Expected: '
168			'(config, command_name, logger)')
169			self.config_hooks.append(func)
170			return self.config_hooks[-1]
171
172			# =========================== Public Interface ============================
173
174			def add_config(self, cfg_or_file=None, **kw_conf):
175			"""
176			Add a configuration entry to this ingredient/experiment.
177
178			Can be called with a filename, a dictionary xor with keyword arguments.
179			Supported formats for the config-file so far are: ``json``, ``pickle``
180			and ``yaml``.
181
182			The resulting dictionary will be converted into a
183			:class:`~sacred.config_scope.ConfigDict`.
184
185			:param cfg_or_file: Configuration dictionary of filename of config file
186			to add to this ingredient/experiment.
187			:type cfg_or_file: dict or str
188			:param kw_conf: Configuration entries to be added to this
189			ingredient/experiment.
190			"""
191			self.configurations.append(self._create_config_dict(cfg_or_file,
192			kw_conf))
193
194			def _add_named_config(self, name, conf):
195			if name in self.named_configs:
196			raise KeyError('Configuration name "{}" already in use!')
197			self.named_configs[name] = conf
198
199			@staticmethod
200			def _create_config_dict(cfg_or_file, kw_conf):
201			if cfg_or_file is not None and kw_conf:
202			raise ValueError("cannot combine keyword config with "
203			"positional argument")
204			if cfg_or_file is None:
205			if not kw_conf:
206			raise ValueError("attempted to add empty config")
207			return ConfigDict(kw_conf)
208			elif isinstance(cfg_or_file, dict):
209			return ConfigDict(cfg_or_file)
210			elif isinstance(cfg_or_file, basestring):
211			if not os.path.exists(cfg_or_file):
212			raise IOError('File not found {}'.format(cfg_or_file))
213			abspath = os.path.abspath(cfg_or_file)
214			return ConfigDict(load_config_file(abspath))
215			else:
216			raise TypeError("Invalid argument type {}"
217			.format(type(cfg_or_file)))
218
219			def add_named_config(self, name, cfg_or_file=None, **kw_conf):
220			"""
221			Add a named configuration entry to this ingredient/experiment.
222
223			Can be called with a filename, a dictionary xor with keyword arguments.
224			Supported formats for the config-file so far are: ``json``, ``pickle``
225			and ``yaml``.
226
227			The resulting dictionary will be converted into a
228			:class:`~sacred.config_scope.ConfigDict`.
229
230			See :ref:`named_configurations`
231
232			:param name: name of the configuration
233			:type name: str
234			:param cfg_or_file: Configuration dictionary of filename of config file
235			to add to this ingredient/experiment.
236			:type cfg_or_file: dict or str
237			:param kw_conf: Configuration entries to be added to this
238			ingredient/experiment.
239			"""
240			self._add_named_config(name, self._create_config_dict(cfg_or_file,
241			kw_conf))
242
243			def add_source_file(self, filename):
244			"""
245			Add a file as source dependency to this experiment/ingredient.
246
247			:param filename: filename of the source to be added as dependency
248			:type filename: str
249			"""
250			self.sources.add(Source.create(filename))
251
252			def add_package_dependency(self, package_name, version):
253			"""
254			Add a package to the list of dependencies.
255
256			:param package_name: The name of the package dependency
257			:type package_name: str
258			:param version: The (minimum) version of the package
259			:type version: str
260			"""
261			if not PEP440_VERSION_PATTERN.match(version):
262			raise ValueError('Invalid Version: "{}"'.format(version))
263			self.dependencies.add(PackageDependency(package_name, version))
264
265			def gather_commands(self):
266			for cmd_name, cmd in self.commands.items():
267			yield self.path + '.' + cmd_name, cmd
268
269			for ingred in self.ingredients:
270			for cmd_name, cmd in ingred.gather_commands():
271			yield cmd_name, cmd
272
273			def get_experiment_info(self):
274			"""Get a dictionary with information about this experiment.
275
276			Contains:
277			* name: the name
278			* sources: a list of sources (filename, md5)
279			* dependencies: a list of package dependencies (name, version)
280
281			:return: experiment information
282			:rtype: dict
283			"""
284			dependencies = set()
285			sources = set()
286			for ing, _ in self.traverse_ingredients():
287			dependencies \|= ing.dependencies
288			sources \|= ing.sources
289
290			for dep in dependencies:
291			dep.fill_missing_version()
292
293			return dict(
294			name=self.path,
295			base_dir=self.base_dir,
296			sources=[s.to_json(self.base_dir) for s in sorted(sources)],
297			dependencies=[d.to_json() for d in sorted(dependencies)],
298			repositories=collect_repositories(sources)
299			)
300
301			def traverse_ingredients(self):
302			if self._is_traversing:
303			raise CircularDependencyError()
304			else:
305			self._is_traversing = True
306			yield self, 0
307			for ingredient in self.ingredients:
308			for ingred, depth in ingredient.traverse_ingredients():
309			yield ingred, depth + 1
310			self._is_traversing = False
311

IDSIA / sacred

Push — master ( dbc38f...56accc )

Ingredient._create_run_for_command() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like