Ingredient.add_source_file() - Code Metrics - IDSIA/sacred - Measure and Improve Code Quality continuously with Scrutinizer

Ingredient.add_source_file() A
last analyzed 2018-07-01 19:23 UTC

↳ Parent: Ingredient

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
c	0
b	0
f	0
dl	0
loc	8
rs	10

#!/usr/bin/env python
# coding=utf-8
from __future__ import division, print_function, unicode_literals

import inspect
import os.path

from collections import OrderedDict

from sacred.config import (ConfigDict, ConfigScope, create_captured_function,
                           load_config_file)
from sacred.dependencies import (PEP440_VERSION_PATTERN, PackageDependency,
                                 Source, gather_sources_and_dependencies)
from sacred.utils import (CircularDependencyError, optional_kwargs_decorator,
                          basestring)

__all__ = ('Ingredient',)


def collect_repositories(sources):
    return [{'url': s.repo, 'commit': s.commit, 'dirty': s.is_dirty}
            for s in sources if s.repo]


class Ingredient(object):
    """
    Ingredients are reusable parts of experiments.

    Each Ingredient can have its own configuration (visible as an entry in the
    parents configuration), named configurations, captured functions and
    commands.

    Ingredients can themselves use ingredients.
    """

    def __init__(self, path, ingredients=(), interactive=False,
                 _caller_globals=None, base_dir=None):
        self.path = path
        self.config_hooks = []
        self.configurations = []
        self.named_configs = dict()
        self.ingredients = list(ingredients)
        self.logger = None
        self.captured_functions = []
        self.post_run_hooks = []
        self.pre_run_hooks = []
        self._is_traversing = False
        self.commands = OrderedDict()
        # capture some context information
        _caller_globals = _caller_globals or inspect.stack()[1][0].f_globals
        mainfile_dir = os.path.dirname(_caller_globals.get('__file__', '.'))
        self.base_dir = os.path.abspath(base_dir or mainfile_dir)
        self.doc = _caller_globals.get('__doc__', "")
        self.mainfile, self.sources, self.dependencies = \
            gather_sources_and_dependencies(_caller_globals, self.base_dir)
        if self.mainfile is None and not interactive:
            raise RuntimeError("Defining an experiment in interactive mode! "
                               "The sourcecode cannot be stored and the "
                               "experiment won't be reproducible. If you still"
                               " want to run it pass interactive=True")

    # =========================== Decorators ==================================
    @optional_kwargs_decorator
    def capture(self, function=None, prefix=None):
        """
        Decorator to turn a function into a captured function.

        The missing arguments of captured functions are automatically filled
        from the configuration if possible.
        See :ref:`captured_functions` for more information.

        If a ``prefix`` is specified, the search for suitable
        entries is performed in the corresponding subtree of the configuration.
        """
        if function in self.captured_functions:
            return function
        captured_function = create_captured_function(function, prefix=prefix)
        self.captured_functions.append(captured_function)
        return captured_function

    @optional_kwargs_decorator
    def pre_run_hook(self, func, prefix=None):
        """
        Decorator to add a pre-run hook to this ingredient.

        Pre-run hooks are captured functions that are run, just before the
        main function is executed.
        """
        cf = self.capture(func, prefix=prefix)
        self.pre_run_hooks.append(cf)
        return cf

    @optional_kwargs_decorator
    def post_run_hook(self, func, prefix=None):
        """
        Decorator to add a post-run hook to this ingredient.

        Post-run hooks are captured functions that are run, just after the
        main function is executed.
        """
        cf = self.capture(func, prefix=prefix)
        self.post_run_hooks.append(cf)
        return cf

    @optional_kwargs_decorator
    def command(self, function=None, prefix=None, unobserved=False):
        """
        Decorator to define a new command for this Ingredient or Experiment.

        The name of the command will be the name of the function. It can be
        called from the command-line or by using the run_command function.

        Commands are automatically also captured functions.

        The command can be given a prefix, to restrict its configuration space
        to a subtree. (see ``capture`` for more information)

        A command can be made unobserved (i.e. ignoring all observers) by
        passing the unobserved=True keyword argument.
        """
        captured_f = self.capture(function, prefix=prefix)
        captured_f.unobserved = unobserved
        self.commands[function.__name__] = captured_f
        return captured_f

    def config(self, function):
        """
        Decorator to add a function to the configuration of the Experiment.

        The decorated function is turned into a
        :class:`~sacred.config_scope.ConfigScope` and added to the
        Ingredient/Experiment.

        When the experiment is run, this function will also be executed and
        all json-serializable local variables inside it will end up as entries
        in the configuration of the experiment.
        """
        self.configurations.append(ConfigScope(function))
        return self.configurations[-1]

    def named_config(self, func):
        """
        Decorator to turn a function into a named configuration.

        See :ref:`named_configurations`.
        """
        config_scope = ConfigScope(func)
        self._add_named_config(func.__name__, config_scope)
        return config_scope

    def config_hook(self, func):
        """
        Decorator to add a config hook to this ingredient.

        Config hooks need to be a function that takes 3 parameters and returns
        a dictionary:
        (config, command_name, logger) --> dict

        Config hooks are run after the configuration of this Ingredient, but
        before any further ingredient-configurations are run.
        The dictionary returned by a config hook is used to update the
        config updates.
        Note that they are not restricted to the local namespace of the
        ingredient.
        """
        argspec = inspect.getargspec(func)
        args = ['config', 'command_name', 'logger']
        if not (argspec.args == args and argspec.varargs is None and
                argspec.keywords is None and argspec.defaults is None):
            raise ValueError('Wrong signature for config_hook. Expected: '
                             '(config, command_name, logger)')
        self.config_hooks.append(func)
        return self.config_hooks[-1]

    # =========================== Public Interface ============================

    def add_config(self, cfg_or_file=None, **kw_conf):
        """
        Add a configuration entry to this ingredient/experiment.

        Can be called with a filename, a dictionary xor with keyword arguments.
        Supported formats for the config-file so far are: ``json``, ``pickle``
        and ``yaml``.

        The resulting dictionary will be converted into a
         :class:`~sacred.config_scope.ConfigDict`.

        :param cfg_or_file: Configuration dictionary of filename of config file
                            to add to this ingredient/experiment.
        :type cfg_or_file: dict or str
        :param kw_conf: Configuration entries to be added to this
                        ingredient/experiment.
        """
        self.configurations.append(self._create_config_dict(cfg_or_file,
                                                            kw_conf))

    def _add_named_config(self, name, conf):
        if name in self.named_configs:
            raise KeyError('Configuration name "{}" already in use!'
                           .format(name))
        self.named_configs[name] = conf

    @staticmethod
    def _create_config_dict(cfg_or_file, kw_conf):
        if cfg_or_file is not None and kw_conf:
            raise ValueError("cannot combine keyword config with "
                             "positional argument")
        if cfg_or_file is None:
            if not kw_conf:
                raise ValueError("attempted to add empty config")
            return ConfigDict(kw_conf)
        elif isinstance(cfg_or_file, dict):
            return ConfigDict(cfg_or_file)
        elif isinstance(cfg_or_file, basestring):
            if not os.path.exists(cfg_or_file):
                raise IOError('File not found {}'.format(cfg_or_file))
            abspath = os.path.abspath(cfg_or_file)
            return ConfigDict(load_config_file(abspath))
        else:
            raise TypeError("Invalid argument type {}"
                            .format(type(cfg_or_file)))

    def add_named_config(self, name, cfg_or_file=None, **kw_conf):
        """
        Add a **named** configuration entry to this ingredient/experiment.

        Can be called with a filename, a dictionary xor with keyword arguments.
        Supported formats for the config-file so far are: ``json``, ``pickle``
        and ``yaml``.

        The resulting dictionary will be converted into a
         :class:`~sacred.config_scope.ConfigDict`.

        See :ref:`named_configurations`

        :param name: name of the configuration
        :type name: str
        :param cfg_or_file: Configuration dictionary of filename of config file
                            to add to this ingredient/experiment.
        :type cfg_or_file: dict or str
        :param kw_conf: Configuration entries to be added to this
                        ingredient/experiment.
        """
        self._add_named_config(name, self._create_config_dict(cfg_or_file,
                                                              kw_conf))

    def add_source_file(self, filename):
        """
        Add a file as source dependency to this experiment/ingredient.

        :param filename: filename of the source to be added as dependency
        :type filename: str
        """
        self.sources.add(Source.create(filename))

    def add_package_dependency(self, package_name, version):
        """
        Add a package to the list of dependencies.

        :param package_name: The name of the package dependency
        :type package_name: str
        :param version: The (minimum) version of the package
        :type version: str
        """
        if not PEP440_VERSION_PATTERN.match(version):
            raise ValueError('Invalid Version: "{}"'.format(version))
        self.dependencies.add(PackageDependency(package_name, version))

    def gather_commands(self):
        """Collect all commands from this ingredient and its sub-ingredients.

        Yields
        ------
        cmd_name: str
            The full (dotted) name of the command.
        cmd: function
            The corresponding captured function.
        """
        for cmd_name, cmd in self.commands.items():
            yield self.path + '.' + cmd_name, cmd

        for ingred in self.ingredients:
            for cmd_name, cmd in ingred.gather_commands():
                yield cmd_name, cmd

    def get_experiment_info(self):
        """Get a dictionary with information about this experiment.

        Contains:
          * *name*: the name
          * *sources*: a list of sources (filename, md5)
          * *dependencies*: a list of package dependencies (name, version)

        :return: experiment information
        :rtype: dict
        """
        dependencies = set()
        sources = set()
        for ing, _ in self.traverse_ingredients():
            dependencies |= ing.dependencies
            sources |= ing.sources

        for dep in dependencies:
            dep.fill_missing_version()

        mainfile = (self.mainfile.to_json(self.base_dir)[0]
                    if self.mainfile else None)

        def name_lower(d):
            return d.name.lower()

        return dict(
            name=self.path,
            base_dir=self.base_dir,
            sources=[s.to_json(self.base_dir) for s in sorted(sources)],
            dependencies=[d.to_json()
                          for d in sorted(dependencies, key=name_lower)],
            repositories=collect_repositories(sources),
            mainfile=mainfile
        )

    def traverse_ingredients(self):
        """Recursively traverse this ingredient and its sub-ingredients.
        Yields
        ------
        ingredient: sacred.Ingredient
            The ingredient as traversed in preorder.
        depth: int
            The depth of the ingredient starting from 0.

        Raises
        ------
        CircularDependencyError:
            If a circular structure among ingredients was detected.
        """
        if self._is_traversing:
            raise CircularDependencyError()
        else:
            self._is_traversing = True
        yield self, 0
        for ingredient in self.ingredients:
            for ingred, depth in ingredient.traverse_ingredients():
                yield ingred, depth + 1
        self._is_traversing = False


1			#!/usr/bin/env python
2			# coding=utf-8
3			from __future__ import division, print_function, unicode_literals
4
5			import inspect
6			import os.path
7
8			from collections import OrderedDict
9
10			from sacred.config import (ConfigDict, ConfigScope, create_captured_function,
11			load_config_file)
12			from sacred.dependencies import (PEP440_VERSION_PATTERN, PackageDependency,
13			Source, gather_sources_and_dependencies)
14			from sacred.utils import (CircularDependencyError, optional_kwargs_decorator,
15			basestring)
16
17			__all__ = ('Ingredient',)
18
19
20			def collect_repositories(sources):
21			return [{'url': s.repo, 'commit': s.commit, 'dirty': s.is_dirty}
22			for s in sources if s.repo]
23
24
25			class Ingredient(object):
26			"""
27			Ingredients are reusable parts of experiments.
28
29			Each Ingredient can have its own configuration (visible as an entry in the
30			parents configuration), named configurations, captured functions and
31			commands.
32
33			Ingredients can themselves use ingredients.
34			"""
35
36			def __init__(self, path, ingredients=(), interactive=False,
37			_caller_globals=None, base_dir=None):
38			self.path = path
39			self.config_hooks = []
40			self.configurations = []
41			self.named_configs = dict()
42			self.ingredients = list(ingredients)
43			self.logger = None
44			self.captured_functions = []
45			self.post_run_hooks = []
46			self.pre_run_hooks = []
47			self._is_traversing = False
48			self.commands = OrderedDict()
49			# capture some context information
50			_caller_globals = _caller_globals or inspect.stack()[1][0].f_globals
51			mainfile_dir = os.path.dirname(_caller_globals.get('__file__', '.'))
52			self.base_dir = os.path.abspath(base_dir or mainfile_dir)
53			self.doc = _caller_globals.get('__doc__', "")
54			self.mainfile, self.sources, self.dependencies = \
55			gather_sources_and_dependencies(_caller_globals, self.base_dir)
56			if self.mainfile is None and not interactive:
57			raise RuntimeError("Defining an experiment in interactive mode! "
58			"The sourcecode cannot be stored and the "
59			"experiment won't be reproducible. If you still"
60			" want to run it pass interactive=True")
61
62			# =========================== Decorators ==================================
63			@optional_kwargs_decorator
64			def capture(self, function=None, prefix=None):
65			"""
66			Decorator to turn a function into a captured function.
67
68			The missing arguments of captured functions are automatically filled
69			from the configuration if possible.
70			See :ref:`captured_functions` for more information.
71
72			If a ``prefix`` is specified, the search for suitable
73			entries is performed in the corresponding subtree of the configuration.
74			"""
75			if function in self.captured_functions:
76			return function
77			captured_function = create_captured_function(function, prefix=prefix)
78			self.captured_functions.append(captured_function)
79			return captured_function
80
81			@optional_kwargs_decorator
82			def pre_run_hook(self, func, prefix=None):
83			"""
84			Decorator to add a pre-run hook to this ingredient.
85
86			Pre-run hooks are captured functions that are run, just before the
87			main function is executed.
88			"""
89			cf = self.capture(func, prefix=prefix)
90			self.pre_run_hooks.append(cf)
91			return cf
92
93			@optional_kwargs_decorator
94			def post_run_hook(self, func, prefix=None):
95			"""
96			Decorator to add a post-run hook to this ingredient.
97
98			Post-run hooks are captured functions that are run, just after the
99			main function is executed.
100			"""
101			cf = self.capture(func, prefix=prefix)
102			self.post_run_hooks.append(cf)
103			return cf
104
105			@optional_kwargs_decorator
106			def command(self, function=None, prefix=None, unobserved=False):
107			"""
108			Decorator to define a new command for this Ingredient or Experiment.
109
110			The name of the command will be the name of the function. It can be
111			called from the command-line or by using the run_command function.
112
113			Commands are automatically also captured functions.
114
115			The command can be given a prefix, to restrict its configuration space
116			to a subtree. (see ``capture`` for more information)
117
118			A command can be made unobserved (i.e. ignoring all observers) by
119			passing the unobserved=True keyword argument.
120			"""
121			captured_f = self.capture(function, prefix=prefix)
122			captured_f.unobserved = unobserved
123			self.commands[function.__name__] = captured_f
124			return captured_f
125
126			def config(self, function):
127			"""
128			Decorator to add a function to the configuration of the Experiment.
129
130			The decorated function is turned into a
131			:class:`~sacred.config_scope.ConfigScope` and added to the
132			Ingredient/Experiment.
133
134			When the experiment is run, this function will also be executed and
135			all json-serializable local variables inside it will end up as entries
136			in the configuration of the experiment.
137			"""
138			self.configurations.append(ConfigScope(function))
139			return self.configurations[-1]
140
141			def named_config(self, func):
142			"""
143			Decorator to turn a function into a named configuration.
144
145			See :ref:`named_configurations`.
146			"""
147			config_scope = ConfigScope(func)
148			self._add_named_config(func.__name__, config_scope)
149			return config_scope
150
151			def config_hook(self, func):
152			"""
153			Decorator to add a config hook to this ingredient.
154
155			Config hooks need to be a function that takes 3 parameters and returns
156			a dictionary:
157			(config, command_name, logger) --> dict
158
159			Config hooks are run after the configuration of this Ingredient, but
160			before any further ingredient-configurations are run.
161			The dictionary returned by a config hook is used to update the
162			config updates.
163			Note that they are not restricted to the local namespace of the
164			ingredient.
165			"""
166			argspec = inspect.getargspec(func)
167			args = ['config', 'command_name', 'logger']
168			if not (argspec.args == args and argspec.varargs is None and
169			argspec.keywords is None and argspec.defaults is None):
170			raise ValueError('Wrong signature for config_hook. Expected: '
171			'(config, command_name, logger)')
172			self.config_hooks.append(func)
173			return self.config_hooks[-1]
174
175			# =========================== Public Interface ============================
176
177			def add_config(self, cfg_or_file=None, **kw_conf):
178			"""
179			Add a configuration entry to this ingredient/experiment.
180
181			Can be called with a filename, a dictionary xor with keyword arguments.
182			Supported formats for the config-file so far are: ``json``, ``pickle``
183			and ``yaml``.
184
185			The resulting dictionary will be converted into a
186			:class:`~sacred.config_scope.ConfigDict`.
187
188			:param cfg_or_file: Configuration dictionary of filename of config file
189			to add to this ingredient/experiment.
190			:type cfg_or_file: dict or str
191			:param kw_conf: Configuration entries to be added to this
192			ingredient/experiment.
193			"""
194			self.configurations.append(self._create_config_dict(cfg_or_file,
195			kw_conf))
196
197			def _add_named_config(self, name, conf):
198			if name in self.named_configs:
199			raise KeyError('Configuration name "{}" already in use!'
200			.format(name))
201			self.named_configs[name] = conf
202
203			@staticmethod
204			def _create_config_dict(cfg_or_file, kw_conf):
205			if cfg_or_file is not None and kw_conf:
206			raise ValueError("cannot combine keyword config with "
207			"positional argument")
208			if cfg_or_file is None:
209			if not kw_conf:
210			raise ValueError("attempted to add empty config")
211			return ConfigDict(kw_conf)
212			elif isinstance(cfg_or_file, dict):
213			return ConfigDict(cfg_or_file)
214			elif isinstance(cfg_or_file, basestring):
215			if not os.path.exists(cfg_or_file):
216			raise IOError('File not found {}'.format(cfg_or_file))
217			abspath = os.path.abspath(cfg_or_file)
218			return ConfigDict(load_config_file(abspath))
219			else:
220			raise TypeError("Invalid argument type {}"
221			.format(type(cfg_or_file)))
222
223			def add_named_config(self, name, cfg_or_file=None, **kw_conf):
224			"""
225			Add a named configuration entry to this ingredient/experiment.
226
227			Can be called with a filename, a dictionary xor with keyword arguments.
228			Supported formats for the config-file so far are: ``json``, ``pickle``
229			and ``yaml``.
230
231			The resulting dictionary will be converted into a
232			:class:`~sacred.config_scope.ConfigDict`.
233
234			See :ref:`named_configurations`
235
236			:param name: name of the configuration
237			:type name: str
238			:param cfg_or_file: Configuration dictionary of filename of config file
239			to add to this ingredient/experiment.
240			:type cfg_or_file: dict or str
241			:param kw_conf: Configuration entries to be added to this
242			ingredient/experiment.
243			"""
244			self._add_named_config(name, self._create_config_dict(cfg_or_file,
245			kw_conf))
246
247			def add_source_file(self, filename):
248			"""
249			Add a file as source dependency to this experiment/ingredient.
250
251			:param filename: filename of the source to be added as dependency
252			:type filename: str
253			"""
254			self.sources.add(Source.create(filename))
255
256			def add_package_dependency(self, package_name, version):
257			"""
258			Add a package to the list of dependencies.
259
260			:param package_name: The name of the package dependency
261			:type package_name: str
262			:param version: The (minimum) version of the package
263			:type version: str
264			"""
265			if not PEP440_VERSION_PATTERN.match(version):
266			raise ValueError('Invalid Version: "{}"'.format(version))
267			self.dependencies.add(PackageDependency(package_name, version))
268
269			def gather_commands(self):
270			"""Collect all commands from this ingredient and its sub-ingredients.
271
272			Yields
273			------
274			cmd_name: str
275			The full (dotted) name of the command.
276			cmd: function
277			The corresponding captured function.
278			"""
279			for cmd_name, cmd in self.commands.items():
280			yield self.path + '.' + cmd_name, cmd
281
282			for ingred in self.ingredients:
283			for cmd_name, cmd in ingred.gather_commands():
284			yield cmd_name, cmd
285
286			def get_experiment_info(self):
287			"""Get a dictionary with information about this experiment.
288
289			Contains:
290			* name: the name
291			* sources: a list of sources (filename, md5)
292			* dependencies: a list of package dependencies (name, version)
293
294			:return: experiment information
295			:rtype: dict
296			"""
297			dependencies = set()
298			sources = set()
299			for ing, _ in self.traverse_ingredients():
300			dependencies \|= ing.dependencies
301			sources \|= ing.sources
302
303			for dep in dependencies:
304			dep.fill_missing_version()
305
306			mainfile = (self.mainfile.to_json(self.base_dir)[0]
307			if self.mainfile else None)
308
309			def name_lower(d):
310			return d.name.lower()
311
312			return dict(
313			name=self.path,
314			base_dir=self.base_dir,
315			sources=[s.to_json(self.base_dir) for s in sorted(sources)],
316			dependencies=[d.to_json()
317			for d in sorted(dependencies, key=name_lower)],
318			repositories=collect_repositories(sources),
319			mainfile=mainfile
320			)
321
322			def traverse_ingredients(self):
323			"""Recursively traverse this ingredient and its sub-ingredients.
324			Yields
325			------
326			ingredient: sacred.Ingredient
327			The ingredient as traversed in preorder.
328			depth: int
329			The depth of the ingredient starting from 0.
330
331			Raises
332			------
333			CircularDependencyError:
334			If a circular structure among ingredients was detected.
335			"""
336			if self._is_traversing:
337			raise CircularDependencyError()
338			else:
339			self._is_traversing = True
340			yield self, 0
341			for ingredient in self.ingredients:
342			for ingred, depth in ingredient.traverse_ingredients():
343			yield ingred, depth + 1
344			self._is_traversing = False
345

IDSIA / sacred

Ingredient.add_source_file() A last analyzed 2018-07-01 19:23 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

Ingredient.add_source_file() A
last analyzed 2018-07-01 19:23 UTC