Experiment.log_scalar() - Code Metrics - Inspection of "Feature/data logger" - IDSIA/sacred - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#184)

by Martin

created 2017-06-28 16:35 UTC

Experiment.log_scalar() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
c	0
b	0
f	0
dl	0
loc	17
rs	9.4285

#!/usr/bin/env python
# coding=utf-8
"""This module defines the Experiment class, which is central to sacred."""
from __future__ import division, print_function, unicode_literals

import inspect
import os.path
import shlex
import sys
from collections import OrderedDict

from sacred.arg_parser import get_config_updates, parse_args
from sacred.commandline_options import gather_command_line_options, ForceOption
from sacred.commands import print_config, print_dependencies, save_config
from sacred.config.signature import Signature
from sacred.ingredient import Ingredient
from sacred.initialize import create_run
from sacred.optional import basestring
from sacred.utils import print_filtered_stacktrace

__sacred__ = True  # marks files that should be filtered from stack traces

__all__ = ('Experiment',)


class Experiment(Ingredient):
    """
    The central class for each experiment in Sacred.

    It manages the configuration, the main function, captured methods,
    observers, commands, and further ingredients.

    An Experiment instance should be created as one of the first
    things in any experiment-file.
    """

    def __init__(self, name=None, ingredients=(), interactive=False):
        """
        Create a new experiment with the given name and optional ingredients.

        Parameters
        ----------
        name : str, optional
            Optional name of this experiment, defaults to the filename.
            (Required in interactive mode)

        ingredients : list[sacred.Ingredient], optional
            A list of ingredients to be used with this experiment.

        interactive : bool, optional
            If set to True will allow the experiment to be run in interactive
            mode (e.g. IPython or Jupyter notebooks).
            However, this mode is discouraged since it won't allow storing the
            source-code or reliable reproduction of the runs.
        """
        caller_globals = inspect.stack()[1][0].f_globals
        if name is None:
            if interactive:
                raise RuntimeError('name is required in interactive mode.')
            mainfile = caller_globals.get('__file__')
            if mainfile is None:
                raise RuntimeError('No main-file found. Are you running in '
                                   'interactive mode? If so please provide a '
                                   'name and set interactive=True.')
            name = os.path.basename(mainfile)
            if name.endswith('.py'):
                name = name[:-3]
            elif name.endswith('.pyc'):
                name = name[:-4]
        super(Experiment, self).__init__(path=name,
                                         ingredients=ingredients,
                                         interactive=interactive,
                                         _caller_globals=caller_globals)
        self.default_command = ""
        self.command(print_config, unobserved=True)
        self.command(print_dependencies, unobserved=True)
        self.command(save_config, unobserved=True)
        self.observers = []
        self.current_run = None
        self.captured_out_filter = None
        """Filter function to be applied to captured output of a run"""
        self.option_hooks = []

    # =========================== Decorators ==================================

    def main(self, function):
        """
        Decorator to define the main function of the experiment.

        The main function of an experiment is the default command that is being
        run when no command is specified, or when calling the run() method.

        Usually it is more convenient to use ``automain`` instead.
        """
        captured = self.command(function)
        self.default_command = captured.__name__
        return captured

    def automain(self, function):
        """
        Decorator that defines *and runs* the main function of the experiment.

        The decorated function is marked as the default command for this
        experiment, and the command-line interface is automatically run when
        the file is executed.

        The method decorated by this should be last in the file because is
        equivalent to:

        .. code-block:: python

            @ex.main
            def my_main():
                pass

            if __name__ == '__main__':
                ex.run_commandline()
        """
        captured = self.main(function)
        if function.__module__ == '__main__':
            # Ensure that automain is not used in interactive mode.
            import inspect
            main_filename = inspect.getfile(function)
            if (main_filename == '<stdin>' or
                    (main_filename.startswith('<ipython-input-') and
                     main_filename.endswith('>'))):
                raise RuntimeError('Cannot use @ex.automain decorator in '
                                   'interactive mode. Use @ex.main instead.')

            self.run_commandline()
        return captured

    def option_hook(self, function):
        """
        Decorator for adding an option hook function.

        An option hook is a function that is called right before a run
        is created. It receives (and potentially modifies) the options
        dictionary. That is, the dictionary of commandline options used for
        this run.

        .. note::
            The decorated function MUST have an argument called options.

            The options also contain ``'COMMAND'`` and ``'UPDATE'`` entries,
            but changing them has no effect. Only modification on
            flags (entries starting with ``'--'``) are considered.
        """
        sig = Signature(function)
        if "options" not in sig.arguments:
            raise KeyError("option_hook functions must have an argument called"
                           " 'options', but got {}".format(sig.arguments))
        self.option_hooks.append(function)
        return function

    # =========================== Public Interface ============================

    def run(self, command_name=None, config_updates=None, named_configs=(),
            meta_info=None, options=None):
        """
        Run the main function of the experiment or a given command.

        Parameters
        ----------
        command_name : str, optional
            Name of the command to be run. Defaults to main function.

        config_updates : dict, optional
            Changes to the configuration as a nested dictionary

        named_configs : list[str], optional
            list of names of named_configs to use

        meta_info : dict, optional
            Additional meta information for this run.

        options : dict, optional
            Dictionary of options to use

        Returns
        -------
        sacred.run.Run
            the Run object corresponding to the finished run
        """
        run = self._create_run(command_name, config_updates, named_configs,
                               meta_info, options)
        run()
        return run

    def run_command(self, command_name, config_updates=None,
                    named_configs=(), args=(), meta_info=None):
        """Run the command with the given name.

        .. note:: Deprecated in Sacred 0.7
            run_command() will be removed in Sacred 1.0.
            It is replaced by run() which can now also handle command_names.
        """
        import warnings
        warnings.warn("run_command is deprecated. Use run instead",
                      DeprecationWarning)
        return self.run(command_name, config_updates, named_configs, meta_info,
                        args)

    def run_commandline(self, argv=None):
        """
        Run the command-line interface of this experiment.

        If ``argv`` is omitted it defaults to ``sys.argv``.

        Parameters
        ----------
        argv : list[str] or str, optional
            Command-line as string or list of strings like ``sys.argv``.

        Returns
        -------
        sacred.run.Run
            The Run object corresponding to the finished run.
        """
        if argv is None:
            argv = sys.argv
        elif isinstance(argv, basestring):
            argv = shlex.split(argv)
        else:
            if not isinstance(argv, (list, tuple)):
                raise ValueError("argv must be str or list, but was {}"
                                 .format(type(argv)))
            if not all([isinstance(a, basestring) for a in argv]):
                problems = [a for a in argv if not isinstance(a, basestring)]
                raise ValueError("argv must be list of str but contained the "
                                 "following elements: {}".format(problems))

        all_commands = self.gather_commands()

        args = parse_args(argv,
                          description=self.doc,
                          commands=OrderedDict(all_commands))
        config_updates, named_configs = get_config_updates(args['UPDATE'])
        cmd_name = args.get('COMMAND') or self.default_command

        try:
            return self.run(cmd_name, config_updates, named_configs, {}, args)
        except Exception:
            if not self.current_run or self.current_run.debug:
                raise
            elif self.current_run.pdb:
                import traceback
                import pdb
                traceback.print_exception(*sys.exc_info())
                pdb.post_mortem()
            else:
                print_filtered_stacktrace()

    def open_resource(self, filename, mode='r'):
        """Open a file and also save it as a resource.

        Opens a file, reports it to the observers as a resource, and returns
        the opened file.

        In Sacred terminology a resource is a file that the experiment needed
        to access during a run. In case of a MongoObserver that means making
        sure the file is stored in the database (but avoiding duplicates) along
        its path and md5 sum.

        This function can only be called during a run, and just calls the
        :py:meth:`sacred.run.Run.open_resource` method.

        Parameters
        ----------
        filename: str
            name of the file that should be opened
        mode : str
            mode that file will be open

        Returns
        -------
        file
            the opened file-object
        """
        assert self.current_run is not None, "Can only be called during a run."
        return self.current_run.open_resource(filename, mode)

    def add_resource(self, filename):
        """Add a file as a resource.

        In Sacred terminology a resource is a file that the experiment needed
        to access during a run. In case of a MongoObserver that means making
        sure the file is stored in the database (but avoiding duplicates) along
        its path and md5 sum.

        This function can only be called during a run, and just calls the
        :py:meth:`sacred.run.Run.add_resource` method.

        Parameters
        ----------
        filename : str
            name of the file to be stored as a resource
        """
        assert self.current_run is not None, "Can only be called during a run."
        self.current_run.add_resource(filename)

    def add_artifact(self, filename, name=None):
        """Add a file as an artifact.

        In Sacred terminology an artifact is a file produced by the experiment
        run. In case of a MongoObserver that means storing the file in the
        database.

        This function can only be called during a run, and just calls the
        :py:meth:`sacred.run.Run.add_artifact` method.

        Parameters
        ----------
        filename : str
            name of the file to be stored as artifact
        name : str, optional
            optionally set the name of the artifact.
            Defaults to the relative file-path.
        """
        assert self.current_run is not None, "Can only be called during a run."
        self.current_run.add_artifact(filename, name)

    @property
    def info(self):
        """Access the info-dict for storing custom information.

        Only works during a run and is essentially a shortcut to:

        .. code-block:: python

            @ex.capture
            def my_captured_function(_run):
                # [...]
                _run.info   # == ex.info
        """
        return self.current_run.info

    def log_scalar(self, name, value, step=None):
        """
        Add a new measurement.

        The measurement will be processed by the MongoDB* observer
        during a heartbeat event.
        Other observers are not yet supported.

        :param metric_name: The name of the metric, e.g. training.loss
        :param value: The measured value
        :param step: The step number (integer), e.g. the iteration number
                    If not specified, an internal counter for each metric
                    is used, incremented by one.
        """
        # Method added in change https://github.com/chovanecm/sacred/issues/4
        # The same as Run.log_scalar
        return self.current_run.log_scalar(name, value, step)

    def gather_commands(self):
        """Iterator over all commands of this experiment.

        Also recursively collects all commands from ingredients.

        Yields
        ------
        (str, function)
            A tuple consisting of the (dotted) command-name and the
            corresponding captured function.
        """
        for cmd_name, cmd in self.commands.items():
            yield cmd_name, cmd

        for ingred in self.ingredients:
            for cmd_name, cmd in ingred.gather_commands():
                yield cmd_name, cmd

    def get_default_options(self):
        """Get a dictionary of default options as used with run.

        Returns
        -------
        dict
            A dictionary containing option keys of the form '--beat_interval'.
            Their values are boolean if the option is a flag, otherwise None or
            its default value.
        """
        all_commands = self.gather_commands()
        args = parse_args(["dummy"],
                          description=self.doc,
                          commands=OrderedDict(all_commands))
        return {k: v for k, v in args.items() if k.startswith('--')}

    # =========================== Internal Interface ==========================

    def _create_run(self, command_name=None, config_updates=None,
                    named_configs=(), meta_info=None, options=None):
        command_name = command_name or self.default_command
        if command_name is None:
            raise RuntimeError('No command found to be run. Specify a command '
                               'or define a main function.')

        default_options = self.get_default_options()
        if options:
            default_options.update(options)
        options = default_options

        # call option hooks
        for oh in self.option_hooks:
            oh(options=options)

        run = create_run(self, command_name, config_updates,
                         named_configs=named_configs,
                         force=options.get(ForceOption.get_flag(), False))
        run.meta_info['command'] = command_name
        run.meta_info['options'] = options

        if meta_info:
            run.meta_info.update(meta_info)

        for option in gather_command_line_options():
            option_value = options.get(option.get_flag(), False)
            if option_value:
                option.apply(option_value, run)

        self.current_run = run
        return run


1			#!/usr/bin/env python
2			# coding=utf-8
3			"""This module defines the Experiment class, which is central to sacred."""
4			from __future__ import division, print_function, unicode_literals
5
6			import inspect
7			import os.path
8			import shlex
9			import sys
10			from collections import OrderedDict
11
12			from sacred.arg_parser import get_config_updates, parse_args
13			from sacred.commandline_options import gather_command_line_options, ForceOption
14			from sacred.commands import print_config, print_dependencies, save_config
15			from sacred.config.signature import Signature
16			from sacred.ingredient import Ingredient
17			from sacred.initialize import create_run
18			from sacred.optional import basestring
19			from sacred.utils import print_filtered_stacktrace
20
21			__sacred__ = True # marks files that should be filtered from stack traces
22
23			__all__ = ('Experiment',)
24
25
26			class Experiment(Ingredient):
27			"""
28			The central class for each experiment in Sacred.
29
30			It manages the configuration, the main function, captured methods,
31			observers, commands, and further ingredients.
32
33			An Experiment instance should be created as one of the first
34			things in any experiment-file.
35			"""
36
37			def __init__(self, name=None, ingredients=(), interactive=False):
38			"""
39			Create a new experiment with the given name and optional ingredients.
40
41			Parameters
42			----------
43			name : str, optional
44			Optional name of this experiment, defaults to the filename.
45			(Required in interactive mode)
46
47			ingredients : list[sacred.Ingredient], optional
48			A list of ingredients to be used with this experiment.
49
50			interactive : bool, optional
51			If set to True will allow the experiment to be run in interactive
52			mode (e.g. IPython or Jupyter notebooks).
53			However, this mode is discouraged since it won't allow storing the
54			source-code or reliable reproduction of the runs.
55			"""
56			caller_globals = inspect.stack()[1][0].f_globals
57			if name is None:
58			if interactive:
59			raise RuntimeError('name is required in interactive mode.')
60			mainfile = caller_globals.get('__file__')
61			if mainfile is None:
62			raise RuntimeError('No main-file found. Are you running in '
63			'interactive mode? If so please provide a '
64			'name and set interactive=True.')
65			name = os.path.basename(mainfile)
66			if name.endswith('.py'):
67			name = name[:-3]
68			elif name.endswith('.pyc'):
69			name = name[:-4]
70			super(Experiment, self).__init__(path=name,
71			ingredients=ingredients,
72			interactive=interactive,
73			_caller_globals=caller_globals)
74			self.default_command = ""
75			self.command(print_config, unobserved=True)
76			self.command(print_dependencies, unobserved=True)
77			self.command(save_config, unobserved=True)
78			self.observers = []
79			self.current_run = None
80			self.captured_out_filter = None
81			"""Filter function to be applied to captured output of a run"""
82			self.option_hooks = []
83
84			# =========================== Decorators ==================================
85
86			def main(self, function):
87			"""
88			Decorator to define the main function of the experiment.
89
90			The main function of an experiment is the default command that is being
91			run when no command is specified, or when calling the run() method.
92
93			Usually it is more convenient to use ``automain`` instead.
94			"""
95			captured = self.command(function)
96			self.default_command = captured.__name__
97			return captured
98
99			def automain(self, function):
100			"""
101			Decorator that defines and runs the main function of the experiment.
102
103			The decorated function is marked as the default command for this
104			experiment, and the command-line interface is automatically run when
105			the file is executed.
106
107			The method decorated by this should be last in the file because is
108			equivalent to:
109
110			.. code-block:: python
111
112			@ex.main
113			def my_main():
114			pass
115
116			if __name__ == '__main__':
117			ex.run_commandline()
118			"""
119			captured = self.main(function)
120			if function.__module__ == '__main__':
121			# Ensure that automain is not used in interactive mode.
122			import inspect
123			main_filename = inspect.getfile(function)
124			if (main_filename == '<stdin>' or
125			(main_filename.startswith('<ipython-input-') and
126			main_filename.endswith('>'))):
127			raise RuntimeError('Cannot use @ex.automain decorator in '
128			'interactive mode. Use @ex.main instead.')
129
130			self.run_commandline()
131			return captured
132
133			def option_hook(self, function):
134			"""
135			Decorator for adding an option hook function.
136
137			An option hook is a function that is called right before a run
138			is created. It receives (and potentially modifies) the options
139			dictionary. That is, the dictionary of commandline options used for
140			this run.
141
142			.. note::
143			The decorated function MUST have an argument called options.
144
145			The options also contain ``'COMMAND'`` and ``'UPDATE'`` entries,
146			but changing them has no effect. Only modification on
147			flags (entries starting with ``'--'``) are considered.
148			"""
149			sig = Signature(function)
150			if "options" not in sig.arguments:
151			raise KeyError("option_hook functions must have an argument called"
152			" 'options', but got {}".format(sig.arguments))
153			self.option_hooks.append(function)
154			return function
155
156			# =========================== Public Interface ============================
157
158			def run(self, command_name=None, config_updates=None, named_configs=(),
159			meta_info=None, options=None):
160			"""
161			Run the main function of the experiment or a given command.
162
163			Parameters
164			----------
165			command_name : str, optional
166			Name of the command to be run. Defaults to main function.
167
168			config_updates : dict, optional
169			Changes to the configuration as a nested dictionary
170
171			named_configs : list[str], optional
172			list of names of named_configs to use
173
174			meta_info : dict, optional
175			Additional meta information for this run.
176
177			options : dict, optional
178			Dictionary of options to use
179
180			Returns
181			-------
182			sacred.run.Run
183			the Run object corresponding to the finished run
184			"""
185			run = self._create_run(command_name, config_updates, named_configs,
186			meta_info, options)
187			run()
188			return run
189
190			def run_command(self, command_name, config_updates=None,
191			named_configs=(), args=(), meta_info=None):
192			"""Run the command with the given name.
193
194			.. note:: Deprecated in Sacred 0.7
195			run_command() will be removed in Sacred 1.0.
196			It is replaced by run() which can now also handle command_names.
197			"""
198			import warnings
199			warnings.warn("run_command is deprecated. Use run instead",
200			DeprecationWarning)
201			return self.run(command_name, config_updates, named_configs, meta_info,
202			args)
203
204			def run_commandline(self, argv=None):
205			"""
206			Run the command-line interface of this experiment.
207
208			If ``argv`` is omitted it defaults to ``sys.argv``.
209
210			Parameters
211			----------
212			argv : list[str] or str, optional
213			Command-line as string or list of strings like ``sys.argv``.
214
215			Returns
216			-------
217			sacred.run.Run
218			The Run object corresponding to the finished run.
219			"""
220			if argv is None:
221			argv = sys.argv
222			elif isinstance(argv, basestring):
223			argv = shlex.split(argv)
224			else:
225			if not isinstance(argv, (list, tuple)):
226			raise ValueError("argv must be str or list, but was {}"
227			.format(type(argv)))
228			if not all([isinstance(a, basestring) for a in argv]):
229			problems = [a for a in argv if not isinstance(a, basestring)]
230			raise ValueError("argv must be list of str but contained the "
231			"following elements: {}".format(problems))
232
233			all_commands = self.gather_commands()
234
235			args = parse_args(argv,
236			description=self.doc,
237			commands=OrderedDict(all_commands))
238			config_updates, named_configs = get_config_updates(args['UPDATE'])
239			cmd_name = args.get('COMMAND') or self.default_command
240
241			try:
242			return self.run(cmd_name, config_updates, named_configs, {}, args)
243			except Exception:
244			if not self.current_run or self.current_run.debug:
245			raise
246			elif self.current_run.pdb:
247			import traceback
248			import pdb
249			traceback.print_exception(*sys.exc_info())
250			pdb.post_mortem()
251			else:
252			print_filtered_stacktrace()
253
254			def open_resource(self, filename, mode='r'):
255			"""Open a file and also save it as a resource.
256
257			Opens a file, reports it to the observers as a resource, and returns
258			the opened file.
259
260			In Sacred terminology a resource is a file that the experiment needed
261			to access during a run. In case of a MongoObserver that means making
262			sure the file is stored in the database (but avoiding duplicates) along
263			its path and md5 sum.
264
265			This function can only be called during a run, and just calls the
266			:py:meth:`sacred.run.Run.open_resource` method.
267
268			Parameters
269			----------
270			filename: str
271			name of the file that should be opened
272			mode : str
273			mode that file will be open
274
275			Returns
276			-------
277			file
278			the opened file-object
279			"""
280			assert self.current_run is not None, "Can only be called during a run."
281			return self.current_run.open_resource(filename, mode)
282
283			def add_resource(self, filename):
284			"""Add a file as a resource.
285
286			In Sacred terminology a resource is a file that the experiment needed
287			to access during a run. In case of a MongoObserver that means making
288			sure the file is stored in the database (but avoiding duplicates) along
289			its path and md5 sum.
290
291			This function can only be called during a run, and just calls the
292			:py:meth:`sacred.run.Run.add_resource` method.
293
294			Parameters
295			----------
296			filename : str
297			name of the file to be stored as a resource
298			"""
299			assert self.current_run is not None, "Can only be called during a run."
300			self.current_run.add_resource(filename)
301
302			def add_artifact(self, filename, name=None):
303			"""Add a file as an artifact.
304
305			In Sacred terminology an artifact is a file produced by the experiment
306			run. In case of a MongoObserver that means storing the file in the
307			database.
308
309			This function can only be called during a run, and just calls the
310			:py:meth:`sacred.run.Run.add_artifact` method.
311
312			Parameters
313			----------
314			filename : str
315			name of the file to be stored as artifact
316			name : str, optional
317			optionally set the name of the artifact.
318			Defaults to the relative file-path.
319			"""
320			assert self.current_run is not None, "Can only be called during a run."
321			self.current_run.add_artifact(filename, name)
322
323			@property
324			def info(self):
325			"""Access the info-dict for storing custom information.
326
327			Only works during a run and is essentially a shortcut to:
328
329			.. code-block:: python
330
331			@ex.capture
332			def my_captured_function(_run):
333			# [...]
334			_run.info # == ex.info
335			"""
336			return self.current_run.info
337
338			def log_scalar(self, name, value, step=None):
339			"""
340			Add a new measurement.
341
342			The measurement will be processed by the MongoDB* observer
343			during a heartbeat event.
344			Other observers are not yet supported.
345
346			:param metric_name: The name of the metric, e.g. training.loss
347			:param value: The measured value
348			:param step: The step number (integer), e.g. the iteration number
349			If not specified, an internal counter for each metric
350			is used, incremented by one.
351			"""
352			# Method added in change https://github.com/chovanecm/sacred/issues/4
353			# The same as Run.log_scalar
354			return self.current_run.log_scalar(name, value, step)
355
356			def gather_commands(self):
357			"""Iterator over all commands of this experiment.
358
359			Also recursively collects all commands from ingredients.
360
361			Yields
362			------
363			(str, function)
364			A tuple consisting of the (dotted) command-name and the
365			corresponding captured function.
366			"""
367			for cmd_name, cmd in self.commands.items():
368			yield cmd_name, cmd
369
370			for ingred in self.ingredients:
371			for cmd_name, cmd in ingred.gather_commands():
372			yield cmd_name, cmd
373
374			def get_default_options(self):
375			"""Get a dictionary of default options as used with run.
376
377			Returns
378			-------
379			dict
380			A dictionary containing option keys of the form '--beat_interval'.
381			Their values are boolean if the option is a flag, otherwise None or
382			its default value.
383			"""
384			all_commands = self.gather_commands()
385			args = parse_args(["dummy"],
386			description=self.doc,
387			commands=OrderedDict(all_commands))
388			return {k: v for k, v in args.items() if k.startswith('--')}
389
390			# =========================== Internal Interface ==========================
391
392			def _create_run(self, command_name=None, config_updates=None,
393			named_configs=(), meta_info=None, options=None):
394			command_name = command_name or self.default_command
395			if command_name is None:
396			raise RuntimeError('No command found to be run. Specify a command '
397			'or define a main function.')
398
399			default_options = self.get_default_options()
400			if options:
401			default_options.update(options)
402			options = default_options
403
404			# call option hooks
405			for oh in self.option_hooks:
406			oh(options=options)
407
408			run = create_run(self, command_name, config_updates,
409			named_configs=named_configs,
410			force=options.get(ForceOption.get_flag(), False))
411			run.meta_info['command'] = command_name
412			run.meta_info['options'] = options
413
414			if meta_info:
415			run.meta_info.update(meta_info)
416
417			for option in gather_command_line_options():
418			option_value = options.get(option.get_flag(), False)
419			if option_value:
420			option.apply(option_value, run)
421
422			self.current_run = run
423			return run
424

IDSIA / sacred

Pull Request — master (#184)

Experiment.log_scalar() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like