|
1
|
|
|
#!/usr/bin/env python |
|
2
|
|
|
# coding=utf-8 |
|
3
|
|
|
"""Defines the stock-commands that every sacred experiment ships with.""" |
|
4
|
|
|
from __future__ import division, print_function, unicode_literals |
|
5
|
|
|
|
|
6
|
|
|
import pprint |
|
7
|
|
|
import pydoc |
|
8
|
|
|
import re |
|
9
|
|
|
from collections import namedtuple |
|
10
|
|
|
|
|
11
|
|
|
from sacred.config import save_config_file |
|
12
|
|
|
from sacred.serializer import flatten |
|
13
|
|
|
from sacred.utils import PATHCHANGE, iterate_flattened_separately |
|
14
|
|
|
|
|
15
|
|
|
__sacred__ = True # marks files that should be filtered from stack traces |
|
16
|
|
|
|
|
17
|
|
|
__all__ = ('print_config', 'print_dependencies', 'save_config', |
|
18
|
|
|
'help_for_command') |
|
19
|
|
|
|
|
20
|
|
|
BLUE = '\033[94m' |
|
21
|
|
|
GREEN = '\033[92m' |
|
22
|
|
|
RED = '\033[91m' |
|
23
|
|
|
GREY = '\033[90m' |
|
24
|
|
|
ENDC = '\033[0m' |
|
25
|
|
|
|
|
26
|
|
|
LEGEND = '(' + BLUE + 'modified' + ENDC +\ |
|
27
|
|
|
', ' + GREEN + 'added' + ENDC +\ |
|
28
|
|
|
', ' + RED + 'typechanged' + ENDC +\ |
|
29
|
|
|
', ' + GREY + 'doc' + ENDC + ')' |
|
30
|
|
|
|
|
31
|
|
|
ConfigEntry = namedtuple('ConfigEntry', |
|
32
|
|
|
'key value added modified typechanged doc') |
|
33
|
|
|
PathEntry = namedtuple('PathEntry', 'key added modified typechanged doc') |
|
34
|
|
|
|
|
35
|
|
|
|
|
36
|
|
|
def _non_unicode_repr(objekt, context, maxlevels, level): |
|
37
|
|
|
""" |
|
38
|
|
|
Used to override the pprint format method to get rid of unicode prefixes. |
|
39
|
|
|
|
|
40
|
|
|
E.g.: 'John' instead of u'John'. |
|
41
|
|
|
""" |
|
42
|
|
|
repr_string, isreadable, isrecursive = pprint._safe_repr(objekt, context, |
|
43
|
|
|
maxlevels, level) |
|
44
|
|
|
if repr_string.startswith('u"') or repr_string.startswith("u'"): |
|
45
|
|
|
repr_string = repr_string[1:] |
|
46
|
|
|
return repr_string, isreadable, isrecursive |
|
47
|
|
|
|
|
48
|
|
|
|
|
49
|
|
|
PRINTER = pprint.PrettyPrinter() |
|
50
|
|
|
PRINTER.format = _non_unicode_repr |
|
51
|
|
|
|
|
52
|
|
|
|
|
53
|
|
|
def print_config(_run): |
|
54
|
|
|
""" |
|
55
|
|
|
Print the updated configuration and exit. |
|
56
|
|
|
|
|
57
|
|
|
Text is highlighted: |
|
58
|
|
|
green: value modified |
|
59
|
|
|
blue: value added |
|
60
|
|
|
red: value modified but type changed |
|
61
|
|
|
""" |
|
62
|
|
|
final_config = _run.config |
|
63
|
|
|
config_mods = _run.config_modifications |
|
64
|
|
|
print(_format_config(final_config, config_mods)) |
|
65
|
|
|
|
|
66
|
|
|
|
|
67
|
|
|
def help_for_command(command): |
|
68
|
|
|
"""Get the help text (signature + docstring) for a command (function).""" |
|
69
|
|
|
help_text = pydoc.text.document(command) |
|
70
|
|
|
# remove backspaces |
|
71
|
|
|
return re.subn('.\\x08', '', help_text)[0] |
|
72
|
|
|
|
|
73
|
|
|
|
|
74
|
|
|
def print_dependencies(_run): |
|
75
|
|
|
"""Print the detected source-files and dependencies.""" |
|
76
|
|
|
print('Dependencies:') |
|
77
|
|
|
for dep in _run.experiment_info['dependencies']: |
|
78
|
|
|
pack, _, version = dep.partition('==') |
|
79
|
|
|
print(' {:<20} == {}'.format(pack, version)) |
|
80
|
|
|
|
|
81
|
|
|
print('\nSources:') |
|
82
|
|
|
for source, digest in _run.experiment_info['sources']: |
|
83
|
|
|
print(' {:<43} {}'.format(source, digest)) |
|
84
|
|
|
|
|
85
|
|
|
if _run.experiment_info['repositories']: |
|
86
|
|
|
repos = _run.experiment_info['repositories'] |
|
87
|
|
|
print('\nVersion Control:') |
|
88
|
|
|
for repo in repos: |
|
89
|
|
|
mod = RED + 'M' if repo['dirty'] else ' ' |
|
90
|
|
|
print('{} {:<43} {}'.format(mod, repo['url'], repo['commit']) + |
|
91
|
|
|
ENDC) |
|
92
|
|
|
print('') |
|
93
|
|
|
|
|
94
|
|
|
|
|
95
|
|
|
def save_config(_config, _log, config_filename='config.json'): |
|
96
|
|
|
""" |
|
97
|
|
|
Store the updated configuration in a file. |
|
98
|
|
|
|
|
99
|
|
|
By default uses the filename "config.json", but that can be changed by |
|
100
|
|
|
setting the config_filename config entry. |
|
101
|
|
|
""" |
|
102
|
|
|
if 'config_filename' in _config: |
|
103
|
|
|
del _config['config_filename'] |
|
104
|
|
|
_log.info('Saving config to "{}"'.format(config_filename)) |
|
105
|
|
|
save_config_file(flatten(_config), config_filename) |
|
106
|
|
|
|
|
107
|
|
|
|
|
108
|
|
|
def _iterate_marked(cfg, config_mods): |
|
109
|
|
|
for path, value in iterate_flattened_separately(cfg, ['__doc__']): |
|
110
|
|
|
if value is PATHCHANGE: |
|
111
|
|
|
yield path, PathEntry( |
|
112
|
|
|
key=path.rpartition('.')[2], |
|
113
|
|
|
added=path in config_mods.added, |
|
114
|
|
|
modified=path in config_mods.modified, |
|
115
|
|
|
typechanged=config_mods.typechanged.get(path), |
|
116
|
|
|
doc=config_mods.docs.get(path)) |
|
117
|
|
|
else: |
|
118
|
|
|
yield path, ConfigEntry( |
|
119
|
|
|
key=path.rpartition('.')[2], |
|
120
|
|
|
value=value, |
|
121
|
|
|
added=path in config_mods.added, |
|
122
|
|
|
modified=path in config_mods.modified, |
|
123
|
|
|
typechanged=config_mods.typechanged.get(path), |
|
124
|
|
|
doc=config_mods.docs.get(path)) |
|
125
|
|
|
|
|
126
|
|
|
|
|
127
|
|
|
def _format_entry(indent, entry): |
|
128
|
|
|
color = "" |
|
129
|
|
|
indent = ' ' * indent |
|
130
|
|
|
if entry.typechanged: |
|
131
|
|
|
color = RED |
|
132
|
|
|
elif entry.added: |
|
133
|
|
|
color = GREEN |
|
134
|
|
|
elif entry.modified: |
|
135
|
|
|
color = BLUE |
|
136
|
|
|
if entry.key == '__doc__': |
|
137
|
|
|
color = GREY |
|
138
|
|
|
doc_string = entry.value.replace('\n', '\n' + indent) |
|
139
|
|
|
assign = '{}"""{}"""'.format(indent, doc_string) |
|
140
|
|
|
elif isinstance(entry, ConfigEntry): |
|
141
|
|
|
assign = indent + entry.key + " = " + PRINTER.pformat(entry.value) |
|
142
|
|
|
else: # isinstance(entry, PathEntry): |
|
143
|
|
|
assign = indent + entry.key + ":" |
|
144
|
|
|
if entry.doc: |
|
145
|
|
|
doc_string = GREY + '# ' + entry.doc + ENDC |
|
146
|
|
|
if len(assign) <= 35: |
|
147
|
|
|
assign = "{:<35} {}".format(assign, doc_string) |
|
148
|
|
|
else: |
|
149
|
|
|
assign += ' ' + doc_string |
|
150
|
|
|
end = ENDC if color else "" |
|
151
|
|
|
return color + assign + end |
|
152
|
|
|
|
|
153
|
|
|
|
|
154
|
|
|
def _format_config(cfg, config_mods): |
|
155
|
|
|
lines = ['Configuration ' + LEGEND + ':'] |
|
156
|
|
|
for path, entry in _iterate_marked(cfg, config_mods): |
|
157
|
|
|
indent = 2 + 2 * path.count('.') |
|
158
|
|
|
lines.append(_format_entry(indent, entry)) |
|
159
|
|
|
return "\n".join(lines) |
|
160
|
|
|
|
|
161
|
|
|
|
|
162
|
|
|
def _write_file(base_dir, filename, content, mode='t'): |
|
163
|
|
|
full_name = os.path.join(base_dir, filename) |
|
164
|
|
|
os.makedirs(os.path.dirname(full_name), exist_ok=True) |
|
165
|
|
|
with open(full_name, 'w' + mode) as f: |
|
166
|
|
|
f.write(content) |
|
167
|
|
|
|
|
168
|
|
|
|
|
169
|
|
|
def _get_truncated_python_version(host_info): |
|
170
|
|
|
version = parse_version(host_info['python_version']) |
|
171
|
|
|
return '{}.{}'.format(*version._version.release[:2]) |
|
172
|
|
|
|
|
173
|
|
|
|
IDSIA /
sacred
