|
1
|
|
|
#!/usr/bin/env python |
|
2
|
|
|
# coding=utf-8 |
|
3
|
|
|
from __future__ import division, print_function, unicode_literals |
|
4
|
|
|
|
|
5
|
|
|
from sacred.observers.base import RunObserver |
|
6
|
|
|
from sacred.config.config_files import load_config_file |
|
7
|
|
|
import json |
|
8
|
|
|
|
|
9
|
|
|
|
|
10
|
|
|
DEFAULT_SLACK_PRIORITY = 10 |
|
11
|
|
|
|
|
12
|
|
|
|
|
13
|
|
|
# http://stackoverflow.com/questions/538666/python-format-timedelta-to-string |
|
14
|
|
|
def td_format(td_object): |
|
15
|
|
|
seconds = int(td_object.total_seconds()) |
|
16
|
|
|
if seconds == 0: |
|
17
|
|
|
return "less than a second" |
|
18
|
|
|
|
|
19
|
|
|
periods = [ |
|
20
|
|
|
('year', 60 * 60 * 24 * 365), |
|
21
|
|
|
('month', 60 * 60 * 24 * 30), |
|
22
|
|
|
('day', 60 * 60 * 24), |
|
23
|
|
|
('hour', 60 * 60), |
|
24
|
|
|
('minute', 60), |
|
25
|
|
|
('second', 1) |
|
26
|
|
|
] |
|
27
|
|
|
|
|
28
|
|
|
strings = [] |
|
29
|
|
|
for period_name, period_seconds in periods: |
|
30
|
|
|
if seconds >= period_seconds: |
|
31
|
|
|
period_value, seconds = divmod(seconds, period_seconds) |
|
32
|
|
|
if period_value == 1: |
|
33
|
|
|
strings.append("%s %s" % (period_value, period_name)) |
|
34
|
|
|
else: |
|
35
|
|
|
strings.append("%s %ss" % (period_value, period_name)) |
|
36
|
|
|
|
|
37
|
|
|
return ", ".join(strings) |
|
38
|
|
|
|
|
39
|
|
|
|
|
40
|
|
|
class SlackObserver(RunObserver): |
|
41
|
|
|
"""Sends a message to Slack upon completion/failing of an experiment.""" |
|
42
|
|
|
|
|
43
|
|
|
@classmethod |
|
44
|
|
|
def from_config(cls, filename): |
|
45
|
|
|
""" |
|
46
|
|
|
Create a SlackObserver from a given configuration file. |
|
47
|
|
|
|
|
48
|
|
|
The file can be in any format supported by Sacred |
|
49
|
|
|
(.json, .pickle, [.yaml]). |
|
50
|
|
|
It has to specify a ``webhook_url`` and can optionally set |
|
51
|
|
|
``bot_name``, ``icon``, ``completed_text``, ``interrupted_text``, and |
|
52
|
|
|
``failed_text``. |
|
53
|
|
|
""" |
|
54
|
|
|
d = load_config_file(filename) |
|
55
|
|
|
obs = None |
|
56
|
|
|
if 'webhook_url' in d: |
|
57
|
|
|
obs = cls(d['webhook_url']) |
|
58
|
|
|
else: |
|
59
|
|
|
raise ValueError("Slack configuration file must contain " |
|
60
|
|
|
"an entry for 'webhook_url'!") |
|
61
|
|
|
for k in ['completed_text', 'interrupted_text', 'failed_text', |
|
62
|
|
|
'bot_name', 'icon']: |
|
63
|
|
|
if k in d: |
|
64
|
|
|
setattr(obs, k, d[k]) |
|
65
|
|
|
return obs |
|
66
|
|
|
|
|
67
|
|
|
def __init__(self, webhook_url, bot_name="sacred-bot", icon=":angel:", |
|
68
|
|
|
priority=DEFAULT_SLACK_PRIORITY): |
|
69
|
|
|
self.webhook_url = webhook_url |
|
70
|
|
|
self.bot_name = bot_name |
|
71
|
|
|
self.icon = icon |
|
72
|
|
|
self.completed_text = ":white_check_mark: *{experiment[name]}* " \ |
|
73
|
|
|
"completed after _{elapsed_time}_ with result=`{result}`" |
|
74
|
|
|
self.interrupted_text = ":warning: *{experiment[name]}* " \ |
|
75
|
|
|
"interrupted after _{elapsed_time}_" |
|
76
|
|
|
self.failed_text = ":x: *{experiment[name]}* failed after " \ |
|
77
|
|
|
"_{elapsed_time}_ with `{error}`" |
|
78
|
|
|
self.run = None |
|
79
|
|
|
self.priority = priority |
|
80
|
|
|
|
|
81
|
|
|
def started_event(self, ex_info, command, host_info, start_time, config, |
|
82
|
|
|
meta_info, _id): |
|
83
|
|
|
self.run = { |
|
84
|
|
|
'_id': _id, |
|
85
|
|
|
'config': config, |
|
86
|
|
|
'start_time': start_time, |
|
87
|
|
|
'experiment': ex_info, |
|
88
|
|
|
'command': command, |
|
89
|
|
|
'host_info': host_info, |
|
90
|
|
|
} |
|
91
|
|
|
|
|
92
|
|
|
def get_completed_text(self): |
|
93
|
|
|
return self.completed_text.format(**self.run) |
|
94
|
|
|
|
|
95
|
|
|
def get_interrupted_text(self): |
|
96
|
|
|
return self.interrupted_text.format(**self.run) |
|
97
|
|
|
|
|
98
|
|
|
def get_failed_text(self): |
|
99
|
|
|
return self.failed_text.format(**self.run) |
|
100
|
|
|
|
|
101
|
|
View Code Duplication |
def completed_event(self, stop_time, result): |
|
|
|
|
|
|
102
|
|
|
import requests |
|
103
|
|
|
if self.completed_text is None: |
|
104
|
|
|
return |
|
105
|
|
|
|
|
106
|
|
|
self.run['result'] = result |
|
107
|
|
|
self.run['stop_time'] = stop_time |
|
108
|
|
|
self.run['elapsed_time'] = td_format(stop_time - |
|
109
|
|
|
self.run['start_time']) |
|
110
|
|
|
|
|
111
|
|
|
data = { |
|
112
|
|
|
"username": self.bot_name, |
|
113
|
|
|
"icon_emoji": self.icon, |
|
114
|
|
|
"text": self.get_completed_text() |
|
115
|
|
|
} |
|
116
|
|
|
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} |
|
117
|
|
|
requests.post(self.webhook_url, data=json.dumps(data), headers=headers) |
|
118
|
|
|
|
|
119
|
|
View Code Duplication |
def interrupted_event(self, interrupt_time, status): |
|
|
|
|
|
|
120
|
|
|
import requests |
|
121
|
|
|
if self.interrupted_text is None: |
|
122
|
|
|
return |
|
123
|
|
|
|
|
124
|
|
|
self.run['status'] = status |
|
125
|
|
|
self.run['interrupt_time'] = interrupt_time |
|
126
|
|
|
self.run['elapsed_time'] = td_format(interrupt_time - |
|
127
|
|
|
self.run['start_time']) |
|
128
|
|
|
|
|
129
|
|
|
data = { |
|
130
|
|
|
"username": self.bot_name, |
|
131
|
|
|
"icon_emoji": self.icon, |
|
132
|
|
|
"text": self.get_interrupted_text() |
|
133
|
|
|
} |
|
134
|
|
|
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} |
|
135
|
|
|
requests.post(self.webhook_url, data=json.dumps(data), headers=headers) |
|
136
|
|
|
|
|
137
|
|
|
def failed_event(self, fail_time, fail_trace): |
|
138
|
|
|
import requests |
|
139
|
|
|
if self.failed_text is None: |
|
140
|
|
|
return |
|
141
|
|
|
|
|
142
|
|
|
self.run['fail_trace'] = fail_trace |
|
143
|
|
|
self.run['error'] = fail_trace[-1].strip() |
|
144
|
|
|
self.run['fail_time'] = fail_time |
|
145
|
|
|
self.run['elapsed_time'] = td_format(fail_time - |
|
146
|
|
|
self.run['start_time']) |
|
147
|
|
|
|
|
148
|
|
|
data = { |
|
149
|
|
|
"username": self.bot_name, |
|
150
|
|
|
"icon_emoji": self.icon, |
|
151
|
|
|
"text": self.get_failed_text() |
|
152
|
|
|
} |
|
153
|
|
|
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} |
|
154
|
|
|
requests.post(self.webhook_url, data=json.dumps(data), headers=headers) |
|
155
|
|
|
|
IDSIA /
sacred
Loading history...