1
|
|
|
import json |
2
|
|
|
import logging |
3
|
|
|
import os |
4
|
|
|
import sys |
5
|
|
|
from io import BytesIO |
6
|
|
|
from io import StringIO |
7
|
|
|
from pathlib import Path |
8
|
|
|
|
9
|
|
|
import py |
10
|
|
|
import pytest |
11
|
|
|
from freezegun import freeze_time |
12
|
|
|
|
13
|
|
|
from pytest_benchmark import plugin |
14
|
|
|
from pytest_benchmark.plugin import BenchmarkSession |
15
|
|
|
from pytest_benchmark.plugin import pytest_benchmark_compare_machine_info |
16
|
|
|
from pytest_benchmark.plugin import pytest_benchmark_generate_json |
17
|
|
|
from pytest_benchmark.plugin import pytest_benchmark_group_stats |
18
|
|
|
from pytest_benchmark.session import PerformanceRegression |
19
|
|
|
from pytest_benchmark.stats import normalize_stats |
20
|
|
|
from pytest_benchmark.storage.file import FileStorage |
21
|
|
|
from pytest_benchmark.utils import NAME_FORMATTERS |
22
|
|
|
from pytest_benchmark.utils import DifferenceRegressionCheck |
23
|
|
|
from pytest_benchmark.utils import PercentageRegressionCheck |
24
|
|
|
from pytest_benchmark.utils import get_machine_id |
25
|
|
|
|
26
|
|
|
pytest_plugins = "pytester" |
27
|
|
|
|
28
|
|
|
|
29
|
|
|
THIS = py.path.local(__file__) |
30
|
|
|
STORAGE = THIS.dirpath(THIS.purebasename) |
31
|
|
|
|
32
|
|
|
JSON_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open()) |
33
|
|
|
JSON_DATA["machine_info"] = {'foo': 'bar'} |
34
|
|
|
JSON_DATA["commit_info"] = {'foo': 'bar'} |
35
|
|
|
list(normalize_stats(bench['stats']) for bench in JSON_DATA["benchmarks"]) |
36
|
|
|
|
37
|
|
|
|
38
|
|
|
class Namespace(object): |
39
|
|
|
def __init__(self, **kwargs): |
40
|
|
|
self.__dict__.update(kwargs) |
41
|
|
|
|
42
|
|
|
def __getitem__(self, item): |
43
|
|
|
return self.__dict__[item] |
44
|
|
|
|
45
|
|
|
def getoption(self, item, default=None): |
46
|
|
|
try: |
47
|
|
|
return self[item] |
48
|
|
|
except KeyError: |
49
|
|
|
return default |
50
|
|
|
|
51
|
|
|
|
52
|
|
|
class LooseFileLike(BytesIO): |
53
|
|
|
def close(self): |
54
|
|
|
value = self.getvalue() |
55
|
|
|
super(LooseFileLike, self).close() |
56
|
|
|
self.getvalue = lambda: value |
57
|
|
|
|
58
|
|
|
|
59
|
|
|
class MockSession(BenchmarkSession): |
60
|
|
|
def __init__(self, name_format): |
61
|
|
|
self.histogram = True |
62
|
|
|
self.verbose = False |
63
|
|
|
self.benchmarks = [] |
64
|
|
|
self.performance_regressions = [] |
65
|
|
|
self.sort = u"min" |
66
|
|
|
self.compare = '0001' |
67
|
|
|
logger = logging.getLogger(__name__) |
68
|
|
|
self.logger = Namespace( |
69
|
|
|
debug=lambda *args, **_kwargs: logger.debug(*args), |
70
|
|
|
info=lambda *args, **_kwargs: logger.info(*args), |
71
|
|
|
warn=lambda *args, **_kwargs: logger.warn(*args), |
72
|
|
|
error=lambda *args, **_kwargs: logger.error(*args), |
73
|
|
|
) |
74
|
|
|
self.machine_id = "FoobarOS" |
75
|
|
|
self.machine_info = {'foo': 'bar'} |
76
|
|
|
self.save = self.autosave = self.json = False |
77
|
|
|
self.name_format = NAME_FORMATTERS[name_format] |
78
|
|
|
self.options = { |
79
|
|
|
'min_rounds': 123, |
80
|
|
|
'min_time': 234, |
81
|
|
|
'max_time': 345, |
82
|
|
|
'use_cprofile': False, |
83
|
|
|
} |
84
|
|
|
self.cprofile_sort_by = 'cumtime' |
85
|
|
|
self.compare_fail = [] |
86
|
|
|
self.config = Namespace(hook=Namespace( |
87
|
|
|
pytest_benchmark_group_stats=pytest_benchmark_group_stats, |
88
|
|
|
pytest_benchmark_generate_machine_info=lambda **kwargs: {'foo': 'bar'}, |
89
|
|
|
pytest_benchmark_update_machine_info=lambda **kwargs: None, |
90
|
|
|
pytest_benchmark_compare_machine_info=pytest_benchmark_compare_machine_info, |
91
|
|
|
pytest_benchmark_generate_json=pytest_benchmark_generate_json, |
92
|
|
|
pytest_benchmark_update_json=lambda **kwargs: None, |
93
|
|
|
pytest_benchmark_generate_commit_info=lambda **kwargs: {'foo': 'bar'}, |
94
|
|
|
pytest_benchmark_update_commit_info=lambda **kwargs: None, |
95
|
|
|
)) |
96
|
|
|
self.storage = FileStorage(str(STORAGE), default_machine_id=get_machine_id(), logger=self.logger) |
97
|
|
|
self.group_by = 'group' |
98
|
|
|
self.columns = ['min', 'max', 'mean', 'stddev', 'median', 'iqr', |
99
|
|
|
'outliers', 'rounds', 'iterations', 'ops'] |
100
|
|
|
for bench_file, data in reversed(list(self.storage.load("[0-9][0-9][0-9][0-9]_*"))): |
101
|
|
|
self.benchmarks.extend( |
102
|
|
|
Namespace( |
103
|
|
|
as_dict=lambda include_data=False, stats=True, flat=False, _bench=bench, cprofile='cumtime': |
104
|
|
|
dict(_bench, **_bench["stats"]) if flat else dict(_bench), |
105
|
|
|
name=bench['name'], |
106
|
|
|
fullname=bench['fullname'], |
107
|
|
|
group=bench['group'], |
108
|
|
|
options=bench['options'], |
109
|
|
|
has_error=False, |
110
|
|
|
params=None, |
111
|
|
|
**bench['stats'] |
112
|
|
|
) |
113
|
|
|
for bench in data['benchmarks'] |
114
|
|
|
) |
115
|
|
|
break |
116
|
|
|
|
117
|
|
|
|
118
|
|
|
try: |
119
|
|
|
text_type = unicode |
120
|
|
|
except NameError: |
121
|
|
|
text_type = str |
122
|
|
|
|
123
|
|
|
|
124
|
|
|
def force_text(text): |
125
|
|
|
if isinstance(text, text_type): |
126
|
|
|
return text |
127
|
|
|
else: |
128
|
|
|
return text.decode('utf-8') |
129
|
|
|
|
130
|
|
|
|
131
|
|
|
def force_bytes(text): |
132
|
|
|
if isinstance(text, text_type): |
133
|
|
|
return text.encode('utf-8') |
134
|
|
|
else: |
135
|
|
|
return text |
136
|
|
|
|
137
|
|
|
|
138
|
|
|
@pytest.fixture(params=['short', 'normal', 'long']) |
139
|
|
|
def name_format(request): |
140
|
|
|
return request.param |
141
|
|
|
|
142
|
|
View Code Duplication |
|
|
|
|
|
143
|
|
|
@pytest.fixture |
144
|
|
|
def sess(request, name_format): |
145
|
|
|
return MockSession(name_format) |
146
|
|
|
|
147
|
|
|
|
148
|
|
|
def make_logger(sess): |
149
|
|
|
output = StringIO() |
150
|
|
|
sess.logger = Namespace( |
151
|
|
|
warn=lambda code, text, **opts: output.write(u"%s: %s %s\n" % (code, force_text(text), opts)), |
152
|
|
|
info=lambda text, **opts: output.write(force_text(text) + u'\n'), |
153
|
|
|
error=lambda text: output.write(force_text(text) + u'\n'), |
154
|
|
|
) |
155
|
|
|
sess.storage.logger = Namespace( |
156
|
|
|
warn=lambda code, text, **opts: output.write(u"%s: %s %s\n" % (code, force_text(text), opts)), |
157
|
|
|
info=lambda text, **opts: output.write(force_text(text) + u'\n'), |
158
|
|
|
error=lambda text: output.write(force_text(text) + u'\n'), |
159
|
|
|
) |
160
|
|
|
return output |
161
|
|
|
|
162
|
|
|
|
163
|
|
|
def test_rendering(sess): |
164
|
|
|
output = make_logger(sess) |
165
|
|
|
sess.histogram = os.path.join('docs', 'sample') |
166
|
|
|
sess.compare = '*/*' |
167
|
|
|
sess.sort = 'name' |
168
|
|
|
sess.handle_loading() |
169
|
|
|
sess.finish() |
170
|
|
|
sess.display(Namespace( |
171
|
|
|
ensure_newline=lambda: None, |
172
|
|
View Code Duplication |
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
|
|
|
|
173
|
|
|
write=lambda text, **opts: output.write(force_text(text)), |
174
|
|
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
175
|
|
|
)) |
176
|
|
|
|
177
|
|
|
|
178
|
|
|
def test_regression_checks(sess, name_format): |
179
|
|
|
output = make_logger(sess) |
180
|
|
|
sess.handle_loading() |
181
|
|
|
sess.performance_regressions = [] |
182
|
|
|
sess.compare_fail = [ |
183
|
|
|
PercentageRegressionCheck("stddev", 5), |
184
|
|
|
DifferenceRegressionCheck("max", 0.000001) |
185
|
|
|
] |
186
|
|
|
sess.finish() |
187
|
|
|
pytest.raises(PerformanceRegression, sess.display, Namespace( |
188
|
|
|
ensure_newline=lambda: None, |
189
|
|
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
190
|
|
|
write=lambda text, **opts: output.write(force_text(text)), |
191
|
|
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
192
|
|
|
)) |
193
|
|
|
print(output.getvalue()) |
194
|
|
|
assert sess.performance_regressions == { |
195
|
|
|
'normal': [ |
196
|
|
|
('test_xfast_parametrized[0] (0001_b87b9aa)', |
197
|
|
|
"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
198
|
|
|
('test_xfast_parametrized[0] (0001_b87b9aa)', |
199
|
|
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
200
|
|
|
], |
201
|
|
|
'short': [ |
202
|
|
|
('xfast_parametrized[0] (0001)', |
203
|
|
|
"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
204
|
|
|
('xfast_parametrized[0] (0001)', |
205
|
|
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
206
|
|
|
], |
207
|
|
|
'long': [ |
208
|
|
|
('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)', |
209
|
|
|
"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
210
|
|
|
('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)', |
211
|
|
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
212
|
|
|
], |
213
|
|
|
}[name_format] |
214
|
|
|
output = make_logger(sess) |
215
|
|
|
pytest.raises(PerformanceRegression, sess.check_regressions) |
216
|
|
|
print(output.getvalue()) |
217
|
|
|
assert output.getvalue() == { |
218
|
|
|
'short': """Performance has regressed: |
219
|
|
|
\txfast_parametrized[0] (0001) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000 |
220
|
|
|
\txfast_parametrized[0] (0001) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000 |
221
|
|
|
""", |
222
|
|
|
'normal': """Performance has regressed: |
223
|
|
|
\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000 |
224
|
|
|
\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000 |
225
|
|
|
""", |
226
|
|
|
'long': """Performance has regressed: |
227
|
|
View Code Duplication |
\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000 |
|
|
|
|
228
|
|
|
\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000 |
229
|
|
|
""" |
230
|
|
|
}[name_format] |
231
|
|
|
|
232
|
|
|
|
233
|
|
|
@pytest.mark.skipif(sys.version_info[:2] < (2, 7), |
234
|
|
|
reason="Something weird going on, see: https://bugs.python.org/issue4482") |
235
|
|
|
def test_regression_checks_inf(sess, name_format): |
236
|
|
|
output = make_logger(sess) |
237
|
|
|
sess.compare = '0002' |
238
|
|
|
sess.handle_loading() |
239
|
|
|
sess.performance_regressions = [] |
240
|
|
|
sess.compare_fail = [ |
241
|
|
|
PercentageRegressionCheck("stddev", 5), |
242
|
|
|
DifferenceRegressionCheck("max", 0.000001) |
243
|
|
|
] |
244
|
|
|
sess.finish() |
245
|
|
|
pytest.raises(PerformanceRegression, sess.display, Namespace( |
246
|
|
|
ensure_newline=lambda: None, |
247
|
|
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
248
|
|
|
write=lambda text, **opts: output.write(force_text(text)), |
249
|
|
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
250
|
|
|
)) |
251
|
|
|
print(output.getvalue()) |
252
|
|
|
assert sess.performance_regressions == { |
253
|
|
|
'normal': [ |
254
|
|
|
('test_xfast_parametrized[0] (0002_b87b9aa)', |
255
|
|
|
"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
256
|
|
|
('test_xfast_parametrized[0] (0002_b87b9aa)', |
257
|
|
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000") |
258
|
|
|
], |
259
|
|
|
'short': [ |
260
|
|
|
('xfast_parametrized[0] (0002)', |
261
|
|
|
"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
262
|
|
|
('xfast_parametrized[0] (0002)', |
263
|
|
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000") |
264
|
|
|
], |
265
|
|
|
'long': [ |
266
|
|
|
('tests/test_normal.py::test_xfast_parametrized[0] ' |
267
|
|
|
'(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)', |
268
|
|
|
"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
269
|
|
|
('tests/test_normal.py::test_xfast_parametrized[0] ' |
270
|
|
|
'(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)', |
271
|
|
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > " |
272
|
|
|
'0.000001000') |
273
|
|
|
] |
274
|
|
|
}[name_format] |
275
|
|
|
output = make_logger(sess) |
276
|
|
|
pytest.raises(PerformanceRegression, sess.check_regressions) |
277
|
|
|
print(output.getvalue()) |
278
|
|
|
assert output.getvalue() == { |
279
|
|
|
'short': """Performance has regressed: |
280
|
|
|
\txfast_parametrized[0] (0002) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000 |
281
|
|
|
\txfast_parametrized[0] (0002) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000 |
282
|
|
|
""", |
283
|
|
|
'normal': """Performance has regressed: |
284
|
|
|
\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000 |
285
|
|
|
\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000 |
286
|
|
|
""", |
287
|
|
|
'long': """Performance has regressed: |
288
|
|
View Code Duplication |
\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000 |
|
|
|
|
289
|
|
|
\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000 |
290
|
|
|
""" |
291
|
|
|
}[name_format] |
292
|
|
|
|
293
|
|
|
|
294
|
|
|
def test_compare_1(sess, LineMatcher): |
295
|
|
|
output = make_logger(sess) |
296
|
|
|
sess.handle_loading() |
297
|
|
|
sess.finish() |
298
|
|
|
sess.display(Namespace( |
299
|
|
|
ensure_newline=lambda: None, |
300
|
|
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
301
|
|
|
write=lambda text, **opts: output.write(force_text(text)), |
302
|
|
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
303
|
|
|
)) |
304
|
|
|
print(output.getvalue()) |
305
|
|
|
LineMatcher(output.getvalue().splitlines()).fnmatch_lines([ |
306
|
|
|
'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}', |
307
|
|
|
'Comparing against benchmarks from: 0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted' |
308
|
|
|
'-changes.json', |
309
|
|
|
'', |
310
|
|
|
'*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*', |
311
|
|
|
'Name (time in ns) * Min *Max Mean StdDev Median IQR Outliers Rounds Iterations OPS (Mops/s) *', |
312
|
|
|
'-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------*', |
313
|
|
|
'*xfast_parametrized[[]0[]] (0001*) 217.3145 (1.0) 11*447.3891 (1.0) 262.2408 (1.00) 214.0442 (1.0) 220.1664 (1.00) 38.2154 (2.03) 90;1878 9987 418 3.8133 (1.00)*', |
314
|
|
|
'*xfast_parametrized[[]0[]] (NOW) * 217.9511 (1.00) 13*290.0380 (1.16) 261.2051 (1.0) 263.9842 (1.23) 220.1638 (1.0) 18.8080 (1.0) 160;1726 9710 431 3.8284 (1.0)*', |
315
|
|
|
'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*', |
316
|
|
View Code Duplication |
'Legend:', |
|
|
|
|
317
|
|
|
' Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.', |
318
|
|
|
' OPS: Operations Per Second, computed as 1 / Mean', |
319
|
|
|
]) |
320
|
|
|
|
321
|
|
|
|
322
|
|
|
def test_compare_2(sess, LineMatcher): |
323
|
|
|
output = make_logger(sess) |
324
|
|
|
sess.compare = '0002' |
325
|
|
|
sess.handle_loading() |
326
|
|
|
sess.finish() |
327
|
|
|
sess.display(Namespace( |
328
|
|
|
ensure_newline=lambda: None, |
329
|
|
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
330
|
|
|
section=lambda line, **opts: output.write(force_text(line) + u'\n'), |
331
|
|
|
write=lambda text, **opts: output.write(force_text(text)), |
332
|
|
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
333
|
|
|
)) |
334
|
|
|
print(output.getvalue()) |
335
|
|
|
LineMatcher(output.getvalue().splitlines()).fnmatch_lines([ |
336
|
|
|
'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}', |
337
|
|
|
'Comparing against benchmarks from: 0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes.json', |
338
|
|
|
'', |
339
|
|
|
'*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*', |
340
|
|
|
'Name (time in ns) * Min *Max Mean StdDev Median IQR Outliers Rounds Iterations OPS (Mops/s)*', |
341
|
|
|
'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*', |
342
|
|
|
'*xfast_parametrized[[]0[]] (0002*) 216.9028 (1.0) 7*739.2997 (1.0) 254.0585 (1.0) 0.0000 (1.0) 219.8103 (1.0) 27.3309 (1.45) 235;1688 11009 410 3.9361 (1.0)*', |
343
|
|
|
'*xfast_parametrized[[]0[]] (NOW) * 217.9511 (1.00) 13*290.0380 (1.72) 261.2051 (1.03) 263.9842 (inf) 220.1638 (1.00) 18.8080 (1.0) 160;1726 9710 431 3.8284 (0.97)*', |
344
|
|
|
'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*', |
345
|
|
|
'Legend:', |
346
|
|
|
' Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.', |
347
|
|
|
' OPS: Operations Per Second, computed as 1 / Mean', |
348
|
|
|
]) |
349
|
|
|
|
350
|
|
|
@freeze_time("2015-08-15T00:04:18.687119") |
351
|
|
|
def test_save_json(sess, tmpdir, monkeypatch): |
352
|
|
|
monkeypatch.setattr(plugin, '__version__', '2.5.0') |
353
|
|
|
sess.save = False |
354
|
|
|
sess.autosave = False |
355
|
|
|
sess.json = LooseFileLike() |
356
|
|
View Code Duplication |
sess.save_data = False |
|
|
|
|
357
|
|
|
sess.handle_saving() |
358
|
|
|
assert tmpdir.listdir() == [] |
359
|
|
|
assert json.loads(sess.json.getvalue().decode()) == JSON_DATA |
360
|
|
|
|
361
|
|
|
|
362
|
|
|
@freeze_time("2015-08-15T00:04:18.687119") |
363
|
|
|
def test_save_with_name(sess, tmpdir, monkeypatch): |
364
|
|
|
monkeypatch.setattr(plugin, '__version__', '2.5.0') |
365
|
|
|
sess.save = 'foobar' |
366
|
|
|
sess.autosave = True |
367
|
|
|
sess.json = None |
368
|
|
|
sess.save_data = False |
369
|
|
|
sess.storage.path = Path(str(tmpdir)) |
370
|
|
|
sess.handle_saving() |
371
|
|
View Code Duplication |
files = list(Path(str(tmpdir)).rglob('*.json')) |
|
|
|
|
372
|
|
|
print(files) |
373
|
|
|
assert len(files) == 1 |
374
|
|
|
assert json.load(files[0].open('rU')) == JSON_DATA |
375
|
|
|
|
376
|
|
|
|
377
|
|
|
@freeze_time("2015-08-15T00:04:18.687119") |
378
|
|
|
def test_save_no_name(sess, tmpdir, monkeypatch): |
379
|
|
|
monkeypatch.setattr(plugin, '__version__', '2.5.0') |
380
|
|
|
sess.save = True |
381
|
|
|
sess.autosave = True |
382
|
|
|
sess.json = None |
383
|
|
|
sess.save_data = False |
384
|
|
|
sess.storage.path = Path(str(tmpdir)) |
385
|
|
|
sess.handle_saving() |
386
|
|
|
files = list(Path(str(tmpdir)).rglob('*.json')) |
387
|
|
|
assert len(files) == 1 |
388
|
|
|
assert json.load(files[0].open('rU')) == JSON_DATA |
389
|
|
|
|
390
|
|
|
|
391
|
|
|
@freeze_time("2015-08-15T00:04:18.687119") |
392
|
|
|
def test_save_with_error(sess, tmpdir, monkeypatch): |
393
|
|
|
monkeypatch.setattr(plugin, '__version__', '2.5.0') |
394
|
|
|
sess.save = True |
395
|
|
|
sess.autosave = True |
396
|
|
|
sess.json = None |
397
|
|
|
sess.save_data = False |
398
|
|
|
sess.storage.path = Path(str(tmpdir)) |
399
|
|
|
for bench in sess.benchmarks: |
400
|
|
|
bench.has_error = True |
401
|
|
|
sess.handle_saving() |
402
|
|
|
files = list(Path(str(tmpdir)).rglob('*.json')) |
403
|
|
|
assert len(files) == 1 |
404
|
|
|
assert json.load(files[0].open('rU')) == { |
405
|
|
|
'benchmarks': [], |
406
|
|
|
'commit_info': {'foo': 'bar'}, |
407
|
|
|
'datetime': '2015-08-15T00:04:18.687119', |
408
|
|
|
'machine_info': {'foo': 'bar'}, |
409
|
|
|
'version': '2.5.0' |
410
|
|
|
} |
411
|
|
|
|
412
|
|
|
|
413
|
|
|
@freeze_time("2015-08-15T00:04:18.687119") |
414
|
|
|
def test_autosave(sess, tmpdir, monkeypatch): |
415
|
|
|
monkeypatch.setattr(plugin, '__version__', '2.5.0') |
416
|
|
|
sess.save = False |
417
|
|
|
sess.autosave = True |
418
|
|
|
sess.json = None |
419
|
|
|
sess.save_data = False |
420
|
|
|
sess.storage.path = Path(str(tmpdir)) |
421
|
|
|
sess.handle_saving() |
422
|
|
|
files = list(Path(str(tmpdir)).rglob('*.json')) |
423
|
|
|
assert len(files) == 1 |
424
|
|
|
assert json.load(files[0].open('rU')) == JSON_DATA |
425
|
|
|
|