1
|
|
|
from __future__ import division |
2
|
|
|
from __future__ import print_function |
3
|
|
|
|
4
|
|
|
import pytest |
5
|
|
|
|
6
|
|
|
from .fixture import statistics |
7
|
|
|
from .fixture import statistics_error |
8
|
|
|
from .logger import Logger |
9
|
|
|
from .table import TableResults |
10
|
|
|
from .utils import NAME_FORMATTERS |
11
|
|
|
from .utils import SecondsDecimal |
12
|
|
|
from .utils import first_or_value |
13
|
|
|
from .utils import get_machine_id |
14
|
|
|
from .utils import load_storage |
15
|
|
|
from .utils import load_timer |
16
|
|
|
from .utils import safe_dumps |
17
|
|
|
from .utils import short_filename |
18
|
|
|
|
19
|
|
|
|
20
|
|
|
class PerformanceRegression(pytest.UsageError): |
21
|
|
|
pass |
22
|
|
|
|
23
|
|
|
|
24
|
|
|
class BenchmarkSession(object): |
25
|
|
|
compared_mapping = None |
26
|
|
|
groups = None |
27
|
|
|
|
28
|
|
|
def __init__(self, config): |
29
|
|
|
self.verbose = config.getoption("benchmark_verbose") |
30
|
|
|
self.logger = Logger(self.verbose, config) |
31
|
|
|
self.config = config |
32
|
|
|
self.performance_regressions = [] |
33
|
|
|
self.benchmarks = [] |
34
|
|
|
self.machine_id = get_machine_id() |
35
|
|
|
self.storage = load_storage( |
36
|
|
|
config.getoption("benchmark_storage"), |
37
|
|
|
logger=self.logger, |
38
|
|
|
default_machine_id=self.machine_id, |
39
|
|
|
netrc=config.getoption("benchmark_netrc") |
40
|
|
|
) |
41
|
|
|
self.options = dict( |
42
|
|
|
min_time=SecondsDecimal(config.getoption("benchmark_min_time")), |
43
|
|
|
min_rounds=config.getoption("benchmark_min_rounds"), |
44
|
|
|
max_time=SecondsDecimal(config.getoption("benchmark_max_time")), |
45
|
|
|
timer=load_timer(config.getoption("benchmark_timer")), |
46
|
|
|
calibration_precision=config.getoption("benchmark_calibration_precision"), |
47
|
|
|
disable_gc=config.getoption("benchmark_disable_gc"), |
48
|
|
|
warmup=config.getoption("benchmark_warmup"), |
49
|
|
|
warmup_iterations=config.getoption("benchmark_warmup_iterations"), |
50
|
|
|
use_cprofile=bool(config.getoption("benchmark_cprofile")), |
51
|
|
|
) |
52
|
|
|
self.skip = config.getoption("benchmark_skip") |
53
|
|
|
self.disabled = config.getoption("benchmark_disable") and not config.getoption("benchmark_enable") |
54
|
|
|
self.cprofile_sort_by = config.getoption("benchmark_cprofile") |
55
|
|
|
|
56
|
|
|
if config.getoption("dist", "no") != "no" and not self.skip: |
57
|
|
|
self.logger.warn( |
58
|
|
|
"BENCHMARK-U2", |
59
|
|
|
"Benchmarks are automatically disabled because xdist plugin is active." |
60
|
|
|
"Benchmarks cannot be performed reliably in a parallelized environment.", |
61
|
|
|
fslocation="::" |
62
|
|
|
) |
63
|
|
|
self.disabled = True |
64
|
|
|
if hasattr(config, "slaveinput"): |
65
|
|
|
self.disabled = True |
66
|
|
|
if not statistics: |
67
|
|
|
self.logger.warn( |
68
|
|
|
"BENCHMARK-U3", |
69
|
|
|
"Benchmarks are automatically disabled because we could not import `statistics`\n\n%s" % |
70
|
|
|
statistics_error, |
71
|
|
|
fslocation="::" |
72
|
|
|
) |
73
|
|
|
self.disabled = True |
74
|
|
|
|
75
|
|
|
self.only = config.getoption("benchmark_only") |
76
|
|
|
self.sort = config.getoption("benchmark_sort") |
77
|
|
|
self.columns = config.getoption("benchmark_columns") |
78
|
|
|
if self.skip and self.only: |
79
|
|
|
raise pytest.UsageError("Can't have both --benchmark-only and --benchmark-skip options.") |
80
|
|
|
if self.disabled and self.only: |
81
|
|
|
raise pytest.UsageError( |
82
|
|
|
"Can't have both --benchmark-only and --benchmark-disable options. Note that --benchmark-disable is " |
83
|
|
|
"automatically activated if xdist is on or you're missing the statistics dependency.") |
84
|
|
|
self.group_by = config.getoption("benchmark_group_by") |
85
|
|
|
self.save = config.getoption("benchmark_save") |
86
|
|
|
self.autosave = config.getoption("benchmark_autosave") |
87
|
|
|
self.save_data = config.getoption("benchmark_save_data") |
88
|
|
|
self.json = config.getoption("benchmark_json") |
89
|
|
|
self.compare = config.getoption("benchmark_compare") |
90
|
|
|
self.compare_fail = config.getoption("benchmark_compare_fail") |
91
|
|
|
self.name_format = NAME_FORMATTERS[config.getoption("benchmark_name")] |
92
|
|
|
|
93
|
|
|
self.histogram = first_or_value(config.getoption("benchmark_histogram"), False) |
94
|
|
|
|
95
|
|
|
def get_machine_info(self): |
96
|
|
|
obj = self.config.hook.pytest_benchmark_generate_machine_info(config=self.config) |
97
|
|
|
self.config.hook.pytest_benchmark_update_machine_info( |
98
|
|
|
config=self.config, |
99
|
|
|
machine_info=obj |
100
|
|
|
) |
101
|
|
|
return obj |
102
|
|
|
|
103
|
|
|
def prepare_benchmarks(self): |
104
|
|
|
for bench in self.benchmarks: |
105
|
|
|
if bench: |
106
|
|
|
compared = False |
107
|
|
|
for path, compared_mapping in self.compared_mapping.items(): |
108
|
|
|
if bench.fullname in compared_mapping: |
109
|
|
|
compared = compared_mapping[bench.fullname] |
110
|
|
|
source = short_filename(path, self.machine_id) |
111
|
|
|
flat_bench = bench.as_dict(include_data=False, stats=False, cprofile=self.cprofile_sort_by) |
112
|
|
|
flat_bench.update(compared["stats"]) |
113
|
|
|
flat_bench["path"] = str(path) |
114
|
|
|
flat_bench["source"] = source |
115
|
|
|
if self.compare_fail: |
116
|
|
|
for check in self.compare_fail: |
117
|
|
|
fail = check.fails(bench, flat_bench) |
118
|
|
|
if fail: |
119
|
|
|
self.performance_regressions.append((self.name_format(flat_bench), fail)) |
120
|
|
|
yield flat_bench |
121
|
|
|
flat_bench = bench.as_dict(include_data=False, flat=True, cprofile=self.cprofile_sort_by) |
122
|
|
|
flat_bench["path"] = None |
123
|
|
|
flat_bench["source"] = compared and "NOW" |
124
|
|
|
yield flat_bench |
125
|
|
|
|
126
|
|
|
def save_json(self, output_json): |
127
|
|
|
with self.json as fh: |
128
|
|
|
fh.write(safe_dumps(output_json, ensure_ascii=True, indent=4).encode()) |
129
|
|
|
self.logger.info("Wrote benchmark data in: %s" % self.json, purple=True) |
130
|
|
|
|
131
|
|
|
def handle_saving(self): |
132
|
|
|
save = self.save or self.autosave |
133
|
|
|
if save or self.json: |
134
|
|
|
if not self.benchmarks: |
135
|
|
|
self.logger.warn("BENCHMARK-U2", "Not saving anything, no benchmarks have been run!") |
136
|
|
|
return |
137
|
|
|
machine_info = self.get_machine_info() |
138
|
|
|
commit_info = self.config.hook.pytest_benchmark_generate_commit_info(config=self.config) |
139
|
|
|
self.config.hook.pytest_benchmark_update_commit_info(config=self.config, commit_info=commit_info) |
140
|
|
|
|
141
|
|
|
if self.json: |
142
|
|
|
output_json = self.config.hook.pytest_benchmark_generate_json( |
143
|
|
|
config=self.config, |
144
|
|
|
benchmarks=self.benchmarks, |
145
|
|
|
include_data=True, |
146
|
|
|
machine_info=machine_info, |
147
|
|
|
commit_info=commit_info, |
148
|
|
|
) |
149
|
|
|
self.config.hook.pytest_benchmark_update_json( |
150
|
|
|
config=self.config, |
151
|
|
|
benchmarks=self.benchmarks, |
152
|
|
|
output_json=output_json, |
153
|
|
|
) |
154
|
|
|
self.save_json(output_json) |
155
|
|
|
|
156
|
|
|
if save: |
157
|
|
|
output_json = self.config.hook.pytest_benchmark_generate_json( |
158
|
|
|
config=self.config, |
159
|
|
|
benchmarks=self.benchmarks, |
160
|
|
|
include_data=self.save_data, |
161
|
|
|
machine_info=machine_info, |
162
|
|
|
commit_info=commit_info, |
163
|
|
|
) |
164
|
|
|
self.config.hook.pytest_benchmark_update_json( |
165
|
|
|
config=self.config, |
166
|
|
|
benchmarks=self.benchmarks, |
167
|
|
|
output_json=output_json, |
168
|
|
|
) |
169
|
|
|
self.storage.save(output_json, save) |
170
|
|
|
|
171
|
|
|
def handle_loading(self): |
172
|
|
|
compared_mapping = {} |
173
|
|
|
if self.compare: |
174
|
|
|
if self.compare is True: |
175
|
|
|
compared_benchmarks = list(self.storage.load())[-1:] |
176
|
|
|
else: |
177
|
|
|
compared_benchmarks = list(self.storage.load(self.compare)) |
178
|
|
|
|
179
|
|
|
if not compared_benchmarks: |
180
|
|
|
msg = "Can't compare. No benchmark files in %r" % str(self.storage) |
181
|
|
|
if self.compare is True: |
182
|
|
|
msg += ". Can't load the previous benchmark." |
183
|
|
|
code = "BENCHMARK-C2" |
184
|
|
|
else: |
185
|
|
|
msg += " match %r." % self.compare |
186
|
|
|
code = "BENCHMARK-C1" |
187
|
|
|
self.logger.warn(code, msg, fslocation=self.storage.location) |
188
|
|
|
|
189
|
|
|
machine_info = self.get_machine_info() |
190
|
|
|
for path, compared_benchmark in compared_benchmarks: |
191
|
|
|
self.config.hook.pytest_benchmark_compare_machine_info( |
192
|
|
|
config=self.config, |
193
|
|
|
benchmarksession=self, |
194
|
|
|
machine_info=machine_info, |
195
|
|
|
compared_benchmark=compared_benchmark, |
196
|
|
|
) |
197
|
|
|
compared_mapping[path] = dict( |
198
|
|
|
(bench['fullname'], bench) for bench in compared_benchmark['benchmarks'] |
199
|
|
|
) |
200
|
|
|
self.logger.info("Comparing against benchmarks from: %s" % path, newline=False) |
201
|
|
|
self.compared_mapping = compared_mapping |
202
|
|
|
|
203
|
|
|
def finish(self): |
204
|
|
|
self.handle_saving() |
205
|
|
|
prepared_benchmarks = list(self.prepare_benchmarks()) |
206
|
|
|
if prepared_benchmarks: |
207
|
|
|
self.groups = self.config.hook.pytest_benchmark_group_stats( |
208
|
|
|
config=self.config, |
209
|
|
|
benchmarks=prepared_benchmarks, |
210
|
|
|
group_by=self.group_by |
211
|
|
|
) |
212
|
|
|
|
213
|
|
|
def display(self, tr): |
214
|
|
|
if not self.groups: |
215
|
|
|
return |
216
|
|
|
|
217
|
|
|
tr.ensure_newline() |
218
|
|
|
results_table = TableResults( |
219
|
|
|
columns=self.columns, |
220
|
|
|
sort=self.sort, |
221
|
|
|
histogram=self.histogram, |
222
|
|
|
name_format=self.name_format, |
223
|
|
|
logger=self.logger |
224
|
|
|
) |
225
|
|
|
results_table.display(tr, self.groups) |
226
|
|
|
self.check_regressions() |
227
|
|
|
self.display_cprofile(tr) |
228
|
|
|
|
229
|
|
|
def check_regressions(self): |
230
|
|
|
if self.compare_fail and not self.compared_mapping: |
231
|
|
|
raise pytest.UsageError("--benchmark-compare-fail requires valid --benchmark-compare.") |
232
|
|
|
|
233
|
|
|
if self.performance_regressions: |
234
|
|
|
self.logger.error("Performance has regressed:\n%s" % "\n".join( |
235
|
|
|
"\t%s - %s" % line for line in self.performance_regressions |
236
|
|
|
)) |
237
|
|
|
raise PerformanceRegression("Performance has regressed.") |
238
|
|
|
|
239
|
|
|
def display_cprofile(self, tr): |
240
|
|
|
if self.options["use_cprofile"]: |
241
|
|
|
tr.section("cProfile information") |
242
|
|
|
tr.write_line("Time in s") |
243
|
|
|
for group in self.groups: |
244
|
|
|
group_name, benchmarks = group |
245
|
|
|
for benchmark in benchmarks: |
246
|
|
|
tr.write(benchmark["fullname"], yellow=True) |
247
|
|
|
if benchmark["source"]: |
248
|
|
|
tr.write_line(" ({})".format((benchmark["source"]))) |
249
|
|
|
else: |
250
|
|
|
tr.write("\n") |
251
|
|
|
tr.write_line("ncalls\ttottime\tpercall\tcumtime\tpercall\tfilename:lineno(function)") |
252
|
|
|
for function_info in benchmark["cprofile"]: |
253
|
|
|
line = ("{ncalls_recursion}\t{tottime:.{prec}f}\t{tottime_per:.{prec}f}\t{cumtime:.{prec}f}" |
254
|
|
|
"\t{cumtime_per:.{prec}f}\t{function_name}").format( |
255
|
|
|
prec=4, **function_info) |
256
|
|
|
tr.write_line(line) |
257
|
|
|
tr.write("\n") |
258
|
|
|
|