1
|
|
|
from __future__ import division |
2
|
|
|
from __future__ import print_function |
3
|
|
|
|
4
|
|
|
import argparse |
5
|
|
|
import operator |
6
|
|
|
import platform |
7
|
|
|
import sys |
8
|
|
|
import traceback |
9
|
|
|
from collections import defaultdict |
10
|
|
|
from datetime import datetime |
11
|
|
|
|
12
|
|
|
import pytest |
13
|
|
|
|
14
|
|
|
from . import __version__ |
15
|
|
|
from .fixture import BenchmarkFixture |
16
|
|
|
from .session import BenchmarkSession |
17
|
|
|
from .session import PerformanceRegression |
18
|
|
|
from .timers import default_timer |
19
|
|
|
from .utils import NameWrapper |
20
|
|
|
from .utils import format_dict |
21
|
|
|
from .utils import get_commit_info |
22
|
|
|
from .utils import get_current_time |
23
|
|
|
from .utils import get_tag |
24
|
|
|
from .utils import parse_columns |
25
|
|
|
from .utils import parse_compare_fail |
26
|
|
|
from .utils import parse_name_format |
27
|
|
|
from .utils import parse_rounds |
28
|
|
|
from .utils import parse_save |
29
|
|
|
from .utils import parse_seconds |
30
|
|
|
from .utils import parse_sort |
31
|
|
|
from .utils import parse_timer |
32
|
|
|
from .utils import parse_warmup |
33
|
|
|
|
34
|
|
|
|
35
|
|
|
def pytest_report_header(config): |
36
|
|
|
bs = config._benchmarksession |
37
|
|
|
|
38
|
|
|
return ("benchmark: {version} (defaults:" |
39
|
|
|
" timer={timer}" |
40
|
|
|
" disable_gc={0[disable_gc]}" |
41
|
|
|
" min_rounds={0[min_rounds]}" |
42
|
|
|
" min_time={0[min_time]}" |
43
|
|
|
" max_time={0[max_time]}" |
44
|
|
|
" calibration_precision={0[calibration_precision]}" |
45
|
|
|
" warmup={0[warmup]}" |
46
|
|
|
" warmup_iterations={0[warmup_iterations]}" |
47
|
|
|
")").format( |
48
|
|
|
bs.options, |
49
|
|
|
version=__version__, |
50
|
|
|
timer=bs.options.get("timer"), |
51
|
|
|
) |
52
|
|
|
|
53
|
|
|
|
54
|
|
|
def add_display_options(addoption, prefix="benchmark-"): |
55
|
|
|
addoption( |
56
|
|
|
"--{0}sort".format(prefix), |
57
|
|
|
metavar="COL", type=parse_sort, default="min", |
58
|
|
|
help="Column to sort on. Can be one of: 'min', 'max', 'mean', 'stddev', " |
59
|
|
|
"'name', 'fullname'. Default: %(default)r" |
60
|
|
|
) |
61
|
|
|
addoption( |
62
|
|
|
"--{0}group-by".format(prefix), |
63
|
|
|
metavar="LABEL", default="group", |
64
|
|
|
help="How to group tests. Can be one of: 'group', 'name', 'fullname', 'func', 'fullfunc', " |
65
|
|
|
"'param' or 'param:NAME', where NAME is the name passed to @pytest.parametrize." |
66
|
|
|
" Default: %(default)r" |
67
|
|
|
) |
68
|
|
|
addoption( |
69
|
|
|
"--{0}columns".format(prefix), |
70
|
|
|
metavar="LABELS", type=parse_columns, |
71
|
|
|
default=["min", "max", "mean", "stddev", "median", "iqr", "outliers", "ops", "rounds", "iterations"], |
72
|
|
|
help="Comma-separated list of columns to show in the result table. Default: " |
73
|
|
|
"'min, max, mean, stddev, median, iqr, outliers, rounds, iterations'" |
74
|
|
|
) |
75
|
|
|
addoption( |
76
|
|
|
"--{0}name".format(prefix), |
77
|
|
|
metavar="FORMAT", type=parse_name_format, |
78
|
|
|
default="normal", |
79
|
|
|
help="How to format names in results. Can be one of 'short', 'normal', 'long'. Default: %(default)r" |
80
|
|
|
) |
81
|
|
|
|
82
|
|
|
|
83
|
|
|
def add_histogram_options(addoption, prefix="benchmark-"): |
84
|
|
|
filename_prefix = "benchmark_%s" % get_current_time() |
85
|
|
|
addoption( |
86
|
|
|
"--{0}histogram".format(prefix), |
87
|
|
|
action="append", metavar="FILENAME-PREFIX", nargs="?", default=[], const=filename_prefix, |
88
|
|
|
help="Plot graphs of min/max/avg/stddev over time in FILENAME-PREFIX-test_name.svg. If FILENAME-PREFIX contains" |
89
|
|
|
" slashes ('/') then directories will be created. Default: %r" % filename_prefix |
90
|
|
|
) |
91
|
|
|
|
92
|
|
|
|
93
|
|
|
def add_csv_options(addoption, prefix="benchmark-"): |
94
|
|
|
filename_prefix = "benchmark_%s" % get_current_time() |
95
|
|
|
addoption( |
96
|
|
|
"--{0}csv".format(prefix), |
97
|
|
|
action="append", metavar="FILENAME", nargs="?", default=[], const=filename_prefix, |
98
|
|
|
help="Save a csv report. If FILENAME contains" |
99
|
|
|
" slashes ('/') then directories will be created. Default: %r" % filename_prefix |
100
|
|
|
) |
101
|
|
|
|
102
|
|
|
|
103
|
|
|
def add_global_options(addoption, prefix="benchmark-"): |
104
|
|
|
addoption( |
105
|
|
|
"--{0}storage".format(prefix), *[] if prefix else ['-s'], |
106
|
|
|
metavar="URI", default="file://./.benchmarks", |
107
|
|
|
help="Specify a path to store the runs as uri in form file://path or" |
108
|
|
|
" elasticsearch+http[s]://host1,host2/[index/doctype?project_name=Project] " |
109
|
|
|
"(when --benchmark-save or --benchmark-autosave are used). For backwards compatibility unexpected values " |
110
|
|
|
"are converted to file://<value>. Default: %(default)r." |
111
|
|
|
) |
112
|
|
|
addoption( |
113
|
|
|
"--{0}netrc".format(prefix), |
114
|
|
|
nargs="?", default='', const='~/.netrc', |
115
|
|
|
help="Load elasticsearch credentials from a netrc file. Default: %(default)r.", |
116
|
|
|
) |
117
|
|
|
addoption( |
118
|
|
|
"--{0}verbose".format(prefix), *[] if prefix else ['-v'], |
119
|
|
|
action="store_true", default=False, |
120
|
|
|
help="Dump diagnostic and progress information." |
121
|
|
|
) |
122
|
|
|
|
123
|
|
|
|
124
|
|
|
def pytest_addoption(parser): |
125
|
|
|
group = parser.getgroup("benchmark") |
126
|
|
|
group.addoption( |
127
|
|
|
"--benchmark-min-time", |
128
|
|
|
metavar="SECONDS", type=parse_seconds, default="0.000005", |
129
|
|
|
help="Minimum time per round in seconds. Default: %(default)r" |
130
|
|
|
) |
131
|
|
|
group.addoption( |
132
|
|
|
"--benchmark-max-time", |
133
|
|
|
metavar="SECONDS", type=parse_seconds, default="1.0", |
134
|
|
|
help="Maximum run time per test - it will be repeated until this total time is reached. It may be " |
135
|
|
|
"exceeded if test function is very slow or --benchmark-min-rounds is large (it takes precedence). " |
136
|
|
|
"Default: %(default)r" |
137
|
|
|
) |
138
|
|
|
group.addoption( |
139
|
|
|
"--benchmark-min-rounds", |
140
|
|
|
metavar="NUM", type=parse_rounds, default=5, |
141
|
|
|
help="Minimum rounds, even if total time would exceed `--max-time`. Default: %(default)r" |
142
|
|
|
) |
143
|
|
|
group.addoption( |
144
|
|
|
"--benchmark-timer", |
145
|
|
|
metavar="FUNC", type=parse_timer, default=str(NameWrapper(default_timer)), |
146
|
|
|
help="Timer to use when measuring time. Default: %(default)r" |
147
|
|
|
) |
148
|
|
|
group.addoption( |
149
|
|
|
"--benchmark-calibration-precision", |
150
|
|
|
metavar="NUM", type=int, default=10, |
151
|
|
|
help="Precision to use when calibrating number of iterations. Precision of 10 will make the timer look 10 times" |
152
|
|
|
" more accurate, at a cost of less precise measure of deviations. Default: %(default)r" |
153
|
|
|
) |
154
|
|
|
group.addoption( |
155
|
|
|
"--benchmark-warmup", |
156
|
|
|
metavar="KIND", nargs="?", default=parse_warmup("auto"), type=parse_warmup, |
157
|
|
|
help="Activates warmup. Will run the test function up to number of times in the calibration phase. " |
158
|
|
|
"See `--benchmark-warmup-iterations`. Note: Even the warmup phase obeys --benchmark-max-time. " |
159
|
|
|
"Available KIND: 'auto', 'off', 'on'. Default: 'auto' (automatically activate on PyPy)." |
160
|
|
|
) |
161
|
|
|
group.addoption( |
162
|
|
|
"--benchmark-warmup-iterations", |
163
|
|
|
metavar="NUM", type=int, default=100000, |
164
|
|
|
help="Max number of iterations to run in the warmup phase. Default: %(default)r" |
165
|
|
|
) |
166
|
|
|
group.addoption( |
167
|
|
|
"--benchmark-disable-gc", |
168
|
|
|
action="store_true", default=False, |
169
|
|
|
help="Disable GC during benchmarks." |
170
|
|
|
) |
171
|
|
|
group.addoption( |
172
|
|
|
"--benchmark-skip", |
173
|
|
|
action="store_true", default=False, |
174
|
|
|
help="Skip running any tests that contain benchmarks." |
175
|
|
|
) |
176
|
|
|
group.addoption( |
177
|
|
|
"--benchmark-disable", |
178
|
|
|
action="store_true", default=False, |
179
|
|
|
help="Disable benchmarks. Benchmarked functions are only ran once and no stats are reported. Use this is you " |
180
|
|
|
"want to run the test but don't do any benchmarking." |
181
|
|
|
) |
182
|
|
|
group.addoption( |
183
|
|
|
"--benchmark-enable", |
184
|
|
|
action="store_true", default=False, |
185
|
|
|
help="Forcibly enable benchmarks. Use this option to override --benchmark-disable (in case you have it in " |
186
|
|
|
"pytest configuration)." |
187
|
|
|
) |
188
|
|
|
group.addoption( |
189
|
|
|
"--benchmark-only", |
190
|
|
|
action="store_true", default=False, |
191
|
|
|
help="Only run benchmarks." |
192
|
|
|
) |
193
|
|
|
group.addoption( |
194
|
|
|
"--benchmark-save", |
195
|
|
|
metavar="NAME", type=parse_save, |
196
|
|
|
help="Save the current run into 'STORAGE-PATH/counter_NAME.json'." |
197
|
|
|
) |
198
|
|
|
tag = get_tag() |
199
|
|
|
group.addoption( |
200
|
|
|
"--benchmark-autosave", |
201
|
|
|
action='store_const', const=tag, |
202
|
|
|
help="Autosave the current run into 'STORAGE-PATH/counter_%s.json" % tag, |
203
|
|
|
) |
204
|
|
|
group.addoption( |
205
|
|
|
"--benchmark-save-data", |
206
|
|
|
action="store_true", |
207
|
|
|
help="Use this to make --benchmark-save and --benchmark-autosave include all the timing data," |
208
|
|
|
" not just the stats.", |
209
|
|
|
) |
210
|
|
|
group.addoption( |
211
|
|
|
"--benchmark-json", |
212
|
|
|
metavar="PATH", type=argparse.FileType('wb'), |
213
|
|
|
help="Dump a JSON report into PATH. " |
214
|
|
|
"Note that this will include the complete data (all the timings, not just the stats)." |
215
|
|
|
) |
216
|
|
|
group.addoption( |
217
|
|
|
"--benchmark-compare", |
218
|
|
|
metavar="NUM|_ID", nargs="?", default=[], const=True, |
219
|
|
|
help="Compare the current run against run NUM (or prefix of _id in elasticsearch) or the latest " |
220
|
|
|
"saved run if unspecified." |
221
|
|
|
) |
222
|
|
|
group.addoption( |
223
|
|
|
"--benchmark-compare-fail", |
224
|
|
|
metavar="EXPR", nargs="+", type=parse_compare_fail, |
225
|
|
|
help="Fail test if performance regresses according to given EXPR" |
226
|
|
|
" (eg: min:5%% or mean:0.001 for number of seconds). Can be used multiple times." |
227
|
|
|
) |
228
|
|
|
group.addoption( |
229
|
|
|
"--benchmark-cprofile", |
230
|
|
|
metavar="COLUMN", default=None, |
231
|
|
|
choices=['ncalls_recursion', 'ncalls', 'tottime', 'tottime_per', 'cumtime', 'cumtime_per', 'function_name'], |
232
|
|
|
help="If specified measure one run with cProfile and stores 10 top functions." |
233
|
|
|
" Argument is a column to sort by. Available columns: 'ncallls_recursion'," |
234
|
|
|
" 'ncalls', 'tottime', 'tottime_per', 'cumtime', 'cumtime_per', 'function_name'." |
235
|
|
|
) |
236
|
|
|
add_global_options(group.addoption) |
237
|
|
|
add_display_options(group.addoption) |
238
|
|
|
add_histogram_options(group.addoption) |
239
|
|
|
|
240
|
|
|
|
241
|
|
|
def pytest_addhooks(pluginmanager): |
242
|
|
|
from . import hookspec |
243
|
|
|
|
244
|
|
|
method = getattr(pluginmanager, "add_hookspecs", None) |
245
|
|
|
if method is None: |
246
|
|
|
method = pluginmanager.addhooks |
247
|
|
|
method(hookspec) |
248
|
|
|
|
249
|
|
|
|
250
|
|
|
def pytest_benchmark_compare_machine_info(config, benchmarksession, machine_info, compared_benchmark): |
251
|
|
|
machine_info = format_dict(machine_info) |
252
|
|
|
compared_machine_info = format_dict(compared_benchmark["machine_info"]) |
253
|
|
|
|
254
|
|
|
if compared_machine_info != machine_info: |
255
|
|
|
benchmarksession.logger.warn( |
256
|
|
|
"BENCHMARK-C6", |
257
|
|
|
"Benchmark machine_info is different. Current: %s VS saved: %s." % ( |
258
|
|
|
machine_info, |
259
|
|
|
compared_machine_info, |
260
|
|
|
), |
261
|
|
|
fslocation=benchmarksession.storage.location |
262
|
|
|
) |
263
|
|
|
|
264
|
|
|
|
265
|
|
|
if hasattr(pytest, 'hookimpl'): |
266
|
|
|
_hookwrapper = pytest.hookimpl(hookwrapper=True) |
267
|
|
|
else: |
268
|
|
|
_hookwrapper = pytest.mark.hookwrapper |
269
|
|
|
|
270
|
|
|
|
271
|
|
|
@_hookwrapper |
272
|
|
|
def pytest_runtest_call(item): |
273
|
|
|
bs = item.config._benchmarksession |
274
|
|
|
fixure = hasattr(item, "funcargs") and item.funcargs.get("benchmark") |
275
|
|
|
if isinstance(fixure, BenchmarkFixture): |
276
|
|
|
if bs.skip: |
277
|
|
|
pytest.skip("Skipping benchmark (--benchmark-skip active).") |
278
|
|
|
else: |
279
|
|
|
yield |
280
|
|
|
else: |
281
|
|
|
if bs.only: |
282
|
|
|
pytest.skip("Skipping non-benchmark (--benchmark-only active).") |
283
|
|
|
else: |
284
|
|
|
yield |
285
|
|
|
|
286
|
|
|
|
287
|
|
|
def pytest_benchmark_group_stats(config, benchmarks, group_by): |
288
|
|
|
groups = defaultdict(list) |
289
|
|
|
for bench in benchmarks: |
290
|
|
|
key = () |
291
|
|
|
for grouping in group_by.split(','): |
292
|
|
|
if grouping == "group": |
293
|
|
|
key += bench["group"], |
294
|
|
|
elif grouping == "name": |
295
|
|
|
key += bench["name"], |
296
|
|
|
elif grouping == "func": |
297
|
|
|
key += bench["name"].split("[")[0], |
298
|
|
|
elif grouping == "fullname": |
299
|
|
|
key += bench["fullname"], |
300
|
|
|
elif grouping == "fullfunc": |
301
|
|
|
key += bench["fullname"].split("[")[0], |
302
|
|
|
elif grouping == "param": |
303
|
|
|
key += bench["param"], |
304
|
|
|
elif grouping.startswith("param:"): |
305
|
|
|
param_name = grouping[len("param:"):] |
306
|
|
|
key += '%s=%s' % (param_name, bench["params"][param_name]), |
307
|
|
|
else: |
308
|
|
|
raise NotImplementedError("Unsupported grouping %r." % group_by) |
309
|
|
|
groups[' '.join(str(p) for p in key if p) or None].append(bench) |
310
|
|
|
|
311
|
|
|
for grouped_benchmarks in groups.values(): |
312
|
|
|
grouped_benchmarks.sort(key=operator.itemgetter("fullname" if "full" in group_by else "name")) |
313
|
|
|
return sorted(groups.items(), key=lambda pair: pair[0] or "") |
314
|
|
|
|
315
|
|
|
|
316
|
|
|
@_hookwrapper |
317
|
|
|
def pytest_sessionfinish(session, exitstatus): |
318
|
|
|
session.config._benchmarksession.finish() |
319
|
|
|
yield |
320
|
|
|
|
321
|
|
|
|
322
|
|
|
def pytest_terminal_summary(terminalreporter): |
323
|
|
|
try: |
324
|
|
|
terminalreporter.config._benchmarksession.display(terminalreporter) |
325
|
|
|
except PerformanceRegression: |
326
|
|
|
raise |
327
|
|
|
except Exception: |
328
|
|
|
terminalreporter.config._benchmarksession.logger.error("\n%s" % traceback.format_exc()) |
329
|
|
|
raise |
330
|
|
|
|
331
|
|
|
|
332
|
|
|
def get_cpu_info(): |
333
|
|
|
import cpuinfo |
334
|
|
|
all_info = cpuinfo.get_cpu_info() |
335
|
|
|
all_info = all_info or {} |
336
|
|
|
info = {} |
337
|
|
|
for key in ('vendor_id', 'hardware', 'brand'): |
338
|
|
|
info[key] = all_info.get(key, 'unknown') |
339
|
|
|
return info |
340
|
|
|
|
341
|
|
|
|
342
|
|
|
def pytest_benchmark_generate_machine_info(): |
343
|
|
|
python_implementation = platform.python_implementation() |
344
|
|
|
python_implementation_version = platform.python_version() |
345
|
|
|
if python_implementation == 'PyPy': |
346
|
|
|
python_implementation_version = '%d.%d.%d' % sys.pypy_version_info[:3] |
347
|
|
|
if sys.pypy_version_info.releaselevel != 'final': |
348
|
|
|
python_implementation_version += '-%s%d' % sys.pypy_version_info[3:] |
349
|
|
|
return { |
350
|
|
|
"node": platform.node(), |
351
|
|
|
"processor": platform.processor(), |
352
|
|
|
"machine": platform.machine(), |
353
|
|
|
"python_compiler": platform.python_compiler(), |
354
|
|
|
"python_implementation": python_implementation, |
355
|
|
|
"python_implementation_version": python_implementation_version, |
356
|
|
|
"python_version": platform.python_version(), |
357
|
|
|
"python_build": platform.python_build(), |
358
|
|
|
"release": platform.release(), |
359
|
|
|
"system": platform.system(), |
360
|
|
|
"cpu": get_cpu_info(), |
361
|
|
|
} |
362
|
|
|
|
363
|
|
|
|
364
|
|
|
def pytest_benchmark_generate_commit_info(config): |
365
|
|
|
return get_commit_info(config.getoption("benchmark_project_name", None)) |
366
|
|
|
|
367
|
|
|
|
368
|
|
|
def pytest_benchmark_generate_json(config, benchmarks, include_data, machine_info, commit_info): |
369
|
|
|
benchmarks_json = [] |
370
|
|
|
output_json = { |
371
|
|
|
"machine_info": machine_info, |
372
|
|
|
"commit_info": commit_info, |
373
|
|
|
"benchmarks": benchmarks_json, |
374
|
|
|
"datetime": datetime.utcnow().isoformat(), |
375
|
|
|
"version": __version__, |
376
|
|
|
} |
377
|
|
|
for bench in benchmarks: |
378
|
|
|
if not bench.has_error: |
379
|
|
|
benchmarks_json.append(bench.as_dict(include_data=include_data)) |
380
|
|
|
return output_json |
381
|
|
|
|
382
|
|
|
|
383
|
|
|
@pytest.fixture(scope="function") |
384
|
|
|
def benchmark(request): |
385
|
|
|
bs = request.config._benchmarksession |
386
|
|
|
|
387
|
|
|
if bs.skip: |
388
|
|
|
pytest.skip("Benchmarks are skipped (--benchmark-skip was used).") |
389
|
|
|
else: |
390
|
|
|
node = request.node |
391
|
|
|
marker = node.get_marker("benchmark") |
392
|
|
|
options = marker.kwargs if marker else {} |
393
|
|
|
if "timer" in options: |
394
|
|
|
options["timer"] = NameWrapper(options["timer"]) |
395
|
|
|
fixture = BenchmarkFixture( |
396
|
|
|
node, |
397
|
|
|
add_stats=bs.benchmarks.append, |
398
|
|
|
logger=bs.logger, |
399
|
|
|
warner=request.node.warn, |
400
|
|
|
disabled=bs.disabled, |
401
|
|
|
**dict(bs.options, **options) |
402
|
|
|
) |
403
|
|
|
request.addfinalizer(fixture._cleanup) |
404
|
|
|
return fixture |
405
|
|
|
|
406
|
|
|
|
407
|
|
|
@pytest.fixture(scope="function") |
408
|
|
|
def benchmark_weave(benchmark): |
409
|
|
|
return benchmark.weave |
410
|
|
|
|
411
|
|
|
|
412
|
|
|
def pytest_runtest_setup(item): |
413
|
|
|
marker = item.get_marker("benchmark") |
414
|
|
|
if marker: |
415
|
|
|
if marker.args: |
416
|
|
|
raise ValueError("benchmark mark can't have positional arguments.") |
417
|
|
|
for name in marker.kwargs: |
418
|
|
|
if name not in ( |
419
|
|
|
"max_time", "min_rounds", "min_time", "timer", "group", "disable_gc", "warmup", |
420
|
|
|
"warmup_iterations", "calibration_precision"): |
421
|
|
|
raise ValueError("benchmark mark can't have %r keyword argument." % name) |
422
|
|
|
|
423
|
|
|
|
424
|
|
|
@pytest.mark.trylast # force the other plugins to initialise, fixes issue with capture not being properly initialised |
425
|
|
|
def pytest_configure(config): |
426
|
|
|
config.addinivalue_line("markers", "benchmark: mark a test with custom benchmark settings.") |
427
|
|
|
bs = config._benchmarksession = BenchmarkSession(config) |
428
|
|
|
bs.handle_loading() |
429
|
|
|
config.pluginmanager.register(bs, "pytest-benchmark") |
430
|
|
|
|