|
@@ 309-332 (lines=24) @@
|
| 306 |
|
'-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------*', |
| 307 |
|
'*xfast_parametrized[[]0[]] (0001*) 217.3145 (1.0) 11*447.3891 (1.0) 262.2408 (1.00) 214.0442 (1.0) 220.1664 (1.00) 38.2154 (2.03) 90;1878 9987 418 3.8133 (1.00)*', |
| 308 |
|
'*xfast_parametrized[[]0[]] (NOW) * 217.9511 (1.00) 13*290.0380 (1.16) 261.2051 (1.0) 263.9842 (1.23) 220.1638 (1.0) 18.8080 (1.0) 160;1726 9710 431 3.8284 (1.0)*', |
| 309 |
|
'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*', |
| 310 |
|
'Legend:', |
| 311 |
|
' Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.', |
| 312 |
|
' OPS: Operations Per Second, computed as 1 / Mean', |
| 313 |
|
]) |
| 314 |
|
|
| 315 |
|
|
| 316 |
|
def test_compare_2(sess, LineMatcher): |
| 317 |
|
output = make_logger(sess) |
| 318 |
|
sess.compare = '0002' |
| 319 |
|
sess.handle_loading() |
| 320 |
|
sess.finish() |
| 321 |
|
sess.display(Namespace( |
| 322 |
|
ensure_newline=lambda: None, |
| 323 |
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
| 324 |
|
section=lambda line, **opts: output.write(force_text(line) + u'\n'), |
| 325 |
|
write=lambda text, **opts: output.write(force_text(text)), |
| 326 |
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
| 327 |
|
)) |
| 328 |
|
print(output.getvalue()) |
| 329 |
|
LineMatcher(output.getvalue().splitlines()).fnmatch_lines([ |
| 330 |
|
'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}', |
| 331 |
|
'Comparing against benchmarks from: 0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes.json', |
| 332 |
|
'', |
| 333 |
|
'*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*', |
| 334 |
|
'Name (time in ns) * Min *Max Mean StdDev Median IQR Outliers Rounds Iterations OPS (Mops/s)*', |
| 335 |
|
'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*', |
|
@@ 283-305 (lines=23) @@
|
| 280 |
|
""", |
| 281 |
|
'long': """Performance has regressed: |
| 282 |
|
\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000 |
| 283 |
|
\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000 |
| 284 |
|
""" |
| 285 |
|
}[name_format] |
| 286 |
|
|
| 287 |
|
|
| 288 |
|
def test_compare_1(sess, LineMatcher): |
| 289 |
|
output = make_logger(sess) |
| 290 |
|
sess.handle_loading() |
| 291 |
|
sess.finish() |
| 292 |
|
sess.display(Namespace( |
| 293 |
|
ensure_newline=lambda: None, |
| 294 |
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
| 295 |
|
write=lambda text, **opts: output.write(force_text(text)), |
| 296 |
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
| 297 |
|
)) |
| 298 |
|
print(output.getvalue()) |
| 299 |
|
LineMatcher(output.getvalue().splitlines()).fnmatch_lines([ |
| 300 |
|
'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}', |
| 301 |
|
'Comparing against benchmarks from: 0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted' |
| 302 |
|
'-changes.json', |
| 303 |
|
'', |
| 304 |
|
'*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*', |
| 305 |
|
'Name (time in ns) * Min *Max Mean StdDev Median IQR Outliers Rounds Iterations OPS (Mops/s) *', |
| 306 |
|
'-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------*', |
| 307 |
|
'*xfast_parametrized[[]0[]] (0001*) 217.3145 (1.0) 11*447.3891 (1.0) 262.2408 (1.00) 214.0442 (1.0) 220.1664 (1.00) 38.2154 (2.03) 90;1878 9987 418 3.8133 (1.00)*', |
| 308 |
|
'*xfast_parametrized[[]0[]] (NOW) * 217.9511 (1.00) 13*290.0380 (1.16) 261.2051 (1.0) 263.9842 (1.23) 220.1638 (1.0) 18.8080 (1.0) 160;1726 9710 431 3.8284 (1.0)*', |
ionelmc /
pytest-benchmark
