|
@@ 227-272 (lines=46) @@
|
| 224 |
|
\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000 |
| 225 |
|
""", |
| 226 |
|
'long': """Performance has regressed: |
| 227 |
|
\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000 |
| 228 |
|
\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000 |
| 229 |
|
""" |
| 230 |
|
}[name_format] |
| 231 |
|
|
| 232 |
|
|
| 233 |
|
@pytest.mark.skipif(sys.version_info[:2] < (2, 7), |
| 234 |
|
reason="Something weird going on, see: https://bugs.python.org/issue4482") |
| 235 |
|
def test_regression_checks_inf(sess, name_format): |
| 236 |
|
output = make_logger(sess) |
| 237 |
|
sess.compare = '0002' |
| 238 |
|
sess.handle_loading() |
| 239 |
|
sess.performance_regressions = [] |
| 240 |
|
sess.compare_fail = [ |
| 241 |
|
PercentageRegressionCheck("stddev", 5), |
| 242 |
|
DifferenceRegressionCheck("max", 0.000001) |
| 243 |
|
] |
| 244 |
|
sess.finish() |
| 245 |
|
pytest.raises(PerformanceRegression, sess.display, Namespace( |
| 246 |
|
ensure_newline=lambda: None, |
| 247 |
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
| 248 |
|
write=lambda text, **opts: output.write(force_text(text)), |
| 249 |
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
| 250 |
|
)) |
| 251 |
|
print(output.getvalue()) |
| 252 |
|
assert sess.performance_regressions == { |
| 253 |
|
'normal': [ |
| 254 |
|
('test_xfast_parametrized[0] (0002_b87b9aa)', |
| 255 |
|
"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
| 256 |
|
('test_xfast_parametrized[0] (0002_b87b9aa)', |
| 257 |
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000") |
| 258 |
|
], |
| 259 |
|
'short': [ |
| 260 |
|
('xfast_parametrized[0] (0002)', |
| 261 |
|
"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
| 262 |
|
('xfast_parametrized[0] (0002)', |
| 263 |
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000") |
| 264 |
|
], |
| 265 |
|
'long': [ |
| 266 |
|
('tests/test_normal.py::test_xfast_parametrized[0] ' |
| 267 |
|
'(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)', |
| 268 |
|
"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
| 269 |
|
('tests/test_normal.py::test_xfast_parametrized[0] ' |
| 270 |
|
'(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)', |
| 271 |
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > " |
| 272 |
|
'0.000001000') |
| 273 |
|
] |
| 274 |
|
}[name_format] |
| 275 |
|
output = make_logger(sess) |
|
@@ 172-211 (lines=40) @@
|
| 169 |
|
sess.finish() |
| 170 |
|
sess.display(Namespace( |
| 171 |
|
ensure_newline=lambda: None, |
| 172 |
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
| 173 |
|
write=lambda text, **opts: output.write(force_text(text)), |
| 174 |
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
| 175 |
|
)) |
| 176 |
|
|
| 177 |
|
|
| 178 |
|
def test_regression_checks(sess, name_format): |
| 179 |
|
output = make_logger(sess) |
| 180 |
|
sess.handle_loading() |
| 181 |
|
sess.performance_regressions = [] |
| 182 |
|
sess.compare_fail = [ |
| 183 |
|
PercentageRegressionCheck("stddev", 5), |
| 184 |
|
DifferenceRegressionCheck("max", 0.000001) |
| 185 |
|
] |
| 186 |
|
sess.finish() |
| 187 |
|
pytest.raises(PerformanceRegression, sess.display, Namespace( |
| 188 |
|
ensure_newline=lambda: None, |
| 189 |
|
write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
| 190 |
|
write=lambda text, **opts: output.write(force_text(text)), |
| 191 |
|
rewrite=lambda text, **opts: output.write(force_text(text)), |
| 192 |
|
)) |
| 193 |
|
print(output.getvalue()) |
| 194 |
|
assert sess.performance_regressions == { |
| 195 |
|
'normal': [ |
| 196 |
|
('test_xfast_parametrized[0] (0001_b87b9aa)', |
| 197 |
|
"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
| 198 |
|
('test_xfast_parametrized[0] (0001_b87b9aa)', |
| 199 |
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
| 200 |
|
], |
| 201 |
|
'short': [ |
| 202 |
|
('xfast_parametrized[0] (0001)', |
| 203 |
|
"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
| 204 |
|
('xfast_parametrized[0] (0001)', |
| 205 |
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
| 206 |
|
], |
| 207 |
|
'long': [ |
| 208 |
|
('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)', |
| 209 |
|
"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
| 210 |
|
('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)', |
| 211 |
|
"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
| 212 |
|
], |
| 213 |
|
}[name_format] |
| 214 |
|
output = make_logger(sess) |
ionelmc /
pytest-benchmark
