@@ 222-267 (lines=46) @@ | ||
219 | """, |
|
220 | 'long': """Performance has regressed: |
|
221 | \ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000 |
|
222 | \ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000 |
|
223 | """ |
|
224 | }[name_format] |
|
225 | ||
226 | ||
227 | @pytest.mark.skipif(sys.version_info[:2] < (2, 7), |
|
228 | reason="Something weird going on, see: https://bugs.python.org/issue4482") |
|
229 | def test_regression_checks_inf(sess, name_format): |
|
230 | output = make_logger(sess) |
|
231 | sess.compare = '0002' |
|
232 | sess.handle_loading() |
|
233 | sess.performance_regressions = [] |
|
234 | sess.compare_fail = [ |
|
235 | PercentageRegressionCheck("stddev", 5), |
|
236 | DifferenceRegressionCheck("max", 0.000001) |
|
237 | ] |
|
238 | sess.finish() |
|
239 | pytest.raises(PerformanceRegression, sess.display, Namespace( |
|
240 | ensure_newline=lambda: None, |
|
241 | write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
|
242 | write=lambda text, **opts: output.write(force_text(text)), |
|
243 | rewrite=lambda text, **opts: output.write(force_text(text)), |
|
244 | )) |
|
245 | print(output.getvalue()) |
|
246 | assert sess.performance_regressions == { |
|
247 | 'normal': [ |
|
248 | ('test_xfast_parametrized[0] (0002_b87b9aa)', |
|
249 | "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
|
250 | ('test_xfast_parametrized[0] (0002_b87b9aa)', |
|
251 | "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000") |
|
252 | ], |
|
253 | 'short': [ |
|
254 | ('xfast_parametrized[0] (0002)', |
|
255 | "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
|
256 | ('xfast_parametrized[0] (0002)', |
|
257 | "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000") |
|
258 | ], |
|
259 | 'long': [ |
|
260 | ('tests/test_normal.py::test_xfast_parametrized[0] ' |
|
261 | '(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)', |
|
262 | "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"), |
|
263 | ('tests/test_normal.py::test_xfast_parametrized[0] ' |
|
264 | '(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)', |
|
265 | "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > " |
|
266 | '0.000001000') |
|
267 | ] |
|
268 | }[name_format] |
|
269 | output = make_logger(sess) |
|
270 | pytest.raises(PerformanceRegression, sess.check_regressions) |
|
@@ 167-206 (lines=40) @@ | ||
164 | sess.display(Namespace( |
|
165 | ensure_newline=lambda: None, |
|
166 | write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
|
167 | write=lambda text, **opts: output.write(force_text(text)), |
|
168 | rewrite=lambda text, **opts: output.write(force_text(text)), |
|
169 | )) |
|
170 | ||
171 | ||
172 | def test_regression_checks(sess, name_format): |
|
173 | output = make_logger(sess) |
|
174 | sess.handle_loading() |
|
175 | sess.performance_regressions = [] |
|
176 | sess.compare_fail = [ |
|
177 | PercentageRegressionCheck("stddev", 5), |
|
178 | DifferenceRegressionCheck("max", 0.000001) |
|
179 | ] |
|
180 | sess.finish() |
|
181 | pytest.raises(PerformanceRegression, sess.display, Namespace( |
|
182 | ensure_newline=lambda: None, |
|
183 | write_line=lambda line, **opts: output.write(force_text(line) + u'\n'), |
|
184 | write=lambda text, **opts: output.write(force_text(text)), |
|
185 | rewrite=lambda text, **opts: output.write(force_text(text)), |
|
186 | )) |
|
187 | print(output.getvalue()) |
|
188 | assert sess.performance_regressions == { |
|
189 | 'normal': [ |
|
190 | ('test_xfast_parametrized[0] (0001_b87b9aa)', |
|
191 | "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
|
192 | ('test_xfast_parametrized[0] (0001_b87b9aa)', |
|
193 | "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
|
194 | ], |
|
195 | 'short': [ |
|
196 | ('xfast_parametrized[0] (0001)', |
|
197 | "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
|
198 | ('xfast_parametrized[0] (0001)', |
|
199 | "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
|
200 | ], |
|
201 | 'long': [ |
|
202 | ('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)', |
|
203 | "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"), |
|
204 | ('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)', |
|
205 | "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000") |
|
206 | ], |
|
207 | }[name_format] |
|
208 | output = make_logger(sess) |
|
209 | pytest.raises(PerformanceRegression, sess.check_regressions) |