ionelmc / pytest-benchmark

        '*xfast_parametrized[[]0[]] (0001*)     217.3145 (1.0)      11*447.3891 (1.0)      262.2408 (1.00)     214.0442 (1.0)      220.1664 (1.00)     38.2154 (2.03)      90;1878    9987         418        3.8133 (1.00)*',

        '*xfast_parametrized[[]0[]] (NOW) *     217.9511 (1.00)     13*290.0380 (1.16)     261.2051 (1.0)      263.9842 (1.23)     220.1638 (1.0)      18.8080 (1.0)      160;1726    9710         431        3.8284 (1.0)*',

        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',

        'Legend:',

        '  Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',

        '  OPS: Operations Per Second, computed as 1 / Mean',

    ])

def test_compare_2(sess, LineMatcher):

    output = make_logger(sess)

    sess.compare = '0002'

    sess.handle_loading()

    sess.finish()

    sess.display(Namespace(

        ensure_newline=lambda: None,

        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),

        section=lambda line, **opts: output.write(force_text(line) + u'\n'),

        write=lambda text, **opts: output.write(force_text(text)),

        rewrite=lambda text, **opts: output.write(force_text(text)),

    ))

    print(output.getvalue())

    LineMatcher(output.getvalue().splitlines()).fnmatch_lines([

        'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',

        'Comparing against benchmarks from: 0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes.json',

        '',

        '*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*',

        'Name (time in ns)            *         Min                 *Max                Mean              StdDev              Median                IQR            Outliers  Rounds  Iterations  OPS (Mops/s)*',

        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',

        '*xfast_parametrized[[]0[]] (0002*)     216.9028 (1.0)       7*739.2997 (1.0)      254.0585 (1.0)        0.0000 (1.0)      219.8103 (1.0)      27.3309 (1.45)     235;1688   11009         410        3.9361 (1.0)*',

\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000

""",

        'long': """Performance has regressed:

\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000

\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000

"""

    }[name_format]

def test_compare_1(sess, LineMatcher):

    output = make_logger(sess)

    sess.handle_loading()

    sess.finish()

    sess.display(Namespace(

        ensure_newline=lambda: None,

        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),

        write=lambda text, **opts: output.write(force_text(text)),

        rewrite=lambda text, **opts: output.write(force_text(text)),

    ))

    print(output.getvalue())

    LineMatcher(output.getvalue().splitlines()).fnmatch_lines([

        'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',

        'Comparing against benchmarks from: 0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted'

        '-changes.json',

        '',

        '*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*',

        'Name (time in ns)               *      Min                 *Max                Mean              StdDev              Median                IQR            Outliers  Rounds  Iterations  OPS (Mops/s) *',

        '-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------*',

        '*xfast_parametrized[[]0[]] (0001*)     217.3145 (1.0)      11*447.3891 (1.0)      262.2408 (1.00)     214.0442 (1.0)      220.1664 (1.00)     38.2154 (2.03)      90;1878    9987         418        3.8133 (1.00)*',