Namespace.getoption() - Code Metrics - Inspection of "Merge pull request #59 from qntln/feature-cprofile" - ionelmc/pytest-benchmark - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( e21d19...ee2f36 )

by Ionel Cristian

created 2016-10-09 12:09 UTC

Namespace.getoption() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	2
c	1
b	0
f	0
dl	0
loc	5
rs	9.4285

import json
import logging
import os
import sys
from io import BytesIO
from io import StringIO

import py
import pytest
from freezegun import freeze_time
from pathlib import Path

from pytest_benchmark import plugin
from pytest_benchmark.plugin import BenchmarkSession
from pytest_benchmark.plugin import pytest_benchmark_compare_machine_info
from pytest_benchmark.plugin import pytest_benchmark_generate_json
from pytest_benchmark.plugin import pytest_benchmark_group_stats
from pytest_benchmark.session import PerformanceRegression
from pytest_benchmark.storage import Storage
from pytest_benchmark.utils import NAME_FORMATTERS
from pytest_benchmark.utils import DifferenceRegressionCheck
from pytest_benchmark.utils import PercentageRegressionCheck
from pytest_benchmark.utils import get_machine_id


pytest_plugins = "pytester"


THIS = py.path.local(__file__)
STORAGE = THIS.dirpath(THIS.purebasename)

SAVE_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open())
JSON_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open())
SAVE_DATA["machine_info"] = JSON_DATA["machine_info"] = {'foo': 'bar'}
SAVE_DATA["commit_info"] = JSON_DATA["commit_info"] = {'foo': 'bar'}


class Namespace(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

    def __getitem__(self, item):
        return self.__dict__[item]

    def getoption(self, item, default=None):
        try:
            return self[item]
        except KeyError:
            return default


class LooseFileLike(BytesIO):
    def close(self):
        value = self.getvalue()
        super(LooseFileLike, self).close()
        self.getvalue = lambda: value


class MockSession(BenchmarkSession):
    def __init__(self, name_format):
        self.histogram = True
        self.storage = Storage(str(STORAGE), default_machine_id=get_machine_id(), logger=None)
        self.benchmarks = []
        self.performance_regressions = []
        self.sort = u"min"
        self.compare = '0001'
        self.logger = logging.getLogger(__name__)
        self.machine_id = "FoobarOS"
        self.machine_info = {'foo': 'bar'}
        self.name_format = NAME_FORMATTERS[name_format]
        self.save = self.autosave = self.json = False
        self.options = {
            'min_rounds': 123,
            'min_time': 234,
            'max_time': 345,
            'use_cprofile': False,
        }
        self.cprofile_sort_by = 'cumtime'
        self.compare_fail = []
        self.config = Namespace(hook=Namespace(
            pytest_benchmark_group_stats=pytest_benchmark_group_stats,
            pytest_benchmark_generate_machine_info=lambda **kwargs: {'foo': 'bar'},
            pytest_benchmark_update_machine_info=lambda **kwargs: None,
            pytest_benchmark_compare_machine_info=pytest_benchmark_compare_machine_info,
            pytest_benchmark_generate_json=pytest_benchmark_generate_json,
            pytest_benchmark_update_json=lambda **kwargs: None,
            pytest_benchmark_generate_commit_info=lambda **kwargs: {'foo': 'bar'},
            pytest_benchmark_update_commit_info=lambda **kwargs: None,
        ))
        self.group_by = 'group'
        self.columns = ['min', 'max', 'mean', 'stddev', 'median', 'iqr',
                        'outliers', 'rounds', 'iterations']
        for bench_file in reversed(self.storage.query("[0-9][0-9][0-9][0-9]_*")):
            with bench_file.open('rU') as fh:
                data = json.load(fh)
            self.benchmarks.extend(
                Namespace(
                    as_dict=lambda include_data=False, stats=True, flat=False, _bench=bench, cprofile='cumtime':
                        dict(_bench, **_bench["stats"]) if flat else dict(_bench),
                    name=bench['name'],
                    fullname=bench['fullname'],
                    group=bench['group'],
                    options=bench['options'],
                    has_error=False,
                    params=None,
                    **bench['stats']
                )
                for bench in data['benchmarks']
            )
            break


try:
    text_type = unicode
except NameError:
    text_type = str


def force_text(text):
    if isinstance(text, text_type):
        return text
    else:
        return text.decode('utf-8')


def force_bytes(text):
    if isinstance(text, text_type):
        return text.encode('utf-8')
    else:
        return text


@pytest.fixture(params=['short', 'normal', 'long'])
def name_format(request):
    return request.param


@pytest.fixture
def sess(request, name_format):
    return MockSession(name_format)


def make_logger(sess):
    output = StringIO()
    sess.logger = Namespace(
        warn=lambda code, text, **opts: output.write(u"%s: %s %s\n" % (code, force_text(text), opts)),
        info=lambda text, **opts: output.write(force_text(text) + u'\n'),
        error=lambda text: output.write(force_text(text) + u'\n'),
    )
    return output


def test_rendering(sess):
    output = make_logger(sess)
    sess.histogram = os.path.join('docs', 'sample')
    sess.compare = '*/*'
    sess.sort = 'name'
    sess.finish()
    sess.display(Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))


def test_regression_checks(sess, name_format):

    output = make_logger(sess)
    sess.handle_loading()
    sess.performance_regressions = []
    sess.compare_fail = [
        PercentageRegressionCheck("stddev", 5),
        DifferenceRegressionCheck("max", 0.000001)
    ]
    sess.finish()
    pytest.raises(PerformanceRegression, sess.display, Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))
    print(output.getvalue())
    assert sess.performance_regressions == {
        'normal': [
            ('test_xfast_parametrized[0] (0001_b87b9aa)',
             "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
            ('test_xfast_parametrized[0] (0001_b87b9aa)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
        ],
        'short': [
            ('xfast_parametrized[0] (0001)',
             "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
            ('xfast_parametrized[0] (0001)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
        ],
        'long': [
            ('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)',
             "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
            ('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
        ],
    }[name_format]
    output = make_logger(sess)
    pytest.raises(PerformanceRegression, sess.check_regressions)
    print(output.getvalue())
    assert output.getvalue() == {
        'short': """Performance has regressed:
\txfast_parametrized[0] (0001) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
\txfast_parametrized[0] (0001) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
""",
        'normal': """Performance has regressed:
\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
""",
        'long': """Performance has regressed:
\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
"""
    }[name_format]


@pytest.mark.skipif(sys.version_info[:2] < (2, 7),

                    reason="Something weird going on, see: https://bugs.python.org/issue4482")
def test_regression_checks_inf(sess, name_format):
    output = make_logger(sess)
    sess.compare = '0002'
    sess.handle_loading()
    sess.performance_regressions = []
    sess.compare_fail = [
        PercentageRegressionCheck("stddev", 5),
        DifferenceRegressionCheck("max", 0.000001)
    ]
    sess.finish()
    pytest.raises(PerformanceRegression, sess.display, Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))
    print(output.getvalue())
    assert sess.performance_regressions == {
        'normal': [
            ('test_xfast_parametrized[0] (0002_b87b9aa)',
             "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
            ('test_xfast_parametrized[0] (0002_b87b9aa)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000")
        ],
        'short': [
            ('xfast_parametrized[0] (0002)',
             "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
            ('xfast_parametrized[0] (0002)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000")
        ],
        'long': [
            ('tests/test_normal.py::test_xfast_parametrized[0] '
             '(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)',
             "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
            ('tests/test_normal.py::test_xfast_parametrized[0] '
             '(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > "
             '0.000001000')
        ]
    }[name_format]
    output = make_logger(sess)
    pytest.raises(PerformanceRegression, sess.check_regressions)
    print(output.getvalue())
    assert output.getvalue() == {
        'short': """Performance has regressed:
\txfast_parametrized[0] (0002) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
\txfast_parametrized[0] (0002) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
""",
        'normal': """Performance has regressed:
\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
""",
        'long': """Performance has regressed:
\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
"""
    }[name_format]


def test_compare_1(sess, LineMatcher):

    output = make_logger(sess)
    sess.handle_loading()
    sess.finish()
    sess.display(Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))
    print(output.getvalue())
    LineMatcher(output.getvalue().splitlines()).fnmatch_lines([
        'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',
        'Comparing against benchmarks from: 0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted'
        '-changes.json',
        '',
        '*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*',
        'Name (time in ns)               *      Min                 *Max                Mean              StdDev              Median                IQR            Outliers(*)  Rounds  Iterations',
        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
        '*xfast_parametrized[[]0[]] (0001*)     217.3145 (1.0)      11*447.3891 (1.0)      262.2408 (1.00)     214.0442 (1.0)      220.1664 (1.00)     38.2154 (2.03)         90;1878    9987         418',
        '*xfast_parametrized[[]0[]] (NOW) *     217.9511 (1.00)     13*290.0380 (1.16)     261.2051 (1.0)      263.9842 (1.23)     220.1638 (1.0)      18.8080 (1.0)         160;1726    9710         431',
        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
        '(*) Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',
    ])


def test_compare_2(sess, LineMatcher):

    output = make_logger(sess)
    sess.compare = '0002'
    sess.handle_loading()
    sess.finish()
    sess.display(Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        section=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))
    print(output.getvalue())
    LineMatcher(output.getvalue().splitlines()).fnmatch_lines([
        'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',
        'Comparing against benchmarks from: 0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes.json',
        '',
        '*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*',
        'Name (time in ns)            *         Min                 *Max                Mean              StdDev              Median                IQR            Outliers(*)  Rounds  Iterations',
        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
        '*xfast_parametrized[[]0[]] (0002*)     216.9028 (1.0)       7*739.2997 (1.0)      254.0585 (1.0)        0.0000 (1.0)      219.8103 (1.0)      27.3309 (1.45)        235;1688   11009         410',
        '*xfast_parametrized[[]0[]] (NOW) *     217.9511 (1.00)     13*290.0380 (1.72)     261.2051 (1.03)     263.9842 (inf)      220.1638 (1.00)     18.8080 (1.0)         160;1726    9710         431',
        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
        '(*) Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',
    ])

@freeze_time("2015-08-15T00:04:18.687119")
def test_save_json(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = False
    sess.autosave = False
    sess.json = LooseFileLike()
    sess.save_data = False
    sess.handle_saving()
    assert tmpdir.listdir() == []
    assert json.loads(sess.json.getvalue().decode()) == JSON_DATA


@freeze_time("2015-08-15T00:04:18.687119")

def test_save_with_name(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = 'foobar'
    sess.autosave = True
    sess.json = None
    sess.save_data = False
    sess.storage.path = Path(str(tmpdir))
    sess.handle_saving()
    files = list(Path(str(tmpdir)).rglob('*.json'))
    assert len(files) == 1
    assert json.load(files[0].open('rU')) == SAVE_DATA


@freeze_time("2015-08-15T00:04:18.687119")

def test_save_no_name(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = True
    sess.autosave = True
    sess.json = None
    sess.save_data = False
    sess.storage.path = Path(str(tmpdir))
    sess.handle_saving()
    files = list(Path(str(tmpdir)).rglob('*.json'))
    assert len(files) == 1
    assert json.load(files[0].open('rU')) == SAVE_DATA


@freeze_time("2015-08-15T00:04:18.687119")

def test_save_with_error(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = True
    sess.autosave = True
    sess.json = None
    sess.save_data = False
    sess.storage.path = Path(str(tmpdir))
    for bench in sess.benchmarks:
        bench.has_error = True
    sess.handle_saving()
    files = list(Path(str(tmpdir)).rglob('*.json'))
    assert len(files) == 1
    assert json.load(files[0].open('rU')) == {
        'benchmarks': [],
        'commit_info': {'foo': 'bar'},
        'datetime': '2015-08-15T00:04:18.687119',
        'machine_info': {'foo': 'bar'},
        'version': '2.5.0'
    }


@freeze_time("2015-08-15T00:04:18.687119")

def test_autosave(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = False
    sess.autosave = True
    sess.json = None
    sess.save_data = False
    sess.storage.path = Path(str(tmpdir))
    sess.handle_saving()
    files = list(Path(str(tmpdir)).rglob('*.json'))
    assert len(files) == 1
    assert json.load(files[0].open('rU')) == SAVE_DATA


1		import json
2		import logging
3		import os
4		import sys
5		from io import BytesIO
6		from io import StringIO
7
8		import py
9		import pytest
10		from freezegun import freeze_time
11		from pathlib import Path
12
13		from pytest_benchmark import plugin
14		from pytest_benchmark.plugin import BenchmarkSession
15		from pytest_benchmark.plugin import pytest_benchmark_compare_machine_info
16		from pytest_benchmark.plugin import pytest_benchmark_generate_json
17		from pytest_benchmark.plugin import pytest_benchmark_group_stats
18		from pytest_benchmark.session import PerformanceRegression
19		from pytest_benchmark.storage import Storage
20		from pytest_benchmark.utils import NAME_FORMATTERS
21		from pytest_benchmark.utils import DifferenceRegressionCheck
22		from pytest_benchmark.utils import PercentageRegressionCheck
23		from pytest_benchmark.utils import get_machine_id
24
25
26		pytest_plugins = "pytester"
27
28
29		THIS = py.path.local(__file__)
30		STORAGE = THIS.dirpath(THIS.purebasename)
31
32		SAVE_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open())
33		JSON_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open())
34		SAVE_DATA["machine_info"] = JSON_DATA["machine_info"] = {'foo': 'bar'}
35		SAVE_DATA["commit_info"] = JSON_DATA["commit_info"] = {'foo': 'bar'}
36
37
38		class Namespace(object):
39		def __init__(self, **kwargs):
40		self.__dict__.update(kwargs)
41
42		def __getitem__(self, item):
43		return self.__dict__[item]
44
45		def getoption(self, item, default=None):
46		try:
47		return self[item]
48		except KeyError:
49		return default
50
51
52		class LooseFileLike(BytesIO):
53		def close(self):
54		value = self.getvalue()
55		super(LooseFileLike, self).close()
56		self.getvalue = lambda: value
57
58
59		class MockSession(BenchmarkSession):
60		def __init__(self, name_format):
61		self.histogram = True
62		self.storage = Storage(str(STORAGE), default_machine_id=get_machine_id(), logger=None)
63		self.benchmarks = []
64		self.performance_regressions = []
65		self.sort = u"min"
66		self.compare = '0001'
67		self.logger = logging.getLogger(__name__)
68		self.machine_id = "FoobarOS"
69		self.machine_info = {'foo': 'bar'}
70		self.name_format = NAME_FORMATTERS[name_format]
71		self.save = self.autosave = self.json = False
72		self.options = {
73		'min_rounds': 123,
74		'min_time': 234,
75		'max_time': 345,
76		'use_cprofile': False,
77		}
78		self.cprofile_sort_by = 'cumtime'
79		self.compare_fail = []
80		self.config = Namespace(hook=Namespace(
81		pytest_benchmark_group_stats=pytest_benchmark_group_stats,
82		pytest_benchmark_generate_machine_info=lambda **kwargs: {'foo': 'bar'},
83		pytest_benchmark_update_machine_info=lambda **kwargs: None,
84		pytest_benchmark_compare_machine_info=pytest_benchmark_compare_machine_info,
85		pytest_benchmark_generate_json=pytest_benchmark_generate_json,
86		pytest_benchmark_update_json=lambda **kwargs: None,
87		pytest_benchmark_generate_commit_info=lambda **kwargs: {'foo': 'bar'},
88		pytest_benchmark_update_commit_info=lambda **kwargs: None,
89		))
90		self.group_by = 'group'
91		self.columns = ['min', 'max', 'mean', 'stddev', 'median', 'iqr',
92		'outliers', 'rounds', 'iterations']
93		for bench_file in reversed(self.storage.query("[0-9][0-9][0-9][0-9]_*")):
94		with bench_file.open('rU') as fh:
95		data = json.load(fh)
96		self.benchmarks.extend(
97		Namespace(
98		as_dict=lambda include_data=False, stats=True, flat=False, _bench=bench, cprofile='cumtime':
99		dict(_bench, **_bench["stats"]) if flat else dict(_bench),
100		name=bench['name'],
101		fullname=bench['fullname'],
102		group=bench['group'],
103		options=bench['options'],
104		has_error=False,
105		params=None,
106		**bench['stats']
107		)
108		for bench in data['benchmarks']
109		)
110		break
111
112
113		try:
114		text_type = unicode
115		except NameError:
116		text_type = str
117
118
119		def force_text(text):
120		if isinstance(text, text_type):
121		return text
122		else:
123		return text.decode('utf-8')
124
125
126		def force_bytes(text):
127		if isinstance(text, text_type):
128		return text.encode('utf-8')
129		else:
130		return text
131
132
133		@pytest.fixture(params=['short', 'normal', 'long'])
134		def name_format(request):
135		return request.param
136
137
138		@pytest.fixture
139		def sess(request, name_format):
140		return MockSession(name_format)
141
142
143		def make_logger(sess):
144		output = StringIO()
145		sess.logger = Namespace(
146		warn=lambda code, text, **opts: output.write(u"%s: %s %s\n" % (code, force_text(text), opts)),
147		info=lambda text, **opts: output.write(force_text(text) + u'\n'),
148		error=lambda text: output.write(force_text(text) + u'\n'),
149		)
150		return output
151
152
153		def test_rendering(sess):
154		output = make_logger(sess)
155		sess.histogram = os.path.join('docs', 'sample')
156		sess.compare = '/'
157		sess.sort = 'name'
158		sess.finish()
159		sess.display(Namespace(
160		ensure_newline=lambda: None,
161		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
162		write=lambda text, **opts: output.write(force_text(text)),
163		rewrite=lambda text, **opts: output.write(force_text(text)),
164		))
165
166
167	View Code Duplication	def test_regression_checks(sess, name_format):
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
168		output = make_logger(sess)
169		sess.handle_loading()
170		sess.performance_regressions = []
171		sess.compare_fail = [
172		PercentageRegressionCheck("stddev", 5),
173		DifferenceRegressionCheck("max", 0.000001)
174		]
175		sess.finish()
176		pytest.raises(PerformanceRegression, sess.display, Namespace(
177		ensure_newline=lambda: None,
178		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
179		write=lambda text, **opts: output.write(force_text(text)),
180		rewrite=lambda text, **opts: output.write(force_text(text)),
181		))
182		print(output.getvalue())
183		assert sess.performance_regressions == {
184		'normal': [
185		('test_xfast_parametrized[0] (0001_b87b9aa)',
186		"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
187		('test_xfast_parametrized[0] (0001_b87b9aa)',
188		"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
189		],
190		'short': [
191		('xfast_parametrized[0] (0001)',
192		"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
193		('xfast_parametrized[0] (0001)',
194		"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
195		],
196		'long': [
197		('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)',
198		"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
199		('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)',
200		"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
201		],
202		}[name_format]
203		output = make_logger(sess)
204		pytest.raises(PerformanceRegression, sess.check_regressions)
205		print(output.getvalue())
206		assert output.getvalue() == {
207		'short': """Performance has regressed:
208		\txfast_parametrized[0] (0001) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
209		\txfast_parametrized[0] (0001) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
210		""",
211		'normal': """Performance has regressed:
212		\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
213		\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
214		""",
215		'long': """Performance has regressed:
216		\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
217		\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
218		"""
219		}[name_format]
220
221
222	View Code Duplication	@pytest.mark.skipif(sys.version_info[:2] < (2, 7),
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
223		reason="Something weird going on, see: https://bugs.python.org/issue4482")
224		def test_regression_checks_inf(sess, name_format):
225		output = make_logger(sess)
226		sess.compare = '0002'
227		sess.handle_loading()
228		sess.performance_regressions = []
229		sess.compare_fail = [
230		PercentageRegressionCheck("stddev", 5),
231		DifferenceRegressionCheck("max", 0.000001)
232		]
233		sess.finish()
234		pytest.raises(PerformanceRegression, sess.display, Namespace(
235		ensure_newline=lambda: None,
236		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
237		write=lambda text, **opts: output.write(force_text(text)),
238		rewrite=lambda text, **opts: output.write(force_text(text)),
239		))
240		print(output.getvalue())
241		assert sess.performance_regressions == {
242		'normal': [
243		('test_xfast_parametrized[0] (0002_b87b9aa)',
244		"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
245		('test_xfast_parametrized[0] (0002_b87b9aa)',
246		"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000")
247		],
248		'short': [
249		('xfast_parametrized[0] (0002)',
250		"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
251		('xfast_parametrized[0] (0002)',
252		"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000")
253		],
254		'long': [
255		('tests/test_normal.py::test_xfast_parametrized[0] '
256		'(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)',
257		"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
258		('tests/test_normal.py::test_xfast_parametrized[0] '
259		'(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)',
260		"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > "
261		'0.000001000')
262		]
263		}[name_format]
264		output = make_logger(sess)
265		pytest.raises(PerformanceRegression, sess.check_regressions)
266		print(output.getvalue())
267		assert output.getvalue() == {
268		'short': """Performance has regressed:
269		\txfast_parametrized[0] (0002) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
270		\txfast_parametrized[0] (0002) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
271		""",
272		'normal': """Performance has regressed:
273		\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
274		\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
275		""",
276		'long': """Performance has regressed:
277		\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
278		\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
279		"""
280		}[name_format]
281
282
283	View Code Duplication	def test_compare_1(sess, LineMatcher):
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
284		output = make_logger(sess)
285		sess.handle_loading()
286		sess.finish()
287		sess.display(Namespace(
288		ensure_newline=lambda: None,
289		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
290		write=lambda text, **opts: output.write(force_text(text)),
291		rewrite=lambda text, **opts: output.write(force_text(text)),
292		))
293		print(output.getvalue())
294		LineMatcher(output.getvalue().splitlines()).fnmatch_lines([
295		'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',
296		'Comparing against benchmarks from: 0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted'
297		'-changes.json',
298		'',
299		'------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------',
300		'Name (time in ns) * Min Max Mean StdDev Median IQR Outliers() Rounds Iterations',
301		'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
302		'xfast_parametrized[[]0[]] (0001) 217.3145 (1.0) 11*447.3891 (1.0) 262.2408 (1.00) 214.0442 (1.0) 220.1664 (1.00) 38.2154 (2.03) 90;1878 9987 418',
303		'xfast_parametrized[[]0[]] (NOW) 217.9511 (1.00) 13*290.0380 (1.16) 261.2051 (1.0) 263.9842 (1.23) 220.1638 (1.0) 18.8080 (1.0) 160;1726 9710 431',
304		'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
305		'(*) Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',
306		])
307
308
309	View Code Duplication	def test_compare_2(sess, LineMatcher):
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
310		output = make_logger(sess)
311		sess.compare = '0002'
312		sess.handle_loading()
313		sess.finish()
314		sess.display(Namespace(
315		ensure_newline=lambda: None,
316		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
317		section=lambda line, **opts: output.write(force_text(line) + u'\n'),
318		write=lambda text, **opts: output.write(force_text(text)),
319		rewrite=lambda text, **opts: output.write(force_text(text)),
320		))
321		print(output.getvalue())
322		LineMatcher(output.getvalue().splitlines()).fnmatch_lines([
323		'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',
324		'Comparing against benchmarks from: 0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes.json',
325		'',
326		'------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------',
327		'Name (time in ns) * Min Max Mean StdDev Median IQR Outliers() Rounds Iterations',
328		'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
329		'xfast_parametrized[[]0[]] (0002) 216.9028 (1.0) 7*739.2997 (1.0) 254.0585 (1.0) 0.0000 (1.0) 219.8103 (1.0) 27.3309 (1.45) 235;1688 11009 410',
330		'xfast_parametrized[[]0[]] (NOW) 217.9511 (1.00) 13*290.0380 (1.72) 261.2051 (1.03) 263.9842 (inf) 220.1638 (1.00) 18.8080 (1.0) 160;1726 9710 431',
331		'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
332		'(*) Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',
333		])
334
335		@freeze_time("2015-08-15T00:04:18.687119")
336		def test_save_json(sess, tmpdir, monkeypatch):
337		monkeypatch.setattr(plugin, '__version__', '2.5.0')
338		sess.save = False
339		sess.autosave = False
340		sess.json = LooseFileLike()
341		sess.save_data = False
342		sess.handle_saving()
343		assert tmpdir.listdir() == []
344		assert json.loads(sess.json.getvalue().decode()) == JSON_DATA
345
346
347	View Code Duplication	@freeze_time("2015-08-15T00:04:18.687119")
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
348		def test_save_with_name(sess, tmpdir, monkeypatch):
349		monkeypatch.setattr(plugin, '__version__', '2.5.0')
350		sess.save = 'foobar'
351		sess.autosave = True
352		sess.json = None
353		sess.save_data = False
354		sess.storage.path = Path(str(tmpdir))
355		sess.handle_saving()
356		files = list(Path(str(tmpdir)).rglob('*.json'))
357		assert len(files) == 1
358		assert json.load(files[0].open('rU')) == SAVE_DATA
359
360
361	View Code Duplication	@freeze_time("2015-08-15T00:04:18.687119")
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
362		def test_save_no_name(sess, tmpdir, monkeypatch):
363		monkeypatch.setattr(plugin, '__version__', '2.5.0')
364		sess.save = True
365		sess.autosave = True
366		sess.json = None
367		sess.save_data = False
368		sess.storage.path = Path(str(tmpdir))
369		sess.handle_saving()
370		files = list(Path(str(tmpdir)).rglob('*.json'))
371		assert len(files) == 1
372		assert json.load(files[0].open('rU')) == SAVE_DATA
373
374
375	View Code Duplication	@freeze_time("2015-08-15T00:04:18.687119")
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
376		def test_save_with_error(sess, tmpdir, monkeypatch):
377		monkeypatch.setattr(plugin, '__version__', '2.5.0')
378		sess.save = True
379		sess.autosave = True
380		sess.json = None
381		sess.save_data = False
382		sess.storage.path = Path(str(tmpdir))
383		for bench in sess.benchmarks:
384		bench.has_error = True
385		sess.handle_saving()
386		files = list(Path(str(tmpdir)).rglob('*.json'))
387		assert len(files) == 1
388		assert json.load(files[0].open('rU')) == {
389		'benchmarks': [],
390		'commit_info': {'foo': 'bar'},
391		'datetime': '2015-08-15T00:04:18.687119',
392		'machine_info': {'foo': 'bar'},
393		'version': '2.5.0'
394		}
395
396
397	View Code Duplication	@freeze_time("2015-08-15T00:04:18.687119")
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
398		def test_autosave(sess, tmpdir, monkeypatch):
399		monkeypatch.setattr(plugin, '__version__', '2.5.0')
400		sess.save = False
401		sess.autosave = True
402		sess.json = None
403		sess.save_data = False
404		sess.storage.path = Path(str(tmpdir))
405		sess.handle_saving()
406		files = list(Path(str(tmpdir)).rglob('*.json'))
407		assert len(files) == 1
408		assert json.load(files[0].open('rU')) == SAVE_DATA
409

ionelmc / pytest-benchmark

Push — master ( e21d19...ee2f36 )

Namespace.getoption() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like