make_logger() - Code Metrics - Inspection of "Elasticsearch report backend" - ionelmc/pytest-benchmark - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#58)

unknown

created 2016-10-11 11:14 UTC

make_logger() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	13
Ratio	100 %

Importance

Changes	4
Bugs	0	Features	0

Metric	Value
cc	7
c	4
b	0
f	0
dl	13
loc	13
rs	7.3333

import json
import logging
import os
import sys
from io import BytesIO
from io import StringIO
from pathlib import Path

import py
import pytest
from freezegun import freeze_time

from pytest_benchmark import plugin
from pytest_benchmark.plugin import BenchmarkSession
from pytest_benchmark.plugin import pytest_benchmark_compare_machine_info
from pytest_benchmark.plugin import pytest_benchmark_generate_json
from pytest_benchmark.plugin import pytest_benchmark_group_stats
from pytest_benchmark.session import PerformanceRegression
from pytest_benchmark.storage.file import FileStorage
from pytest_benchmark.utils import NAME_FORMATTERS
from pytest_benchmark.utils import DifferenceRegressionCheck
from pytest_benchmark.utils import PercentageRegressionCheck
from pytest_benchmark.utils import get_machine_id

pytest_plugins = "pytester"


THIS = py.path.local(__file__)
STORAGE = THIS.dirpath(THIS.purebasename)

SAVE_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open())
JSON_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open())
SAVE_DATA["machine_info"] = JSON_DATA["machine_info"] = {'foo': 'bar'}
SAVE_DATA["commit_info"] = JSON_DATA["commit_info"] = {'foo': 'bar'}


class Namespace(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

    def __getitem__(self, item):
        return self.__dict__[item]

    def getoption(self, item, default=None):
        try:
            return self[item]
        except KeyError:
            return default


class LooseFileLike(BytesIO):
    def close(self):
        value = self.getvalue()
        super(LooseFileLike, self).close()
        self.getvalue = lambda: value


class MockSession(BenchmarkSession):
    def __init__(self, name_format):
        self.histogram = True
        self.verbose = False
        self.benchmarks = []
        self.performance_regressions = []
        self.sort = u"min"
        self.compare = '0001'
        self.logger = logging.getLogger(__name__)
        self.machine_id = "FoobarOS"
        self.machine_info = {'foo': 'bar'}
        self.save = self.autosave = self.json = False
        self.name_format = NAME_FORMATTERS[name_format]
        self.options = {
            'min_rounds': 123,
            'min_time': 234,
            'max_time': 345,
            'use_cprofile': False,
        }
        self.cprofile_sort_by = 'cumtime'
        self.compare_fail = []
        self.config = Namespace(hook=Namespace(
            pytest_benchmark_group_stats=pytest_benchmark_group_stats,
            pytest_benchmark_generate_machine_info=lambda **kwargs: {'foo': 'bar'},
            pytest_benchmark_update_machine_info=lambda **kwargs: None,
            pytest_benchmark_compare_machine_info=pytest_benchmark_compare_machine_info,
            pytest_benchmark_generate_json=pytest_benchmark_generate_json,
            pytest_benchmark_update_json=lambda **kwargs: None,
            pytest_benchmark_generate_commit_info=lambda **kwargs: {'foo': 'bar'},
            pytest_benchmark_update_commit_info=lambda **kwargs: None,
        ))
        self.storage = FileStorage(str(STORAGE), default_machine_id=get_machine_id(), logger=self.logger)
        self.group_by = 'group'
        self.columns = ['min', 'max', 'mean', 'stddev', 'median', 'iqr',
                        'outliers', 'rounds', 'iterations']
        for bench_file in reversed(self.storage.query("[0-9][0-9][0-9][0-9]_*")):
            with bench_file.open('rU') as fh:
                data = json.load(fh)
            self.benchmarks.extend(
                Namespace(
                    as_dict=lambda include_data=False, stats=True, flat=False, _bench=bench, cprofile='cumtime':
                        dict(_bench, **_bench["stats"]) if flat else dict(_bench),
                    name=bench['name'],
                    fullname=bench['fullname'],
                    group=bench['group'],
                    options=bench['options'],
                    has_error=False,
                    params=None,
                    **bench['stats']
                )
                for bench in data['benchmarks']
            )
            break


try:
    text_type = unicode
except NameError:
    text_type = str


def force_text(text):
    if isinstance(text, text_type):
        return text
    else:
        return text.decode('utf-8')


def force_bytes(text):
    if isinstance(text, text_type):
        return text.encode('utf-8')
    else:
        return text


@pytest.fixture(params=['short', 'normal', 'long'])
def name_format(request):
    return request.param


@pytest.fixture
def sess(request, name_format):
    return MockSession(name_format)


def make_logger(sess):

    output = StringIO()
    sess.logger = Namespace(
        warn=lambda code, text, **opts: output.write(u"%s: %s %s\n" % (code, force_text(text), opts)),
        info=lambda text, **opts: output.write(force_text(text) + u'\n'),
        error=lambda text: output.write(force_text(text) + u'\n'),
    )
    sess.storage.logger = Namespace(
        warn=lambda code, text, **opts: output.write(u"%s: %s %s\n" % (code, force_text(text), opts)),
        info=lambda text, **opts: output.write(force_text(text) + u'\n'),
        error=lambda text: output.write(force_text(text) + u'\n'),
    )
    return output


def test_rendering(sess):
    output = make_logger(sess)
    sess.histogram = os.path.join('docs', 'sample')
    sess.compare = '*/*'
    sess.sort = 'name'
    sess.finish()
    sess.display(Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),

        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))


def test_regression_checks(sess, name_format):
    output = make_logger(sess)
    sess.handle_loading()
    sess.performance_regressions = []
    sess.compare_fail = [
        PercentageRegressionCheck("stddev", 5),
        DifferenceRegressionCheck("max", 0.000001)
    ]
    sess.finish()
    pytest.raises(PerformanceRegression, sess.display, Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))
    print(output.getvalue())
    assert sess.performance_regressions == {
        'normal': [
            ('test_xfast_parametrized[0] (0001_b87b9aa)',
             "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
            ('test_xfast_parametrized[0] (0001_b87b9aa)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
        ],
        'short': [
            ('xfast_parametrized[0] (0001)',
             "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
            ('xfast_parametrized[0] (0001)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
        ],
        'long': [
            ('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)',
             "Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
            ('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
        ],
    }[name_format]
    output = make_logger(sess)
    pytest.raises(PerformanceRegression, sess.check_regressions)
    print(output.getvalue())
    assert output.getvalue() == {
        'short': """Performance has regressed:
\txfast_parametrized[0] (0001) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
\txfast_parametrized[0] (0001) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
""",
        'normal': """Performance has regressed:
\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
""",
        'long': """Performance has regressed:
\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000

"""
    }[name_format]


@pytest.mark.skipif(sys.version_info[:2] < (2, 7),
                    reason="Something weird going on, see: https://bugs.python.org/issue4482")
def test_regression_checks_inf(sess, name_format):
    output = make_logger(sess)
    sess.compare = '0002'
    sess.handle_loading()
    sess.performance_regressions = []
    sess.compare_fail = [
        PercentageRegressionCheck("stddev", 5),
        DifferenceRegressionCheck("max", 0.000001)
    ]
    sess.finish()
    pytest.raises(PerformanceRegression, sess.display, Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))
    print(output.getvalue())
    assert sess.performance_regressions == {
        'normal': [
            ('test_xfast_parametrized[0] (0002_b87b9aa)',
             "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
            ('test_xfast_parametrized[0] (0002_b87b9aa)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000")
        ],
        'short': [
            ('xfast_parametrized[0] (0002)',
             "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
            ('xfast_parametrized[0] (0002)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000")
        ],
        'long': [
            ('tests/test_normal.py::test_xfast_parametrized[0] '
             '(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)',
             "Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
            ('tests/test_normal.py::test_xfast_parametrized[0] '
             '(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)',
             "Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > "
             '0.000001000')
        ]
    }[name_format]
    output = make_logger(sess)
    pytest.raises(PerformanceRegression, sess.check_regressions)
    print(output.getvalue())
    assert output.getvalue() == {
        'short': """Performance has regressed:
\txfast_parametrized[0] (0002) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
\txfast_parametrized[0] (0002) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
""",
        'normal': """Performance has regressed:
\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
""",
        'long': """Performance has regressed:
\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000

"""
    }[name_format]


def test_compare_1(sess, LineMatcher):
    output = make_logger(sess)
    sess.handle_loading()
    sess.finish()
    sess.display(Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))
    print(output.getvalue())
    LineMatcher(output.getvalue().splitlines()).fnmatch_lines([
        'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',
        'Comparing against benchmarks from: 0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted'
        '-changes.json',
        '',
        '*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*',
        'Name (time in ns)               *      Min                 *Max                Mean              StdDev              Median                IQR            Outliers(*)  Rounds  Iterations',
        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
        '*xfast_parametrized[[]0[]] (0001*)     217.3145 (1.0)      11*447.3891 (1.0)      262.2408 (1.00)     214.0442 (1.0)      220.1664 (1.00)     38.2154 (2.03)         90;1878    9987         418',
        '*xfast_parametrized[[]0[]] (NOW) *     217.9511 (1.00)     13*290.0380 (1.16)     261.2051 (1.0)      263.9842 (1.23)     220.1638 (1.0)      18.8080 (1.0)         160;1726    9710         431',
        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',

        '(*) Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',
    ])


def test_compare_2(sess, LineMatcher):
    output = make_logger(sess)
    sess.compare = '0002'
    sess.handle_loading()
    sess.finish()
    sess.display(Namespace(
        ensure_newline=lambda: None,
        write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
        section=lambda line, **opts: output.write(force_text(line) + u'\n'),
        write=lambda text, **opts: output.write(force_text(text)),
        rewrite=lambda text, **opts: output.write(force_text(text)),
    ))
    print(output.getvalue())
    LineMatcher(output.getvalue().splitlines()).fnmatch_lines([
        'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',
        'Comparing against benchmarks from: 0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes.json',
        '',
        '*------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------*',
        'Name (time in ns)            *         Min                 *Max                Mean              StdDev              Median                IQR            Outliers(*)  Rounds  Iterations',
        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
        '*xfast_parametrized[[]0[]] (0002*)     216.9028 (1.0)       7*739.2997 (1.0)      254.0585 (1.0)        0.0000 (1.0)      219.8103 (1.0)      27.3309 (1.45)        235;1688   11009         410',
        '*xfast_parametrized[[]0[]] (NOW) *     217.9511 (1.00)     13*290.0380 (1.72)     261.2051 (1.03)     263.9842 (inf)      220.1638 (1.00)     18.8080 (1.0)         160;1726    9710         431',
        '--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
        '(*) Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',
    ])

@freeze_time("2015-08-15T00:04:18.687119")
def test_save_json(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = False
    sess.autosave = False
    sess.json = LooseFileLike()
    sess.save_data = False
    sess.handle_saving()

    assert tmpdir.listdir() == []
    assert json.loads(sess.json.getvalue().decode()) == JSON_DATA


@freeze_time("2015-08-15T00:04:18.687119")
def test_save_with_name(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = 'foobar'
    sess.autosave = True
    sess.json = None
    sess.save_data = False
    sess.storage.path = Path(str(tmpdir))
    sess.handle_saving()
    files = list(Path(str(tmpdir)).rglob('*.json'))
    print(files)
    assert len(files) == 1
    assert json.load(files[0].open('rU')) == SAVE_DATA


@freeze_time("2015-08-15T00:04:18.687119")
def test_save_no_name(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = True
    sess.autosave = True
    sess.json = None
    sess.save_data = False
    sess.storage.path = Path(str(tmpdir))
    sess.handle_saving()

    files = list(Path(str(tmpdir)).rglob('*.json'))
    assert len(files) == 1
    assert json.load(files[0].open('rU')) == SAVE_DATA


@freeze_time("2015-08-15T00:04:18.687119")
def test_save_with_error(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = True
    sess.autosave = True
    sess.json = None
    sess.save_data = False
    sess.storage.path = Path(str(tmpdir))
    for bench in sess.benchmarks:
        bench.has_error = True
    sess.handle_saving()
    files = list(Path(str(tmpdir)).rglob('*.json'))
    assert len(files) == 1
    assert json.load(files[0].open('rU')) == {
        'benchmarks': [],
        'commit_info': {'foo': 'bar'},
        'datetime': '2015-08-15T00:04:18.687119',
        'machine_info': {'foo': 'bar'},
        'version': '2.5.0'
    }


@freeze_time("2015-08-15T00:04:18.687119")
def test_autosave(sess, tmpdir, monkeypatch):
    monkeypatch.setattr(plugin, '__version__', '2.5.0')
    sess.save = False
    sess.autosave = True
    sess.json = None
    sess.save_data = False
    sess.storage.path = Path(str(tmpdir))
    sess.handle_saving()
    files = list(Path(str(tmpdir)).rglob('*.json'))
    assert len(files) == 1
    assert json.load(files[0].open('rU')) == SAVE_DATA


1		import json
2		import logging
3		import os
4		import sys
5		from io import BytesIO
6		from io import StringIO
7		from pathlib import Path
8
9		import py
10		import pytest
11		from freezegun import freeze_time
12
13		from pytest_benchmark import plugin
14		from pytest_benchmark.plugin import BenchmarkSession
15		from pytest_benchmark.plugin import pytest_benchmark_compare_machine_info
16		from pytest_benchmark.plugin import pytest_benchmark_generate_json
17		from pytest_benchmark.plugin import pytest_benchmark_group_stats
18		from pytest_benchmark.session import PerformanceRegression
19		from pytest_benchmark.storage.file import FileStorage
20		from pytest_benchmark.utils import NAME_FORMATTERS
21		from pytest_benchmark.utils import DifferenceRegressionCheck
22		from pytest_benchmark.utils import PercentageRegressionCheck
23		from pytest_benchmark.utils import get_machine_id
24
25		pytest_plugins = "pytester"
26
27
28		THIS = py.path.local(__file__)
29		STORAGE = THIS.dirpath(THIS.purebasename)
30
31		SAVE_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open())
32		JSON_DATA = json.load(STORAGE.listdir('0030_*.json')[0].open())
33		SAVE_DATA["machine_info"] = JSON_DATA["machine_info"] = {'foo': 'bar'}
34		SAVE_DATA["commit_info"] = JSON_DATA["commit_info"] = {'foo': 'bar'}
35
36
37		class Namespace(object):
38		def __init__(self, **kwargs):
39		self.__dict__.update(kwargs)
40
41		def __getitem__(self, item):
42		return self.__dict__[item]
43
44		def getoption(self, item, default=None):
45		try:
46		return self[item]
47		except KeyError:
48		return default
49
50
51		class LooseFileLike(BytesIO):
52		def close(self):
53		value = self.getvalue()
54		super(LooseFileLike, self).close()
55		self.getvalue = lambda: value
56
57
58		class MockSession(BenchmarkSession):
59		def __init__(self, name_format):
60		self.histogram = True
61		self.verbose = False
62		self.benchmarks = []
63		self.performance_regressions = []
64		self.sort = u"min"
65		self.compare = '0001'
66		self.logger = logging.getLogger(__name__)
67		self.machine_id = "FoobarOS"
68		self.machine_info = {'foo': 'bar'}
69		self.save = self.autosave = self.json = False
70		self.name_format = NAME_FORMATTERS[name_format]
71		self.options = {
72		'min_rounds': 123,
73		'min_time': 234,
74		'max_time': 345,
75		'use_cprofile': False,
76		}
77		self.cprofile_sort_by = 'cumtime'
78		self.compare_fail = []
79		self.config = Namespace(hook=Namespace(
80		pytest_benchmark_group_stats=pytest_benchmark_group_stats,
81		pytest_benchmark_generate_machine_info=lambda **kwargs: {'foo': 'bar'},
82		pytest_benchmark_update_machine_info=lambda **kwargs: None,
83		pytest_benchmark_compare_machine_info=pytest_benchmark_compare_machine_info,
84		pytest_benchmark_generate_json=pytest_benchmark_generate_json,
85		pytest_benchmark_update_json=lambda **kwargs: None,
86		pytest_benchmark_generate_commit_info=lambda **kwargs: {'foo': 'bar'},
87		pytest_benchmark_update_commit_info=lambda **kwargs: None,
88		))
89		self.storage = FileStorage(str(STORAGE), default_machine_id=get_machine_id(), logger=self.logger)
90		self.group_by = 'group'
91		self.columns = ['min', 'max', 'mean', 'stddev', 'median', 'iqr',
92		'outliers', 'rounds', 'iterations']
93		for bench_file in reversed(self.storage.query("[0-9][0-9][0-9][0-9]_*")):
94		with bench_file.open('rU') as fh:
95		data = json.load(fh)
96		self.benchmarks.extend(
97		Namespace(
98		as_dict=lambda include_data=False, stats=True, flat=False, _bench=bench, cprofile='cumtime':
99		dict(_bench, **_bench["stats"]) if flat else dict(_bench),
100		name=bench['name'],
101		fullname=bench['fullname'],
102		group=bench['group'],
103		options=bench['options'],
104		has_error=False,
105		params=None,
106		**bench['stats']
107		)
108		for bench in data['benchmarks']
109		)
110		break
111
112
113		try:
114		text_type = unicode
115		except NameError:
116		text_type = str
117
118
119		def force_text(text):
120		if isinstance(text, text_type):
121		return text
122		else:
123		return text.decode('utf-8')
124
125
126		def force_bytes(text):
127		if isinstance(text, text_type):
128		return text.encode('utf-8')
129		else:
130		return text
131
132
133		@pytest.fixture(params=['short', 'normal', 'long'])
134		def name_format(request):
135		return request.param
136
137
138		@pytest.fixture
139		def sess(request, name_format):
140		return MockSession(name_format)
141
142
143	View Code Duplication	def make_logger(sess):
		0 ignored issues – show Duplication introduced 2016-09-09 08:37 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
144		output = StringIO()
145		sess.logger = Namespace(
146		warn=lambda code, text, **opts: output.write(u"%s: %s %s\n" % (code, force_text(text), opts)),
147		info=lambda text, **opts: output.write(force_text(text) + u'\n'),
148		error=lambda text: output.write(force_text(text) + u'\n'),
149		)
150		sess.storage.logger = Namespace(
151		warn=lambda code, text, **opts: output.write(u"%s: %s %s\n" % (code, force_text(text), opts)),
152		info=lambda text, **opts: output.write(force_text(text) + u'\n'),
153		error=lambda text: output.write(force_text(text) + u'\n'),
154		)
155		return output
156
157
158		def test_rendering(sess):
159		output = make_logger(sess)
160		sess.histogram = os.path.join('docs', 'sample')
161		sess.compare = '/'
162		sess.sort = 'name'
163		sess.finish()
164		sess.display(Namespace(
165		ensure_newline=lambda: None,
166		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
167	View Code Duplication	write=lambda text, **opts: output.write(force_text(text)),
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
168		rewrite=lambda text, **opts: output.write(force_text(text)),
169		))
170
171
172		def test_regression_checks(sess, name_format):
173		output = make_logger(sess)
174		sess.handle_loading()
175		sess.performance_regressions = []
176		sess.compare_fail = [
177		PercentageRegressionCheck("stddev", 5),
178		DifferenceRegressionCheck("max", 0.000001)
179		]
180		sess.finish()
181		pytest.raises(PerformanceRegression, sess.display, Namespace(
182		ensure_newline=lambda: None,
183		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
184		write=lambda text, **opts: output.write(force_text(text)),
185		rewrite=lambda text, **opts: output.write(force_text(text)),
186		))
187		print(output.getvalue())
188		assert sess.performance_regressions == {
189		'normal': [
190		('test_xfast_parametrized[0] (0001_b87b9aa)',
191		"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
192		('test_xfast_parametrized[0] (0001_b87b9aa)',
193		"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
194		],
195		'short': [
196		('xfast_parametrized[0] (0001)',
197		"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
198		('xfast_parametrized[0] (0001)',
199		"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
200		],
201		'long': [
202		('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)',
203		"Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000"),
204		('tests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes)',
205		"Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000")
206		],
207		}[name_format]
208		output = make_logger(sess)
209		pytest.raises(PerformanceRegression, sess.check_regressions)
210		print(output.getvalue())
211		assert output.getvalue() == {
212		'short': """Performance has regressed:
213		\txfast_parametrized[0] (0001) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
214		\txfast_parametrized[0] (0001) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
215		""",
216		'normal': """Performance has regressed:
217		\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
218		\ttest_xfast_parametrized[0] (0001_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
219		""",
220		'long': """Performance has regressed:
221		\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: 23.331641765 > 5.000000000
222	View Code Duplication	\ttests/test_normal.py::test_xfast_parametrized[0] (0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000001843 > 0.000001000
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
223		"""
224		}[name_format]
225
226
227		@pytest.mark.skipif(sys.version_info[:2] < (2, 7),
228		reason="Something weird going on, see: https://bugs.python.org/issue4482")
229		def test_regression_checks_inf(sess, name_format):
230		output = make_logger(sess)
231		sess.compare = '0002'
232		sess.handle_loading()
233		sess.performance_regressions = []
234		sess.compare_fail = [
235		PercentageRegressionCheck("stddev", 5),
236		DifferenceRegressionCheck("max", 0.000001)
237		]
238		sess.finish()
239		pytest.raises(PerformanceRegression, sess.display, Namespace(
240		ensure_newline=lambda: None,
241		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
242		write=lambda text, **opts: output.write(force_text(text)),
243		rewrite=lambda text, **opts: output.write(force_text(text)),
244		))
245		print(output.getvalue())
246		assert sess.performance_regressions == {
247		'normal': [
248		('test_xfast_parametrized[0] (0002_b87b9aa)',
249		"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
250		('test_xfast_parametrized[0] (0002_b87b9aa)',
251		"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000")
252		],
253		'short': [
254		('xfast_parametrized[0] (0002)',
255		"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
256		('xfast_parametrized[0] (0002)',
257		"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000")
258		],
259		'long': [
260		('tests/test_normal.py::test_xfast_parametrized[0] '
261		'(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)',
262		"Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000"),
263		('tests/test_normal.py::test_xfast_parametrized[0] '
264		'(0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes)',
265		"Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > "
266		'0.000001000')
267		]
268		}[name_format]
269		output = make_logger(sess)
270		pytest.raises(PerformanceRegression, sess.check_regressions)
271		print(output.getvalue())
272		assert output.getvalue() == {
273		'short': """Performance has regressed:
274		\txfast_parametrized[0] (0002) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
275		\txfast_parametrized[0] (0002) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
276		""",
277		'normal': """Performance has regressed:
278		\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
279		\ttest_xfast_parametrized[0] (0002_b87b9aa) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
280		""",
281		'long': """Performance has regressed:
282		\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'stddev' has failed PercentageRegressionCheck: inf > 5.000000000
283	View Code Duplication	\ttests/test_normal.py::test_xfast_parametrized[0] (0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes) - Field 'max' has failed DifferenceRegressionCheck: 0.000005551 > 0.000001000
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
284		"""
285		}[name_format]
286
287
288		def test_compare_1(sess, LineMatcher):
289		output = make_logger(sess)
290		sess.handle_loading()
291		sess.finish()
292		sess.display(Namespace(
293		ensure_newline=lambda: None,
294		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
295		write=lambda text, **opts: output.write(force_text(text)),
296		rewrite=lambda text, **opts: output.write(force_text(text)),
297		))
298		print(output.getvalue())
299		LineMatcher(output.getvalue().splitlines()).fnmatch_lines([
300		'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',
301		'Comparing against benchmarks from: 0001_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190343_uncommitted'
302		'-changes.json',
303		'',
304		'------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------',
305		'Name (time in ns) * Min Max Mean StdDev Median IQR Outliers() Rounds Iterations',
306		'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
307		'xfast_parametrized[[]0[]] (0001) 217.3145 (1.0) 11*447.3891 (1.0) 262.2408 (1.00) 214.0442 (1.0) 220.1664 (1.00) 38.2154 (2.03) 90;1878 9987 418',
308		'xfast_parametrized[[]0[]] (NOW) 217.9511 (1.00) 13*290.0380 (1.16) 261.2051 (1.0) 263.9842 (1.23) 220.1638 (1.0) 18.8080 (1.0) 160;1726 9710 431',
309	View Code Duplication	'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
310		'(*) Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',
311		])
312
313
314		def test_compare_2(sess, LineMatcher):
315		output = make_logger(sess)
316		sess.compare = '0002'
317		sess.handle_loading()
318		sess.finish()
319		sess.display(Namespace(
320		ensure_newline=lambda: None,
321		write_line=lambda line, **opts: output.write(force_text(line) + u'\n'),
322		section=lambda line, **opts: output.write(force_text(line) + u'\n'),
323		write=lambda text, **opts: output.write(force_text(text)),
324		rewrite=lambda text, **opts: output.write(force_text(text)),
325		))
326		print(output.getvalue())
327		LineMatcher(output.getvalue().splitlines()).fnmatch_lines([
328		'BENCHMARK-C6: Benchmark machine_info is different. Current: {foo: "bar"} VS saved: {machine: "x86_64", node: "minibox", processor: "x86_64", python_compiler: "GCC 4.6.3", python_implementation: "CPython", python_version: "2.7.3", release: "3.13.0-55-generic", system: "Linux"}. {\'fslocation\': \'tests*test_storage\'}',
329		'Comparing against benchmarks from: 0002_b87b9aae14ff14a7887a6bbaa9731b9a8760555d_20150814_190348_uncommitted-changes.json',
330		'',
331		'------------------------------------------------------------------------ benchmark: 2 tests -----------------------------------------------------------------------',
332		'Name (time in ns) * Min Max Mean StdDev Median IQR Outliers() Rounds Iterations',
333		'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
334		'xfast_parametrized[[]0[]] (0002) 216.9028 (1.0) 7*739.2997 (1.0) 254.0585 (1.0) 0.0000 (1.0) 219.8103 (1.0) 27.3309 (1.45) 235;1688 11009 410',
335		'xfast_parametrized[[]0[]] (NOW) 217.9511 (1.00) 13*290.0380 (1.72) 261.2051 (1.03) 263.9842 (inf) 220.1638 (1.00) 18.8080 (1.0) 160;1726 9710 431',
336		'--------------------------------------------------------------------------------------------------------------------------------------------------------------------*',
337		'(*) Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.',
338		])
339
340		@freeze_time("2015-08-15T00:04:18.687119")
341		def test_save_json(sess, tmpdir, monkeypatch):
342		monkeypatch.setattr(plugin, '__version__', '2.5.0')
343		sess.save = False
344		sess.autosave = False
345		sess.json = LooseFileLike()
346		sess.save_data = False
347	View Code Duplication	sess.handle_saving()
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
348		assert tmpdir.listdir() == []
349		assert json.loads(sess.json.getvalue().decode()) == JSON_DATA
350
351
352		@freeze_time("2015-08-15T00:04:18.687119")
353		def test_save_with_name(sess, tmpdir, monkeypatch):
354		monkeypatch.setattr(plugin, '__version__', '2.5.0')
355		sess.save = 'foobar'
356		sess.autosave = True
357		sess.json = None
358		sess.save_data = False
359		sess.storage.path = Path(str(tmpdir))
360		sess.handle_saving()
361		files = list(Path(str(tmpdir)).rglob('*.json'))
362		print(files)
363		assert len(files) == 1
364		assert json.load(files[0].open('rU')) == SAVE_DATA
365
366
367		@freeze_time("2015-08-15T00:04:18.687119")
368		def test_save_no_name(sess, tmpdir, monkeypatch):
369		monkeypatch.setattr(plugin, '__version__', '2.5.0')
370		sess.save = True
371		sess.autosave = True
372		sess.json = None
373		sess.save_data = False
374		sess.storage.path = Path(str(tmpdir))
375	View Code Duplication	sess.handle_saving()
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
376		files = list(Path(str(tmpdir)).rglob('*.json'))
377		assert len(files) == 1
378		assert json.load(files[0].open('rU')) == SAVE_DATA
379
380
381		@freeze_time("2015-08-15T00:04:18.687119")
382		def test_save_with_error(sess, tmpdir, monkeypatch):
383		monkeypatch.setattr(plugin, '__version__', '2.5.0')
384		sess.save = True
385		sess.autosave = True
386		sess.json = None
387		sess.save_data = False
388		sess.storage.path = Path(str(tmpdir))
389		for bench in sess.benchmarks:
390		bench.has_error = True
391		sess.handle_saving()
392		files = list(Path(str(tmpdir)).rglob('*.json'))
393		assert len(files) == 1
394		assert json.load(files[0].open('rU')) == {
395		'benchmarks': [],
396		'commit_info': {'foo': 'bar'},
397		'datetime': '2015-08-15T00:04:18.687119',
398		'machine_info': {'foo': 'bar'},
399		'version': '2.5.0'
400		}
401
402
403		@freeze_time("2015-08-15T00:04:18.687119")
404		def test_autosave(sess, tmpdir, monkeypatch):
405		monkeypatch.setattr(plugin, '__version__', '2.5.0')
406		sess.save = False
407		sess.autosave = True
408		sess.json = None
409		sess.save_data = False
410		sess.storage.path = Path(str(tmpdir))
411		sess.handle_saving()
412		files = list(Path(str(tmpdir)).rglob('*.json'))
413		assert len(files) == 1
414		assert json.load(files[0].open('rU')) == SAVE_DATA
415

ionelmc / pytest-benchmark

Pull Request — master (#58)

make_logger() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like