BenchmarkSession.display_cprofile() - Code Metrics - Inspection of "Feature cProfile" - ionelmc/pytest-benchmark - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#59)

unknown

created 2016-09-12 09:37 UTC

BenchmarkSession.display_cprofile() B

↳ Parent: BenchmarkSession

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	6
c	1
b	0
f	0
dl	0
loc	17
rs	8

from __future__ import division
from __future__ import print_function

import pytest

from .fixture import statistics
from .fixture import statistics_error
from .logger import Logger
from .storage import Storage
from .table import TableResults
from .utils import NAME_FORMATTERS
from .utils import SecondsDecimal
from .utils import cached_property
from .utils import first_or_value
from .utils import get_machine_id
from .utils import load_timer
from .utils import safe_dumps
from .utils import short_filename


class PerformanceRegression(pytest.UsageError):
    pass


class BenchmarkSession(object):
    compared_mapping = None
    groups = None

    def __init__(self, config):
        self.verbose = config.getoption("benchmark_verbose")
        self.logger = Logger(self.verbose, config)
        self.config = config
        self.performance_regressions = []
        self.benchmarks = []
        self.machine_id = get_machine_id()

        self.options = dict(
            min_time=SecondsDecimal(config.getoption("benchmark_min_time")),
            min_rounds=config.getoption("benchmark_min_rounds"),
            max_time=SecondsDecimal(config.getoption("benchmark_max_time")),
            timer=load_timer(config.getoption("benchmark_timer")),
            calibration_precision=config.getoption("benchmark_calibration_precision"),
            disable_gc=config.getoption("benchmark_disable_gc"),
            warmup=config.getoption("benchmark_warmup"),
            warmup_iterations=config.getoption("benchmark_warmup_iterations"),
            use_cprofile=bool(config.getoption("benchmark_cprofile")),
        )
        self.skip = config.getoption("benchmark_skip")
        self.disabled = config.getoption("benchmark_disable") and not config.getoption("benchmark_enable")
        self.cprofile_sort_by = config.getoption("benchmark_cprofile")

        if config.getoption("dist", "no") != "no" and not self.skip:
            self.logger.warn(
                "BENCHMARK-U2",
                "Benchmarks are automatically disabled because xdist plugin is active."
                "Benchmarks cannot be performed reliably in a parallelized environment.",
                fslocation="::"
            )
            self.disabled = True
        if hasattr(config, "slaveinput"):
            self.disabled = True
        if not statistics:
            self.logger.warn(
                "BENCHMARK-U3",
                "Benchmarks are automatically disabled because we could not import `statistics`\n\n%s" %
                statistics_error,
                fslocation="::"
            )
            self.disabled = True

        self.only = config.getoption("benchmark_only")
        self.sort = config.getoption("benchmark_sort")
        self.columns = config.getoption("benchmark_columns")
        if self.skip and self.only:
            raise pytest.UsageError("Can't have both --benchmark-only and --benchmark-skip options.")
        if self.disabled and self.only:
            raise pytest.UsageError(
                "Can't have both --benchmark-only and --benchmark-disable options. Note that --benchmark-disable is "
                "automatically activated if xdist is on or you're missing the statistics dependency.")
        self.group_by = config.getoption("benchmark_group_by")
        self.save = config.getoption("benchmark_save")
        self.autosave = config.getoption("benchmark_autosave")
        self.save_data = config.getoption("benchmark_save_data")
        self.json = config.getoption("benchmark_json")
        self.compare = config.getoption("benchmark_compare")
        self.compare_fail = config.getoption("benchmark_compare_fail")
        self.name_format = NAME_FORMATTERS[config.getoption("benchmark_name")]

        self.storage = Storage(config.getoption("benchmark_storage"),
                               default_machine_id=self.machine_id, logger=self.logger)
        self.histogram = first_or_value(config.getoption("benchmark_histogram"), False)

    @cached_property
    def machine_info(self):
        obj = self.config.hook.pytest_benchmark_generate_machine_info(config=self.config)
        self.config.hook.pytest_benchmark_update_machine_info(
            config=self.config,
            machine_info=obj
        )
        return obj

    def prepare_benchmarks(self):
        for bench in self.benchmarks:
            if bench:
                compared = False
                for path, compared_mapping in self.compared_mapping.items():
                    if bench.fullname in compared_mapping:
                        compared = compared_mapping[bench.fullname]
                        source = short_filename(path, self.machine_id)
                        flat_bench = bench.as_dict(include_data=False, stats=False,
                                                   cprofile_sort_by=self.cprofile_sort_by,
                                                   cprofile_all_columns=False)
                        flat_bench.update(compared["stats"])
                        flat_bench["path"] = str(path)
                        flat_bench["source"] = source
                        if self.compare_fail:
                            for check in self.compare_fail:
                                fail = check.fails(bench, flat_bench)
                                if fail:
                                    self.performance_regressions.append((self.name_format(flat_bench), fail))
                        yield flat_bench
                flat_bench = bench.as_dict(include_data=False, flat=True,
                                           cprofile_sort_by=self.cprofile_sort_by,
                                           cprofile_all_columns=False)
                flat_bench["path"] = None
                flat_bench["source"] = compared and "NOW"
                yield flat_bench

    @property
    def next_num(self):
        files = self.storage.query("[0-9][0-9][0-9][0-9]_*")
        files.sort(reverse=True)
        if not files:
            return "0001"
        for f in files:
            try:
                return "%04i" % (int(str(f.name).split('_')[0]) + 1)
            except ValueError:
                raise

    def handle_saving(self):
        save = self.benchmarks and self.save or self.autosave
        if save or self.json:
            commit_info = self.config.hook.pytest_benchmark_generate_commit_info(config=self.config)
            self.config.hook.pytest_benchmark_update_commit_info(config=self.config, commit_info=commit_info)

        if self.json:

            output_json = self.config.hook.pytest_benchmark_generate_json(
                config=self.config,
                benchmarks=self.benchmarks,
                include_data=True,
                machine_info=self.machine_info,
                commit_info=commit_info,
            )
            self.config.hook.pytest_benchmark_update_json(
                config=self.config,
                benchmarks=self.benchmarks,
                output_json=output_json,
            )
            with self.json as fh:
                fh.write(safe_dumps(output_json, ensure_ascii=True, indent=4).encode())
            self.logger.info("Wrote benchmark data in: %s" % self.json, purple=True)

        if save:

            output_json = self.config.hook.pytest_benchmark_generate_json(
                config=self.config,
                benchmarks=self.benchmarks,
                include_data=self.save_data,
                machine_info=self.machine_info,
                commit_info=commit_info,
            )
            self.config.hook.pytest_benchmark_update_json(
                config=self.config,
                benchmarks=self.benchmarks,
                output_json=output_json,
            )
            output_file = self.storage.get("%s_%s.json" % (self.next_num, save))
            assert not output_file.exists()

            with output_file.open('wb') as fh:
                fh.write(safe_dumps(output_json, ensure_ascii=True, indent=4).encode())
            self.logger.info("Saved benchmark data in: %s" % output_file)

    def handle_loading(self):
        self.compared_mapping = {}
        if self.compare:
            if self.compare is True:
                compared_benchmarks = list(self.storage.load('[0-9][0-9][0-9][0-9]_'))[-1:]
            else:
                compared_benchmarks = list(self.storage.load(self.compare))

            if not compared_benchmarks:
                msg = "Can't compare. No benchmark files in %r" % str(self.storage)
                if self.compare is True:
                    msg += ". Can't load the previous benchmark."
                    code = "BENCHMARK-C2"
                else:
                    msg += " match %r." % self.compare
                    code = "BENCHMARK-C1"
                self.logger.warn(code, msg, fslocation=self.storage.location)

            for path, compared_benchmark in compared_benchmarks:
                self.config.hook.pytest_benchmark_compare_machine_info(
                    config=self.config,
                    benchmarksession=self,
                    machine_info=self.machine_info,
                    compared_benchmark=compared_benchmark,
                )
                self.compared_mapping[path] = dict(
                    (bench['fullname'], bench) for bench in compared_benchmark['benchmarks']
                )
                self.logger.info("Comparing against benchmarks from: %s" % path)

    def finish(self):
        self.handle_saving()
        self.handle_loading()
        prepared_benchmarks = list(self.prepare_benchmarks())
        if prepared_benchmarks:
            self.groups = self.config.hook.pytest_benchmark_group_stats(
                config=self.config,
                benchmarks=prepared_benchmarks,
                group_by=self.group_by
            )

    def display(self, tr):
        if not self.groups:
            return

        tr.ensure_newline()
        results_table = TableResults(
            columns=self.columns,
            sort=self.sort,
            histogram=self.histogram,
            name_format=self.name_format,
            logger=self.logger
        )
        results_table.display(tr, self.groups)
        self.check_regressions()
        self.display_cprofile(tr)

    def check_regressions(self):
        if self.compare_fail and not self.compared_mapping:
            raise pytest.UsageError("--benchmark-compare-fail requires valid --benchmark-compare.")

        if self.performance_regressions:
            self.logger.error("Performance has regressed:\n%s" % "\n".join(
                "\t%s - %s" % line for line in self.performance_regressions
            ))
            raise PerformanceRegression("Performance has regressed.")

    def display_cprofile(self, tr):
        if self.options["use_cprofile"]:
            tr.section("cProfile information")
            tr.write_line("Time in s")
            for group in self.groups:
                group_name, benchmarks = group
                for benchmark in benchmarks:
                    tr.write(benchmark["fullname"], yellow=True)
                    if benchmark["source"]:
                        tr.write_line(" ({})".format((benchmark["source"])))
                    else:
                        tr.write("\n")
                    tr.write_line("ncalls\ttottime\tpercall\tcumtime\tpercall\tfilename:lineno(function)")
                    for function_info in benchmark["cprofile"]:
                        line = "{ncalls_recursion}\t{tottime:.{prec}f}\t{tottime_per:.{prec}f}\t{cumtime:.{prec}f}\t{cumtime_per:.{prec}f}\t{function_name}".format(prec=4, **function_info)
                        tr.write_line(line)
                    tr.write("\n")


1		from __future__ import division
2		from __future__ import print_function
3
4		import pytest
5
6		from .fixture import statistics
7		from .fixture import statistics_error
8		from .logger import Logger
9		from .storage import Storage
10		from .table import TableResults
11		from .utils import NAME_FORMATTERS
12		from .utils import SecondsDecimal
13		from .utils import cached_property
14		from .utils import first_or_value
15		from .utils import get_machine_id
16		from .utils import load_timer
17		from .utils import safe_dumps
18		from .utils import short_filename
19
20
21		class PerformanceRegression(pytest.UsageError):
22		pass
23
24
25		class BenchmarkSession(object):
26		compared_mapping = None
27		groups = None
28
29		def __init__(self, config):
30		self.verbose = config.getoption("benchmark_verbose")
31		self.logger = Logger(self.verbose, config)
32		self.config = config
33		self.performance_regressions = []
34		self.benchmarks = []
35		self.machine_id = get_machine_id()
36
37		self.options = dict(
38		min_time=SecondsDecimal(config.getoption("benchmark_min_time")),
39		min_rounds=config.getoption("benchmark_min_rounds"),
40		max_time=SecondsDecimal(config.getoption("benchmark_max_time")),
41		timer=load_timer(config.getoption("benchmark_timer")),
42		calibration_precision=config.getoption("benchmark_calibration_precision"),
43		disable_gc=config.getoption("benchmark_disable_gc"),
44		warmup=config.getoption("benchmark_warmup"),
45		warmup_iterations=config.getoption("benchmark_warmup_iterations"),
46		use_cprofile=bool(config.getoption("benchmark_cprofile")),
47		)
48		self.skip = config.getoption("benchmark_skip")
49		self.disabled = config.getoption("benchmark_disable") and not config.getoption("benchmark_enable")
50		self.cprofile_sort_by = config.getoption("benchmark_cprofile")
51
52		if config.getoption("dist", "no") != "no" and not self.skip:
53		self.logger.warn(
54		"BENCHMARK-U2",
55		"Benchmarks are automatically disabled because xdist plugin is active."
56		"Benchmarks cannot be performed reliably in a parallelized environment.",
57		fslocation="::"
58		)
59		self.disabled = True
60		if hasattr(config, "slaveinput"):
61		self.disabled = True
62		if not statistics:
63		self.logger.warn(
64		"BENCHMARK-U3",
65		"Benchmarks are automatically disabled because we could not import `statistics`\n\n%s" %
66		statistics_error,
67		fslocation="::"
68		)
69		self.disabled = True
70
71		self.only = config.getoption("benchmark_only")
72		self.sort = config.getoption("benchmark_sort")
73		self.columns = config.getoption("benchmark_columns")
74		if self.skip and self.only:
75		raise pytest.UsageError("Can't have both --benchmark-only and --benchmark-skip options.")
76		if self.disabled and self.only:
77		raise pytest.UsageError(
78		"Can't have both --benchmark-only and --benchmark-disable options. Note that --benchmark-disable is "
79		"automatically activated if xdist is on or you're missing the statistics dependency.")
80		self.group_by = config.getoption("benchmark_group_by")
81		self.save = config.getoption("benchmark_save")
82		self.autosave = config.getoption("benchmark_autosave")
83		self.save_data = config.getoption("benchmark_save_data")
84		self.json = config.getoption("benchmark_json")
85		self.compare = config.getoption("benchmark_compare")
86		self.compare_fail = config.getoption("benchmark_compare_fail")
87		self.name_format = NAME_FORMATTERS[config.getoption("benchmark_name")]
88
89		self.storage = Storage(config.getoption("benchmark_storage"),
90		default_machine_id=self.machine_id, logger=self.logger)
91		self.histogram = first_or_value(config.getoption("benchmark_histogram"), False)
92
93		@cached_property
94		def machine_info(self):
95		obj = self.config.hook.pytest_benchmark_generate_machine_info(config=self.config)
96		self.config.hook.pytest_benchmark_update_machine_info(
97		config=self.config,
98		machine_info=obj
99		)
100		return obj
101
102		def prepare_benchmarks(self):
103		for bench in self.benchmarks:
104		if bench:
105		compared = False
106		for path, compared_mapping in self.compared_mapping.items():
107		if bench.fullname in compared_mapping:
108		compared = compared_mapping[bench.fullname]
109		source = short_filename(path, self.machine_id)
110		flat_bench = bench.as_dict(include_data=False, stats=False,
111		cprofile_sort_by=self.cprofile_sort_by,
112		cprofile_all_columns=False)
113		flat_bench.update(compared["stats"])
114		flat_bench["path"] = str(path)
115		flat_bench["source"] = source
116		if self.compare_fail:
117		for check in self.compare_fail:
118		fail = check.fails(bench, flat_bench)
119		if fail:
120		self.performance_regressions.append((self.name_format(flat_bench), fail))
121		yield flat_bench
122		flat_bench = bench.as_dict(include_data=False, flat=True,
123		cprofile_sort_by=self.cprofile_sort_by,
124		cprofile_all_columns=False)
125		flat_bench["path"] = None
126		flat_bench["source"] = compared and "NOW"
127		yield flat_bench
128
129		@property
130		def next_num(self):
131		files = self.storage.query("[0-9][0-9][0-9][0-9]_*")
132		files.sort(reverse=True)
133		if not files:
134		return "0001"
135		for f in files:
136		try:
137		return "%04i" % (int(str(f.name).split('_')[0]) + 1)
138		except ValueError:
139		raise
140
141		def handle_saving(self):
142		save = self.benchmarks and self.save or self.autosave
143		if save or self.json:
144		commit_info = self.config.hook.pytest_benchmark_generate_commit_info(config=self.config)
145		self.config.hook.pytest_benchmark_update_commit_info(config=self.config, commit_info=commit_info)
146
147	View Code Duplication	if self.json:
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
148		output_json = self.config.hook.pytest_benchmark_generate_json(
149		config=self.config,
150		benchmarks=self.benchmarks,
151		include_data=True,
152		machine_info=self.machine_info,
153		commit_info=commit_info,
154		)
155		self.config.hook.pytest_benchmark_update_json(
156		config=self.config,
157		benchmarks=self.benchmarks,
158		output_json=output_json,
159		)
160		with self.json as fh:
161		fh.write(safe_dumps(output_json, ensure_ascii=True, indent=4).encode())
162		self.logger.info("Wrote benchmark data in: %s" % self.json, purple=True)
163
164	View Code Duplication	if save:
		0 ignored issues – show Duplication introduced 2016-05-26 23:02 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
165		output_json = self.config.hook.pytest_benchmark_generate_json(
166		config=self.config,
167		benchmarks=self.benchmarks,
168		include_data=self.save_data,
169		machine_info=self.machine_info,
170		commit_info=commit_info,
171		)
172		self.config.hook.pytest_benchmark_update_json(
173		config=self.config,
174		benchmarks=self.benchmarks,
175		output_json=output_json,
176		)
177		output_file = self.storage.get("%s_%s.json" % (self.next_num, save))
178		assert not output_file.exists()
179
180		with output_file.open('wb') as fh:
181		fh.write(safe_dumps(output_json, ensure_ascii=True, indent=4).encode())
182		self.logger.info("Saved benchmark data in: %s" % output_file)
183
184		def handle_loading(self):
185		self.compared_mapping = {}
186		if self.compare:
187		if self.compare is True:
188		compared_benchmarks = list(self.storage.load('[0-9][0-9][0-9][0-9]_'))[-1:]
189		else:
190		compared_benchmarks = list(self.storage.load(self.compare))
191
192		if not compared_benchmarks:
193		msg = "Can't compare. No benchmark files in %r" % str(self.storage)
194		if self.compare is True:
195		msg += ". Can't load the previous benchmark."
196		code = "BENCHMARK-C2"
197		else:
198		msg += " match %r." % self.compare
199		code = "BENCHMARK-C1"
200		self.logger.warn(code, msg, fslocation=self.storage.location)
201
202		for path, compared_benchmark in compared_benchmarks:
203		self.config.hook.pytest_benchmark_compare_machine_info(
204		config=self.config,
205		benchmarksession=self,
206		machine_info=self.machine_info,
207		compared_benchmark=compared_benchmark,
208		)
209		self.compared_mapping[path] = dict(
210		(bench['fullname'], bench) for bench in compared_benchmark['benchmarks']
211		)
212		self.logger.info("Comparing against benchmarks from: %s" % path)
213
214		def finish(self):
215		self.handle_saving()
216		self.handle_loading()
217		prepared_benchmarks = list(self.prepare_benchmarks())
218		if prepared_benchmarks:
219		self.groups = self.config.hook.pytest_benchmark_group_stats(
220		config=self.config,
221		benchmarks=prepared_benchmarks,
222		group_by=self.group_by
223		)
224
225		def display(self, tr):
226		if not self.groups:
227		return
228
229		tr.ensure_newline()
230		results_table = TableResults(
231		columns=self.columns,
232		sort=self.sort,
233		histogram=self.histogram,
234		name_format=self.name_format,
235		logger=self.logger
236		)
237		results_table.display(tr, self.groups)
238		self.check_regressions()
239		self.display_cprofile(tr)
240
241		def check_regressions(self):
242		if self.compare_fail and not self.compared_mapping:
243		raise pytest.UsageError("--benchmark-compare-fail requires valid --benchmark-compare.")
244
245		if self.performance_regressions:
246		self.logger.error("Performance has regressed:\n%s" % "\n".join(
247		"\t%s - %s" % line for line in self.performance_regressions
248		))
249		raise PerformanceRegression("Performance has regressed.")
250
251		def display_cprofile(self, tr):
252		if self.options["use_cprofile"]:
253		tr.section("cProfile information")
254		tr.write_line("Time in s")
255		for group in self.groups:
256		group_name, benchmarks = group
257		for benchmark in benchmarks:
258		tr.write(benchmark["fullname"], yellow=True)
259		if benchmark["source"]:
260		tr.write_line(" ({})".format((benchmark["source"])))
261		else:
262		tr.write("\n")
263		tr.write_line("ncalls\ttottime\tpercall\tcumtime\tpercall\tfilename:lineno(function)")
264		for function_info in benchmark["cprofile"]:
265		line = "{ncalls_recursion}\t{tottime:.{prec}f}\t{tottime_per:.{prec}f}\t{cumtime:.{prec}f}\t{cumtime_per:.{prec}f}\t{function_name}".format(prec=4, **function_info)
266		tr.write_line(line)
267		tr.write("\n")
268

ionelmc / pytest-benchmark

Pull Request — master (#59)

BenchmarkSession.display_cprofile() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like