1
|
|
|
from __future__ import division |
2
|
|
|
from __future__ import print_function |
3
|
|
|
|
4
|
|
|
import statistics |
5
|
|
|
from bisect import bisect_left |
6
|
|
|
from bisect import bisect_right |
7
|
|
|
|
8
|
|
|
from .utils import cached_property |
9
|
|
|
from .utils import funcname |
10
|
|
|
|
11
|
|
|
|
12
|
|
|
class Stats(object): |
13
|
|
|
fields = ( |
14
|
|
|
"min", "max", "mean", "stddev", "rounds", "median", "iqr", "q1", "q3", "iqr_outliers", "stddev_outliers", |
15
|
|
|
"outliers", "ld15iqr", "hd15iqr" |
16
|
|
|
) |
17
|
|
|
|
18
|
|
|
def __init__(self): |
19
|
|
|
self.data = [] |
20
|
|
|
|
21
|
|
|
def __bool__(self): |
22
|
|
|
return bool(self.data) |
23
|
|
|
|
24
|
|
|
def __nonzero__(self): |
25
|
|
|
return bool(self.data) |
26
|
|
|
|
27
|
|
|
def as_dict(self): |
28
|
|
|
return dict( |
29
|
|
|
(field, getattr(self, field)) |
30
|
|
|
for field in self.fields |
31
|
|
|
) |
32
|
|
|
|
33
|
|
|
def update(self, duration): |
34
|
|
|
self.data.append(duration) |
35
|
|
|
|
36
|
|
|
@cached_property |
37
|
|
|
def sorted_data(self): |
38
|
|
|
return sorted(self.data) |
39
|
|
|
|
40
|
|
|
@cached_property |
41
|
|
|
def total(self): |
42
|
|
|
return sum(self.data) |
43
|
|
|
|
44
|
|
|
@cached_property |
45
|
|
|
def min(self): |
46
|
|
|
return min(self.data) |
47
|
|
|
|
48
|
|
|
@cached_property |
49
|
|
|
def max(self): |
50
|
|
|
return max(self.data) |
51
|
|
|
|
52
|
|
|
@cached_property |
53
|
|
|
def mean(self): |
54
|
|
|
return statistics.mean(self.data) |
55
|
|
|
|
56
|
|
|
@cached_property |
57
|
|
|
def stddev(self): |
58
|
|
|
if len(self.data) > 1: |
59
|
|
|
return statistics.stdev(self.data) |
60
|
|
|
else: |
61
|
|
|
return 0 |
62
|
|
|
|
63
|
|
|
@property |
64
|
|
|
def stddev_outliers(self): |
65
|
|
|
""" |
66
|
|
|
Count of StdDev outliers: what's beyond (Mean - StdDev, Mean - StdDev) |
67
|
|
|
""" |
68
|
|
|
count = 0 |
69
|
|
|
q0 = self.mean - self.stddev |
70
|
|
|
q4 = self.mean + self.stddev |
71
|
|
|
for val in self.data: |
72
|
|
|
if val < q0 or val > q4: |
73
|
|
|
count += 1 |
74
|
|
|
return count |
75
|
|
|
|
76
|
|
|
@cached_property |
77
|
|
|
def rounds(self): |
78
|
|
|
return len(self.data) |
79
|
|
|
|
80
|
|
|
@cached_property |
81
|
|
|
def median(self): |
82
|
|
|
return statistics.median(self.data) |
83
|
|
|
|
84
|
|
|
@cached_property |
85
|
|
|
def ld15iqr(self): |
86
|
|
|
""" |
87
|
|
|
Tukey-style Lowest Datum within 1.5 IQR under Q1. |
88
|
|
|
""" |
89
|
|
|
if len(self.data) == 1: |
90
|
|
|
return self.data[0] |
91
|
|
|
else: |
92
|
|
|
return self.sorted_data[bisect_left(self.sorted_data, self.q1 - 1.5 * self.iqr)] |
93
|
|
|
|
94
|
|
|
@cached_property |
95
|
|
|
def hd15iqr(self): |
96
|
|
|
""" |
97
|
|
|
Tukey-style Highest Datum within 1.5 IQR over Q3. |
98
|
|
|
""" |
99
|
|
|
if len(self.data) == 1: |
100
|
|
|
return self.data[0] |
101
|
|
|
else: |
102
|
|
|
pos = bisect_right(self.sorted_data, self.q3 + 1.5 * self.iqr) |
103
|
|
|
if pos == len(self.data): |
104
|
|
|
return self.sorted_data[-1] |
105
|
|
|
else: |
106
|
|
|
return self.sorted_data[pos] |
107
|
|
|
|
108
|
|
View Code Duplication |
@cached_property |
|
|
|
|
109
|
|
|
def q1(self): |
110
|
|
|
rounds = self.rounds |
111
|
|
|
data = self.sorted_data |
112
|
|
|
|
113
|
|
|
# See: https://en.wikipedia.org/wiki/Quartile#Computing_methods |
114
|
|
|
if rounds == 1: |
115
|
|
|
return data[0] |
116
|
|
|
elif rounds % 2: # Method 3 |
117
|
|
|
n, q = rounds // 4, rounds % 4 |
118
|
|
|
if q == 1: |
119
|
|
|
return 0.25 * data[n - 1] + 0.75 * data[n] |
120
|
|
|
else: |
121
|
|
|
return 0.75 * data[n] + 0.25 * data[n + 1] |
122
|
|
|
else: # Method 2 |
123
|
|
|
return statistics.median(data[:rounds // 2]) |
124
|
|
|
|
125
|
|
View Code Duplication |
@cached_property |
|
|
|
|
126
|
|
|
def q3(self): |
127
|
|
|
rounds = self.rounds |
128
|
|
|
data = self.sorted_data |
129
|
|
|
|
130
|
|
|
# See: https://en.wikipedia.org/wiki/Quartile#Computing_methods |
131
|
|
|
if rounds == 1: |
132
|
|
|
return data[0] |
133
|
|
|
elif rounds % 2: # Method 3 |
134
|
|
|
n, q = rounds // 4, rounds % 4 |
135
|
|
|
if q == 1: |
136
|
|
|
return 0.75 * data[3 * n] + 0.25 * data[3 * n + 1] |
137
|
|
|
else: |
138
|
|
|
return 0.25 * data[3 * n + 1] + 0.75 * data[3 * n + 2] |
139
|
|
|
else: # Method 2 |
140
|
|
|
return statistics.median(data[rounds // 2:]) |
141
|
|
|
|
142
|
|
|
@cached_property |
143
|
|
|
def iqr(self): |
144
|
|
|
return self.q3 - self.q1 |
145
|
|
|
|
146
|
|
|
@property |
147
|
|
|
def iqr_outliers(self): |
148
|
|
|
""" |
149
|
|
|
Count of Tukey outliers: what's beyond (Q1 - 1.5IQR, Q3 + 1.5IQR) |
150
|
|
|
""" |
151
|
|
|
count = 0 |
152
|
|
|
q0 = self.q1 - 1.5 * self.iqr |
153
|
|
|
q4 = self.q3 + 1.5 * self.iqr |
154
|
|
|
for val in self.data: |
155
|
|
|
if val < q0 or val > q4: |
156
|
|
|
count += 1 |
157
|
|
|
return count |
158
|
|
|
|
159
|
|
|
@cached_property |
160
|
|
|
def outliers(self): |
161
|
|
|
return "%s;%s" % (self.stddev_outliers, self.iqr_outliers) |
162
|
|
|
|
163
|
|
|
|
164
|
|
|
class BenchmarkStats(object): |
165
|
|
|
def __init__(self, fixture, iterations, options): |
166
|
|
|
self.name = fixture.name |
167
|
|
|
self.fullname = fixture.fullname |
168
|
|
|
self.group = fixture.group |
169
|
|
|
self.param = fixture.param |
170
|
|
|
self.params = fixture.params |
171
|
|
|
|
172
|
|
|
self.iterations = iterations |
173
|
|
|
self.stats = Stats() |
174
|
|
|
self.options = options |
175
|
|
|
self.fixture = fixture |
176
|
|
|
|
177
|
|
|
def __bool__(self): |
178
|
|
|
return bool(self.stats) |
179
|
|
|
|
180
|
|
|
def __nonzero__(self): |
181
|
|
|
return bool(self.stats) |
182
|
|
|
|
183
|
|
|
def get(self, key, default=None): |
184
|
|
|
try: |
185
|
|
|
return getattr(self.stats, key) |
186
|
|
|
except AttributeError: |
187
|
|
|
return getattr(self, key, default) |
188
|
|
|
|
189
|
|
|
def __getitem__(self, key): |
190
|
|
|
try: |
191
|
|
|
return getattr(self.stats, key) |
192
|
|
|
except AttributeError: |
193
|
|
|
return getattr(self, key) |
194
|
|
|
|
195
|
|
|
@property |
196
|
|
|
def has_error(self): |
197
|
|
|
return self.fixture.has_error |
198
|
|
|
|
199
|
|
|
def as_dict(self, include_data=True, flat=False, stats=True): |
200
|
|
|
result = { |
201
|
|
|
"group": self.group, |
202
|
|
|
"name": self.name, |
203
|
|
|
"fullname": self.fullname, |
204
|
|
|
"params": self.params, |
205
|
|
|
"param": self.param, |
206
|
|
|
"options": dict( |
207
|
|
|
(k, funcname(v) if callable(v) else v) for k, v in self.options.items() |
208
|
|
|
) |
209
|
|
|
} |
210
|
|
|
if stats: |
211
|
|
|
stats = self.stats.as_dict() |
212
|
|
|
if include_data: |
213
|
|
|
stats["data"] = self.stats.data |
214
|
|
|
stats["iterations"] = self.iterations |
215
|
|
|
if flat: |
216
|
|
|
result.update(stats) |
217
|
|
|
else: |
218
|
|
|
result["stats"] = stats |
219
|
|
|
return result |
220
|
|
|
|
221
|
|
|
def update(self, duration): |
222
|
|
|
self.stats.update(duration / self.iterations) |
223
|
|
|
|