CounterManager - Code Metrics - binux/pyspider - Measure and Improve Code Quality continuously with Scrutinizer

CounterManager B
last analyzed 2018-06-14 19:08 UTC

↳ Parent: Project

Complexity

Total Complexity

Size/Duplication

Total Lines	113
Duplicated Lines	0 %

Importance

Changes	1
Bugs	1	Features	0

Metric	Value
dl	0
loc	113
rs	8.6
c	1
b	1
f	0
wmc	37

12 Methods

Rating	Name	Size	Complexity
A	__init__()	4	1
A	event()	9	4
A	to_dict()	12	4
A	load()	9	3
A	value()	10	4
A	__iter__()	2	1
A	keys()	5	3
A	__len__()	2	1
A	trim()	5	3
A	__delitem__()	8	4
A	dump()	9	3
B	__getitem__()	16	6

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Author: Binux<[email protected]>
#         http://binux.me
# Created on 2012-11-14 17:09:50

from __future__ import unicode_literals, division, absolute_import

import time
import logging
from collections import deque
try:
    from UserDict import DictMixin
except ImportError:
    from collections import Mapping as DictMixin

import six
from six import iteritems
from six.moves import cPickle


class BaseCounter(object):

    def __init__(self):
        pass

    def event(self, value=1):
        """Fire a event."""
        raise NotImplementedError

    def value(self, value):
        """Set counter value."""
        raise NotImplementedError

    @property
    def avg(self):
        """Get average value"""
        raise NotImplementedError

    @property
    def sum(self):
        """Get sum of counter"""
        raise NotImplementedError

    def empty(self):
        """Clear counter"""
        raise NotImplementedError


class TotalCounter(BaseCounter):
    """Total counter"""

    def __init__(self):
        super(TotalCounter, self).__init__()
        self.cnt = 0

    def event(self, value=1):
        self.cnt += value

    def value(self, value):
        self.cnt = value

    @property
    def avg(self):
        return self.cnt

    @property
    def sum(self):
        return self.cnt

    def empty(self):
        return self.cnt == 0


class AverageWindowCounter(BaseCounter):
    """
    Record last N(window) value
    """

    def __init__(self, window_size=300):
        super(AverageWindowCounter, self).__init__()
        self.window_size = window_size
        self.values = deque(maxlen=window_size)

    def event(self, value=1):
        self.values.append(value)

    value = event

    @property
    def avg(self):
        return self.sum / len(self.values)

    @property
    def sum(self):
        return sum(self.values)

    def empty(self):
        if not self.values:
            return True


class TimebaseAverageEventCounter(BaseCounter):
    """
    Record last window_size * window_interval seconds event.

    records will trim ever window_interval seconds
    """

    def __init__(self, window_size=30, window_interval=10):
        super(TimebaseAverageEventCounter, self).__init__()
        self.max_window_size = window_size
        self.window_size = 0
        self.window_interval = window_interval
        self.values = deque(maxlen=window_size)
        self.events = deque(maxlen=window_size)
        self.times = deque(maxlen=window_size)

        self.cache_value = 0
        self.cache_event = 0
        self.cache_start = None
        self._first_data_time = None

    def event(self, value=1):
        now = time.time()
        if self._first_data_time is None:
            self._first_data_time = now

        if self.cache_start is None:
            self.cache_value = value
            self.cache_event = 1
            self.cache_start = now
        elif now - self.cache_start > self.window_interval:
            self.values.append(self.cache_value)
            self.events.append(self.cache_event)
            self.times.append(self.cache_start)
            self.on_append(self.cache_value, self.cache_start)
            self.cache_value = value
            self.cache_event = 1
            self.cache_start = now
        else:
            self.cache_value += value
            self.cache_event += 1
        return self

    def value(self, value):
        self.cache_value = value

    def _trim_window(self):

        now = time.time()
        if self.cache_start and now - self.cache_start > self.window_interval:
            self.values.append(self.cache_value)
            self.events.append(self.cache_event)
            self.times.append(self.cache_start)
            self.on_append(self.cache_value, self.cache_start)
            self.cache_value = 0
            self.cache_start = None

        if self.window_size != self.max_window_size and self._first_data_time is not None:
            time_passed = now - self._first_data_time
            self.window_size = min(self.max_window_size, time_passed / self.window_interval)
        window_limit = now - self.window_size * self.window_interval
        while self.times and self.times[0] < window_limit:
            self.times.popleft()
            self.events.popleft()
            self.values.popleft()

    @property
    def avg(self):
        events = (sum(self.events) + self.cache_event)
        if not events:
            return 0
        return float(self.sum) / events

    @property
    def sum(self):
        self._trim_window()
        return sum(self.values) + self.cache_value

    def empty(self):
        self._trim_window()
        if not self.values and not self.cache_start:
            return True

    def on_append(self, value, time):
        pass


class TimebaseAverageWindowCounter(BaseCounter):
    """
    Record last window_size * window_interval seconds values.

    records will trim ever window_interval seconds
    """

    def __init__(self, window_size=30, window_interval=10):
        super(TimebaseAverageWindowCounter, self).__init__()
        self.max_window_size = window_size
        self.window_size = 0
        self.window_interval = window_interval
        self.values = deque(maxlen=window_size)
        self.times = deque(maxlen=window_size)

        self.cache_value = 0
        self.cache_start = None
        self._first_data_time = None

    def event(self, value=1):
        now = time.time()
        if self._first_data_time is None:
            self._first_data_time = now

        if self.cache_start is None:
            self.cache_value = value
            self.cache_start = now
        elif now - self.cache_start > self.window_interval:
            self.values.append(self.cache_value)
            self.times.append(self.cache_start)
            self.on_append(self.cache_value, self.cache_start)
            self.cache_value = value
            self.cache_start = now
        else:
            self.cache_value += value
        return self

    def value(self, value):
        self.cache_value = value

    def _trim_window(self):

        now = time.time()
        if self.cache_start and now - self.cache_start > self.window_interval:
            self.values.append(self.cache_value)
            self.times.append(self.cache_start)
            self.on_append(self.cache_value, self.cache_start)
            self.cache_value = 0
            self.cache_start = None

        if self.window_size != self.max_window_size and self._first_data_time is not None:
            time_passed = now - self._first_data_time
            self.window_size = min(self.max_window_size, time_passed / self.window_interval)
        window_limit = now - self.window_size * self.window_interval
        while self.times and self.times[0] < window_limit:
            self.times.popleft()
            self.values.popleft()

    @property
    def avg(self):
        sum = float(self.sum)
        if not self.window_size:
            return 0
        return sum / self.window_size / self.window_interval

    @property
    def sum(self):
        self._trim_window()
        return sum(self.values) + self.cache_value

    def empty(self):
        self._trim_window()
        if not self.values and not self.cache_start:
            return True

    def on_append(self, value, time):
        pass


class CounterValue(DictMixin):
    """
    A dict like value item for CounterManager.
    """

    def __init__(self, manager, keys):
        self.manager = manager
        self._keys = keys

    def __getitem__(self, key):
        if key == '__value__':
            key = self._keys
            return self.manager.counters[key]
        else:
            key = self._keys + (key, )

        available_keys = []
        for _key in list(self.manager.counters.keys()):
            if _key[:len(key)] == key:
                available_keys.append(_key)

        if len(available_keys) == 0:
            raise KeyError
        elif len(available_keys) == 1:
            if available_keys[0] == key:
                return self.manager.counters.get(key)
            else:
                return CounterValue(self.manager, key)
        else:
            return CounterValue(self.manager, key)

    def __len__(self):
        return len(self.keys())

    def __iter__(self):
        return iter(self.keys())

    def __contains__(self, key):
        return key in self.keys()

    def keys(self):
        result = set()
        for key in list(self.manager.counters.keys()):
            if key[:len(self._keys)] == self._keys:
                key = key[len(self._keys):]
                result.add(key[0] if key else '__value__')
        return result

    def to_dict(self, get_value=None):
        """Dump counters as a dict"""
        result = {}
        for key, value in iteritems(self):
            if isinstance(value, BaseCounter):
                if get_value is not None:
                    value = getattr(value, get_value)
                result[key] = value
            else:
                result[key] = value.to_dict(get_value)
        return result


class CounterManager(DictMixin):
    """
    A dict like counter manager.

    When using a tuple as event key, say: ('foo', 'bar'), You can visite counter
    with manager['foo']['bar'].  Or get all counters which first element is 'foo'
    by manager['foo'].

    It's useful for a group of counters.
    """

    def __init__(self, cls=TimebaseAverageWindowCounter):
        """init manager with Counter cls"""
        self.cls = cls
        self.counters = {}

    def event(self, key, value=1):
        """Fire a event of a counter by counter key"""
        if isinstance(key, six.string_types):
            key = (key, )
        assert isinstance(key, tuple), "event key type error"
        if key not in self.counters:
            self.counters[key] = self.cls()
        self.counters[key].event(value)
        return self

    def value(self, key, value=1):
        """Set value of a counter by counter key"""
        if isinstance(key, six.string_types):
            key = (key, )
        # assert all(isinstance(k, six.string_types) for k in key)
        assert isinstance(key, tuple), "event key type error"
        if key not in self.counters:
            self.counters[key] = self.cls()
        self.counters[key].value(value)
        return self

    def trim(self):
        """Clear not used counters"""
        for key, value in list(iteritems(self.counters)):
            if value.empty():
                del self.counters[key]

    def __getitem__(self, key):
        key = (key, )
        available_keys = []
        for _key in list(self.counters.keys()):
            if _key[:len(key)] == key:
                available_keys.append(_key)

        if len(available_keys) == 0:
            raise KeyError
        elif len(available_keys) == 1:
            if available_keys[0] == key:
                return self.counters.get(key)
            else:
                return CounterValue(self, key)
        else:
            return CounterValue(self, key)

    def __delitem__(self, key):
        key = (key, )
        available_keys = []
        for _key in list(self.counters.keys()):
            if _key[:len(key)] == key:
                available_keys.append(_key)
        for _key in available_keys:
            del self.counters[_key]

    def __iter__(self):
        return iter(self.keys())

    def __len__(self):
        return len(self.keys())

    def keys(self):
        result = set()
        for key in self.counters.keys():
            result.add(key[0] if key else ())
        return result

    def to_dict(self, get_value=None):
        """Dump counters as a dict"""
        self.trim()
        result = {}
        for key, value in iteritems(self.counters):
            if get_value is not None:
                value = getattr(value, get_value)
            r = result
            for _key in key[:-1]:
                r = r.setdefault(_key, {})
            r[key[-1]] = value
        return result

    def dump(self, filename):
        """Dump counters to file"""
        try:
            with open(filename, 'wb') as fp:
                cPickle.dump(self.counters, fp)
        except Exception as e:
            logging.warning("can't dump counter to file %s: %s", filename, e)
            return False
        return True

    def load(self, filename):
        """Load counters to file"""
        try:
            with open(filename, 'rb') as fp:
                self.counters = cPickle.load(fp)
        except:
            logging.debug("can't load counter from file: %s", filename)
            return False
        return True


1		#!/usr/bin/env python
2		# -- encoding: utf-8 --
3		# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
4		# Author: Binux<[email protected]>
5		# http://binux.me
6		# Created on 2012-11-14 17:09:50
7
8		from __future__ import unicode_literals, division, absolute_import
9
10		import time
11		import logging
12		from collections import deque
13		try:
14		from UserDict import DictMixin
15		except ImportError:
16		from collections import Mapping as DictMixin
17
18		import six
19		from six import iteritems
20		from six.moves import cPickle
21
22
23		class BaseCounter(object):
24
25		def __init__(self):
26		pass
27
28		def event(self, value=1):
29		"""Fire a event."""
30		raise NotImplementedError
31
32		def value(self, value):
33		"""Set counter value."""
34		raise NotImplementedError
35
36		@property
37		def avg(self):
38		"""Get average value"""
39		raise NotImplementedError
40
41		@property
42		def sum(self):
43		"""Get sum of counter"""
44		raise NotImplementedError
45
46		def empty(self):
47		"""Clear counter"""
48		raise NotImplementedError
49
50
51		class TotalCounter(BaseCounter):
52		"""Total counter"""
53
54		def __init__(self):
55		super(TotalCounter, self).__init__()
56		self.cnt = 0
57
58		def event(self, value=1):
59		self.cnt += value
60
61		def value(self, value):
62		self.cnt = value
63
64		@property
65		def avg(self):
66		return self.cnt
67
68		@property
69		def sum(self):
70		return self.cnt
71
72		def empty(self):
73		return self.cnt == 0
74
75
76		class AverageWindowCounter(BaseCounter):
77		"""
78		Record last N(window) value
79		"""
80
81		def __init__(self, window_size=300):
82		super(AverageWindowCounter, self).__init__()
83		self.window_size = window_size
84		self.values = deque(maxlen=window_size)
85
86		def event(self, value=1):
87		self.values.append(value)
88
89		value = event
90
91		@property
92		def avg(self):
93		return self.sum / len(self.values)
94
95		@property
96		def sum(self):
97		return sum(self.values)
98
99		def empty(self):
100		if not self.values:
101		return True
102
103
104		class TimebaseAverageEventCounter(BaseCounter):
105		"""
106		Record last window_size * window_interval seconds event.
107
108		records will trim ever window_interval seconds
109		"""
110
111		def __init__(self, window_size=30, window_interval=10):
112		super(TimebaseAverageEventCounter, self).__init__()
113		self.max_window_size = window_size
114		self.window_size = 0
115		self.window_interval = window_interval
116		self.values = deque(maxlen=window_size)
117		self.events = deque(maxlen=window_size)
118		self.times = deque(maxlen=window_size)
119
120		self.cache_value = 0
121		self.cache_event = 0
122		self.cache_start = None
123		self._first_data_time = None
124
125		def event(self, value=1):
126		now = time.time()
127		if self._first_data_time is None:
128		self._first_data_time = now
129
130		if self.cache_start is None:
131		self.cache_value = value
132		self.cache_event = 1
133		self.cache_start = now
134		elif now - self.cache_start > self.window_interval:
135		self.values.append(self.cache_value)
136		self.events.append(self.cache_event)
137		self.times.append(self.cache_start)
138		self.on_append(self.cache_value, self.cache_start)
139		self.cache_value = value
140		self.cache_event = 1
141		self.cache_start = now
142		else:
143		self.cache_value += value
144		self.cache_event += 1
145		return self
146
147		def value(self, value):
148		self.cache_value = value
149
150	View Code Duplication	def _trim_window(self):
		0 ignored issues – show Duplication introduced 2018-02-28 12:59 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
151		now = time.time()
152		if self.cache_start and now - self.cache_start > self.window_interval:
153		self.values.append(self.cache_value)
154		self.events.append(self.cache_event)
155		self.times.append(self.cache_start)
156		self.on_append(self.cache_value, self.cache_start)
157		self.cache_value = 0
158		self.cache_start = None
159
160		if self.window_size != self.max_window_size and self._first_data_time is not None:
161		time_passed = now - self._first_data_time
162		self.window_size = min(self.max_window_size, time_passed / self.window_interval)
163		window_limit = now - self.window_size * self.window_interval
164		while self.times and self.times[0] < window_limit:
165		self.times.popleft()
166		self.events.popleft()
167		self.values.popleft()
168
169		@property
170		def avg(self):
171		events = (sum(self.events) + self.cache_event)
172		if not events:
173		return 0
174		return float(self.sum) / events
175
176		@property
177		def sum(self):
178		self._trim_window()
179		return sum(self.values) + self.cache_value
180
181		def empty(self):
182		self._trim_window()
183		if not self.values and not self.cache_start:
184		return True
185
186		def on_append(self, value, time):
187		pass
188
189
190		class TimebaseAverageWindowCounter(BaseCounter):
191		"""
192		Record last window_size * window_interval seconds values.
193
194		records will trim ever window_interval seconds
195		"""
196
197		def __init__(self, window_size=30, window_interval=10):
198		super(TimebaseAverageWindowCounter, self).__init__()
199		self.max_window_size = window_size
200		self.window_size = 0
201		self.window_interval = window_interval
202		self.values = deque(maxlen=window_size)
203		self.times = deque(maxlen=window_size)
204
205		self.cache_value = 0
206		self.cache_start = None
207		self._first_data_time = None
208
209		def event(self, value=1):
210		now = time.time()
211		if self._first_data_time is None:
212		self._first_data_time = now
213
214		if self.cache_start is None:
215		self.cache_value = value
216		self.cache_start = now
217		elif now - self.cache_start > self.window_interval:
218		self.values.append(self.cache_value)
219		self.times.append(self.cache_start)
220		self.on_append(self.cache_value, self.cache_start)
221		self.cache_value = value
222		self.cache_start = now
223		else:
224		self.cache_value += value
225		return self
226
227		def value(self, value):
228		self.cache_value = value
229
230	View Code Duplication	def _trim_window(self):
		0 ignored issues – show Duplication introduced 2018-02-28 12:59 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
231		now = time.time()
232		if self.cache_start and now - self.cache_start > self.window_interval:
233		self.values.append(self.cache_value)
234		self.times.append(self.cache_start)
235		self.on_append(self.cache_value, self.cache_start)
236		self.cache_value = 0
237		self.cache_start = None
238
239		if self.window_size != self.max_window_size and self._first_data_time is not None:
240		time_passed = now - self._first_data_time
241		self.window_size = min(self.max_window_size, time_passed / self.window_interval)
242		window_limit = now - self.window_size * self.window_interval
243		while self.times and self.times[0] < window_limit:
244		self.times.popleft()
245		self.values.popleft()
246
247		@property
248		def avg(self):
249		sum = float(self.sum)
250		if not self.window_size:
251		return 0
252		return sum / self.window_size / self.window_interval
253
254		@property
255		def sum(self):
256		self._trim_window()
257		return sum(self.values) + self.cache_value
258
259		def empty(self):
260		self._trim_window()
261		if not self.values and not self.cache_start:
262		return True
263
264		def on_append(self, value, time):
265		pass
266
267
268		class CounterValue(DictMixin):
269		"""
270		A dict like value item for CounterManager.
271		"""
272
273		def __init__(self, manager, keys):
274		self.manager = manager
275		self._keys = keys
276
277		def __getitem__(self, key):
278		if key == '__value__':
279		key = self._keys
280		return self.manager.counters[key]
281		else:
282		key = self._keys + (key, )
283
284		available_keys = []
285		for _key in list(self.manager.counters.keys()):
286		if _key[:len(key)] == key:
287		available_keys.append(_key)
288
289		if len(available_keys) == 0:
290		raise KeyError
291		elif len(available_keys) == 1:
292		if available_keys[0] == key:
293		return self.manager.counters.get(key)
294		else:
295		return CounterValue(self.manager, key)
296		else:
297		return CounterValue(self.manager, key)
298
299		def __len__(self):
300		return len(self.keys())
301
302		def __iter__(self):
303		return iter(self.keys())
304
305		def __contains__(self, key):
306		return key in self.keys()
307
308		def keys(self):
309		result = set()
310		for key in list(self.manager.counters.keys()):
311		if key[:len(self._keys)] == self._keys:
312		key = key[len(self._keys):]
313		result.add(key[0] if key else '__value__')
314		return result
315
316		def to_dict(self, get_value=None):
317		"""Dump counters as a dict"""
318		result = {}
319		for key, value in iteritems(self):
320		if isinstance(value, BaseCounter):
321		if get_value is not None:
322		value = getattr(value, get_value)
323		result[key] = value
324		else:
325		result[key] = value.to_dict(get_value)
326		return result
327
328
329		class CounterManager(DictMixin):
330		"""
331		A dict like counter manager.
332
333		When using a tuple as event key, say: ('foo', 'bar'), You can visite counter
334		with manager['foo']['bar']. Or get all counters which first element is 'foo'
335		by manager['foo'].
336
337		It's useful for a group of counters.
338		"""
339
340		def __init__(self, cls=TimebaseAverageWindowCounter):
341		"""init manager with Counter cls"""
342		self.cls = cls
343		self.counters = {}
344
345		def event(self, key, value=1):
346		"""Fire a event of a counter by counter key"""
347		if isinstance(key, six.string_types):
348		key = (key, )
349		assert isinstance(key, tuple), "event key type error"
350		if key not in self.counters:
351		self.counters[key] = self.cls()
352		self.counters[key].event(value)
353		return self
354
355		def value(self, key, value=1):
356		"""Set value of a counter by counter key"""
357		if isinstance(key, six.string_types):
358		key = (key, )
359		# assert all(isinstance(k, six.string_types) for k in key)
360		assert isinstance(key, tuple), "event key type error"
361		if key not in self.counters:
362		self.counters[key] = self.cls()
363		self.counters[key].value(value)
364		return self
365
366		def trim(self):
367		"""Clear not used counters"""
368		for key, value in list(iteritems(self.counters)):
369		if value.empty():
370		del self.counters[key]
371
372		def __getitem__(self, key):
373		key = (key, )
374		available_keys = []
375		for _key in list(self.counters.keys()):
376		if _key[:len(key)] == key:
377		available_keys.append(_key)
378
379		if len(available_keys) == 0:
380		raise KeyError
381		elif len(available_keys) == 1:
382		if available_keys[0] == key:
383		return self.counters.get(key)
384		else:
385		return CounterValue(self, key)
386		else:
387		return CounterValue(self, key)
388
389		def __delitem__(self, key):
390		key = (key, )
391		available_keys = []
392		for _key in list(self.counters.keys()):
393		if _key[:len(key)] == key:
394		available_keys.append(_key)
395		for _key in available_keys:
396		del self.counters[_key]
397
398		def __iter__(self):
399		return iter(self.keys())
400
401		def __len__(self):
402		return len(self.keys())
403
404		def keys(self):
405		result = set()
406		for key in self.counters.keys():
407		result.add(key[0] if key else ())
408		return result
409
410		def to_dict(self, get_value=None):
411		"""Dump counters as a dict"""
412		self.trim()
413		result = {}
414		for key, value in iteritems(self.counters):
415		if get_value is not None:
416		value = getattr(value, get_value)
417		r = result
418		for _key in key[:-1]:
419		r = r.setdefault(_key, {})
420		r[key[-1]] = value
421		return result
422
423		def dump(self, filename):
424		"""Dump counters to file"""
425		try:
426		with open(filename, 'wb') as fp:
427		cPickle.dump(self.counters, fp)
428		except Exception as e:
429		logging.warning("can't dump counter to file %s: %s", filename, e)
430		return False
431		return True
432
433		def load(self, filename):
434		"""Load counters to file"""
435		try:
436		with open(filename, 'rb') as fp:
437		self.counters = cPickle.load(fp)
438		except:
439		logging.debug("can't load counter from file: %s", filename)
440		return False
441		return True
442

binux / pyspider

CounterManager B last analyzed 2018-06-14 19:08 UTC

Complexity

Size/Duplication

Importance

12 Methods

Duplication Side-by-Side

Filter issues like

CounterManager B
last analyzed 2018-06-14 19:08 UTC