MultiCache.dec() - Code Metrics - Inspection of "ongoing werkzeug-cache compatible cache implementa..." - ergoithz/browsepy - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — 5.2-unstable ( ee49a9 )

by Felipe A.

created 2016-12-02 14:30 UTC

MultiCache.dec() A

↳ Parent: MultiCache

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	2
c	0
b	0
f	0
dl	0
loc	5
rs	9.4285


import time
import threading
import collections

from werkzeug.contrib.cache import BaseCache


NOT_FOUND = object()


class HashTuple(tuple):
    '''
    Tuple object with cached __hash__ and __repr__.
    '''

    def __hash__(self):
        '''
        Self-caching hash method.

        x.__hash__() <==> hash(x)
        '''
        res = tuple.__hash__(self)
        self.__hash__ = lambda x: res
        return res

    def __repr__(self):
        '''
        Self-caching representation method.

        x.__repr__() <==> repr(x)
        '''
        res = (
            '<{0.__class__.__module__}.{0.__class__.__name__} {1} {2}>'
            ).format(self, hash(self), tuple(self))
        self.__repr__ = lambda x: res
        return res


class MemoizeManager(object):
    '''
    Cache manager object exposed via :attr:`cache` of memoized functions (see
    :meth:`BaseCache.memoize`).
    '''
    hashtuple_class = HashTuple
    NOT_CACHED = object()

    def __init__(self, cache):
        self._hits = 0
        self._misses = 0
        self._cache = cache

    @property
    def hits(self):
        '''
        :returns: number of times result was cached
        :rtype: integer
        '''
        return self._hits

    @property
    def misses(self):
        '''
        :returns: number of times result was not cached
        :rtype: integer
        '''
        return self._missses

    @classmethod
    def make_key(cls, args, kwargs, tuple=tuple):
        '''
        Make a cache key from optionally typed positional and keyword arguments

        The key is constructed in a way that is flat as possible rather than
        as a nested structure that would take more memory.

        If there is only a single argument and its data type is known to cache
        its hash value, then that argument is returned without a wrapper.  This
        saves space and improves lookup speed.
        '''
        key_args = tuple([(arg, arg.__class__) for arg in args])
        key_kwargs = tuple([
            (key, value, value.__class__)
            for key, value in sorted(kwargs.items())
            ])
        return cls.hashtuple_class((key_args, key_kwargs))

    def run(self, fnc, *args, **kwargs):
        key = self.make_key(args, kwargs)
        result = self._cache.get(key, self.NOT_CACHED)
        if result is not self.NOT_CACHED:
            self._hits += 1
            return result
        self._misses += 1
        self._cache[key] = result = fnc(*args, **kwargs)
        return result

    def pop(self, *args, **kwargs):
        key = self.make_key(args, kwargs)
        return self._cache.pop(key, None)

    def clear(self):
        '''
        Clear both cache and statistics.
        '''
        self._hits = 0
        self._misses = 0
        self._cache.clear()


class SafeSimpleCache(BaseCache):
    '''
    Simple memory cache for simgle process environments. Thread safe using
    :attr:`lock_class` (threading.Lock)

    Cache object evicting least recently used objects (LRU) when maximum cache
    size is reached, so keys could be discarded independently of their
    timeout.

    '''
    lock_class = threading.Lock
    time_func = time.time

    @property
    def full(self):
        '''
        :returns: True if size reached maxsize, False otherwise
        :rtype: bool
        '''
        return self._full

    @property
    def size(self):
        '''
        :returns: current size of result cache
        :rtype: int
        '''
        return len(self._cache)

    @property
    def maxsize(self):
        '''
        :returns: maximum size of result cache
        :rtype: int
        '''
        return self._maxsize

    def __init__(self, default_timeout=300, maxsize=1024):
        '''
        :param fnc:
        :type fnc: collections.abc.Callable
        :param maxsize: optional maximum cache size (defaults to 1024)
        :type maxsize: int
        '''
        self._default_timeout = default_timeout
        self._cache = {}
        self._full = False
        self._maxsize = maxsize
        self._lock = self.lock_class()
        self._lru_key = None

    def _extract(self, link):
        '''
        Pop given link from circular list.
        '''
        PREV, NEXT = 0, 1
        prev = link[PREV]
        next = link[NEXT]
        prev[NEXT] = next
        next[PREV] = prev
        return link

    def _insert(self, link):
        '''
        Set given link as mru.
        '''
        PREV, NEXT = 0, 1
        next = self._cache.get(self._lru_key) if self._cache else link
        prev = next[PREV]
        link[:2] = (prev, next)
        prev[NEXT] = link
        next[PREV] = link
        return link

    def _shift(self, pop=False):
        '''
        Transform oldest into newest and return it.
        '''
        NEXT, KEY = 1, 2
        if pop:
            link = self._cache.pop(self._older_key)
        else:
            link = self._cache[self._older_key]
        self._lru_key = link[NEXT][KEY]
        return link

    def _bump(self, link):
        NEXT, KEY = 1, 2
        if link[KEY] == self._lru_key:
            return self._shift()
        if link[NEXT][KEY] == self._lru_key:
            return link
        return self._insert(self._extract(link))

    def _getitem(self, key, default=NOT_FOUND):
        VALUE, EXPIRE = 3, 4
        link = self._cache.get(key)
        if link is not None and link[EXPIRE] < self.time_func():
            self._bump(link)
            return link[VALUE]
        return default

    def _setitem(self, key, value, timeout=None):
        KEY, VALUE = 2, 3
        cache = self._cache
        expire = self.time_func() + (
            self._default_timeout
            if timeout is None else
            timeout
            )
        link = self._cache.get(key)
        if link:
            link = self._bump(link)
            link[VALUE:] = (value, expire)
        elif self._full:
            link = self._shift(pop=True)
            link[KEY:] = (key, value, expire)
            self._cache[key] = link
        else:
            self._cache[key] = self._insert([None, None, key, value, expire])
            self._full = (len(cache) >= self._maxsize)
        return value

    def _popitem(self, key, default=NOT_FOUND):
        VALUE = 3
        link = self._cache.pop(key, default)
        return self._extract(link)[VALUE]

    def add(self, key, value, timeout=None):
        with self._lock:
            if key in self._cache:
                return False
            self._setitem(key, value, timeout)
            return True

    def set(self, key, value, timeout=None):
        with self._lock:
            self._setitem(key, value, timeout)
            return True

    def set_many(self, mapping, timeout=None):
        with self._lock:
            for key, value in mapping.items():
                self._setitem(key, value, timeout)
            return True

    def inc(self, key, delta=1):
        with self._lock:
            return self._setitem(key, self._cache.get(key, 0) + delta)

    def dec(self, key, delta=1):
        return self.inc(key, delta=-delta)

    def delete(self, key):
        with self._lock:
            return self._popitem(key) is not NOT_FOUND

    def delete_many(self, *keys):
        with self._lock:
            return NOT_FOUND not in [self._popitem(key) for key in keys]

    def get(self, key):
        with self._lock:
            return self._getitem(key, None)

    def get_dict(self, *keys):
        with self._lock:
            return {key: self._getitem(key, None) for key in keys}

    def get_many(self, *keys):
        with self._lock:
            return [self._getitem(key, None) for key in keys]

    def has(self, key):
        return key in self._cache

    def clear(self):
        with self._lock:
            self._cache.clear()
            self._hits = 0
            self._misses = 0
            self._full = False
            return True


class MultiCache(BaseCache):
    '''
    Cache object wrapping multiple caches.
    '''
    def __init__(self, backends=None):
        self.backends = [] if backends is None else backends

    def add(self, key, value, timeout=None):
        return all([c.add(key, value, timeout) for c in self.backends])

    def set(self, key, value, timeout=None):
        return all([c.set(key, value, timeout) for c in self.backends])

    def set_many(self, mapping, timeout=None):
        return all([c.set_many(mapping, timeout) for c in self.backends])

    def _sync(self, key, value, results):
        for backend, partial in results.iteritems():
            if partial != value:
                backend.set(key, value)

    def inc(self, key, delta=1):
        results = {c: c.inc(key, delta=delta) for c in self.backends}
        value = results[max(results, key=results.get)]
        self._sync(key, value, results)
        return value

    def dec(self, key, delta=1):
        results = {c: c.dec(key, delta=delta) for c in self.backends}
        value = results[min(results, key=results.get)]
        self._sync(key, value, results)
        return value

    def delete(self, key):
        '''
        Delete key from all caches.

        :param key:
        :type key: string
        :returns wWhether the key existed on any backend and has been deleted.
        :rtype: bool
        '''
        return any([c.delete(key) for c in self.backends])

    def delete_many(self, *keys):
        return any([c.delete_many(*keys) for c in self.backends])

    def get(self, key):
        for backend in self.backends:
            result = backend.get(key)
            if result is not None:
                return result
        return None

    def _getmany(self, keys, ordered=False):
        remaining = set(keys)
        if ordered:
            result = collections.OrderedDict((key, None) for key in keys)
        else:
            result = {key: None for key in keys}
        for backend in self.backends:
            partial = backend.get_dict(*remaining)
            result.update(partial)
            remaining -= {k for k, v in partial.items() if v is not None}
            if not remaining:
                return
        return result

    def get_dict(self, *keys):
        return self._getmany(keys)

    def get_many(self, *keys):
        return list(self._getmany(keys, ordered=True).values())

    def has(self, key):
        return any(c.has(key) for c in self._cache)

    def clear(self):
        return all([c.clear() for c in self.backends])


1
2			import time
3			import threading
4			import collections
5
6			from werkzeug.contrib.cache import BaseCache
7
8
9			NOT_FOUND = object()
10
11
12			class HashTuple(tuple):
13			'''
14			Tuple object with cached __hash__ and __repr__.
15			'''
16
17			def __hash__(self):
18			'''
19			Self-caching hash method.
20
21			x.__hash__() <==> hash(x)
22			'''
23			res = tuple.__hash__(self)
24			self.__hash__ = lambda x: res
25			return res
26
27			def __repr__(self):
28			'''
29			Self-caching representation method.
30
31			x.__repr__() <==> repr(x)
32			'''
33			res = (
34			'<{0.__class__.__module__}.{0.__class__.__name__} {1} {2}>'
35			).format(self, hash(self), tuple(self))
36			self.__repr__ = lambda x: res
37			return res
38
39
40			class MemoizeManager(object):
41			'''
42			Cache manager object exposed via :attr:`cache` of memoized functions (see
43			:meth:`BaseCache.memoize`).
44			'''
45			hashtuple_class = HashTuple
46			NOT_CACHED = object()
47
48			def __init__(self, cache):
49			self._hits = 0
50			self._misses = 0
51			self._cache = cache
52
53			@property
54			def hits(self):
55			'''
56			:returns: number of times result was cached
57			:rtype: integer
58			'''
59			return self._hits
60
61			@property
62			def misses(self):
63			'''
64			:returns: number of times result was not cached
65			:rtype: integer
66			'''
67			return self._missses
68
69			@classmethod
70			def make_key(cls, args, kwargs, tuple=tuple):
71			'''
72			Make a cache key from optionally typed positional and keyword arguments
73
74			The key is constructed in a way that is flat as possible rather than
75			as a nested structure that would take more memory.
76
77			If there is only a single argument and its data type is known to cache
78			its hash value, then that argument is returned without a wrapper. This
79			saves space and improves lookup speed.
80			'''
81			key_args = tuple([(arg, arg.__class__) for arg in args])
82			key_kwargs = tuple([
83			(key, value, value.__class__)
84			for key, value in sorted(kwargs.items())
85			])
86			return cls.hashtuple_class((key_args, key_kwargs))
87
88			def run(self, fnc, args, *kwargs):
89			key = self.make_key(args, kwargs)
90			result = self._cache.get(key, self.NOT_CACHED)
91			if result is not self.NOT_CACHED:
92			self._hits += 1
93			return result
94			self._misses += 1
95			self._cache[key] = result = fnc(args, *kwargs)
96			return result
97
98			def pop(self, args, *kwargs):
99			key = self.make_key(args, kwargs)
100			return self._cache.pop(key, None)
101
102			def clear(self):
103			'''
104			Clear both cache and statistics.
105			'''
106			self._hits = 0
107			self._misses = 0
108			self._cache.clear()
109
110
111			class SafeSimpleCache(BaseCache):
112			'''
113			Simple memory cache for simgle process environments. Thread safe using
114			:attr:`lock_class` (threading.Lock)
115
116			Cache object evicting least recently used objects (LRU) when maximum cache
117			size is reached, so keys could be discarded independently of their
118			timeout.
119
120			'''
121			lock_class = threading.Lock
122			time_func = time.time
123
124			@property
125			def full(self):
126			'''
127			:returns: True if size reached maxsize, False otherwise
128			:rtype: bool
129			'''
130			return self._full
131
132			@property
133			def size(self):
134			'''
135			:returns: current size of result cache
136			:rtype: int
137			'''
138			return len(self._cache)
139
140			@property
141			def maxsize(self):
142			'''
143			:returns: maximum size of result cache
144			:rtype: int
145			'''
146			return self._maxsize
147
148			def __init__(self, default_timeout=300, maxsize=1024):
149			'''
150			:param fnc:
151			:type fnc: collections.abc.Callable
152			:param maxsize: optional maximum cache size (defaults to 1024)
153			:type maxsize: int
154			'''
155			self._default_timeout = default_timeout
156			self._cache = {}
157			self._full = False
158			self._maxsize = maxsize
159			self._lock = self.lock_class()
160			self._lru_key = None
161
162			def _extract(self, link):
163			'''
164			Pop given link from circular list.
165			'''
166			PREV, NEXT = 0, 1
167			prev = link[PREV]
168			next = link[NEXT]
169			prev[NEXT] = next
170			next[PREV] = prev
171			return link
172
173			def _insert(self, link):
174			'''
175			Set given link as mru.
176			'''
177			PREV, NEXT = 0, 1
178			next = self._cache.get(self._lru_key) if self._cache else link
179			prev = next[PREV]
180			link[:2] = (prev, next)
181			prev[NEXT] = link
182			next[PREV] = link
183			return link
184
185			def _shift(self, pop=False):
186			'''
187			Transform oldest into newest and return it.
188			'''
189			NEXT, KEY = 1, 2
190			if pop:
191			link = self._cache.pop(self._older_key)
192			else:
193			link = self._cache[self._older_key]
194			self._lru_key = link[NEXT][KEY]
195			return link
196
197			def _bump(self, link):
198			NEXT, KEY = 1, 2
199			if link[KEY] == self._lru_key:
200			return self._shift()
201			if link[NEXT][KEY] == self._lru_key:
202			return link
203			return self._insert(self._extract(link))
204
205			def _getitem(self, key, default=NOT_FOUND):
206			VALUE, EXPIRE = 3, 4
207			link = self._cache.get(key)
208			if link is not None and link[EXPIRE] < self.time_func():
209			self._bump(link)
210			return link[VALUE]
211			return default
212
213			def _setitem(self, key, value, timeout=None):
214			KEY, VALUE = 2, 3
215			cache = self._cache
216			expire = self.time_func() + (
217			self._default_timeout
218			if timeout is None else
219			timeout
220			)
221			link = self._cache.get(key)
222			if link:
223			link = self._bump(link)
224			link[VALUE:] = (value, expire)
225			elif self._full:
226			link = self._shift(pop=True)
227			link[KEY:] = (key, value, expire)
228			self._cache[key] = link
229			else:
230			self._cache[key] = self._insert([None, None, key, value, expire])
231			self._full = (len(cache) >= self._maxsize)
232			return value
233
234			def _popitem(self, key, default=NOT_FOUND):
235			VALUE = 3
236			link = self._cache.pop(key, default)
237			return self._extract(link)[VALUE]
238
239			def add(self, key, value, timeout=None):
240			with self._lock:
241			if key in self._cache:
242			return False
243			self._setitem(key, value, timeout)
244			return True
245
246			def set(self, key, value, timeout=None):
247			with self._lock:
248			self._setitem(key, value, timeout)
249			return True
250
251			def set_many(self, mapping, timeout=None):
252			with self._lock:
253			for key, value in mapping.items():
254			self._setitem(key, value, timeout)
255			return True
256
257			def inc(self, key, delta=1):
258			with self._lock:
259			return self._setitem(key, self._cache.get(key, 0) + delta)
260
261			def dec(self, key, delta=1):
262			return self.inc(key, delta=-delta)
263
264			def delete(self, key):
265			with self._lock:
266			return self._popitem(key) is not NOT_FOUND
267
268			def delete_many(self, *keys):
269			with self._lock:
270			return NOT_FOUND not in [self._popitem(key) for key in keys]
271
272			def get(self, key):
273			with self._lock:
274			return self._getitem(key, None)
275
276			def get_dict(self, *keys):
277			with self._lock:
278			return {key: self._getitem(key, None) for key in keys}
279
280			def get_many(self, *keys):
281			with self._lock:
282			return [self._getitem(key, None) for key in keys]
283
284			def has(self, key):
285			return key in self._cache
286
287			def clear(self):
288			with self._lock:
289			self._cache.clear()
290			self._hits = 0
291			self._misses = 0
292			self._full = False
293			return True
294
295
296			class MultiCache(BaseCache):
297			'''
298			Cache object wrapping multiple caches.
299			'''
300			def __init__(self, backends=None):
301			self.backends = [] if backends is None else backends
302
303			def add(self, key, value, timeout=None):
304			return all([c.add(key, value, timeout) for c in self.backends])
305
306			def set(self, key, value, timeout=None):
307			return all([c.set(key, value, timeout) for c in self.backends])
308
309			def set_many(self, mapping, timeout=None):
310			return all([c.set_many(mapping, timeout) for c in self.backends])
311
312			def _sync(self, key, value, results):
313			for backend, partial in results.iteritems():
314			if partial != value:
315			backend.set(key, value)
316
317			def inc(self, key, delta=1):
318			results = {c: c.inc(key, delta=delta) for c in self.backends}
319			value = results[max(results, key=results.get)]
320			self._sync(key, value, results)
321			return value
322
323			def dec(self, key, delta=1):
324			results = {c: c.dec(key, delta=delta) for c in self.backends}
325			value = results[min(results, key=results.get)]
326			self._sync(key, value, results)
327			return value
328
329			def delete(self, key):
330			'''
331			Delete key from all caches.
332
333			:param key:
334			:type key: string
335			:returns wWhether the key existed on any backend and has been deleted.
336			:rtype: bool
337			'''
338			return any([c.delete(key) for c in self.backends])
339
340			def delete_many(self, *keys):
341			return any([c.delete_many(*keys) for c in self.backends])
342
343			def get(self, key):
344			for backend in self.backends:
345			result = backend.get(key)
346			if result is not None:
347			return result
348			return None
349
350			def _getmany(self, keys, ordered=False):
351			remaining = set(keys)
352			if ordered:
353			result = collections.OrderedDict((key, None) for key in keys)
354			else:
355			result = {key: None for key in keys}
356			for backend in self.backends:
357			partial = backend.get_dict(*remaining)
358			result.update(partial)
359			remaining -= {k for k, v in partial.items() if v is not None}
360			if not remaining:
361			return
362			return result
363
364			def get_dict(self, *keys):
365			return self._getmany(keys)
366
367			def get_many(self, *keys):
368			return list(self._getmany(keys, ordered=True).values())
369
370			def has(self, key):
371			return any(c.has(key) for c in self._cache)
372
373			def clear(self):
374			return all([c.clear() for c in self.backends])
375

ergoithz / browsepy

Push — 5.2-unstable ( ee49a9 )

MultiCache.dec() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like