dmyersturnbull /
pocketutils
| Total Complexity | 46 |
| Total Lines | 207 |
| Duplicated Lines | 0 % |
| Changes | 0 | ||
Complex classes like pocketutils.misc.mem_cache often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | import operator |
||
|
introduced
by
|
|||
| 2 | import sys |
||
| 3 | from abc import ABC |
||
| 4 | from datetime import datetime |
||
| 5 | from typing import Any, Callable, Dict, Generic, Iterator, Mapping, Optional, Sequence, TypeVar |
||
| 6 | |||
| 7 | from psutil import virtual_memory |
||
| 8 | |||
| 9 | # noinspection PyProtectedMember |
||
| 10 | from pocketutils.core._internal import nicesize |
||
| 11 | |||
| 12 | K = TypeVar("K", covariant=True) |
||
| 13 | V = TypeVar("V", contravariant=True) |
||
| 14 | |||
| 15 | |||
| 16 | class MemCachePolicy(Generic[K, V]): |
||
| 17 | def should_archive(self) -> bool: |
||
| 18 | raise NotImplementedError() |
||
| 19 | |||
| 20 | def can_archive(self) -> bool: |
||
| 21 | raise NotImplementedError() |
||
| 22 | |||
| 23 | def reindex(self, items: Mapping[K, V]) -> None: |
||
| 24 | raise NotImplementedError() |
||
| 25 | |||
| 26 | def items(self) -> Iterator[K]: |
||
| 27 | # TODO this is not overloaded!! |
||
| 28 | raise NotImplementedError() |
||
| 29 | |||
| 30 | def accessed(self, key: K) -> None: |
||
| 31 | pass |
||
| 32 | |||
| 33 | def added(self, key: K, value: V) -> None: |
||
| 34 | pass |
||
| 35 | |||
| 36 | def removed(self, key: K) -> None: |
||
| 37 | pass |
||
| 38 | |||
| 39 | |||
| 40 | class MemoryLimitingPolicy(MemCachePolicy, ABC): |
||
| 41 | def __init__( |
||
| 42 | self, |
||
| 43 | max_memory_bytes: Optional[int] = None, |
||
| 44 | max_fraction_available_bytes: Optional[float] = None, |
||
| 45 | ): |
||
| 46 | self._max_memory_bytes = max_memory_bytes |
||
| 47 | self._max_fraction_available_bytes = max_fraction_available_bytes |
||
| 48 | self._total_memory_bytes = 0 |
||
| 49 | self._usage_bytes = {} # type: Dict[K, int] |
||
| 50 | self._last_accessed = {} # type: Dict[K, datetime] |
||
| 51 | self._created = {} # type: Dict[K, datetime] |
||
| 52 | |||
| 53 | def can_archive(self) -> bool: |
||
| 54 | return len(self._last_accessed) > 0 |
||
| 55 | |||
| 56 | def should_archive(self) -> bool: |
||
| 57 | return ( |
||
| 58 | self._max_memory_bytes is not None and self._total_memory_bytes > self._max_memory_bytes |
||
| 59 | ) or ( |
||
| 60 | self._max_fraction_available_bytes is not None |
||
| 61 | and self._total_memory_bytes |
||
| 62 | > virtual_memory().available * self._max_fraction_available_bytes |
||
| 63 | ) |
||
| 64 | |||
| 65 | def reindex(self, items: Mapping[K, V]) -> None: |
||
| 66 | for key in set(self._last_accessed.keys()) - set(items.keys()): |
||
| 67 | if key not in items.keys(): |
||
| 68 | self.removed(key) |
||
| 69 | for key, value in items.items(): |
||
| 70 | self._usage_bytes[key] = sys.getsizeof(value) |
||
| 71 | self._total_memory_bytes = sum(self._usage_bytes.values()) |
||
| 72 | |||
| 73 | def accessed(self, key: K) -> None: |
||
| 74 | self._last_accessed[key] = datetime.now() |
||
| 75 | |||
| 76 | def added(self, key: K, value: V) -> None: |
||
| 77 | now = datetime.now() |
||
| 78 | self._created[key] = now |
||
| 79 | self._last_accessed[key] = now |
||
| 80 | self._usage_bytes[key] = sys.getsizeof(value) |
||
| 81 | self._total_memory_bytes += self._usage_bytes[key] |
||
| 82 | |||
| 83 | def removed(self, key: K) -> None: |
||
| 84 | self._total_memory_bytes -= self._usage_bytes[key] |
||
| 85 | del self._last_accessed[key] |
||
| 86 | del self._created[key] |
||
| 87 | del self._usage_bytes[key] |
||
| 88 | |||
| 89 | def __str__(self): |
||
| 90 | available = virtual_memory().available |
||
| 91 | name = self.__class__.__name__ |
||
| 92 | n_items = len(self._usage_bytes) |
||
| 93 | real = nicesize(self._total_memory_bytes) |
||
| 94 | cap_raw = "-" if self._max_memory_bytes is None else nicesize(self._max_memory_bytes) |
||
| 95 | cap_percent = ( |
||
| 96 | "-" |
||
| 97 | if self._max_fraction_available_bytes is None |
||
| 98 | else nicesize(available * self._max_fraction_available_bytes) |
||
| 99 | ) |
||
| 100 | percent = ( |
||
| 101 | "-" |
||
| 102 | if self._max_fraction_available_bytes is None |
||
| 103 | else round( |
||
| 104 | 100 * self._total_memory_bytes / (available * self._max_fraction_available_bytes), |
||
| 105 | 3, |
||
| 106 | ) |
||
| 107 | ) |
||
| 108 | return f"{name}(n={n_items}, {real}/{cap_raw}, {real}/{cap_percent}={percent}%" |
||
| 109 | |||
| 110 | def __repr__(self): |
||
| 111 | ordered = list(self.items()) |
||
| 112 | ss = [] |
||
| 113 | current_day = None |
||
| 114 | for k in ordered: |
||
| 115 | dt = self._last_accessed[k] |
||
| 116 | if current_day is None or current_day.date() != dt.date(): |
||
| 117 | current_day = dt |
||
| 118 | ss.append("#" + current_day.strftime("%Y-%m-%d") + "...") |
||
| 119 | nice = nicesize(self._usage_bytes[k]) |
||
| 120 | access = self._last_accessed[k].strftime("%H:%M:%S") |
||
| 121 | ss.append(f"{k}:{nice}@{access}") |
||
| 122 | return f"{str(self)}@{hex(id(self))}: [{', '.join(ss)}]" |
||
| 123 | |||
| 124 | |||
| 125 | class MemoryLruPolicy(MemoryLimitingPolicy): |
||
| 126 | def items(self) -> Iterator[K]: |
||
| 127 | return iter([k for k, v in sorted(self._last_accessed.items(), key=operator.itemgetter(1))]) |
||
| 128 | |||
| 129 | |||
| 130 | class MemoryMruPolicy(MemoryLimitingPolicy): |
||
| 131 | def items(self) -> Iterator[K]: |
||
| 132 | rev = reversed(sorted(self._last_accessed.items(), key=operator.itemgetter(1))) |
||
| 133 | return iter([k for k, v in rev]) |
||
| 134 | |||
| 135 | |||
| 136 | class MemCache(Generic[K, V]): |
||
| 137 | def __init__( |
||
| 138 | self, loader: Callable[[K], V], policy: MemCachePolicy, *, log: Callable[[str], Any] |
||
| 139 | ): |
||
| 140 | self._loader = loader |
||
| 141 | self._items = {} # type: Dict[K, V] |
||
| 142 | self._policy = policy |
||
| 143 | self._log = lambda _: None if log is None else log |
||
| 144 | |||
| 145 | def __getitem__(self, key: K) -> V: |
||
| 146 | self._policy.accessed(key) |
||
| 147 | if key in self._items: |
||
| 148 | return self._items[key] |
||
| 149 | else: |
||
| 150 | value = self._loader(key) |
||
| 151 | self._log(f"Loaded {key}") |
||
| 152 | self._items[key] = value |
||
| 153 | self._policy.added(key, value) |
||
| 154 | self.archive() |
||
| 155 | return value |
||
| 156 | |||
| 157 | def __call__(self, key: K) -> V: |
||
| 158 | return self[key] |
||
| 159 | |||
| 160 | def archive(self, at_least: Optional[int] = None) -> Sequence[K]: |
||
| 161 | it = self._policy.items() |
||
| 162 | archived = [] |
||
| 163 | while self._policy.can_archive() and ( |
||
| 164 | at_least is not None and len(archived) < at_least or self._policy.should_archive() |
||
| 165 | ): |
||
| 166 | key = next(it) |
||
| 167 | self._policy.removed(key) |
||
| 168 | del self._items[key] |
||
| 169 | archived.append(key) |
||
| 170 | self._log(f"Archived {len(archived)} items: {archived}") |
||
| 171 | return archived |
||
| 172 | |||
| 173 | def clear(self) -> None: |
||
| 174 | it = self._policy.items() |
||
| 175 | cleared = [] |
||
| 176 | while self._policy.can_archive(): |
||
| 177 | key = next(it) |
||
| 178 | self._policy.removed(key) |
||
| 179 | del self._items[key] |
||
| 180 | cleared.append(key) |
||
| 181 | self._log(f"Cleared {len(cleared)} items: {cleared}") |
||
| 182 | |||
| 183 | def remove(self, key: K) -> None: |
||
| 184 | if key in self: |
||
| 185 | self._policy.removed(key) |
||
| 186 | del self._items[key] |
||
| 187 | |||
| 188 | def __contains__(self, key: K): |
||
| 189 | return key in self._items |
||
| 190 | |||
| 191 | def __delitem__(self, key): |
||
| 192 | self.remove(key) |
||
| 193 | |||
| 194 | def __repr__(self): |
||
| 195 | return str(self) + "@" + hex(id(self)) |
||
| 196 | |||
| 197 | def __str__(self): |
||
| 198 | return f"{self.__class__.__name__}({repr(self._policy)})" |
||
| 199 | |||
| 200 | |||
| 201 | __all__ = [ |
||
| 202 | "MemCachePolicy", |
||
| 203 | "MemCache", |
||
| 204 | "MemoryLimitingPolicy", |
||
| 205 | "MemoryLruPolicy", |
||
| 206 | "MemoryMruPolicy", |
||
| 207 | ] |
||
| 208 |
Loading history...