1
|
|
|
import sys |
|
|
|
|
2
|
|
|
from collections import defaultdict |
3
|
|
|
from typing import ( |
4
|
|
|
Any, |
5
|
|
|
Callable, |
6
|
|
|
Generator, |
7
|
|
|
Iterable, |
8
|
|
|
Iterator, |
9
|
|
|
Mapping, |
10
|
|
|
Optional, |
11
|
|
|
Sequence, |
12
|
|
|
Tuple, |
13
|
|
|
Type, |
14
|
|
|
TypeVar, |
15
|
|
|
Union, |
16
|
|
|
) |
17
|
|
|
|
18
|
|
|
import pandas as pd |
|
|
|
|
19
|
|
|
|
20
|
|
|
from pocketutils.core._internal import nicesize |
|
|
|
|
21
|
|
|
from pocketutils.core.exceptions import RefusingRequestError, XKeyError, XValueError |
|
|
|
|
22
|
|
|
from pocketutils.core.input_output import DevNull |
|
|
|
|
23
|
|
|
from pocketutils.tools.base_tools import BaseTools |
24
|
|
|
|
25
|
|
|
Y = TypeVar("Y") |
|
|
|
|
26
|
|
|
T = TypeVar("T") |
|
|
|
|
27
|
|
|
Z = TypeVar("Z") |
|
|
|
|
28
|
|
|
Q = TypeVar("Q") |
|
|
|
|
29
|
|
|
|
30
|
|
|
|
31
|
|
|
class CommonTools(BaseTools): |
|
|
|
|
32
|
|
|
@classmethod |
33
|
|
|
def limit(cls, items: Iterable[Q], n: int) -> Generator[Q, None, None]: |
|
|
|
|
34
|
|
|
for i, x in zip(range(n), items): |
|
|
|
|
35
|
|
|
yield x |
36
|
|
|
|
37
|
|
|
@classmethod |
38
|
|
|
def is_float(cls, s: Any) -> bool: |
|
|
|
|
39
|
|
|
""" |
40
|
|
|
Returns whether ``float(s)`` succeeds. |
41
|
|
|
""" |
42
|
|
|
try: |
43
|
|
|
float(s) |
44
|
|
|
return True |
45
|
|
|
except ValueError: |
46
|
|
|
return False |
47
|
|
|
|
48
|
|
|
@classmethod |
49
|
|
|
def try_none( |
50
|
|
|
cls, |
|
|
|
|
51
|
|
|
function: Callable[[], T], |
|
|
|
|
52
|
|
|
fail_val: Optional[T] = None, |
|
|
|
|
53
|
|
|
exception=Exception, |
|
|
|
|
54
|
|
|
) -> Optional[T]: |
55
|
|
|
""" |
56
|
|
|
Returns the value of a function or None if it raised an exception. |
57
|
|
|
|
58
|
|
|
Args: |
59
|
|
|
function: Try calling this function |
60
|
|
|
fail_val: Return this value |
61
|
|
|
exception: Restrict caught exceptions to subclasses of this type |
62
|
|
|
""" |
63
|
|
|
# noinspection PyBroadException |
64
|
|
|
try: |
65
|
|
|
return function() |
66
|
|
|
except exception: |
|
|
|
|
67
|
|
|
return fail_val |
68
|
|
|
|
69
|
|
|
@classmethod |
70
|
|
|
def succeeds(cls, function: Callable[[], Any], exception=Exception) -> bool: |
71
|
|
|
"""Returns True iff ``function`` does not raise an error.""" |
72
|
|
|
return cls.try_none(function, exception=exception) is not None |
73
|
|
|
|
74
|
|
|
@classmethod |
75
|
|
|
def or_null(cls, x: Any, dtype=lambda s: s, or_else: Any = None) -> Optional[Any]: |
|
|
|
|
76
|
|
|
""" |
77
|
|
|
Return ``None`` if the operation ``dtype`` on ``x`` failed; returns the result otherwise. |
78
|
|
|
""" |
79
|
|
|
return or_else if cls.is_null(x) else dtype(x) |
80
|
|
|
|
81
|
|
|
@classmethod |
82
|
|
|
def or_raise( |
|
|
|
|
83
|
|
|
cls, |
|
|
|
|
84
|
|
|
x: Any, |
|
|
|
|
85
|
|
|
dtype=lambda s: s, |
|
|
|
|
86
|
|
|
or_else: Union[None, BaseException, Type[BaseException]] = None, |
|
|
|
|
87
|
|
|
) -> Any: |
88
|
|
|
""" |
89
|
|
|
Returns ``dtype(x)`` if ``x`` is not None, or raises ``or_else``. |
90
|
|
|
""" |
91
|
|
|
if or_else is None: |
92
|
|
|
or_else = LookupError(f"Value is {x}") |
93
|
|
|
elif isinstance(or_else, type): |
94
|
|
|
or_else = or_else(f"Value is {x}") |
95
|
|
|
if cls.is_null(x): |
96
|
|
|
raise or_else |
97
|
|
|
return dtype(x) |
98
|
|
|
|
99
|
|
|
@classmethod |
100
|
|
|
def iterator_has_elements(cls, x: Iterator[Any]) -> bool: |
|
|
|
|
101
|
|
|
""" |
102
|
|
|
Returns False iff ``next(x)`` raises a ``StopIteration``. |
103
|
|
|
WARNING: Tries to call ``next(x)``, progressing iterators. Don't use ``x`` after calling this. |
|
|
|
|
104
|
|
|
Note that calling ``iterator_has_elements([5])`` will raise a `TypeError` |
105
|
|
|
|
106
|
|
|
Args: |
107
|
|
|
x: Must be an Iterator |
108
|
|
|
""" |
109
|
|
|
return cls.succeeds(lambda: next(x), StopIteration) |
110
|
|
|
|
111
|
|
|
@classmethod |
112
|
|
|
def is_null(cls, x: Any) -> bool: |
|
|
|
|
113
|
|
|
""" |
114
|
|
|
Returns True if: |
115
|
|
|
- x is ``None`` |
116
|
|
|
- x is ``float('NaN')`` |
117
|
|
|
- ``pd.isna(x)`` |
118
|
|
|
""" |
119
|
|
|
try: |
120
|
|
|
if pd.isna(x): |
121
|
|
|
return True |
122
|
|
|
except (ValueError, TypeError): |
123
|
|
|
pass |
124
|
|
|
return x is None or x == float("NaN") |
125
|
|
|
|
126
|
|
|
@classmethod |
127
|
|
|
def is_empty(cls, x: Any) -> bool: |
|
|
|
|
128
|
|
|
""" |
129
|
|
|
Returns True iff either: |
130
|
|
|
- x is None |
131
|
|
|
- pd.is_na(x) |
132
|
|
|
- x is something with 0 length |
133
|
|
|
- x is iterable and has 0 elements (will call ``__iter__``) |
134
|
|
|
|
135
|
|
|
Raises: |
136
|
|
|
RefusingRequestError If ``x`` is an Iterator. Calling this would empty the iterator, which is dangerous. |
|
|
|
|
137
|
|
|
""" |
138
|
|
|
if isinstance(x, Iterator): |
|
|
|
|
139
|
|
|
raise RefusingRequestError("Do not call is_empty on an iterator.") |
140
|
|
|
try: |
141
|
|
|
if pd.isna(x): |
142
|
|
|
return True |
143
|
|
|
except (ValueError, TypeError): |
144
|
|
|
pass |
145
|
|
|
return ( |
146
|
|
|
x is None |
147
|
|
|
or x == float("NaN") |
148
|
|
|
or hasattr(x, "__len__") |
149
|
|
|
and len(x) == 0 |
150
|
|
|
or hasattr(x, "__iter__") |
151
|
|
|
and len(list(iter(x))) == 0 |
152
|
|
|
) |
153
|
|
|
|
154
|
|
|
@classmethod |
155
|
|
|
def is_probable_null(cls, x: Any) -> bool: |
|
|
|
|
156
|
|
|
""" |
157
|
|
|
Returns True iff either: |
158
|
|
|
- x is None |
159
|
|
|
- pd.isna(x) |
160
|
|
|
- x is something with 0 length |
161
|
|
|
- x is iterable and has 0 elements (will call ``__iter__``) |
162
|
|
|
- a str(x) is 'nan', 'n/a', 'null', or 'none'; case-insensitive |
163
|
|
|
|
164
|
|
|
Things that are **NOT** probable nulls: |
165
|
|
|
- "na" |
166
|
|
|
- 0 |
167
|
|
|
- [None] |
168
|
|
|
|
169
|
|
|
Raises: |
170
|
|
|
TypeError If ``x`` is an Iterator. |
171
|
|
|
Calling this would empty the iterator, which is dangerous. |
172
|
|
|
""" |
173
|
|
|
return cls.is_empty(x) or str(x).lower() in ["nan", "n/a", "null", "none"] |
174
|
|
|
|
175
|
|
|
@classmethod |
176
|
|
|
def unique(cls, sequence: Iterable[T]) -> Sequence[T]: |
177
|
|
|
""" |
178
|
|
|
Returns the unique items in `sequence`, in the order they appear in the iteration. |
179
|
|
|
|
180
|
|
|
Args: |
181
|
|
|
sequence: Any once-iterable sequence |
182
|
|
|
|
183
|
|
|
Returns: |
184
|
|
|
An ordered List of unique elements |
185
|
|
|
""" |
186
|
|
|
seen = set() |
187
|
|
|
return [x for x in sequence if not (x in seen or seen.add(x))] |
188
|
|
|
|
189
|
|
|
@classmethod |
190
|
|
|
def first(cls, collection: Iterable[Any], attr: Optional[str] = None) -> Optional[Any]: |
191
|
|
|
""" |
192
|
|
|
Gets the first element. |
193
|
|
|
|
194
|
|
|
WARNING: Tries to call ``next(x)``, progressing iterators. |
195
|
|
|
|
196
|
|
|
Args: |
197
|
|
|
collection: Any iterable |
198
|
|
|
attr: The name of the attribute that might be defined on the elements, |
199
|
|
|
or None to indicate the elements themselves should be used |
200
|
|
|
|
201
|
|
|
Returns: |
202
|
|
|
- The attribute of the first element if ``attr`` is defined on an element |
203
|
|
|
- None if the the sequence is empty |
204
|
|
|
- None if the sequence has no attribute ``attr`` |
205
|
|
|
""" |
206
|
|
|
try: |
207
|
|
|
# note: calling iter on an iterator creates a view only |
208
|
|
|
x = next(iter(collection)) |
|
|
|
|
209
|
|
|
return x if attr is None else cls.look(x, attr) |
210
|
|
|
except StopIteration: |
211
|
|
|
return None |
212
|
|
|
|
213
|
|
|
@classmethod |
214
|
|
|
def iter_rowcol(cls, n_rows: int, n_cols: int) -> Generator[Tuple[int, int], None, None]: |
215
|
|
|
""" |
216
|
|
|
An iterator over (row column) pairs for a row-first grid traversal. |
217
|
|
|
|
218
|
|
|
Example: |
219
|
|
|
.. code-block:: |
220
|
|
|
it = CommonTools.iter_rowcol(5, 3) |
221
|
|
|
[next(it) for _ in range(5)] # [(0,0),(0,1),(0,2),(1,0),(1,1)] |
222
|
|
|
""" |
223
|
|
|
for i in range(n_rows * n_cols): |
224
|
|
|
yield i // n_cols, i % n_cols |
225
|
|
|
|
226
|
|
|
@classmethod |
227
|
|
|
def multidict( |
228
|
|
|
cls, |
|
|
|
|
229
|
|
|
sequence: Iterable[Z], |
|
|
|
|
230
|
|
|
key_attr: Union[str, Iterable[str], Callable[[Y], Z]], |
|
|
|
|
231
|
|
|
skip_none: bool = False, |
|
|
|
|
232
|
|
|
) -> Mapping[Y, Sequence[Z]]: |
233
|
|
|
""" |
234
|
|
|
Builds a mapping from keys to multiple values. |
235
|
|
|
Builds a mapping of some attribute in ``sequence`` to |
236
|
|
|
the containing elements of ``sequence``. |
237
|
|
|
|
238
|
|
|
Args: |
239
|
|
|
sequence: Any iterable |
240
|
|
|
key_attr: Usually string like 'attr1.attr2'; see `look` |
241
|
|
|
skip_none: If None, raises a `KeyError` if the key is missing for any item; otherwise, skips it |
|
|
|
|
242
|
|
|
""" |
243
|
|
|
dct = defaultdict(lambda: []) |
244
|
|
|
for item in sequence: |
245
|
|
|
v = CommonTools.look(item, key_attr) |
|
|
|
|
246
|
|
|
if not skip_none and v is None: |
247
|
|
|
raise XKeyError(f"No {key_attr} in {item}", key=key_attr) |
248
|
|
|
if v is not None: |
249
|
|
|
dct[v].append(item) |
250
|
|
|
return dct |
251
|
|
|
|
252
|
|
|
@classmethod |
253
|
|
|
def mem_size(cls, obj) -> str: |
254
|
|
|
""" |
255
|
|
|
Returns the size of the object in memory as a human-readable string. |
256
|
|
|
|
257
|
|
|
Args: |
258
|
|
|
obj: Any Python object |
259
|
|
|
|
260
|
|
|
Returns: |
261
|
|
|
A human-readable size with units |
262
|
|
|
""" |
263
|
|
|
return nicesize(sys.getsizeof(obj)) |
264
|
|
|
|
265
|
|
|
@classmethod |
266
|
|
|
def devnull(cls): |
267
|
|
|
""" |
268
|
|
|
Yields a 'writer' that does nothing. |
269
|
|
|
|
270
|
|
|
Example: |
271
|
|
|
>>> with CommonTools.devnull() as devnull: |
272
|
|
|
>>> devnull.write('hello') |
273
|
|
|
""" |
274
|
|
|
yield DevNull() |
275
|
|
|
|
276
|
|
|
@classmethod |
277
|
|
|
def parse_bool(cls, s: str) -> bool: |
|
|
|
|
278
|
|
|
""" |
279
|
|
|
Parses a 'true'/'false' string to a bool, ignoring case. |
280
|
|
|
|
281
|
|
|
Raises: |
282
|
|
|
ValueError: If neither true nor false |
283
|
|
|
""" |
284
|
|
|
if isinstance(s, bool): |
285
|
|
|
return s |
286
|
|
|
if s.lower() == "false": |
287
|
|
|
return False |
288
|
|
|
if s.lower() == "true": |
289
|
|
|
return True |
290
|
|
|
raise XValueError(f"{s} is not true/false", value=s) |
291
|
|
|
|
292
|
|
|
@classmethod |
293
|
|
|
def parse_bool_flex(cls, s: str) -> bool: |
|
|
|
|
294
|
|
|
""" |
295
|
|
|
Parses a 'true'/'false'/'yes'/'no'/... string to a bool, ignoring case. |
296
|
|
|
|
297
|
|
|
Allowed: |
298
|
|
|
- "true", "t", "yes", "y", "1", "+" |
299
|
|
|
- "false", "f", "no", "n", "0", "-" |
300
|
|
|
|
301
|
|
|
Raises: |
302
|
|
|
XValueError: If neither true nor false |
303
|
|
|
""" |
304
|
|
|
mp = { |
|
|
|
|
305
|
|
|
**{v: True for v in {"true", "t", "yes", "y", "1", "+"}}, |
306
|
|
|
**{v: False for v in {"false", "f", "no", "n", "0", "-"}}, |
307
|
|
|
} |
308
|
|
|
v = mp.get(s.lower()) |
|
|
|
|
309
|
|
|
if v is None: |
310
|
|
|
raise XValueError(f"{s.lower()} is not in {','.join(mp.keys())}", value=s) |
311
|
|
|
return v |
312
|
|
|
|
313
|
|
|
|
314
|
|
|
__all__ = ["CommonTools"] |
315
|
|
|
|