Completed
Push — master ( a2eb28...52e99c )
by Satoru
03:28
created

groupby()   A

Complexity

Conditions 1

Size

Total Lines 14

Duplication

Lines 0
Ratio 0 %

Importance

Changes 1
Bugs 0 Features 0
Metric Value
cc 1
c 1
b 0
f 0
dl 0
loc 14
rs 9.4285
1
#
2
# Copyright (C) 2012 - 2018 Satoru SATOH <ssato @ redhat.com>
3
# License: MIT
4
#
5
"""Misc utility routines for anyconfig module.
6
"""
7
from __future__ import absolute_import
8
9
import collections
10
import glob
11
import itertools
12
import os.path
13
import types
14
15
import anyconfig.compat
16
import anyconfig.globals
17
18
from anyconfig.compat import pathlib
19
20
21
def groupby(itr, key_fn=None):
22
    """
23
    An wrapper function around itertools.groupby to sort each results.
24
25
    :param itr: Iterable object, a list/tuple/genrator, etc.
26
    :param key_fn: Key function to sort `itr`.
27
28
    >>> import operator
29
    >>> itr = [("a", 1), ("b", -1), ("c", 1)]
30
    >>> res = groupby(itr, operator.itemgetter(1))
31
    >>> [(key, tuple(grp)) for key, grp in res]
32
    [(-1, (('b', -1),)), (1, (('a', 1), ('c', 1)))]
33
    """
34
    return itertools.groupby(sorted(itr, key=key_fn), key=key_fn)
35
36
37
def get_file_extension(file_path):
38
    """
39
    >>> get_file_extension("/a/b/c")
40
    ''
41
    >>> get_file_extension("/a/b.txt")
42
    'txt'
43
    >>> get_file_extension("/a/b/c.tar.xz")
44
    'xz'
45
    """
46
    _ext = os.path.splitext(file_path)[-1]
47
    if _ext:
48
        return _ext[1:] if _ext.startswith('.') else _ext
49
50
    return ""
51
52
53
def sglob(files_pattern):
54
    """
55
    glob.glob alternative of which results sorted always.
56
    """
57
    return sorted(glob.glob(files_pattern))
58
59
60
def is_iterable(obj):
61
    """
62
    >>> is_iterable([])
63
    True
64
    >>> is_iterable(())
65
    True
66
    >>> is_iterable([x for x in range(10)])
67
    True
68
    >>> is_iterable((1, 2, 3))
69
    True
70
    >>> g = (x for x in range(10))
71
    >>> is_iterable(g)
72
    True
73
    >>> is_iterable("abc")
74
    False
75
    >>> is_iterable(0)
76
    False
77
    >>> is_iterable({})
78
    False
79
    """
80
    return isinstance(obj, (list, tuple, types.GeneratorType)) or \
81
        (not isinstance(obj, (int, str, dict)) and
82
         bool(getattr(obj, "next", False)))
83
84
85
def concat(xss):
86
    """
87
    Concatenates a list of lists.
88
89
    >>> concat([[]])
90
    []
91
    >>> concat((()))
92
    []
93
    >>> concat([[1,2,3],[4,5]])
94
    [1, 2, 3, 4, 5]
95
    >>> concat([[1,2,3],[4,5,[6,7]]])
96
    [1, 2, 3, 4, 5, [6, 7]]
97
    >>> concat(((1,2,3),(4,5,[6,7])))
98
    [1, 2, 3, 4, 5, [6, 7]]
99
    >>> concat(((1,2,3),(4,5,[6,7])))
100
    [1, 2, 3, 4, 5, [6, 7]]
101
    >>> concat((i, i*2) for i in range(3))
102
    [0, 0, 1, 2, 2, 4]
103
    """
104
    return list(anyconfig.compat.from_iterable(xs for xs in xss))
105
106
107
def normpath(path):
108
    """Normalize path.
109
110
    - eliminating double slashes, etc. (os.path.normpath)
111
    - ensure paths contain ~[user]/ expanded.
112
113
    :param path: Path string :: str
114
    """
115
    return os.path.normpath(os.path.expanduser(path) if '~' in path else path)
116
117
118
def is_path(obj):
119
    """
120
    Is given object `obj` a file path?
121
122
    :param obj: file path or something
123
    :return: True if `obj` is a file path
124
    """
125
    return isinstance(obj, anyconfig.compat.STR_TYPES)
126
127
128
def is_path_obj(obj):
129
    """Is given object `input` a pathlib.Path object?
130
131
    :param obj: a pathlib.Path object or something
132
    :return: True if `obj` is a pathlib.Path object
133
134
    >>> from anyconfig.compat import pathlib
135
    >>> if pathlib is not None:
136
    ...      obj = pathlib.Path(__file__)
137
    ...      assert is_path_obj(obj)
138
    >>>
139
    >>> assert not is_path_obj(__file__)
140
    """
141
    return pathlib is not None and isinstance(obj, pathlib.Path)
142
143
144
def is_file_stream(obj):
145
    """Is given object `input` a file stream (file/file-like object)?
146
147
    :param obj: a file / file-like (stream) object or something
148
    :return: True if `obj` is a file stream
149
150
    >>> assert is_file_stream(open(__file__))
151
    >>> assert not is_file_stream(__file__)
152
    """
153
    return getattr(obj, "read", False)
154
155
156
def is_ioinfo(obj, keys=None):
157
    """
158
    :return: True if given `obj` is a 'IOInfo' namedtuple object.
159
160
    >>> assert not is_ioinfo(1)
161
    >>> assert not is_ioinfo("aaa")
162
    >>> assert not is_ioinfo({})
163
    >>> assert not is_ioinfo(('a', 1, {}))
164
165
    >>> inp = anyconfig.globals.IOInfo("/etc/hosts", "path", "/etc/hosts",
166
    ...                                None, open)
167
    >>> assert is_ioinfo(inp)
168
    """
169
    if keys is None:
170
        keys = anyconfig.globals.IOI_KEYS
171
172
    if isinstance(obj, tuple) and getattr(obj, "_asdict", False):
173
        return all(k in obj._asdict() for k in keys)
174
175
    return False
176
177
178
def is_stream_ioinfo(obj):
179
    """
180
    :param obj: IOInfo object or something
181
    :return: True if given IOInfo object `obj` is of file / file-like object
182
183
    >>> ioi = anyconfig.globals.IOInfo(None, anyconfig.globals.IOI_STREAM,
184
    ...                                None, None, None)
185
    >>> assert is_stream_ioinfo(ioi)
186
    >>> assert not is_stream_ioinfo(__file__)
187
    """
188
    return getattr(obj, "type", None) == anyconfig.globals.IOI_STREAM
189
190
191
def is_path_like_object(obj, marker='*'):
192
    """
193
    Is given object `obj` a path string, a pathlib.Path, a file / file-like
194
    (stream) or IOInfo namedtuple object?
195
196
    :param obj:
197
        a path string, pathlib.Path object, a file / file-like or 'IOInfo'
198
        object
199
200
    :return:
201
        True if `obj` is a path string or a pathlib.Path object or a file
202
        (stream) object
203
204
    >>> assert is_path_like_object(__file__)
205
    >>> assert not is_path_like_object("/a/b/c/*.json", '*')
206
207
    >>> from anyconfig.compat import pathlib
208
    >>> if pathlib is not None:
209
    ...      assert is_path_like_object(pathlib.Path("a.ini"))
210
    ...      assert not is_path_like_object(pathlib.Path("x.ini"), 'x')
211
212
    >>> assert is_path_like_object(open(__file__))
213
    """
214
    return ((is_path(obj) and marker not in obj) or
215
            (is_path_obj(obj) and marker not in obj.as_posix()) or
216
            is_file_stream(obj) or is_ioinfo(obj))
217
218
219
def is_paths(maybe_paths, marker='*'):
220
    """
221
    Does given object `maybe_paths` consist of path or path pattern strings?
222
    """
223
    return ((is_path(maybe_paths) and marker in maybe_paths) or  # Path str
224
            (is_path_obj(maybe_paths) and marker in maybe_paths.as_posix()) or
225
            (is_iterable(maybe_paths) and
226
             all(is_path(p) or is_ioinfo(p) for p in maybe_paths)))
227
228
229
def get_path_from_stream(strm):
230
    """
231
    Try to get file path from given file or file-like object `strm`.
232
233
    :param strm: A file or file-like object
234
    :return: Path of given file or file-like object or None
235
    :raises: ValueError
236
237
    >>> assert __file__ == get_path_from_stream(open(__file__, 'r'))
238
    >>> assert get_path_from_stream(anyconfig.compat.StringIO()) is None
239
    >>> get_path_from_stream(__file__)  # doctest: +ELLIPSIS
240
    Traceback (most recent call last):
241
        ...
242
    ValueError: ...
243
    """
244
    if not is_file_stream(strm):
245
        raise ValueError("Given object does not look a file/file-like "
246
                         "object: %r" % strm)
247
248
    path = getattr(strm, "name", None)
249
    if path is not None:
250
        return normpath(path)
251
252
    return None
253
254
255
def _try_to_get_extension(obj):
256
    """
257
    Try to get file extension from given path or file object.
258
259
    :param obj: a file, file-like object or something
260
    :return: File extension or None
261
262
    >>> _try_to_get_extension("a.py")
263
    'py'
264
    """
265
    if is_path(obj):
266
        path = obj
267
268
    elif is_path_obj(obj):
269
        return obj.suffix[1:]
270
271
    elif is_file_stream(obj):
272
        try:
273
            path = get_path_from_stream(obj)
274
        except ValueError:
275
            return None
276
277
    elif is_ioinfo(obj):
278
        path = obj.path
279
280
    else:
281
        return None
282
283
    if path:
284
        return get_file_extension(path)
285
286
    return None
287
288
289
def are_same_file_types(objs):
290
    """
291
    Are given (maybe) file objs same type (extension) ?
292
293
    :param objs: A list of file path or file(-like) objects
294
295
    >>> are_same_file_types([])
296
    False
297
    >>> are_same_file_types(["a.conf"])
298
    True
299
    >>> are_same_file_types(["a.conf", "b.conf"])
300
    True
301
    >>> are_same_file_types(["a.yml", "b.yml"])
302
    True
303
    >>> are_same_file_types(["a.yml", "b.json"])
304
    False
305
    >>> strm = anyconfig.compat.StringIO()
306
    >>> are_same_file_types(["a.yml", "b.yml", strm])
307
    False
308
    """
309
    if not objs:
310
        return False
311
312
    ext = _try_to_get_extension(objs[0])
313
    if ext is None:
314
        return False
315
316
    return all(_try_to_get_extension(p) == ext for p in objs[1:])
317
318
319
def _expand_paths_itr(paths, marker='*'):
320
    """Iterator version of :func:`expand_paths`.
321
    """
322
    for path in paths:
323
        if is_path(path):
324
            if marker in path:  # glob path pattern
325
                for ppath in sglob(path):
326
                    yield ppath
327
            else:
328
                yield path  # a simple file path
329
        elif is_path_obj(path):
330
            if marker in path.as_posix():
331
                for ppath in sglob(path.as_posix()):
332
                    yield normpath(ppath)
333
            else:
334
                yield normpath(path.as_posix())
335
        elif is_ioinfo(path):
336
            yield path.path
337
        else:  # A file or file-like object
338
            yield path
339
340
341
def expand_paths(paths, marker='*'):
342
    """
343
    :param paths:
344
        A glob path pattern string or pathlib.Path object holding such path, or
345
        a list consists of path strings or glob path pattern strings or
346
        pathlib.Path object holding such ones, or file objects
347
    :param marker: Glob marker character or string, e.g. '*'
348
349
    :return: List of path strings
350
351
    >>> expand_paths([])
352
    []
353
    >>> expand_paths("/usr/lib/a/b.conf /etc/a/b.conf /run/a/b.conf".split())
354
    ['/usr/lib/a/b.conf', '/etc/a/b.conf', '/run/a/b.conf']
355
    >>> paths_s = os.path.join(os.path.dirname(__file__), "u*.py")
356
    >>> ref = sglob(paths_s)
357
    >>> assert expand_paths(paths_s) == ref
358
    >>> ref = ["/etc/a.conf"] + ref
359
    >>> assert expand_paths(["/etc/a.conf", paths_s]) == ref
360
    >>> strm = anyconfig.compat.StringIO()
361
    >>> assert expand_paths(["/etc/a.conf", strm]) == ["/etc/a.conf", strm]
362
    """
363
    if is_path(paths) and marker in paths:
364
        return sglob(paths)
365
366
    if is_path_obj(paths) and marker in paths.as_posix():
367
        # TBD: Is it better to return [p :: pathlib.Path] instead?
368
        return [normpath(p) for p in sglob(paths.as_posix())]
369
370
    return list(_expand_paths_itr(paths, marker=marker))
371
372
373
# pylint: disable=unused-argument
374
def noop(val, *args, **kwargs):
375
    """A function does nothing.
376
377
    >>> noop(1)
378
    1
379
    """
380
    # It means nothing but can suppress 'Unused argument' pylint warns.
381
    # (val, args, kwargs)[0]
382
    return val
383
384
385
_LIST_LIKE_TYPES = (collections.Iterable, collections.Sequence)
386
387
388
def is_dict_like(obj):
389
    """
390
    :param obj: Any object behaves like a dict.
391
392
    >>> is_dict_like("a string")
393
    False
394
    >>> is_dict_like({})
395
    True
396
    >>> is_dict_like(anyconfig.compat.OrderedDict((('a', 1), ('b', 2))))
397
    True
398
    """
399
    return isinstance(obj, (dict, collections.Mapping))  # any others?
400
401
402
def is_namedtuple(obj):
403
    """
404
    >>> p0 = collections.namedtuple("Point", "x y")(1, 2)
405
    >>> is_namedtuple(p0)
406
    True
407
    >>> is_namedtuple(tuple(p0))
408
    False
409
    """
410
    return isinstance(obj, tuple) and hasattr(obj, "_asdict")
411
412
413
def is_list_like(obj):
414
    """
415
    >>> is_list_like([])
416
    True
417
    >>> is_list_like(())
418
    True
419
    >>> is_list_like([x for x in range(10)])
420
    True
421
    >>> is_list_like((1, 2, 3))
422
    True
423
    >>> g = (x for x in range(10))
424
    >>> is_list_like(g)
425
    True
426
    >>> is_list_like("abc")
427
    False
428
    >>> is_list_like(0)
429
    False
430
    >>> is_list_like({})
431
    False
432
    """
433
    return isinstance(obj, _LIST_LIKE_TYPES) and \
434
        not (isinstance(obj, anyconfig.compat.STR_TYPES) or is_dict_like(obj))
435
436
437
def filter_options(keys, options):
438
    """
439
    Filter `options` with given `keys`.
440
441
    :param keys: key names of optional keyword arguments
442
    :param options: optional keyword arguments to filter with `keys`
443
444
    >>> filter_options(("aaa", ), dict(aaa=1, bbb=2))
445
    {'aaa': 1}
446
    >>> filter_options(("aaa", ), dict(bbb=2))
447
    {}
448
    """
449
    return dict((k, options[k]) for k in keys if k in options)
450
451
# vim:sw=4:ts=4:et:
452