1
|
|
|
# -*- coding: utf-8 -*- |
2
|
|
|
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- |
3
|
|
|
# vi: set ft=python sts=4 ts=4 sw=4 et: |
4
|
|
|
""" |
5
|
|
|
Crumb class: the smart path model class. |
6
|
|
|
""" |
7
|
|
|
import re |
8
|
|
|
import os.path as op |
9
|
|
|
from copy import deepcopy |
10
|
|
|
from collections import OrderedDict, Mapping, Sequence |
11
|
|
|
from functools import partial |
12
|
|
|
from six import string_types |
13
|
|
|
try: |
14
|
|
|
from pathlib2 import Path |
15
|
|
|
except: |
16
|
|
|
from pathlib import Path |
17
|
|
|
|
18
|
|
|
|
19
|
|
|
from .utils import (list_subpaths, |
20
|
|
|
fnmatch_filter, |
21
|
|
|
regex_match_filter, |
22
|
|
|
) |
23
|
|
|
|
24
|
|
|
#from hansel._utils import deprecated |
25
|
|
|
from ._utils import (_get_path, |
26
|
|
|
_is_crumb_arg, |
27
|
|
|
_replace, |
28
|
|
|
_split_exists, |
29
|
|
|
_split, |
30
|
|
|
_touch, |
31
|
|
|
_arg_params, |
32
|
|
|
_dict_popitems, |
33
|
|
|
has_crumbs, |
34
|
|
|
is_valid, |
35
|
|
|
_arg_format, |
36
|
|
|
) |
37
|
|
|
|
38
|
|
|
|
39
|
|
|
class Crumb(object): |
40
|
|
|
""" The crumb path model class. |
41
|
|
|
Parameters |
42
|
|
|
---------- |
43
|
|
|
crumb_path: str |
44
|
|
|
A file or folder path with crumb arguments. See Examples. |
45
|
|
|
|
46
|
|
|
ignore_list: sequence of str |
47
|
|
|
A list of `fnmatch` patterns of filenames to be ignored. |
48
|
|
|
|
49
|
|
|
regex: str |
50
|
|
|
Choices: 'fnmatch', 're' or 're.ignorecase' |
51
|
|
|
If 'fnmatch' will use fnmatch regular expressions to |
52
|
|
|
match any expression you may have in a crumb argument. |
53
|
|
|
If 're' will use re.match. |
54
|
|
|
If 're.ignorecase' will use re.match and pass re.IGNORE_CASE to re.compile. |
55
|
|
|
|
56
|
|
|
Examples |
57
|
|
|
-------- |
58
|
|
|
>>> crumb = Crumb("{base_dir}/raw/{subject_id}/{session_id}/{modality}/{image}") |
59
|
|
|
>>> cr = Crumb(op.join(op.expanduser('~'), '{user_folder}')) |
60
|
|
|
""" |
61
|
|
|
# symbols indicating start and end of a crumb argument |
62
|
|
|
_start_end_syms = ('{', '}') |
63
|
|
|
_regex_sym = ':' |
64
|
|
|
|
65
|
|
|
# specify partial functions from _utils with _arg_start_sym and _arg_end_sym |
66
|
|
|
# everything would be much simpler if I hardcoded these symbols but I still |
67
|
|
|
# feel that this flexibility is nice to have. |
68
|
|
|
_is_crumb_arg = partial(_is_crumb_arg, start_end_syms=_start_end_syms) |
69
|
|
|
_arg_params = partial(_arg_params, start_end_syms=_start_end_syms, reg_sym=_regex_sym) |
70
|
|
|
_arg_format = partial(_arg_format, start_end_syms=_start_end_syms, reg_sym=_regex_sym) |
71
|
|
|
is_valid = partial(is_valid, start_end_syms=_start_end_syms) |
72
|
|
|
has_crumbs = partial(has_crumbs, start_end_syms=_start_end_syms) |
73
|
|
|
_split = partial(_split, start_end_syms=_start_end_syms) |
74
|
|
|
_touch = partial(_touch, start_end_syms=_start_end_syms) |
75
|
|
|
_split_exists = partial(_split_exists, start_end_syms=_start_end_syms) |
76
|
|
|
|
77
|
|
|
def __init__(self, crumb_path, ignore_list=None, regex='fnmatch'): |
78
|
|
|
self._path = _get_path(crumb_path) |
79
|
|
|
self._argidx = OrderedDict() # in which order the crumb argument appears |
80
|
|
|
self._argval = {} # what is the value of the argument in the current path, if any has been set. |
81
|
|
|
self.patterns = {} # what is the pattern set for the argument, if any. This is left public for the user. |
82
|
|
|
self._re_method = regex |
83
|
|
|
self._re_args = None |
84
|
|
|
|
85
|
|
|
if ignore_list is None: |
86
|
|
|
ignore_list = [] |
87
|
|
|
|
88
|
|
|
self._ignore = ignore_list |
89
|
|
|
self._update() |
90
|
|
|
|
91
|
|
|
@property |
92
|
|
|
def arg_values(self): |
93
|
|
|
""" Return a dict with the arg_names and values of the already replaced crumb arguments.""" |
94
|
|
|
return self._argval |
95
|
|
|
|
96
|
|
|
@property |
97
|
|
|
def path(self): |
98
|
|
|
"""Return the current crumb path string.""" |
99
|
|
|
return self._path |
100
|
|
|
|
101
|
|
|
@path.setter |
102
|
|
|
def path(self, value): |
103
|
|
|
""" Set the current crumb path string and updates the internal members. |
104
|
|
|
Parameters |
105
|
|
|
---------- |
106
|
|
|
value: str |
107
|
|
|
A file or folder path with crumb arguments. See Examples in class docstring. |
108
|
|
|
""" |
109
|
|
|
self._path = value |
110
|
|
|
self._update() |
111
|
|
|
|
112
|
|
|
def _open_arg_items(self): |
113
|
|
|
""" Return an iterator to the crumb _argidx items in `self` that have not been replaced yet. |
114
|
|
|
In the same order as they appear in the crumb path. |
115
|
|
|
|
116
|
|
|
Returns |
117
|
|
|
------- |
118
|
|
|
crumb_args: set of str |
119
|
|
|
|
120
|
|
|
Note |
121
|
|
|
---- |
122
|
|
|
I know that there is shorter/faster ways to program this but I wanted to maintain the |
123
|
|
|
order of the arguments in argidx in the result of this function. |
124
|
|
|
""" |
125
|
|
|
for arg_name, idx in self._argidx.items(): |
126
|
|
|
if arg_name not in self._argval: |
127
|
|
|
yield arg_name, idx |
128
|
|
|
|
129
|
|
|
def has_set(self, arg_name): |
130
|
|
|
""" Return True if the argument `arg_name` has been set to a specific value, |
131
|
|
|
False if it is still a crumb argument.""" |
132
|
|
|
return arg_name not in self.open_args() |
133
|
|
|
|
134
|
|
|
def open_args(self): |
135
|
|
|
""" Return an iterator to the crumb argument names in `self` that have not been replaced yet. |
136
|
|
|
In the same order as they appear in the crumb path. |
137
|
|
|
|
138
|
|
|
Returns |
139
|
|
|
------- |
140
|
|
|
crumb_args: set of str |
141
|
|
|
|
142
|
|
|
Note |
143
|
|
|
---- |
144
|
|
|
I know that there is shorter/faster ways to program this but I wanted to maintain the |
145
|
|
|
order of the arguments in argidx in the result of this function. |
146
|
|
|
""" |
147
|
|
|
for arg_name, _ in self._open_arg_items(): |
148
|
|
|
yield arg_name |
149
|
|
|
|
150
|
|
|
def all_args(self): |
151
|
|
|
""" Return an iterator to all the crumb argument names in `self`, first the open ones and then the |
152
|
|
|
replaced ones. |
153
|
|
|
|
154
|
|
|
Returns |
155
|
|
|
------- |
156
|
|
|
crumb_args: set of str |
157
|
|
|
""" |
158
|
|
|
for arg_name in self._argidx.keys(): |
159
|
|
|
yield arg_name |
160
|
|
|
|
161
|
|
|
def _check(self): |
162
|
|
|
""" Raise ValueError if the path of the Crumb has errors using `self.is_valid`.""" |
163
|
|
|
if not self.is_valid(self._path): |
164
|
|
|
raise ValueError("The current crumb path has errors, got {}.".format(self.path)) |
165
|
|
|
|
166
|
|
|
def _update(self): |
167
|
|
|
""" Clean up, parse the current crumb path and fill the internal |
168
|
|
|
members for functioning.""" |
169
|
|
|
self._clean() |
170
|
|
|
self._check() |
171
|
|
|
self._set_argdicts() |
172
|
|
|
self._set_match_function() |
173
|
|
|
self._set_replace_function() |
174
|
|
|
|
175
|
|
|
def _set_replace_function(self): |
176
|
|
|
""" Set self._replace function as a partial function, adding regex=self.patterns.""" |
177
|
|
|
self._replace = partial(_replace, |
178
|
|
|
start_end_syms=self._start_end_syms, |
179
|
|
|
regexes=self.patterns) |
180
|
|
|
|
181
|
|
|
def _set_match_function(self): |
182
|
|
|
""" Update self._match_filter with a regular expression |
183
|
|
|
matching function depending on the value of self._re_method.""" |
184
|
|
|
if self._re_method == 'fnmatch': |
185
|
|
|
self._match_filter = fnmatch_filter |
186
|
|
|
elif self._re_method == 're': |
187
|
|
|
self._match_filter = regex_match_filter |
188
|
|
|
elif self._re_method == 're.ignorecase': |
189
|
|
|
self._match_filter = regex_match_filter |
190
|
|
|
self._re_args = (re.IGNORECASE, ) |
191
|
|
|
else: |
192
|
|
|
raise ValueError('Expected regex method value to be "fnmatch", "re" or "re.ignorecase"' |
193
|
|
|
', got {}.'.format(self._re_method)) |
194
|
|
|
|
195
|
|
|
def _clean(self): |
196
|
|
|
""" Clean up the private utility members, i.e., _argidx. """ |
197
|
|
|
self._argidx = OrderedDict() |
198
|
|
|
|
199
|
|
|
@classmethod |
200
|
|
|
def copy(cls, crumb): |
201
|
|
|
""" Return a deep copy of the given `crumb`. |
202
|
|
|
Parameters |
203
|
|
|
---------- |
204
|
|
|
crumb: str or Crumb |
205
|
|
|
|
206
|
|
|
Returns |
207
|
|
|
------- |
208
|
|
|
copy: Crumb |
209
|
|
|
""" |
210
|
|
|
if isinstance(crumb, cls): |
211
|
|
|
#nucr = deepcopy(crumb) |
212
|
|
|
nucr = cls(crumb._path, ignore_list=crumb._ignore, regex=crumb._re_method) |
213
|
|
|
nucr._argidx = deepcopy(crumb._argidx) |
214
|
|
|
nucr._argval = deepcopy(crumb._argval) |
215
|
|
|
return nucr |
216
|
|
|
elif isinstance(crumb, string_types): |
217
|
|
|
return cls.from_path(crumb) |
218
|
|
|
else: |
219
|
|
|
raise TypeError("Expected a Crumb or a str to copy, got {}.".format(type(crumb))) |
220
|
|
|
|
221
|
|
|
def _set_argdicts(self): |
222
|
|
|
""" Initialize the self._argidx dict. It holds arg_name -> index. |
223
|
|
|
The index is the position in the whole `_path.split(op.sep)` where each argument is. |
224
|
|
|
""" |
225
|
|
|
fs = self._path_split() |
226
|
|
|
for idx, f in enumerate(fs): |
227
|
|
|
if self._is_crumb_arg(f): |
228
|
|
|
arg_name, arg_regex = self._arg_params(f) |
229
|
|
|
self._argidx[arg_name] = idx |
230
|
|
|
|
231
|
|
|
if arg_regex is not None: |
232
|
|
|
self.patterns[arg_name] = arg_regex |
233
|
|
|
|
234
|
|
|
def _find_arg(self, arg_name): |
235
|
|
|
""" Return the index in the current path of the crumb |
236
|
|
|
argument with name `arg_name`. |
237
|
|
|
""" |
238
|
|
|
return self._argidx.get(arg_name, -1) |
239
|
|
|
|
240
|
|
|
def isabs(self): |
241
|
|
|
""" Return True if the current crumb path has an absolute path, False otherwise. |
242
|
|
|
This means that its path is valid and starts with a `op.sep` character |
243
|
|
|
or hard disk letter. |
244
|
|
|
""" |
245
|
|
|
if not self.is_valid(self._path): |
246
|
|
|
raise ValueError("The given crumb path has errors, got {}.".format(self.path)) |
247
|
|
|
|
248
|
|
|
start_sym, _ = self._start_end_syms |
249
|
|
|
subp = self._path.split(start_sym)[0] |
250
|
|
|
return op.isabs(subp) |
251
|
|
|
|
252
|
|
|
def abspath(self, first_is_basedir=False): |
253
|
|
|
""" Return a copy of `self` with an absolute crumb path. |
254
|
|
|
Add as prefix the absolute path to the current directory if the current |
255
|
|
|
crumb is not absolute. |
256
|
|
|
Parameters |
257
|
|
|
---------- |
258
|
|
|
first_is_basedir: bool |
259
|
|
|
If True and the current crumb path starts with a crumb argument and first_is_basedir, |
260
|
|
|
the first argument will be replaced by the absolute path to the current dir, |
261
|
|
|
otherwise the absolute path to the current dir will be added as a prefix. |
262
|
|
|
|
263
|
|
|
Returns |
264
|
|
|
------- |
265
|
|
|
abs_crumb: Crumb |
266
|
|
|
""" |
267
|
|
|
if not self.is_valid(self._path): |
268
|
|
|
raise ValueError("The given crumb path has errors, got {}.".format(self.path)) |
269
|
|
|
|
270
|
|
|
if self.isabs(): |
271
|
|
|
return deepcopy(self) |
272
|
|
|
|
273
|
|
|
nucr = self.copy(self) |
274
|
|
|
nucr._path = self._abspath(first_is_basedir=first_is_basedir) |
275
|
|
|
return nucr |
276
|
|
|
|
277
|
|
|
def _path_split(self): |
278
|
|
|
return self._path.split(op.sep) |
279
|
|
|
|
280
|
|
|
def _abspath(self, first_is_basedir=False): |
281
|
|
|
""" Return the absolute path of the current crumb path. |
282
|
|
|
Parameters |
283
|
|
|
---------- |
284
|
|
|
first_is_basedir: bool |
285
|
|
|
If True and the current crumb path starts with a crumb argument and first_is_basedir, |
286
|
|
|
the first argument will be replaced by the absolute path to the current dir, |
287
|
|
|
otherwise the absolute path to the current dir will be added as a prefix. |
288
|
|
|
|
289
|
|
|
Returns |
290
|
|
|
------- |
291
|
|
|
abspath: str |
292
|
|
|
""" |
293
|
|
|
if not self.has_crumbs(self._path): |
294
|
|
|
return op.abspath(self._path) |
295
|
|
|
|
296
|
|
|
splt = self._path_split() |
297
|
|
|
path = [] |
298
|
|
|
if self._is_crumb_arg(splt[0]): |
299
|
|
|
path.append(op.abspath(op.curdir)) |
300
|
|
|
|
301
|
|
|
if not first_is_basedir: |
302
|
|
|
path.append(splt[0]) |
303
|
|
|
|
304
|
|
|
if splt[1:]: |
305
|
|
|
path.extend(splt[1:]) |
306
|
|
|
|
307
|
|
|
return op.sep.join(path) |
308
|
|
|
|
309
|
|
|
def split(self): |
310
|
|
|
""" Return a list of sub-strings of the current crumb path where the |
311
|
|
|
path parts are separated from the crumb arguments. |
312
|
|
|
|
313
|
|
|
Returns |
314
|
|
|
------- |
315
|
|
|
crumbs: list of str |
316
|
|
|
""" |
317
|
|
|
return self._split(self._path) |
318
|
|
|
|
319
|
|
|
@classmethod |
320
|
|
|
def from_path(cls, crumb_path): |
321
|
|
|
""" Create an instance of Crumb out of `crumb_path`. |
322
|
|
|
Parameters |
323
|
|
|
---------- |
324
|
|
|
val: str or Crumb or pathlib.Path |
325
|
|
|
|
326
|
|
|
Returns |
327
|
|
|
------- |
328
|
|
|
path: Crumb |
329
|
|
|
""" |
330
|
|
|
if isinstance(crumb_path, (cls, Path)): |
331
|
|
|
return cls.copy(crumb_path) |
332
|
|
|
|
333
|
|
|
if isinstance(crumb_path, string_types): |
334
|
|
|
return cls(crumb_path) |
335
|
|
|
else: |
336
|
|
|
raise TypeError("Expected a `val` to be a `str`, got {}.".format(type(crumb_path))) |
337
|
|
|
|
338
|
|
|
def _last_open_arg(self): |
339
|
|
|
""" Return the name and idx of the last open argument.""" |
340
|
|
|
for arg, idx in reversed(list(self._open_arg_items())): |
341
|
|
|
return arg, idx |
342
|
|
|
|
343
|
|
|
def _first_open_arg(self): |
344
|
|
|
""" Return the name and idx of the first open argument.""" |
345
|
|
|
for arg, idx in self._open_arg_items(): |
346
|
|
|
return arg, idx |
347
|
|
|
|
348
|
|
|
def _is_first_open_arg(self, arg_name): |
349
|
|
|
""" Return True if `arg_name` is the first open argument.""" |
350
|
|
|
# Take into account that self._argidx is OrderedDict |
351
|
|
|
return arg_name == self._first_open_arg()[0] |
352
|
|
|
|
353
|
|
|
def _arg_values(self, arg_name, arg_values=None): |
354
|
|
|
""" Return the existing values in the file system for the crumb argument |
355
|
|
|
with name `arg_name`. |
356
|
|
|
The `arg_values` must be a sequence with the tuples with valid values of the dependent |
357
|
|
|
(previous in the path) crumb arguments. |
358
|
|
|
The format of `arg_values` work in such a way that `self._path.format(dict(arg_values[0]))` |
359
|
|
|
would give me a valid path or crumb. |
360
|
|
|
Parameters |
361
|
|
|
---------- |
362
|
|
|
arg_name: str |
363
|
|
|
|
364
|
|
|
arg_values: list of tuples |
365
|
|
|
|
366
|
|
|
Returns |
367
|
|
|
------- |
368
|
|
|
vals: list of tuples |
369
|
|
|
|
370
|
|
|
Raises |
371
|
|
|
------ |
372
|
|
|
ValueError: if `arg_values` is None and `arg_name` is not the |
373
|
|
|
first crumb argument in self._path |
374
|
|
|
|
375
|
|
|
IOError: if this crosses to any path that is non-existing. |
376
|
|
|
""" |
377
|
|
|
if arg_values is None and not self._is_first_open_arg(arg_name): |
378
|
|
|
raise ValueError("Cannot get the list of values for {} if" |
379
|
|
|
" the previous arguments are not filled" |
380
|
|
|
" in `paths`.".format(arg_name)) |
381
|
|
|
|
382
|
|
|
aidx = self._find_arg(arg_name) |
383
|
|
|
|
384
|
|
|
# check if the path is absolute, do it absolute |
385
|
|
|
apath = self._abspath() |
386
|
|
|
splt = apath.split(op.sep) |
387
|
|
|
|
388
|
|
|
if aidx == len(splt) - 1: # this means we have to list files too |
389
|
|
|
just_dirs = False |
390
|
|
|
else: # this means we have to list folders |
391
|
|
|
just_dirs = True |
392
|
|
|
|
393
|
|
|
vals = [] |
394
|
|
|
if arg_values is None: |
395
|
|
|
base = op.sep.join(splt[:aidx]) |
396
|
|
|
vals = list_subpaths(base, |
397
|
|
|
just_dirs=just_dirs, |
398
|
|
|
ignore=self._ignore, |
399
|
|
|
pattern=self.patterns.get(arg_name, ''), |
400
|
|
|
filter_func=self._match_filter, |
401
|
|
|
filter_args=self._re_args) |
402
|
|
|
|
403
|
|
|
vals = [[(arg_name, val)] for val in vals] |
404
|
|
|
else: |
405
|
|
|
for aval in arg_values: |
406
|
|
|
# create the part of the crumb path that is already specified |
407
|
|
|
path = self._split(self._replace(self._path, |
408
|
|
|
**dict(aval)))[0] |
409
|
|
|
|
410
|
|
|
paths = list_subpaths(path, |
411
|
|
|
just_dirs=just_dirs, |
412
|
|
|
ignore=self._ignore, |
413
|
|
|
pattern=self.patterns.get(arg_name, ''), |
414
|
|
|
filter_func=self._match_filter) |
415
|
|
|
|
416
|
|
|
# extend `val` tuples with the new list of values for `aval` |
417
|
|
|
vals.extend([aval + [(arg_name, sp)] for sp in paths]) |
418
|
|
|
|
419
|
|
|
return vals |
420
|
|
|
|
421
|
|
|
def _check_args(self, arg_names, self_args): |
422
|
|
|
""" Raise a ValueError if `self_args` is empty. |
423
|
|
|
Raise a KeyError if `arg_names` is not a subset of `self_args`. |
424
|
|
|
""" |
425
|
|
|
if not self_args: |
426
|
|
|
raise ValueError('This Crumb has no remaining arguments: {}.'.format(self.path)) |
427
|
|
|
|
428
|
|
|
if not set(arg_names).issubset(set(self_args)): |
429
|
|
|
raise KeyError("Expected `arg_names` to be a subset of ({})," |
430
|
|
|
" got {}.".format(list(self_args), arg_names)) |
431
|
|
|
|
432
|
|
|
def _check_open_args(self, arg_names): |
433
|
|
|
""" Raise a KeyError if any of the arguments in `arg_names` is not a crumb |
434
|
|
|
argument name in `self.path`. |
435
|
|
|
Parameters |
436
|
|
|
---------- |
437
|
|
|
arg_names: sequence of str |
438
|
|
|
Names of crumb arguments |
439
|
|
|
|
440
|
|
|
Raises |
441
|
|
|
------ |
442
|
|
|
KeyError |
443
|
|
|
""" |
444
|
|
|
return self._check_args(arg_names, self_args=self.open_args()) |
445
|
|
|
|
446
|
|
|
def _update_argidx(self, **kwargs): |
447
|
|
|
""" Update the argument index `self._argidx` dictionary taking into account the replacement number of splits.""" |
448
|
|
|
for arg_name, value in kwargs.items(): |
449
|
|
|
n_splits = len(value.split(op.sep)) |
450
|
|
|
|
451
|
|
|
if n_splits < 1: |
452
|
|
|
raise ValueError('Error reading your replacement value "{}" for ' |
453
|
|
|
'crumb argument "{}".'.format(value, arg_name)) |
454
|
|
|
elif n_splits == 1: |
455
|
|
|
continue |
456
|
|
|
|
457
|
|
|
# n_splits > 1, so I have to update the position of the argument children |
458
|
|
|
childs = self._arg_children(arg_name) |
459
|
|
|
for child_name in childs: |
460
|
|
|
self._argidx[child_name] = self._argidx[child_name] + n_splits - 1 |
461
|
|
|
|
462
|
|
|
def set_args(self, **kwargs): |
463
|
|
|
""" Set the crumb arguments in path to the given values in kwargs and update |
464
|
|
|
self accordingly. |
465
|
|
|
Parameters |
466
|
|
|
---------- |
467
|
|
|
kwargs: strings |
468
|
|
|
|
469
|
|
|
Returns |
470
|
|
|
------- |
471
|
|
|
crumb: Crumb |
472
|
|
|
""" |
473
|
|
|
self._check_args(kwargs.keys(), self_args=self.all_args()) |
474
|
|
|
|
475
|
|
|
# ignore for now the arguments that are in argval. |
476
|
|
|
# TODO: never change `_path`, make the `path` property to build up the path on runtime checking argval. |
477
|
|
|
for k in list(kwargs.keys()): |
478
|
|
|
if k in self._argval: |
479
|
|
|
kwargs.pop(k) |
480
|
|
|
|
481
|
|
|
self._path = self._replace(self._path, **kwargs) |
482
|
|
|
self._check() |
483
|
|
|
|
484
|
|
|
self._update_argidx(**kwargs) |
485
|
|
|
_dict_popitems(self.patterns, **kwargs) |
486
|
|
|
self._argval.update(**kwargs) |
487
|
|
|
return self |
488
|
|
|
|
489
|
|
|
def replace(self, **kwargs): |
490
|
|
|
""" Return a copy of self with the crumb arguments in |
491
|
|
|
`kwargs` replaced by its values. |
492
|
|
|
As an analogy to the `str.format` function this function could be called `format`. |
493
|
|
|
Parameters |
494
|
|
|
---------- |
495
|
|
|
kwargs: strings |
496
|
|
|
|
497
|
|
|
Returns |
498
|
|
|
------- |
499
|
|
|
crumb: |
500
|
|
|
""" |
501
|
|
|
cr = self.copy(self) |
502
|
|
|
return cr.set_args(**kwargs) |
503
|
|
|
|
504
|
|
|
def _arg_parents(self, arg_name): |
505
|
|
|
""" Return a subdict with the open arguments name and index in `self._argidx` |
506
|
|
|
that come before `arg_name` in the crumb path. Include `arg_name` himself. |
507
|
|
|
Parameters |
508
|
|
|
---------- |
509
|
|
|
arg_name: str |
510
|
|
|
|
511
|
|
|
Returns |
512
|
|
|
------- |
513
|
|
|
arg_deps: Mapping[str, int] |
514
|
|
|
""" |
515
|
|
|
argidx = self._find_arg(arg_name) |
516
|
|
|
return OrderedDict([(arg, idx) for arg, idx in self._open_arg_items() if idx <= argidx]) |
517
|
|
|
|
518
|
|
|
def _arg_children(self, arg_name): |
519
|
|
|
""" Return a subdict with the open arguments name and index in `self._argidx` |
520
|
|
|
that come AFTER `arg_name` in the crumb path. |
521
|
|
|
Parameters |
522
|
|
|
---------- |
523
|
|
|
arg_name: str |
524
|
|
|
|
525
|
|
|
Returns |
526
|
|
|
------- |
527
|
|
|
arg_deps: Mapping[str, int] |
528
|
|
|
""" |
529
|
|
|
argidx = self._find_arg(arg_name) |
530
|
|
|
return OrderedDict([(arg, idx) for arg, idx in self._open_arg_items() if idx > argidx]) |
531
|
|
|
|
532
|
|
|
def _args_open_parents(self, arg_names): |
533
|
|
|
""" Return the name of the arguments that are dependencies of `arg_names`. |
534
|
|
|
Parameters |
535
|
|
|
---------- |
536
|
|
|
arg_names: Sequence[str] |
537
|
|
|
|
538
|
|
|
Returns |
539
|
|
|
------- |
540
|
|
|
rem_deps: Sequence[str] |
541
|
|
|
""" |
542
|
|
|
started = False |
543
|
|
|
arg_dads = [] |
544
|
|
|
for an in reversed(list(self.open_args())): # take into account that argidx is ordered |
545
|
|
|
if an in arg_names: |
546
|
|
|
started = True |
547
|
|
|
else: |
548
|
|
|
if started: |
549
|
|
|
arg_dads.append(an) |
550
|
|
|
|
551
|
|
|
return list(reversed(arg_dads)) |
552
|
|
|
|
553
|
|
|
def values_map(self, arg_name='', check_exists=False): |
554
|
|
|
""" Return a list of tuples of crumb arguments with their values from the first argument |
555
|
|
|
until `arg_name`. |
556
|
|
|
Parameters |
557
|
|
|
---------- |
558
|
|
|
arg_name: str |
559
|
|
|
If empty will pick the arg_name of the last open argument of the Crumb. |
560
|
|
|
|
561
|
|
|
check_exists: bool |
562
|
|
|
|
563
|
|
|
Returns |
564
|
|
|
------- |
565
|
|
|
values_map: list of lists of 2-tuples |
566
|
|
|
I call values_map what is called `record` in pandas. It is a list of lists of 2-tuples, where each 2-tuple |
567
|
|
|
has the shape (arg_name, arg_value). |
568
|
|
|
""" |
569
|
|
|
if not arg_name: |
570
|
|
|
arg_name, _ = self._last_open_arg() |
571
|
|
|
|
572
|
|
|
arg_deps = self._arg_parents(arg_name) |
573
|
|
|
values_map = None |
574
|
|
|
for arg in arg_deps: |
575
|
|
|
values_map = self._arg_values(arg, values_map) |
576
|
|
|
|
577
|
|
|
if check_exists: |
578
|
|
|
paths = [cr for cr in self.build_paths(values_map, make_crumbs=True)] |
579
|
|
|
values_map_checked = [args for args, path in zip(values_map, paths) if path.exists()] |
580
|
|
|
else: |
581
|
|
|
values_map_checked = values_map |
582
|
|
|
|
583
|
|
|
return sorted(values_map_checked) |
584
|
|
|
|
585
|
|
|
def build_paths(self, values_map, make_crumbs=True): |
586
|
|
|
""" Return a list of paths from each tuple of args from `values_map` |
587
|
|
|
Parameters |
588
|
|
|
---------- |
589
|
|
|
values_map: list of sequences of 2-tuple |
590
|
|
|
Example: [[('subject_id', 'haensel'), ('candy', 'lollipop.png')], |
591
|
|
|
[('subject_id', 'gretel'), ('candy', 'jujube.png')], |
592
|
|
|
] |
593
|
|
|
|
594
|
|
|
make_crumbs: bool |
595
|
|
|
If `make_crumbs` is True will create a Crumb for |
596
|
|
|
each element of the result. |
597
|
|
|
Default: True. |
598
|
|
|
|
599
|
|
|
Returns |
600
|
|
|
------- |
601
|
|
|
paths: list of str or list of Crumb |
602
|
|
|
""" |
603
|
|
|
if make_crumbs: |
604
|
|
|
return [self.replace(**dict(val)) for val in values_map] |
605
|
|
|
else: |
606
|
|
|
return [self._replace(self._path, **dict(val)) for val in values_map] |
607
|
|
|
|
608
|
|
|
def ls(self, arg_name='', fullpath=True, make_crumbs=True, check_exists=False): |
609
|
|
|
""" Return the list of values for the argument crumb `arg_name`. |
610
|
|
|
This will also unfold any other argument crumb that appears before in the |
611
|
|
|
path. |
612
|
|
|
Parameters |
613
|
|
|
---------- |
614
|
|
|
arg_name: str |
615
|
|
|
Name of the argument crumb to be unfolded. |
616
|
|
|
If empty will pick the arg_name of the last open argument of the Crumb. |
617
|
|
|
|
618
|
|
|
fullpath: bool |
619
|
|
|
If True will build the full path of the crumb path, will also append |
620
|
|
|
the rest of crumbs not unfolded. |
621
|
|
|
If False will only return the values for the argument with name |
622
|
|
|
`arg_name`. |
623
|
|
|
|
624
|
|
|
make_crumbs: bool |
625
|
|
|
If `fullpath` and `make_crumbs` is True will create a Crumb for |
626
|
|
|
each element of the result. |
627
|
|
|
|
628
|
|
|
check_exists: bool |
629
|
|
|
If True will return only str, Crumb or Path if it exists |
630
|
|
|
in the file path, otherwise it may create file paths |
631
|
|
|
that don't have to exist. |
632
|
|
|
|
633
|
|
|
Returns |
634
|
|
|
------- |
635
|
|
|
values: list of Crumb or str |
636
|
|
|
|
637
|
|
|
Examples |
638
|
|
|
-------- |
639
|
|
|
>>> cr = Crumb(op.join(op.expanduser('~'), '{user_folder}')) |
640
|
|
|
>>> user_folders = cr.ls('user_folder',fullpath=True,make_crumbs=True) |
641
|
|
|
""" |
642
|
|
|
if not arg_name: |
643
|
|
|
arg_name, _ = self._last_open_arg() |
644
|
|
|
|
645
|
|
|
self._check_open_args([arg_name]) |
646
|
|
|
|
647
|
|
|
start_sym, _ = self._start_end_syms |
648
|
|
|
|
649
|
|
|
# if the first chunk of the path is a parameter, I am not interested in this (for now) |
650
|
|
|
if self._path.startswith(start_sym): |
651
|
|
|
raise NotImplementedError("Cannot list paths that start with an argument. " |
652
|
|
|
"If this is a relative path, use the `abspath()` member function.") |
653
|
|
|
|
654
|
|
|
if make_crumbs and not fullpath: |
655
|
|
|
raise ValueError("`make_crumbs` can only work if `fullpath` is also True.") |
656
|
|
|
|
657
|
|
|
values_map = self.values_map(arg_name, check_exists=check_exists) |
658
|
|
|
|
659
|
|
|
if fullpath: |
660
|
|
|
paths = self.build_paths(values_map, make_crumbs=make_crumbs) |
661
|
|
|
|
662
|
|
|
else: |
663
|
|
|
paths = [dict(val)[arg_name] for val in values_map] |
664
|
|
|
|
665
|
|
|
return sorted(paths) |
666
|
|
|
|
667
|
|
|
def touch(self): |
668
|
|
|
""" Create a leaf directory and all intermediate ones using the non |
669
|
|
|
crumbed part of `crumb_path`. |
670
|
|
|
If the target directory already exists, raise an IOError if exist_ok |
671
|
|
|
is False. Otherwise no exception is raised. |
672
|
|
|
Parameters |
673
|
|
|
---------- |
674
|
|
|
crumb_path: str |
675
|
|
|
|
676
|
|
|
exist_ok: bool |
677
|
|
|
Default = True |
678
|
|
|
|
679
|
|
|
Returns |
680
|
|
|
------- |
681
|
|
|
nupath: str |
682
|
|
|
The new path created. |
683
|
|
|
""" |
684
|
|
|
return self._touch(self._path) |
685
|
|
|
|
686
|
|
|
def joinpath(self, suffix): |
687
|
|
|
""" Return a copy of the current crumb with the `suffix` path appended. |
688
|
|
|
If suffix has crumb arguments, the whole crumb will be updated. |
689
|
|
|
Parameters |
690
|
|
|
---------- |
691
|
|
|
suffix: str |
692
|
|
|
|
693
|
|
|
Returns |
694
|
|
|
------- |
695
|
|
|
cr: Crumb |
696
|
|
|
""" |
697
|
|
|
return Crumb(op.join(self._path, suffix)) |
698
|
|
|
|
699
|
|
|
def exists(self): |
700
|
|
|
""" Return True if the current crumb path is a possibly existing path, |
701
|
|
|
False otherwise. |
702
|
|
|
Returns |
703
|
|
|
------- |
704
|
|
|
exists: bool |
705
|
|
|
""" |
706
|
|
|
if not self.has_crumbs(self._path): |
707
|
|
|
return op.exists(str(self)) or op.islink(str(self)) |
708
|
|
|
|
709
|
|
|
if not op.exists(self.split()[0]): |
710
|
|
|
return False |
711
|
|
|
|
712
|
|
|
last, _ = self._last_open_arg() |
713
|
|
|
|
714
|
|
|
paths = self.ls(last, fullpath=True, make_crumbs=False, check_exists=False) |
715
|
|
|
|
716
|
|
|
return any([self._split_exists(lp) for lp in paths]) |
717
|
|
|
|
718
|
|
|
def has_files(self): |
719
|
|
|
""" Return True if the current crumb path has any file in its |
720
|
|
|
possible paths. |
721
|
|
|
Returns |
722
|
|
|
------- |
723
|
|
|
has_files: bool |
724
|
|
|
""" |
725
|
|
|
if not op.exists(list(self.split())[0]): |
726
|
|
|
return False |
727
|
|
|
|
728
|
|
|
last, _ = self._last_open_arg() |
729
|
|
|
paths = self.ls(last, fullpath=True, make_crumbs=True, check_exists=True) |
730
|
|
|
|
731
|
|
|
return any([op.isfile(str(lp)) for lp in paths]) |
732
|
|
|
|
733
|
|
|
def unfold(self): |
734
|
|
|
""" Return a list of all the existing paths until the last crumb argument. |
735
|
|
|
Returns |
736
|
|
|
------- |
737
|
|
|
paths: list of pathlib.Path |
738
|
|
|
""" |
739
|
|
|
return self.ls(self._last_open_arg()[0], fullpath=True, make_crumbs=True, check_exists=True) |
740
|
|
|
|
741
|
|
|
def __getitem__(self, arg_name): |
742
|
|
|
""" Return the existing values of the crumb argument `arg_name` |
743
|
|
|
without removing duplicates. |
744
|
|
|
Parameters |
745
|
|
|
---------- |
746
|
|
|
arg_name: str |
747
|
|
|
|
748
|
|
|
Returns |
749
|
|
|
------- |
750
|
|
|
values: list of str |
751
|
|
|
""" |
752
|
|
|
if arg_name in self._argval: |
753
|
|
|
return [self._argval[arg_name]] |
754
|
|
|
else: |
755
|
|
|
return self.ls(arg_name, fullpath=False, make_crumbs=False, check_exists=True) |
756
|
|
|
|
757
|
|
|
def __setitem__(self, key, value): |
758
|
|
|
if key not in self._argidx: |
759
|
|
|
raise KeyError("Expected `arg_name` to be one of ({})," |
760
|
|
|
" got {}.".format(list(self.open_args()), key)) |
761
|
|
|
_ = self.set_args(**{key: value}) |
762
|
|
|
|
763
|
|
|
def __ge__(self, other): |
764
|
|
|
return self._path >= str(other) |
765
|
|
|
|
766
|
|
|
def __le__(self, other): |
767
|
|
|
return self._path <= str(other) |
768
|
|
|
|
769
|
|
|
def __gt__(self, other): |
770
|
|
|
return self._path > str(other) |
771
|
|
|
|
772
|
|
|
def __lt__(self, other): |
773
|
|
|
return self._path < str(other) |
774
|
|
|
|
775
|
|
|
def __hash__(self): |
776
|
|
|
return self._path.__hash__() |
777
|
|
|
|
778
|
|
|
def __contains__(self, arg_name): |
779
|
|
|
return arg_name in self.all_args() |
780
|
|
|
|
781
|
|
|
def __repr__(self): |
782
|
|
|
return '{}("{}")'.format(type(self).__name__, self._path) |
783
|
|
|
|
784
|
|
|
def __str__(self): |
785
|
|
|
return str(self._path) |
786
|
|
|
|
787
|
|
|
def __eq__(self, other): |
788
|
|
|
""" Return True if `self` and `other` are equal, False otherwise. |
789
|
|
|
Parameters |
790
|
|
|
---------- |
791
|
|
|
other: Crumb |
792
|
|
|
|
793
|
|
|
Returns |
794
|
|
|
------- |
795
|
|
|
is_equal: bool |
796
|
|
|
""" |
797
|
|
|
if self._path != other._path: |
798
|
|
|
return False |
799
|
|
|
|
800
|
|
|
if self._argidx != other._argidx: |
801
|
|
|
return False |
802
|
|
|
|
803
|
|
|
if self._argval != other._argval: |
804
|
|
|
return False |
805
|
|
|
|
806
|
|
|
if self._ignore != other._ignore: |
807
|
|
|
return False |
808
|
|
|
|
809
|
|
|
return True |
810
|
|
|
|