|
1
|
|
|
# -*- coding: utf-8 -*- |
|
2
|
|
|
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- |
|
3
|
|
|
# vi: set ft=python sts=4 ts=4 sw=4 et: |
|
4
|
|
|
""" |
|
5
|
|
|
Utilities to make crumbs |
|
6
|
|
|
""" |
|
7
|
|
|
import os |
|
8
|
|
|
import os.path as op |
|
9
|
|
|
|
|
10
|
|
|
from collections import Mapping |
|
11
|
|
|
from functools import partial, reduce |
|
12
|
|
|
from itertools import product |
|
13
|
|
|
import operator |
|
14
|
|
|
|
|
15
|
|
|
|
|
16
|
|
|
def remove_duplicates(lst): |
|
17
|
|
|
""" Return a sorted lst of non-duplicated elements from `lst`. |
|
18
|
|
|
|
|
19
|
|
|
Parameters |
|
20
|
|
|
---------- |
|
21
|
|
|
lst: sequence of any |
|
22
|
|
|
|
|
23
|
|
|
Returns |
|
24
|
|
|
------- |
|
25
|
|
|
fslst: |
|
26
|
|
|
Filtered and sorted `lst` with non duplicated elements of `lst`. |
|
27
|
|
|
""" |
|
28
|
|
|
return sorted(list(set(lst))) |
|
29
|
|
|
|
|
30
|
|
|
|
|
31
|
|
|
def list_children(path, just_dirs=False): |
|
32
|
|
|
""" Return the immediate elements (files and folders) in `path`. |
|
33
|
|
|
|
|
34
|
|
|
Parameters |
|
35
|
|
|
---------- |
|
36
|
|
|
path: str |
|
37
|
|
|
|
|
38
|
|
|
just_dirs: bool |
|
39
|
|
|
If True will return only folders. |
|
40
|
|
|
|
|
41
|
|
|
Returns |
|
42
|
|
|
------- |
|
43
|
|
|
paths: list of str |
|
44
|
|
|
""" |
|
45
|
|
|
if not op.exists(path): |
|
46
|
|
|
raise IOError("Expected an existing path, but could not" |
|
47
|
|
|
" find {}.".format(path)) |
|
48
|
|
|
|
|
49
|
|
|
if op.isfile(path): |
|
50
|
|
|
if just_dirs: |
|
51
|
|
|
vals = [] |
|
52
|
|
|
else: |
|
53
|
|
|
vals = [path] |
|
54
|
|
|
else: |
|
55
|
|
|
if just_dirs: # this means we have to list only folders |
|
56
|
|
|
vals = [d for d in os.listdir(path) if op.isdir(op.join(path, d))] |
|
57
|
|
|
else: # this means we have to list files |
|
58
|
|
|
vals = os.listdir(path) |
|
59
|
|
|
|
|
60
|
|
|
return vals |
|
61
|
|
|
|
|
62
|
|
|
|
|
63
|
|
|
class ParameterGrid(object): |
|
64
|
|
|
""" |
|
65
|
|
|
Picked from sklearn: https://github.com/scikit-learn/scikit-learn |
|
66
|
|
|
|
|
67
|
|
|
Grid of parameters with a discrete number of values for each. |
|
68
|
|
|
Can be used to iterate over parameter value combinations with the |
|
69
|
|
|
Python built-in function iter. |
|
70
|
|
|
|
|
71
|
|
|
Read more in the :ref:`User Guide <grid_search>`. |
|
72
|
|
|
Parameters |
|
73
|
|
|
---------- |
|
74
|
|
|
param_grid : dict of string to sequence, or sequence of such |
|
75
|
|
|
The parameter grid to explore, as a dictionary mapping estimator |
|
76
|
|
|
parameters to sequences of allowed values. |
|
77
|
|
|
An empty dict signifies default parameters. |
|
78
|
|
|
A sequence of dicts signifies a sequence of grids to search, and is |
|
79
|
|
|
useful to avoid exploring parameter combinations that make no sense |
|
80
|
|
|
or have no effect. See the examples below. |
|
81
|
|
|
Examples |
|
82
|
|
|
-------- |
|
83
|
|
|
>>> from sklearn.grid_search import ParameterGrid |
|
84
|
|
|
>>> param_grid = {'a': [1, 2], 'b': [True, False]} |
|
85
|
|
|
>>> list(ParameterGrid(param_grid)) == ( |
|
86
|
|
|
... [{'a': 1, 'b': True}, {'a': 1, 'b': False}, |
|
87
|
|
|
... {'a': 2, 'b': True}, {'a': 2, 'b': False}]) |
|
88
|
|
|
True |
|
89
|
|
|
>>> grid = [{'kernel': ['linear']}, {'kernel': ['rbf'], 'gamma': [1, 10]}] |
|
90
|
|
|
>>> list(ParameterGrid(grid)) == [{'kernel': 'linear'}, |
|
91
|
|
|
... {'kernel': 'rbf', 'gamma': 1}, |
|
92
|
|
|
... {'kernel': 'rbf', 'gamma': 10}] |
|
93
|
|
|
True |
|
94
|
|
|
>>> ParameterGrid(grid)[1] == {'kernel': 'rbf', 'gamma': 1} |
|
95
|
|
|
True |
|
96
|
|
|
""" |
|
97
|
|
|
|
|
98
|
|
|
def __init__(self, param_grid): |
|
99
|
|
|
if isinstance(param_grid, Mapping): |
|
100
|
|
|
# wrap dictionary in a singleton list to support either dict |
|
101
|
|
|
# or list of dicts |
|
102
|
|
|
param_grid = [param_grid] |
|
103
|
|
|
self.param_grid = param_grid |
|
104
|
|
|
|
|
105
|
|
|
def __iter__(self): |
|
106
|
|
|
"""Iterate over the points in the grid. |
|
107
|
|
|
Returns |
|
108
|
|
|
------- |
|
109
|
|
|
params : iterator over dict of string to any |
|
110
|
|
|
Yields dictionaries mapping each estimator parameter to one of its |
|
111
|
|
|
allowed values. |
|
112
|
|
|
""" |
|
113
|
|
|
for p in self.param_grid: |
|
114
|
|
|
# Always sort the keys of a dictionary, for reproducibility |
|
115
|
|
|
items = sorted(p.items()) |
|
116
|
|
|
if not items: |
|
117
|
|
|
yield {} |
|
118
|
|
|
else: |
|
119
|
|
|
keys, values = zip(*items) |
|
120
|
|
|
for v in product(*values): |
|
121
|
|
|
params = dict(zip(keys, v)) |
|
122
|
|
|
yield params |
|
123
|
|
|
|
|
124
|
|
|
def __len__(self): |
|
125
|
|
|
"""Number of points on the grid.""" |
|
126
|
|
|
# Product function that can handle iterables (np.product can't). |
|
127
|
|
|
product = partial(reduce, operator.mul) |
|
128
|
|
|
return sum(product(len(v) for v in p.values()) if p else 1 |
|
129
|
|
|
for p in self.param_grid) |
|
130
|
|
|
|
|
131
|
|
|
def __getitem__(self, ind): |
|
132
|
|
|
"""Get the parameters that would be ``ind``th in iteration |
|
133
|
|
|
Parameters |
|
134
|
|
|
---------- |
|
135
|
|
|
ind : int |
|
136
|
|
|
The iteration index |
|
137
|
|
|
Returns |
|
138
|
|
|
------- |
|
139
|
|
|
params : dict of string to any |
|
140
|
|
|
Equal to list(self)[ind] |
|
141
|
|
|
""" |
|
142
|
|
|
# This is used to make discrete sampling without replacement memory |
|
143
|
|
|
# efficient. |
|
144
|
|
|
for sub_grid in self.param_grid: |
|
145
|
|
|
# XXX: could memoize information used here |
|
146
|
|
|
if not sub_grid: |
|
147
|
|
|
if ind == 0: |
|
148
|
|
|
return {} |
|
149
|
|
|
else: |
|
150
|
|
|
ind -= 1 |
|
151
|
|
|
continue |
|
152
|
|
|
|
|
153
|
|
|
# Reverse so most frequent cycling parameter comes first |
|
154
|
|
|
keys, values_lists = zip(*sorted(sub_grid.items())[::-1]) |
|
155
|
|
|
sizes = [len(v_list) for v_list in values_lists] |
|
156
|
|
|
product = partial(reduce, operator.mul) |
|
157
|
|
|
total = product(sizes) |
|
158
|
|
|
|
|
159
|
|
|
if ind >= total: |
|
160
|
|
|
# Try the next grid |
|
161
|
|
|
ind -= total |
|
162
|
|
|
else: |
|
163
|
|
|
out = {} |
|
164
|
|
|
for key, v_list, n in zip(keys, values_lists, sizes): |
|
165
|
|
|
ind, offset = divmod(ind, n) |
|
166
|
|
|
out[key] = v_list[offset] |
|
167
|
|
|
return out |
|
168
|
|
|
|
|
169
|
|
|
raise IndexError('ParameterGrid index out of range') |
|
170
|
|
|
|
alexsavio /
hansel
