1
|
|
|
import inspect |
2
|
|
|
import warnings |
3
|
|
|
from abc import ABCMeta |
4
|
|
|
from collections import OrderedDict |
5
|
|
|
from six import wraps |
|
|
|
|
6
|
|
|
from operator import attrgetter |
7
|
|
|
from types import MethodType |
8
|
|
|
|
9
|
|
|
import six |
10
|
|
|
from six import add_metaclass |
11
|
|
|
from theano import tensor |
12
|
|
|
from theano.gof import Variable |
13
|
|
|
|
14
|
|
|
from blocks.graph import add_annotation, Annotation |
15
|
|
|
from blocks.roles import add_role, PARAMETER, INPUT, OUTPUT |
16
|
|
|
from blocks.utils import dict_union, pack, repr_attrs, reraise_as, unpack |
17
|
|
|
from blocks.utils.containers import AnnotatingList |
18
|
|
|
|
19
|
|
|
BRICK_DELIMITER = '/' |
20
|
|
|
|
21
|
|
|
|
22
|
|
|
def create_unbound_method(func, cls): |
23
|
|
|
"""Create an unbounded method from a function and a class. |
24
|
|
|
|
25
|
|
|
Notes |
26
|
|
|
----- |
27
|
|
|
See https://bitbucket.org/gutworth/six/pull-request/64. |
28
|
|
|
|
29
|
|
|
""" |
30
|
|
|
if six.PY2: |
31
|
|
|
return MethodType(func, None, cls) |
32
|
|
|
if six.PY3: |
33
|
|
|
return func |
34
|
|
|
|
35
|
|
|
# Rename built-in property to avoid conflict with Application.property |
36
|
|
|
property_ = property |
|
|
|
|
37
|
|
|
|
38
|
|
|
|
39
|
|
|
class Parameters(AnnotatingList): |
40
|
|
|
"""Adds the PARAMETER role to parameters automatically.""" |
41
|
|
|
def __init__(self, brick, *args, **kwargs): |
42
|
|
|
self.brick = brick |
43
|
|
|
super(Parameters, self).__init__(*args, **kwargs) |
44
|
|
|
|
45
|
|
|
def _setitem(self, key, value): |
46
|
|
|
if isinstance(value, Variable): |
47
|
|
|
add_role(value, PARAMETER) |
48
|
|
|
add_annotation(value, self.brick) |
49
|
|
|
|
50
|
|
|
|
51
|
|
|
class Children(AnnotatingList): |
52
|
|
|
"""Adds the brick to the list of parents of its children.""" |
53
|
|
|
def __init__(self, brick, *args, **kwargs): |
54
|
|
|
self.brick = brick |
55
|
|
|
super(Children, self).__init__(*args, **kwargs) |
56
|
|
|
|
57
|
|
|
def _setitem(self, key, value): |
58
|
|
|
if value is not None: |
59
|
|
|
value.parents.append(self.brick) |
60
|
|
|
|
61
|
|
|
def _delitem(self, key): |
62
|
|
|
child = self._items[key] |
63
|
|
|
if child is not None: |
64
|
|
|
child.parents.remove(self.brick) |
65
|
|
|
|
66
|
|
|
|
67
|
|
|
class Application(object): |
68
|
|
|
"""An application method belonging to a particular type of brick. |
69
|
|
|
|
70
|
|
|
The application methods of each :class:`Brick` class are automatically |
71
|
|
|
replaced by an instance of :class:`Application`. This allows us to |
72
|
|
|
store metadata about particular application methods (such as their in- |
73
|
|
|
and outputs) easily. |
74
|
|
|
|
75
|
|
|
Attributes |
76
|
|
|
---------- |
77
|
|
|
application : callable |
78
|
|
|
The original (unbounded) application function defined on the |
79
|
|
|
:class:`Brick`. |
80
|
|
|
delegate_function : callable |
81
|
|
|
A function that takes a :class:`Brick` instance as an argument and |
82
|
|
|
returns a :class:`BoundApplication` object to which attribute |
83
|
|
|
requests should be routed. |
84
|
|
|
properties : :obj:`dict` (:obj:`str`, :obj:`callable`) |
85
|
|
|
A dictionary of property getters that should be called when an |
86
|
|
|
attribute with the given name is requested. |
87
|
|
|
instances : :obj:`dict` (:class:`Brick`, :class:`BoundApplication`) |
88
|
|
|
A record of bound application instances created by the descriptor |
89
|
|
|
protocol. |
90
|
|
|
call_stack : :obj:`list` of :class:`Brick` |
91
|
|
|
The call stack of brick application methods. Used to check whether |
92
|
|
|
the current call was made by a parent brick. |
93
|
|
|
brick : type |
94
|
|
|
The brick class to which this instance belongs. |
95
|
|
|
|
96
|
|
|
Raises |
97
|
|
|
------ |
98
|
|
|
ValueError |
99
|
|
|
If a brick's application method is applied by another brick which |
100
|
|
|
does not list the former as a child. |
101
|
|
|
ValueError |
102
|
|
|
If the application method's inputs and/or outputs don't match with |
103
|
|
|
the function signature or the values returned (respectively). |
104
|
|
|
|
105
|
|
|
Notes |
106
|
|
|
----- |
107
|
|
|
When a :class:`Brick` is instantiated and its application method (i.e. |
108
|
|
|
an instance of this class) requested, the descriptor protocol (through |
109
|
|
|
the :meth:`__get__` method) automatically instantiates a |
110
|
|
|
:class:`BoundApplication` class and returns this. This bound |
111
|
|
|
application class can be used to store application information |
112
|
|
|
particular to a brick instance. Any attributes unknown to the bounded |
113
|
|
|
application are automatically routed to the application that |
114
|
|
|
instantiated it. |
115
|
|
|
|
116
|
|
|
""" |
117
|
|
|
call_stack = [] |
118
|
|
|
|
119
|
|
|
def __init__(self, application_function): |
120
|
|
|
self.__doc__ = application_function.__doc__ |
121
|
|
|
self._application_function = application_function |
122
|
|
|
self.application_name = application_function.__name__ |
123
|
|
|
self.delegate_function = None |
124
|
|
|
self.properties = {} |
125
|
|
|
|
126
|
|
|
@property |
127
|
|
|
def application_function(self): |
128
|
|
|
if hasattr(self, '_application_function'): |
129
|
|
|
return self._application_function |
130
|
|
|
return getattr(self.brick, '_' + self.application_name) |
131
|
|
|
|
132
|
|
|
def property(self, name): |
133
|
|
|
"""Decorator to make application properties. |
134
|
|
|
|
135
|
|
|
Parameters |
136
|
|
|
---------- |
137
|
|
|
name : str |
138
|
|
|
The name the property should take. |
139
|
|
|
|
140
|
|
|
Examples |
141
|
|
|
-------- |
142
|
|
|
>>> class Foo(Brick): |
143
|
|
|
... @application |
144
|
|
|
... def apply(self, x): |
145
|
|
|
... return x + 1 |
146
|
|
|
... |
147
|
|
|
... @apply.property('inputs') |
148
|
|
|
... def apply_inputs(self): |
149
|
|
|
... return ['foo', 'bar'] |
150
|
|
|
>>> foo = Foo() |
151
|
|
|
>>> foo.apply.inputs |
152
|
|
|
['foo', 'bar'] |
153
|
|
|
|
154
|
|
|
""" |
155
|
|
|
if not isinstance(name, six.string_types): |
156
|
|
|
raise ValueError |
157
|
|
|
|
158
|
|
|
def wrap_property(application_property): |
159
|
|
|
self.properties[name] = application_property.__name__ |
160
|
|
|
return application_property |
161
|
|
|
return wrap_property |
162
|
|
|
|
163
|
|
|
def delegate(self, f): |
164
|
|
|
"""Decorator to assign a delegate application. |
165
|
|
|
|
166
|
|
|
An application method can assign a delegate application. Whenever |
167
|
|
|
an attribute is not available, it will be requested from the |
168
|
|
|
delegate instead. |
169
|
|
|
|
170
|
|
|
Examples |
171
|
|
|
-------- |
172
|
|
|
>>> class Foo(Brick): |
173
|
|
|
... @application(outputs=['baz']) |
174
|
|
|
... def apply(self, x): |
175
|
|
|
... return x + 1 |
176
|
|
|
... |
177
|
|
|
... @apply.property('inputs') |
178
|
|
|
... def apply_inputs(self): |
179
|
|
|
... return ['foo', 'bar'] |
180
|
|
|
>>> class Bar(Brick): |
181
|
|
|
... def __init__(self, foo): |
182
|
|
|
... self.foo = foo |
183
|
|
|
... |
184
|
|
|
... @application(outputs=['foo']) |
185
|
|
|
... def apply(self, x): |
186
|
|
|
... return x + 1 |
187
|
|
|
... |
188
|
|
|
... @apply.delegate |
189
|
|
|
... def apply_delegate(self): |
190
|
|
|
... return self.foo.apply |
191
|
|
|
>>> foo = Foo() |
192
|
|
|
>>> bar = Bar(foo) |
193
|
|
|
>>> bar.apply.outputs |
194
|
|
|
['foo'] |
195
|
|
|
>>> bar.apply.inputs |
196
|
|
|
['foo', 'bar'] |
197
|
|
|
|
198
|
|
|
""" |
199
|
|
|
self.delegate_function = f.__name__ |
200
|
|
|
return f |
201
|
|
|
|
202
|
|
|
def __get__(self, instance, owner): |
203
|
|
|
"""Instantiate :class:`BoundApplication` for each :class:`Brick`.""" |
204
|
|
|
if instance is None: |
205
|
|
|
return self |
206
|
|
|
if not hasattr(instance, "_bound_applications"): |
207
|
|
|
instance._bound_applications = {} |
|
|
|
|
208
|
|
|
key = "{}.{}".format(self.brick.__name__, self.application_name) |
209
|
|
|
return instance._bound_applications.setdefault( |
|
|
|
|
210
|
|
|
key, BoundApplication(self, instance)) |
211
|
|
|
|
212
|
|
|
def __getattr__(self, name): |
213
|
|
|
# Mimic behavior of properties |
214
|
|
|
if 'properties' in self.__dict__ and name in self.properties: |
215
|
|
|
return property(create_unbound_method( |
216
|
|
|
getattr(self, self.properties[name]), self.brick)) |
217
|
|
|
raise AttributeError |
218
|
|
|
|
219
|
|
|
def __setattr__(self, name, value): |
220
|
|
|
# Mimic behavior of read-only properties |
221
|
|
|
if 'properties' in self.__dict__ and name in self.properties: |
222
|
|
|
raise AttributeError("can't set attribute") |
223
|
|
|
super(Application, self).__setattr__(name, value) |
224
|
|
|
|
225
|
|
|
@property_ |
226
|
|
|
def inputs(self): |
227
|
|
|
return self._inputs |
228
|
|
|
|
229
|
|
|
@inputs.setter |
230
|
|
|
def inputs(self, inputs): |
231
|
|
|
args_names, varargs_name, _, _ = inspect.getargspec( |
232
|
|
|
self.application_function) |
233
|
|
|
if not all(input_ in args_names + [varargs_name] for input_ in inputs): |
234
|
|
|
raise ValueError("Unexpected inputs") |
235
|
|
|
self._inputs = inputs |
236
|
|
|
|
237
|
|
|
@property_ |
238
|
|
|
def name(self): |
239
|
|
|
return self.application_name |
240
|
|
|
|
241
|
|
|
def __call__(self, brick, *args, **kwargs): |
242
|
|
|
if not isinstance(brick, Brick) and six.PY2: |
243
|
|
|
raise TypeError |
244
|
|
|
bound_application = self.__get__(brick, brick.__class__) |
245
|
|
|
return self.apply(bound_application, *args, **kwargs) |
246
|
|
|
|
247
|
|
|
def apply(self, bound_application, *args, **kwargs): |
248
|
|
|
as_dict = kwargs.pop('as_dict', False) |
249
|
|
|
as_list = kwargs.pop('as_list', False) |
250
|
|
|
if as_list and as_dict: |
251
|
|
|
raise ValueError |
252
|
|
|
|
253
|
|
|
brick = bound_application.brick |
254
|
|
|
|
255
|
|
|
# Find the names of the inputs to the application method |
256
|
|
|
args_names, varargs_name, _, _ = inspect.getargspec( |
257
|
|
|
self.application_function) |
258
|
|
|
args_names = args_names[1:] |
259
|
|
|
|
260
|
|
|
# Construct the ApplicationCall, used to store data in for this call |
261
|
|
|
call = ApplicationCall(bound_application) |
262
|
|
|
args = list(args) |
263
|
|
|
if 'application' in args_names: |
264
|
|
|
args.insert(args_names.index('application'), bound_application) |
265
|
|
|
if 'application_call' in args_names: |
266
|
|
|
args.insert(args_names.index('application_call'), call) |
267
|
|
|
|
268
|
|
|
# Allocate before applying, and optionally initialize |
269
|
|
|
if not brick.allocated: |
270
|
|
|
brick.allocate() |
271
|
|
|
|
272
|
|
|
# Annotate all the input variables which are Theano variables |
273
|
|
|
|
274
|
|
|
for i, input_ in enumerate(args): |
275
|
|
|
if isinstance(input_, tensor.Variable): |
276
|
|
|
if i < len(args_names): |
277
|
|
|
name = args_names[i] |
278
|
|
|
else: |
279
|
|
|
name = "{}_{}".format(varargs_name, i - len(args_names)) |
280
|
|
|
args[i] = copy_and_tag(input_, brick, call, INPUT, |
281
|
|
|
self.name, name) |
282
|
|
|
for name, input_ in kwargs.items(): |
283
|
|
|
if isinstance(input_, tensor.Variable): |
284
|
|
|
kwargs[name] = copy_and_tag(input_, brick, call, INPUT, |
285
|
|
|
self.name, name) |
286
|
|
|
|
287
|
|
|
# Run the application method on the annotated variables |
288
|
|
|
last_brick = self.call_stack[-1] if self.call_stack else None |
289
|
|
|
if (last_brick and brick is not last_brick and |
290
|
|
|
brick not in last_brick.children): |
291
|
|
|
warnings.warn('Brick ' + str(self.call_stack[-1]) + ' tries ' |
292
|
|
|
'to call brick ' + str(self.brick) + ' which ' |
293
|
|
|
'is not in the list of its children. This could ' |
294
|
|
|
'be caused because an @application decorator is ' |
295
|
|
|
'missing.') |
296
|
|
|
self.call_stack.append(brick) |
297
|
|
|
try: |
298
|
|
|
outputs = self.application_function(brick, *args, **kwargs) |
299
|
|
|
outputs = pack(outputs) |
300
|
|
|
finally: |
301
|
|
|
self.call_stack.pop() |
302
|
|
|
|
303
|
|
|
# Rename and annotate output variables |
304
|
|
|
for i, output in enumerate(outputs): |
305
|
|
|
if isinstance(output, tensor.Variable): |
306
|
|
|
try: |
307
|
|
|
name = bound_application.outputs[i] |
308
|
|
|
except AttributeError: |
309
|
|
|
name = "output_{}".format(i) |
310
|
|
|
except IndexError: |
311
|
|
|
reraise_as(ValueError("Unexpected outputs")) |
312
|
|
|
# TODO Tag with dimensions, axes, etc. for error-checking |
313
|
|
|
outputs[i] = copy_and_tag(outputs[i], brick, call, |
314
|
|
|
OUTPUT, self.name, name) |
315
|
|
|
|
316
|
|
|
# Return values |
317
|
|
|
if as_list: |
318
|
|
|
return outputs |
319
|
|
|
if as_dict: |
320
|
|
|
return OrderedDict(zip(bound_application.outputs, outputs)) |
321
|
|
|
return unpack(outputs) |
322
|
|
|
|
323
|
|
|
# Application instances are used instead of usual methods in bricks. |
324
|
|
|
# The usual methods are not pickled per-se, similarly to classes |
325
|
|
|
# and modules. Instead, a reference to the method is put into the pickle. |
326
|
|
|
# Here, we ensure the same behaviour for Application instances. |
327
|
|
|
def __reduce__(self): |
328
|
|
|
return (getattr, (self.brick, self.application_name)) |
329
|
|
|
|
330
|
|
|
|
331
|
|
|
class BoundApplication(object): |
332
|
|
|
"""An application method bound to a :class:`Brick` instance.""" |
333
|
|
|
def __init__(self, application, brick): |
|
|
|
|
334
|
|
|
self.application = application |
335
|
|
|
self.brick = brick |
336
|
|
|
|
337
|
|
|
def __getattr__(self, name): |
338
|
|
|
# Prevent infinite loops |
339
|
|
|
if name == 'application': |
340
|
|
|
raise AttributeError |
341
|
|
|
# These always belong to the parent (the unbound application) |
342
|
|
|
if name in ('delegate_function', 'properties'): |
343
|
|
|
return getattr(self.application, name) |
344
|
|
|
if name in self.properties.values(): |
345
|
|
|
return getattr(self.application.brick, name) |
346
|
|
|
if name in self.properties: |
347
|
|
|
return getattr(self, self.properties[name])(self.brick) |
348
|
|
|
# First try the parent (i.e. class level), before trying the delegate |
349
|
|
|
try: |
350
|
|
|
return getattr(self.application, name) |
351
|
|
|
except AttributeError: |
352
|
|
|
if self.delegate_function: |
353
|
|
|
return getattr(getattr(self.brick, |
354
|
|
|
self.delegate_function)(), |
355
|
|
|
name) |
356
|
|
|
raise |
357
|
|
|
|
358
|
|
|
@property |
359
|
|
|
def name(self): |
360
|
|
|
return self.application.name |
361
|
|
|
|
362
|
|
|
def __call__(self, *args, **kwargs): |
363
|
|
|
return self.application.apply(self, *args, **kwargs) |
364
|
|
|
|
365
|
|
|
|
366
|
|
|
def rename_function(function, new_name): |
367
|
|
|
old_name = function.__name__ |
368
|
|
|
function.__name__ = new_name |
369
|
|
|
if six.PY3: |
370
|
|
|
function.__qualname__ = \ |
371
|
|
|
function.__qualname__[:-len(old_name)] + new_name |
372
|
|
|
return function |
373
|
|
|
|
374
|
|
|
|
375
|
|
|
class _Brick(ABCMeta): |
376
|
|
|
"""Metaclass which attaches brick instances to the applications. |
377
|
|
|
|
378
|
|
|
In addition picklability of :class:`Application` objects is ensured. |
379
|
|
|
This means that :class:`Application` objects can not be added to a |
380
|
|
|
brick class after it is created. To allow adding application methods |
381
|
|
|
programatically, the following hook is supported: the class namespace |
382
|
|
|
is searched for `decorators` attribute, which can contain a |
383
|
|
|
list of functions to be applied to the namespace of the class being |
384
|
|
|
created. These functions can arbitratily modify this namespace. |
385
|
|
|
|
386
|
|
|
""" |
387
|
|
|
def __new__(mcs, name, bases, namespace): |
388
|
|
|
decorators = namespace.get('decorators', []) |
389
|
|
|
for decorator in decorators: |
390
|
|
|
decorator(mcs, name, bases, namespace) |
391
|
|
|
for attr in list(namespace.values()): |
392
|
|
|
if (isinstance(attr, Application) and |
393
|
|
|
hasattr(attr, '_application_function')): |
394
|
|
|
namespace['_' + attr.application_name] = \ |
395
|
|
|
rename_function(attr._application_function, |
|
|
|
|
396
|
|
|
'_' + attr.application_name) |
397
|
|
|
del attr._application_function |
398
|
|
|
brick = super(_Brick, mcs).__new__(mcs, name, bases, namespace) |
399
|
|
|
for attr in namespace.values(): |
400
|
|
|
if isinstance(attr, Application): |
401
|
|
|
attr.brick = brick |
402
|
|
|
return brick |
403
|
|
|
|
404
|
|
|
|
405
|
|
|
@add_metaclass(_Brick) |
406
|
|
|
class Brick(Annotation): |
407
|
|
|
"""A brick encapsulates Theano operations with parameters. |
408
|
|
|
|
409
|
|
|
A brick goes through the following stages: |
410
|
|
|
|
411
|
|
|
1. Construction: The call to :meth:`__init__` constructs a |
412
|
|
|
:class:`Brick` instance with a name and creates any child bricks as |
413
|
|
|
well. |
414
|
|
|
2. Allocation of parameters: |
415
|
|
|
|
416
|
|
|
a) Allocation configuration of children: The |
417
|
|
|
:meth:`push_allocation_config` method configures any children of |
418
|
|
|
this block. |
419
|
|
|
b) Allocation: The :meth:`allocate` method allocates the shared |
420
|
|
|
Theano variables required for the parameters. Also allocates |
421
|
|
|
parameters for all children. |
422
|
|
|
|
423
|
|
|
3. The following can be done in either order: |
424
|
|
|
|
425
|
|
|
a) Application: By applying the brick to a set of Theano |
426
|
|
|
variables a part of the computational graph of the final model is |
427
|
|
|
constructed. |
428
|
|
|
b) The initialization of parameters: |
429
|
|
|
|
430
|
|
|
1. Initialization configuration of children: The |
431
|
|
|
:meth:`push_initialization_config` method configures any |
432
|
|
|
children of this block. |
433
|
|
|
2. Initialization: This sets the initial values of the |
434
|
|
|
parameters by a call to :meth:`initialize`, which is needed |
435
|
|
|
to call the final compiled Theano function. Also initializes |
436
|
|
|
all children. |
437
|
|
|
|
438
|
|
|
Not all stages need to be called explicitly. Step 3(a) will |
439
|
|
|
automatically allocate the parameters if needed. Similarly, step |
440
|
|
|
3(b.2) and 2(b) will automatically perform steps 3(b.1) and 2(a) if |
441
|
|
|
needed. They only need to be called separately if greater control is |
442
|
|
|
required. The only two methods which always need to be called are an |
443
|
|
|
application method to construct the computational graph, and the |
444
|
|
|
:meth:`initialize` method in order to initialize the parameters. |
445
|
|
|
|
446
|
|
|
At each different stage, a brick might need a certain set of |
447
|
|
|
configuration settings. All of these settings can be passed to the |
448
|
|
|
:meth:`__init__` constructor. However, by default many bricks support |
449
|
|
|
*lazy initialization*. This means that the configuration settings can |
450
|
|
|
be set later. |
451
|
|
|
|
452
|
|
|
.. note:: |
453
|
|
|
|
454
|
|
|
Some arguments to :meth:`__init__` are *always* required, even when |
455
|
|
|
lazy initialization is enabled. Other arguments must be given before |
456
|
|
|
calling :meth:`allocate`, while others yet only need to be given in |
457
|
|
|
order to call :meth:`initialize`. Always read the documentation of |
458
|
|
|
each brick carefully. |
459
|
|
|
|
460
|
|
|
Lazy initialization can be turned off by setting ``Brick.lazy = |
461
|
|
|
False``. In this case, there is no need to call :meth:`initialize` |
462
|
|
|
manually anymore, but all the configuration must be passed to the |
463
|
|
|
:meth:`__init__` method. |
464
|
|
|
|
465
|
|
|
Parameters |
466
|
|
|
---------- |
467
|
|
|
name : str, optional |
468
|
|
|
The name of this brick. This can be used to filter the application |
469
|
|
|
of certain modifications by brick names. By default, the brick |
470
|
|
|
receives the name of its class (lowercased). |
471
|
|
|
|
472
|
|
|
Attributes |
473
|
|
|
---------- |
474
|
|
|
name : str |
475
|
|
|
The name of this brick. |
476
|
|
|
print_shapes : bool |
477
|
|
|
``False`` by default. If ``True`` it logs the shapes of all the |
478
|
|
|
input and output variables, which can be useful for debugging. |
479
|
|
|
parameters : list of :class:`~tensor.TensorSharedVariable` and ``None`` |
480
|
|
|
After calling the :meth:`allocate` method this attribute will be |
481
|
|
|
populated with the shared variables storing this brick's |
482
|
|
|
parameters. Allows for ``None`` so that parameters can always be |
483
|
|
|
accessed at the same index, even if some parameters are only |
484
|
|
|
defined given a particular configuration. |
485
|
|
|
children : list of bricks |
486
|
|
|
The children of this brick. |
487
|
|
|
allocated : bool |
488
|
|
|
``False`` if :meth:`allocate` has not been called yet. ``True`` |
489
|
|
|
otherwise. |
490
|
|
|
initialized : bool |
491
|
|
|
``False`` if :meth:`allocate` has not been called yet. ``True`` |
492
|
|
|
otherwise. |
493
|
|
|
allocation_config_pushed : bool |
494
|
|
|
``False`` if :meth:`allocate` or :meth:`push_allocation_config` |
495
|
|
|
hasn't been called yet. ``True`` otherwise. |
496
|
|
|
initialization_config_pushed : bool |
497
|
|
|
``False`` if :meth:`initialize` or |
498
|
|
|
:meth:`push_initialization_config` hasn't been called yet. ``True`` |
499
|
|
|
otherwise. |
500
|
|
|
|
501
|
|
|
Notes |
502
|
|
|
----- |
503
|
|
|
To provide support for lazy initialization, apply the :meth:`lazy` |
504
|
|
|
decorator to the :meth:`__init__` method. |
505
|
|
|
|
506
|
|
|
Brick implementations *must* call the :meth:`__init__` constructor of |
507
|
|
|
their parent using `super(BlockImplementation, |
508
|
|
|
self).__init__(**kwargs)` at the *beginning* of the overriding |
509
|
|
|
`__init__`. |
510
|
|
|
|
511
|
|
|
The methods :meth:`_allocate` and :meth:`_initialize` need to be |
512
|
|
|
overridden if the brick needs to allocate shared variables and |
513
|
|
|
initialize their values in order to function. |
514
|
|
|
|
515
|
|
|
A brick can have any number of methods which apply the brick on Theano |
516
|
|
|
variables. These methods should be decorated with the |
517
|
|
|
:func:`application` decorator. |
518
|
|
|
|
519
|
|
|
If a brick has children, they must be listed in the :attr:`children` |
520
|
|
|
attribute. Moreover, if the brick wants to control the configuration of |
521
|
|
|
its children, the :meth:`_push_allocation_config` and |
522
|
|
|
:meth:`_push_initialization_config` methods need to be overridden. |
523
|
|
|
|
524
|
|
|
Examples |
525
|
|
|
-------- |
526
|
|
|
Most bricks have lazy initialization enabled. |
527
|
|
|
|
528
|
|
|
>>> import theano |
529
|
|
|
>>> from blocks.initialization import IsotropicGaussian, Constant |
530
|
|
|
>>> from blocks.bricks import Linear |
531
|
|
|
>>> linear = Linear(input_dim=5, output_dim=3, |
532
|
|
|
... weights_init=IsotropicGaussian(), |
533
|
|
|
... biases_init=Constant(0)) |
534
|
|
|
>>> x = theano.tensor.vector() |
535
|
|
|
>>> linear.apply(x) # Calls linear.allocate() automatically |
536
|
|
|
linear_apply_output |
537
|
|
|
>>> linear.initialize() # Initializes the weight matrix |
538
|
|
|
|
539
|
|
|
""" |
540
|
|
|
#: See :attr:`Brick.print_shapes` |
541
|
|
|
print_shapes = False |
542
|
|
|
|
543
|
|
|
def __init__(self, name=None, children=None): |
544
|
|
|
if name is None: |
545
|
|
|
name = self.__class__.__name__.lower() |
546
|
|
|
|
547
|
|
|
if children is None: |
548
|
|
|
children = [] |
549
|
|
|
|
550
|
|
|
self.name = name |
551
|
|
|
self.children = children |
552
|
|
|
self.parents = [] |
553
|
|
|
|
554
|
|
|
self.allocated = False |
555
|
|
|
self.allocation_config_pushed = False |
556
|
|
|
self.initialized = False |
557
|
|
|
self.initialization_config_pushed = False |
558
|
|
|
super(Brick, self).__init__() |
559
|
|
|
|
560
|
|
|
def __repr__(self): |
561
|
|
|
return repr_attrs(self, 'name') |
562
|
|
|
|
563
|
|
|
@property |
564
|
|
|
def parameters(self): |
565
|
|
|
return self._parameters |
566
|
|
|
|
567
|
|
|
@parameters.setter |
568
|
|
|
def parameters(self, value): |
569
|
|
|
self._parameters = Parameters(self, value) |
570
|
|
|
|
571
|
|
|
@property |
572
|
|
|
def children(self): |
573
|
|
|
return self._children |
574
|
|
|
|
575
|
|
|
@children.setter |
576
|
|
|
def children(self, value): |
577
|
|
|
self._children = Children(self, value) |
578
|
|
|
|
579
|
|
|
def allocate(self): |
580
|
|
|
"""Allocate shared variables for parameters. |
581
|
|
|
|
582
|
|
|
Based on the current configuration of this :class:`Brick` create |
583
|
|
|
Theano shared variables to store the parameters. After allocation, |
584
|
|
|
parameters are accessible through the :attr:`parameters` attribute. |
585
|
|
|
|
586
|
|
|
This method calls the :meth:`allocate` method of all children |
587
|
|
|
first, allowing the :meth:`_allocate` method to override the |
588
|
|
|
parameters of the children if needed. |
589
|
|
|
|
590
|
|
|
Raises |
591
|
|
|
------ |
592
|
|
|
ValueError |
593
|
|
|
If the configuration of this brick is insufficient to determine |
594
|
|
|
the number of parameters or their dimensionality to be |
595
|
|
|
initialized. |
596
|
|
|
|
597
|
|
|
Notes |
598
|
|
|
----- |
599
|
|
|
This method sets the :attr:`parameters` attribute to an empty list. |
600
|
|
|
This is in order to ensure that calls to this method completely |
601
|
|
|
reset the parameters. |
602
|
|
|
|
603
|
|
|
""" |
604
|
|
|
if hasattr(self, 'allocation_args'): |
605
|
|
|
missing_config = [arg for arg in self.allocation_args |
|
|
|
|
606
|
|
|
if getattr(self, arg) is NoneAllocation] |
607
|
|
|
if missing_config: |
608
|
|
|
raise ValueError('allocation config not set: ' |
609
|
|
|
'{}'.format(', '.join(missing_config))) |
610
|
|
|
if not self.allocation_config_pushed: |
611
|
|
|
self.push_allocation_config() |
612
|
|
|
for child in self.children: |
613
|
|
|
child.allocate() |
614
|
|
|
self.parameters = [] |
615
|
|
|
self._allocate() |
616
|
|
|
self.allocated = True |
617
|
|
|
|
618
|
|
|
def _allocate(self): |
619
|
|
|
"""Brick implementation of parameter initialization. |
620
|
|
|
|
621
|
|
|
Implement this if your brick needs to allocate its parameters. |
622
|
|
|
|
623
|
|
|
.. warning:: |
624
|
|
|
|
625
|
|
|
This method should never be called directly. Call |
626
|
|
|
:meth:`initialize` instead. |
627
|
|
|
|
628
|
|
|
""" |
629
|
|
|
pass |
630
|
|
|
|
631
|
|
|
def initialize(self): |
632
|
|
|
"""Initialize parameters. |
633
|
|
|
|
634
|
|
|
Intialize parameters, such as weight matrices and biases. |
635
|
|
|
|
636
|
|
|
Notes |
637
|
|
|
----- |
638
|
|
|
If the brick has not allocated its parameters yet, this method will |
639
|
|
|
call the :meth:`allocate` method in order to do so. |
640
|
|
|
|
641
|
|
|
""" |
642
|
|
|
if hasattr(self, 'initialization_args'): |
643
|
|
|
missing_config = [arg for arg in self.initialization_args |
|
|
|
|
644
|
|
|
if getattr(self, arg) is NoneInitialization] |
645
|
|
|
if missing_config: |
646
|
|
|
raise ValueError('initialization config not set: ' |
647
|
|
|
'{}'.format(', '.join(missing_config))) |
648
|
|
|
if not self.allocated: |
649
|
|
|
self.allocate() |
650
|
|
|
if not self.initialization_config_pushed: |
651
|
|
|
self.push_initialization_config() |
652
|
|
|
for child in self.children: |
653
|
|
|
child.initialize() |
654
|
|
|
self._initialize() |
655
|
|
|
self.initialized = True |
656
|
|
|
|
657
|
|
|
def _initialize(self): |
658
|
|
|
"""Brick implementation of parameter initialization. |
659
|
|
|
|
660
|
|
|
Implement this if your brick needs to initialize its parameters. |
661
|
|
|
|
662
|
|
|
.. warning:: |
663
|
|
|
|
664
|
|
|
This method should never be called directly. Call |
665
|
|
|
:meth:`initialize` instead. |
666
|
|
|
|
667
|
|
|
""" |
668
|
|
|
pass |
669
|
|
|
|
670
|
|
|
def push_allocation_config(self): |
671
|
|
|
"""Push the configuration for allocation to child bricks. |
672
|
|
|
|
673
|
|
|
Bricks can configure their children, based on their own current |
674
|
|
|
configuration. This will be automatically done by a call to |
675
|
|
|
:meth:`allocate`, but if you want to override the configuration of |
676
|
|
|
child bricks manually, then you can call this function manually. |
677
|
|
|
|
678
|
|
|
""" |
679
|
|
|
self._push_allocation_config() |
680
|
|
|
self.allocation_config_pushed = True |
681
|
|
|
for child in self.children: |
682
|
|
|
try: |
683
|
|
|
child.push_allocation_config() |
684
|
|
|
except Exception: |
685
|
|
|
self.allocation_config_pushed = False |
686
|
|
|
raise |
687
|
|
|
|
688
|
|
|
def _push_allocation_config(self): |
689
|
|
|
"""Brick implementation of configuring child before allocation. |
690
|
|
|
|
691
|
|
|
Implement this if your brick needs to set the configuration of its |
692
|
|
|
children before allocation. |
693
|
|
|
|
694
|
|
|
.. warning:: |
695
|
|
|
|
696
|
|
|
This method should never be called directly. Call |
697
|
|
|
:meth:`push_allocation_config` instead. |
698
|
|
|
|
699
|
|
|
""" |
700
|
|
|
pass |
701
|
|
|
|
702
|
|
|
def push_initialization_config(self): |
703
|
|
|
"""Push the configuration for initialization to child bricks. |
704
|
|
|
|
705
|
|
|
Bricks can configure their children, based on their own current |
706
|
|
|
configuration. This will be automatically done by a call to |
707
|
|
|
:meth:`initialize`, but if you want to override the configuration |
708
|
|
|
of child bricks manually, then you can call this function manually. |
709
|
|
|
|
710
|
|
|
""" |
711
|
|
|
self._push_initialization_config() |
712
|
|
|
self.initialization_config_pushed = True |
713
|
|
|
for child in self.children: |
714
|
|
|
try: |
715
|
|
|
child.push_initialization_config() |
716
|
|
|
except Exception: |
717
|
|
|
self.initialization_config_pushed = False |
718
|
|
|
raise |
719
|
|
|
|
720
|
|
|
def _push_initialization_config(self): |
721
|
|
|
"""Brick implementation of configuring child before initialization. |
722
|
|
|
|
723
|
|
|
Implement this if your brick needs to set the configuration of its |
724
|
|
|
children before initialization. |
725
|
|
|
|
726
|
|
|
.. warning:: |
727
|
|
|
|
728
|
|
|
This method should never be called directly. Call |
729
|
|
|
:meth:`push_initialization_config` instead. |
730
|
|
|
|
731
|
|
|
""" |
732
|
|
|
pass |
733
|
|
|
|
734
|
|
|
def get_dim(self, name): |
735
|
|
|
"""Get dimension of an input/output variable of a brick. |
736
|
|
|
|
737
|
|
|
Parameters |
738
|
|
|
---------- |
739
|
|
|
name : str |
740
|
|
|
The name of the variable. |
741
|
|
|
|
742
|
|
|
""" |
743
|
|
|
raise ValueError("No dimension information for {} available" |
744
|
|
|
.format(name)) |
745
|
|
|
|
746
|
|
|
def get_dims(self, names): |
747
|
|
|
"""Get list of dimensions for a set of input/output variables. |
748
|
|
|
|
749
|
|
|
Parameters |
750
|
|
|
---------- |
751
|
|
|
names : list |
752
|
|
|
The variable names. |
753
|
|
|
|
754
|
|
|
Returns |
755
|
|
|
------- |
756
|
|
|
dims : list |
757
|
|
|
The dimensions of the sources. |
758
|
|
|
|
759
|
|
|
""" |
760
|
|
|
return [self.get_dim(name) for name in names] |
761
|
|
|
|
762
|
|
|
def get_unique_path(self): |
763
|
|
|
"""Returns unique path to this brick in the application graph.""" |
764
|
|
|
if self.parents: |
765
|
|
|
parent = min(self.parents, key=attrgetter('name')) |
766
|
|
|
return parent.get_unique_path() + [self] |
767
|
|
|
else: |
768
|
|
|
return [self] |
769
|
|
|
|
770
|
|
|
def get_hierarchical_name(self, parameter, delimiter=BRICK_DELIMITER): |
771
|
|
|
"""Return hierarhical name for a parameter. |
772
|
|
|
|
773
|
|
|
Returns a path of the form ``brick1/brick2/brick3.parameter1``. The |
774
|
|
|
delimiter is configurable. |
775
|
|
|
|
776
|
|
|
Parameters |
777
|
|
|
---------- |
778
|
|
|
delimiter : str |
779
|
|
|
The delimiter used to separate brick names in the path. |
780
|
|
|
|
781
|
|
|
""" |
782
|
|
|
return '{}.{}'.format( |
783
|
|
|
delimiter.join( |
784
|
|
|
[""] + [brick.name for brick in |
785
|
|
|
self.get_unique_path()]), |
786
|
|
|
parameter.name) |
787
|
|
|
|
788
|
|
|
|
789
|
|
|
def args_to_kwargs(args, f): |
790
|
|
|
arg_names, vararg_names, _, _ = inspect.getargspec(f) |
791
|
|
|
return dict((arg_name, arg) for arg_name, arg |
792
|
|
|
in zip(arg_names + [vararg_names], args)) |
793
|
|
|
|
794
|
|
|
|
795
|
|
|
class LazyNone(object): |
796
|
|
|
def __init__(self, name): |
797
|
|
|
self.name = name |
798
|
|
|
|
799
|
|
|
def __repr__(self): |
800
|
|
|
return self.name |
801
|
|
|
|
802
|
|
|
def __bool__(self): |
803
|
|
|
return False |
804
|
|
|
|
805
|
|
|
__nonzero__ = __bool__ |
806
|
|
|
|
807
|
|
|
NoneAllocation = LazyNone('NoneAllocation') |
|
|
|
|
808
|
|
|
NoneInitialization = LazyNone('NoneInitialization') |
|
|
|
|
809
|
|
|
|
810
|
|
|
|
811
|
|
|
def lazy(allocation=None, initialization=None): |
812
|
|
|
"""Makes the initialization lazy. |
813
|
|
|
|
814
|
|
|
This decorator allows the user to define positional arguments which |
815
|
|
|
will not be needed until the allocation or initialization stage of the |
816
|
|
|
brick. If these arguments are not passed, it will automatically replace |
817
|
|
|
them with a custom ``None`` object. It is assumed that the missing |
818
|
|
|
arguments can be set after initialization by setting attributes with |
819
|
|
|
the same name. |
820
|
|
|
|
821
|
|
|
Parameters |
822
|
|
|
---------- |
823
|
|
|
allocation : list |
824
|
|
|
A list of argument names that are needed for allocation. |
825
|
|
|
initialization : list |
826
|
|
|
A list of argument names that are needed for initialization. |
827
|
|
|
|
828
|
|
|
Examples |
829
|
|
|
-------- |
830
|
|
|
>>> class SomeBrick(Brick): |
831
|
|
|
... @lazy(allocation=['a'], initialization=['b']) |
832
|
|
|
... def __init__(self, a, b, c='c', d=None): |
833
|
|
|
... print(a, b, c, d) |
834
|
|
|
>>> brick = SomeBrick('a') |
835
|
|
|
a NoneInitialization c None |
836
|
|
|
>>> brick = SomeBrick(d='d', b='b') |
837
|
|
|
NoneAllocation b c d |
838
|
|
|
|
839
|
|
|
""" |
840
|
|
|
if not allocation: |
841
|
|
|
allocation = [] |
842
|
|
|
if not initialization: |
843
|
|
|
initialization = [] |
844
|
|
|
|
845
|
|
|
def lazy_wrapper(init): |
846
|
|
|
def lazy_init(*args, **kwargs): |
847
|
|
|
self = args[0] |
848
|
|
|
self.allocation_args = (getattr(self, 'allocation_args', |
849
|
|
|
[]) + allocation) |
850
|
|
|
self.initialization_args = (getattr(self, 'initialization_args', |
851
|
|
|
[]) + initialization) |
852
|
|
|
kwargs = dict_union(args_to_kwargs(args, init), kwargs) |
853
|
|
|
for allocation_arg in allocation: |
854
|
|
|
kwargs.setdefault(allocation_arg, NoneAllocation) |
855
|
|
|
for initialization_arg in initialization: |
856
|
|
|
kwargs.setdefault(initialization_arg, NoneInitialization) |
857
|
|
|
return init(**kwargs) |
858
|
|
|
wraps(init)(lazy_init) |
859
|
|
|
return lazy_init |
860
|
|
|
return lazy_wrapper |
861
|
|
|
|
862
|
|
|
|
863
|
|
|
class ApplicationCall(Annotation): |
864
|
|
|
"""A link between the variable tags and bricks. |
865
|
|
|
|
866
|
|
|
The application call can be used to attach to an apply call auxiliary |
867
|
|
|
variables (e.g. monitors or regularizers) that do not form part of the |
868
|
|
|
main computation graph. |
869
|
|
|
|
870
|
|
|
The application call object is created before the call to the |
871
|
|
|
application method and can be accessed by specifying an |
872
|
|
|
application_call argument. |
873
|
|
|
|
874
|
|
|
Also see :class:`.Annotation`. |
875
|
|
|
|
876
|
|
|
Parameters |
877
|
|
|
---------- |
878
|
|
|
application : :class:`BoundApplication` instance |
879
|
|
|
The bound application (i.e. belong to a brick instance) object |
880
|
|
|
being called |
881
|
|
|
|
882
|
|
|
Examples |
883
|
|
|
-------- |
884
|
|
|
>>> class Foo(Brick): |
885
|
|
|
... @application |
886
|
|
|
... def apply(self, x, application_call): |
887
|
|
|
... application_call.add_auxiliary_variable(x.mean()) |
888
|
|
|
... return x + 1 |
889
|
|
|
>>> x = tensor.vector() |
890
|
|
|
>>> y = Foo().apply(x) |
891
|
|
|
>>> from blocks.filter import get_application_call |
892
|
|
|
>>> get_application_call(y) # doctest: +ELLIPSIS |
893
|
|
|
<blocks.bricks.base.ApplicationCall object at ...> |
894
|
|
|
|
895
|
|
|
""" |
896
|
|
|
def __init__(self, application): |
|
|
|
|
897
|
|
|
self.application = application |
898
|
|
|
self.metadata = {} |
899
|
|
|
super(ApplicationCall, self).__init__() |
900
|
|
|
|
901
|
|
|
def add_auxiliary_variable(self, variable, roles=None, name=None): |
902
|
|
|
if name: |
903
|
|
|
variable.name = _variable_name( |
904
|
|
|
self.application.brick.name, self.application.name, name) |
905
|
|
|
variable.tag.name = name |
906
|
|
|
name = None |
907
|
|
|
add_annotation(variable, self.application.brick) |
908
|
|
|
return super(ApplicationCall, self).add_auxiliary_variable( |
909
|
|
|
variable, roles, name) |
910
|
|
|
|
911
|
|
|
|
912
|
|
|
def application(*args, **kwargs): |
913
|
|
|
r"""Decorator for methods that apply a brick to inputs. |
914
|
|
|
|
915
|
|
|
Parameters |
916
|
|
|
---------- |
917
|
|
|
\*args, optional |
918
|
|
|
The application method to wrap. |
919
|
|
|
\*\*kwargs, optional |
920
|
|
|
Attributes to attach to this application. |
921
|
|
|
|
922
|
|
|
Notes |
923
|
|
|
----- |
924
|
|
|
This decorator replaces application methods with :class:`Application` |
925
|
|
|
instances. It also sets the attributes given as keyword arguments to |
926
|
|
|
the decorator. |
927
|
|
|
|
928
|
|
|
Note that this decorator purposely does not wrap the original method |
929
|
|
|
using e.g. :func:`~functools.wraps` or |
930
|
|
|
:func:`~functools.update_wrapper`, since that would make the class |
931
|
|
|
impossible to pickle (see notes at :class:`Application`). |
932
|
|
|
|
933
|
|
|
Examples |
934
|
|
|
-------- |
935
|
|
|
>>> class Foo(Brick): |
936
|
|
|
... @application(inputs=['x'], outputs=['y']) |
937
|
|
|
... def apply(self, x): |
938
|
|
|
... return x + 1 |
939
|
|
|
... @application |
940
|
|
|
... def other_apply(self, x): |
941
|
|
|
... return x - 1 |
942
|
|
|
>>> foo = Foo() |
943
|
|
|
>>> Foo.apply.inputs |
944
|
|
|
['x'] |
945
|
|
|
>>> foo.apply.outputs |
946
|
|
|
['y'] |
947
|
|
|
>>> Foo.other_apply # doctest: +ELLIPSIS |
948
|
|
|
<blocks.bricks.base.Application object at ...> |
949
|
|
|
|
950
|
|
|
""" |
951
|
|
|
if not ((args and not kwargs) or (not args and kwargs)): |
952
|
|
|
raise ValueError |
953
|
|
|
if args: |
954
|
|
|
application_function, = args |
955
|
|
|
application = Application(application_function) |
|
|
|
|
956
|
|
|
return application |
957
|
|
|
else: |
958
|
|
|
def wrap_application(application_function): |
959
|
|
|
application = Application(application_function) |
|
|
|
|
960
|
|
|
for key, value in kwargs.items(): |
961
|
|
|
setattr(application, key, value) |
962
|
|
|
return application |
963
|
|
|
return wrap_application |
964
|
|
|
|
965
|
|
|
|
966
|
|
|
def _variable_name(brick_name, application_name, name): |
967
|
|
|
return "{}_{}_{}".format(brick_name, application_name, name) |
968
|
|
|
|
969
|
|
|
|
970
|
|
|
def copy_and_tag(variable, brick, call, role, application_name, name): |
971
|
|
|
"""Helper method to copy a variable and annotate it.""" |
972
|
|
|
copy = variable.copy() |
973
|
|
|
# Theano name |
974
|
|
|
copy.name = _variable_name(brick.name, application_name, name) |
975
|
|
|
add_annotation(copy, brick) |
976
|
|
|
add_annotation(copy, call) |
977
|
|
|
# Blocks name |
978
|
|
|
copy.tag.name = name |
979
|
|
|
add_role(copy, role) |
980
|
|
|
return copy |
981
|
|
|
|