1
|
|
|
"""Bricks that are interfaces and/or mixins.""" |
2
|
|
|
import numpy |
3
|
|
|
from six import add_metaclass |
4
|
|
|
from theano.sandbox.rng_mrg import MRG_RandomStreams |
5
|
|
|
|
6
|
|
|
from ..config import config |
7
|
|
|
from .base import _Brick, Brick, lazy |
8
|
|
|
|
9
|
|
|
|
10
|
|
|
class ActivationDocumentation(_Brick): |
11
|
|
|
"""Dynamically adds documentation to activations. |
12
|
|
|
|
13
|
|
|
Notes |
14
|
|
|
----- |
15
|
|
|
See http://bugs.python.org/issue12773. |
16
|
|
|
|
17
|
|
|
""" |
18
|
|
|
def __new__(cls, name, bases, classdict): |
|
|
|
|
19
|
|
|
classdict['__doc__'] = \ |
20
|
|
|
"""Elementwise application of {0} function.""".format(name.lower()) |
21
|
|
|
if 'apply' in classdict: |
22
|
|
|
classdict['apply'].__doc__ = \ |
23
|
|
|
"""Apply the {0} function element-wise. |
24
|
|
|
|
25
|
|
|
Parameters |
26
|
|
|
---------- |
27
|
|
|
input_ : :class:`~tensor.TensorVariable` |
28
|
|
|
Theano variable to apply {0} to, element-wise. |
29
|
|
|
|
30
|
|
|
Returns |
31
|
|
|
------- |
32
|
|
|
output : :class:`~tensor.TensorVariable` |
33
|
|
|
The input with the activation function applied. |
34
|
|
|
|
35
|
|
|
""".format(name.lower()) |
36
|
|
|
return super(ActivationDocumentation, cls).__new__(cls, name, bases, |
37
|
|
|
classdict) |
38
|
|
|
|
39
|
|
|
|
40
|
|
|
@add_metaclass(ActivationDocumentation) |
41
|
|
|
class Activation(Brick): |
42
|
|
|
"""A base class for simple, element-wise activation functions. |
43
|
|
|
|
44
|
|
|
This base class ensures that activation functions are automatically |
45
|
|
|
documented using the :class:`ActivationDocumentation` metaclass. |
46
|
|
|
|
47
|
|
|
""" |
48
|
|
|
pass |
49
|
|
|
|
50
|
|
|
|
51
|
|
|
class Feedforward(Brick): |
52
|
|
|
"""Declares an interface for bricks with one input and one output. |
53
|
|
|
|
54
|
|
|
Many bricks have just one input and just one output (activations, |
55
|
|
|
:class:`Linear`, :class:`MLP`). To make such bricks interchangable |
56
|
|
|
in most contexts they should share an interface for configuring |
57
|
|
|
their input and output dimensions. This brick declares such an |
58
|
|
|
interface. |
59
|
|
|
|
60
|
|
|
Attributes |
61
|
|
|
---------- |
62
|
|
|
input_dim : int |
63
|
|
|
The input dimension of the brick. |
64
|
|
|
output_dim : int |
65
|
|
|
The output dimension of the brick. |
66
|
|
|
|
67
|
|
|
""" |
68
|
|
|
def __getattr__(self, name): |
69
|
|
|
message = ("'{}' object does not have an attribute '{}'" |
70
|
|
|
.format(self.__class__.__name__, name)) |
71
|
|
|
if name in ('input_dim', 'output_dim'): |
72
|
|
|
message += (" (which is a part of 'Feedforward' interface it" |
73
|
|
|
" claims to support)") |
74
|
|
|
raise AttributeError(message) |
75
|
|
|
|
76
|
|
|
|
77
|
|
|
class RNGMixin(object): |
78
|
|
|
"""Mixin for initialization random number generators.""" |
79
|
|
|
seed_rng = numpy.random.RandomState(config.default_seed) |
80
|
|
|
|
81
|
|
|
@property |
82
|
|
|
def seed(self): |
83
|
|
|
if getattr(self, '_seed', None) is not None: |
84
|
|
|
return self._seed |
85
|
|
|
else: |
86
|
|
|
self._seed = self.seed_rng.randint( |
87
|
|
|
numpy.iinfo(numpy.int32).max) |
88
|
|
|
return self._seed |
89
|
|
|
|
90
|
|
|
@seed.setter |
91
|
|
|
def seed(self, value): |
92
|
|
|
if hasattr(self, '_seed'): |
93
|
|
|
raise AttributeError("seed already set") |
94
|
|
|
self._seed = value |
95
|
|
|
|
96
|
|
|
@property |
97
|
|
|
def rng(self): |
98
|
|
|
if getattr(self, '_rng', None) is not None: |
99
|
|
|
return self._rng |
100
|
|
|
else: |
101
|
|
|
self._rng = numpy.random.RandomState(self.seed) |
102
|
|
|
return self._rng |
103
|
|
|
|
104
|
|
|
@rng.setter |
105
|
|
|
def rng(self, rng): |
106
|
|
|
self._rng = rng |
107
|
|
|
|
108
|
|
|
|
109
|
|
|
class Initializable(RNGMixin, Brick): |
110
|
|
|
"""Base class for bricks which push parameter initialization. |
111
|
|
|
|
112
|
|
|
Many bricks will initialize children which perform a linear |
113
|
|
|
transformation, often with biases. This brick allows the weights |
114
|
|
|
and biases initialization to be configured in the parent brick and |
115
|
|
|
pushed down the hierarchy. |
116
|
|
|
|
117
|
|
|
Parameters |
118
|
|
|
---------- |
119
|
|
|
weights_init : object |
120
|
|
|
A `NdarrayInitialization` instance which will be used by to |
121
|
|
|
initialize the weight matrix. Required by |
122
|
|
|
:meth:`~.Brick.initialize`. |
123
|
|
|
biases_init : :obj:`object`, optional |
124
|
|
|
A `NdarrayInitialization` instance that will be used to initialize |
125
|
|
|
the biases. Required by :meth:`~.Brick.initialize` when `use_bias` |
126
|
|
|
is `True`. Only supported by bricks for which :attr:`has_biases` is |
127
|
|
|
``True``. |
128
|
|
|
use_bias : :obj:`bool`, optional |
129
|
|
|
Whether to use a bias. Defaults to `True`. Required by |
130
|
|
|
:meth:`~.Brick.initialize`. Only supported by bricks for which |
131
|
|
|
:attr:`has_biases` is ``True``. |
132
|
|
|
rng : :class:`numpy.random.RandomState` |
133
|
|
|
|
134
|
|
|
Attributes |
135
|
|
|
---------- |
136
|
|
|
has_biases : bool |
137
|
|
|
``False`` if the brick does not support biases, and only has |
138
|
|
|
:attr:`weights_init`. For an example of this, see |
139
|
|
|
:class:`.Bidirectional`. If this is ``False``, the brick does not |
140
|
|
|
support the arguments ``biases_init`` or ``use_bias``. |
141
|
|
|
|
142
|
|
|
""" |
143
|
|
|
has_biases = True |
144
|
|
|
|
145
|
|
|
@lazy() |
146
|
|
|
def __init__(self, weights_init=None, biases_init=None, use_bias=None, |
147
|
|
|
seed=None, **kwargs): |
148
|
|
|
super(Initializable, self).__init__(**kwargs) |
149
|
|
|
self.weights_init = weights_init |
150
|
|
|
if self.has_biases: |
151
|
|
|
self.biases_init = biases_init |
152
|
|
|
elif biases_init is not None or not use_bias: |
153
|
|
|
raise ValueError("This brick does not support biases config") |
154
|
|
|
if use_bias is not None: |
155
|
|
|
self.use_bias = use_bias |
156
|
|
|
self.seed = seed |
157
|
|
|
|
158
|
|
|
def _push_initialization_config(self): |
159
|
|
|
for child in self.children: |
160
|
|
|
if isinstance(child, Initializable): |
161
|
|
|
child.rng = self.rng |
162
|
|
|
if self.weights_init: |
163
|
|
|
child.weights_init = self.weights_init |
164
|
|
|
if hasattr(self, 'biases_init') and self.biases_init: |
165
|
|
|
for child in self.children: |
166
|
|
|
if (isinstance(child, Initializable) and |
167
|
|
|
hasattr(child, 'biases_init')): |
168
|
|
|
child.biases_init = self.biases_init |
169
|
|
|
|
170
|
|
|
|
171
|
|
|
class LinearLike(Initializable): |
172
|
|
|
"""Initializable subclass with logic for :class:`Linear`-like classes. |
173
|
|
|
|
174
|
|
|
Notes |
175
|
|
|
----- |
176
|
|
|
Provides `W` and `b` properties that can be overridden in subclasses |
177
|
|
|
to implement pre-application transformations on the weights and |
178
|
|
|
biases. Application methods should refer to ``self.W`` and ``self.b`` |
179
|
|
|
rather than accessing the parameters list directly. |
180
|
|
|
|
181
|
|
|
This assumes a layout of the parameters list with the weights coming |
182
|
|
|
first and biases (if ``use_bias`` is True) coming second. |
183
|
|
|
|
184
|
|
|
""" |
185
|
|
|
@property |
186
|
|
|
def W(self): |
187
|
|
|
return self.parameters[0] |
188
|
|
|
|
189
|
|
|
@property |
190
|
|
|
def b(self): |
191
|
|
|
if getattr(self, 'use_bias', True): |
192
|
|
|
return self.parameters[1] |
193
|
|
|
else: |
194
|
|
|
raise AttributeError('use_bias is False') |
195
|
|
|
|
196
|
|
|
def _initialize(self): |
197
|
|
|
# Use self.parameters[] references in case W and b are overridden |
198
|
|
|
# to return non-shared-variables. |
199
|
|
|
if getattr(self, 'use_bias', True): |
200
|
|
|
self.biases_init.initialize(self.parameters[1], self.rng) |
201
|
|
|
self.weights_init.initialize(self.parameters[0], self.rng) |
202
|
|
|
|
203
|
|
|
|
204
|
|
|
class Random(Brick): |
205
|
|
|
"""A mixin class for Bricks which need Theano RNGs. |
206
|
|
|
|
207
|
|
|
Parameters |
208
|
|
|
---------- |
209
|
|
|
theano_seed : int or list, optional |
210
|
|
|
Seed to use for a |
211
|
|
|
:class:`~theano.sandbox.rng_mrg.MRG_RandomStreams` object. |
212
|
|
|
|
213
|
|
|
""" |
214
|
|
|
seed_rng = numpy.random.RandomState(config.default_seed) |
215
|
|
|
|
216
|
|
|
def __init__(self, theano_seed=None, **kwargs): |
217
|
|
|
super(Random, self).__init__(**kwargs) |
218
|
|
|
self.theano_seed = theano_seed |
219
|
|
|
|
220
|
|
|
@property |
221
|
|
|
def theano_seed(self): |
222
|
|
|
if getattr(self, '_theano_seed', None) is not None: |
223
|
|
|
return self._theano_seed |
224
|
|
|
else: |
225
|
|
|
self._theano_seed = self.seed_rng.randint( |
226
|
|
|
numpy.iinfo(numpy.int32).max) |
227
|
|
|
return self._theano_seed |
228
|
|
|
|
229
|
|
|
@theano_seed.setter |
230
|
|
|
def theano_seed(self, value): |
231
|
|
|
if hasattr(self, '_theano_seed'): |
232
|
|
|
raise AttributeError("seed already set") |
233
|
|
|
self._theano_seed = value |
234
|
|
|
|
235
|
|
|
@property |
236
|
|
|
def theano_rng(self): |
237
|
|
|
"""Returns Brick's Theano RNG, or a default one. |
238
|
|
|
|
239
|
|
|
The default seed can be set through ``blocks.config``. |
240
|
|
|
|
241
|
|
|
""" |
242
|
|
|
if not hasattr(self, '_theano_rng'): |
243
|
|
|
self._theano_rng = MRG_RandomStreams(self.theano_seed) |
244
|
|
|
return self._theano_rng |
245
|
|
|
|
246
|
|
|
@theano_rng.setter |
247
|
|
|
def theano_rng(self, theano_rng): |
248
|
|
|
self._theano_rng = theano_rng |
249
|
|
|
|