1
|
|
|
"""Evaluate Theano variables on auxiliary data and during training.""" |
2
|
|
|
from functools import partial |
3
|
|
|
import logging |
4
|
|
|
from abc import ABCMeta, abstractmethod |
5
|
|
|
|
6
|
|
|
from six import add_metaclass |
7
|
|
|
from theano import tensor |
8
|
|
|
from theano.ifelse import ifelse |
9
|
|
|
|
10
|
|
|
from blocks.utils import shared_like |
11
|
|
|
|
12
|
|
|
logger = logging.getLogger(__name__) |
13
|
|
|
|
14
|
|
|
|
15
|
|
|
@add_metaclass(ABCMeta) |
16
|
|
|
class AggregationScheme(object): |
17
|
|
|
"""How to incrementally evaluate a Theano variable over minibatches. |
18
|
|
|
|
19
|
|
|
An AggregationScheme allocates :class:`Aggregator` that can |
20
|
|
|
incrementally compute the value of a Theano variable on a full dataset |
21
|
|
|
by aggregating partial results computed on multiple batches. |
22
|
|
|
|
23
|
|
|
The AggregationScheme should be attached via the tag |
24
|
|
|
``aggregation_scheme`` to a Theano variable which computes the desired |
25
|
|
|
value on a single batch. |
26
|
|
|
|
27
|
|
|
Parameters |
28
|
|
|
---------- |
29
|
|
|
variable: :class:`~tensor.TensorVariable` |
30
|
|
|
The variable that holds the desired value on a single batch. |
31
|
|
|
|
32
|
|
|
""" |
33
|
|
|
def __init__(self, variable): |
34
|
|
|
self.variable = variable |
35
|
|
|
|
36
|
|
|
@abstractmethod |
37
|
|
|
def get_aggregator(self): |
38
|
|
|
"""Return a new Aggregator for this variable.""" |
39
|
|
|
pass |
40
|
|
|
|
41
|
|
|
|
42
|
|
|
class Aggregator(object): |
43
|
|
|
"""An Aggregator incrementally evaluates a Theano variable on a dataset. |
44
|
|
|
|
45
|
|
|
.. warning:: |
46
|
|
|
The Aggregators should never be created directly. Instead use the |
47
|
|
|
:meth:`AggregationScheme.get_aggregator` method. |
48
|
|
|
|
49
|
|
|
Example usages are: |
50
|
|
|
|
51
|
|
|
* compute the mean of some value over examples, sequence lengths etc. |
52
|
|
|
* track a parameter of a model |
53
|
|
|
* monitor a penalty |
54
|
|
|
|
55
|
|
|
The Aggregator maintains a set of Theano sharer values called |
56
|
|
|
accumulators and specifies how they should be initialized, and |
57
|
|
|
updated with incremental calculations. Finally, it |
58
|
|
|
provides a Theano variable that reads the accumulators |
59
|
|
|
and computes the final value. |
60
|
|
|
|
61
|
|
|
Parameters |
62
|
|
|
---------- |
63
|
|
|
aggregation_scheme : :class:`AggregationScheme` |
64
|
|
|
The aggregation scheme that constructed this Aggregator |
65
|
|
|
initialization_updates : list of Theano updates |
66
|
|
|
Updates that specify how to initialize shared variables of |
67
|
|
|
this Aggregator. *Can only refer to shared variables and |
68
|
|
|
constants.* |
69
|
|
|
accumulation_updates : list of Theano updates |
70
|
|
|
Updates that specify how a new batch of data gets processed |
71
|
|
|
by this Aggregator. *Can refer to model inputs.* |
72
|
|
|
readout_variable : :class:`~tensor.TensorVariable` |
73
|
|
|
Theano variable that holds the final value based on aggregated |
74
|
|
|
partial results. *readout_variable must only consist of shared |
75
|
|
|
variables and constants.* |
76
|
|
|
|
77
|
|
|
Attributes |
78
|
|
|
---------- |
79
|
|
|
All constructor parameters are accessible as attributes. |
80
|
|
|
|
81
|
|
|
""" |
82
|
|
|
def __init__(self, aggregation_scheme, initialization_updates=None, |
83
|
|
|
accumulation_updates=None, readout_variable=None): |
84
|
|
|
self.aggregation_scheme = aggregation_scheme |
85
|
|
|
self.readout_variable = readout_variable |
86
|
|
|
|
87
|
|
|
if initialization_updates is None: |
88
|
|
|
initialization_updates = [] |
89
|
|
|
if accumulation_updates is None: |
90
|
|
|
accumulation_updates = [] |
91
|
|
|
self.initialization_updates = initialization_updates |
92
|
|
|
self.accumulation_updates = accumulation_updates |
93
|
|
|
|
94
|
|
|
|
95
|
|
|
class Mean(AggregationScheme): |
96
|
|
|
"""Aggregation scheme which computes the mean. |
97
|
|
|
|
98
|
|
|
Parameters |
99
|
|
|
---------- |
100
|
|
|
numerator : :class:`~tensor.TensorVariable` |
101
|
|
|
Theano variable for the numerator e.g. the likelihood |
102
|
|
|
denominator : :class:`~tensor.TensorVariable` |
103
|
|
|
Theano variable for the denominator e.g. the batch size |
104
|
|
|
|
105
|
|
|
""" |
106
|
|
|
def __init__(self, numerator, denominator): |
|
|
|
|
107
|
|
|
self.numerator = numerator |
108
|
|
|
self.denominator = denominator |
109
|
|
|
|
110
|
|
|
def get_aggregator(self): |
111
|
|
|
initialized = shared_like(0.) |
112
|
|
|
numerator_acc = shared_like(self.numerator) |
113
|
|
|
denominator_acc = shared_like(self.denominator) |
114
|
|
|
|
115
|
|
|
# Dummy default expression to use as the previously-aggregated |
116
|
|
|
# value, that has the same shape as the new result |
117
|
|
|
numerator_zeros = tensor.as_tensor(self.numerator).zeros_like() |
118
|
|
|
denominator_zeros = tensor.as_tensor(self.denominator).zeros_like() |
119
|
|
|
|
120
|
|
|
conditional_update_num = self.numerator + ifelse(initialized, |
121
|
|
|
numerator_acc, |
122
|
|
|
numerator_zeros) |
123
|
|
|
conditional_update_den = self.denominator + ifelse(initialized, |
124
|
|
|
denominator_acc, |
125
|
|
|
denominator_zeros) |
126
|
|
|
|
127
|
|
|
initialization_updates = [(numerator_acc, |
128
|
|
|
tensor.zeros_like(numerator_acc)), |
129
|
|
|
(denominator_acc, |
130
|
|
|
tensor.zeros_like(denominator_acc)), |
131
|
|
|
(initialized, |
132
|
|
|
tensor.zeros_like(initialized))] |
133
|
|
|
accumulation_updates = [(numerator_acc, |
134
|
|
|
conditional_update_num), |
135
|
|
|
(denominator_acc, |
136
|
|
|
conditional_update_den), |
137
|
|
|
(initialized, tensor.ones_like(initialized))] |
138
|
|
|
aggregator = Aggregator(aggregation_scheme=self, |
139
|
|
|
initialization_updates=initialization_updates, |
140
|
|
|
accumulation_updates=accumulation_updates, |
141
|
|
|
readout_variable=(numerator_acc / |
142
|
|
|
denominator_acc)) |
143
|
|
|
return aggregator |
144
|
|
|
|
145
|
|
|
|
146
|
|
|
def mean(numerator, denominator=1.): |
147
|
|
|
"""Mean of quantity (numerator) over a number (denominator) values.""" |
148
|
|
|
variable = numerator / denominator |
149
|
|
|
variable.tag.aggregation_scheme = Mean(numerator, denominator) |
150
|
|
|
variable.name = numerator.name |
151
|
|
|
return variable |
152
|
|
|
|
153
|
|
|
|
154
|
|
|
class _DataIndependent(AggregationScheme): |
155
|
|
|
"""Dummy aggregation scheme for values that don't depend on data.""" |
156
|
|
|
def get_aggregator(self): |
157
|
|
|
return Aggregator(aggregation_scheme=self, |
158
|
|
|
initialization_updates=[], |
159
|
|
|
accumulation_updates=[], |
160
|
|
|
readout_variable=self.variable) |
161
|
|
|
|
162
|
|
|
|
163
|
|
|
class TakeLast(AggregationScheme): |
164
|
|
|
"""Aggregation scheme which remembers only the last value.""" |
165
|
|
|
def get_aggregator(self): |
166
|
|
|
self.storage = shared_like(self.variable) |
167
|
|
|
return Aggregator(aggregation_scheme=self, |
168
|
|
|
initialization_updates=[ |
169
|
|
|
(self.storage, tensor.zeros_like(self.storage))], |
170
|
|
|
accumulation_updates=[(self.storage, self.variable)], |
171
|
|
|
readout_variable=self.storage) |
172
|
|
|
|
173
|
|
|
|
174
|
|
|
def _simple_aggregation(scheme, variable): |
175
|
|
|
variable = variable.copy(variable.name) |
176
|
|
|
variable.tag.aggregation_scheme = scheme(variable) |
177
|
|
|
return variable |
178
|
|
|
|
179
|
|
|
|
180
|
|
|
take_last = partial(_simple_aggregation, TakeLast) |
|
|
|
|
181
|
|
|
|
182
|
|
|
|
183
|
|
|
class Minimum(AggregationScheme): |
184
|
|
|
"""Aggregation scheme which remembers only the minimum value.""" |
185
|
|
|
def _build_aggregator(self, accumulate_update): |
186
|
|
|
initialized = shared_like(0.) |
187
|
|
|
accumulate = ifelse(initialized, accumulate_update, self.variable) |
188
|
|
|
return Aggregator(aggregation_scheme=self, |
189
|
|
|
initialization_updates=[ |
190
|
|
|
(self.storage, tensor.zeros_like(self.storage)), |
191
|
|
|
(initialized, tensor.zeros_like(initialized)) |
192
|
|
|
], |
193
|
|
|
accumulation_updates=[ |
194
|
|
|
(self.storage, accumulate), |
195
|
|
|
(initialized, tensor.ones_like(initialized)) |
196
|
|
|
], |
197
|
|
|
readout_variable=self.storage) |
198
|
|
|
|
199
|
|
|
def get_aggregator(self): |
200
|
|
|
self.storage = shared_like(self.variable) |
201
|
|
|
return self._build_aggregator(tensor.minimum(self.storage, |
202
|
|
|
self.variable)) |
203
|
|
|
|
204
|
|
|
minimum = partial(_simple_aggregation, Minimum) |
|
|
|
|
205
|
|
|
|
206
|
|
|
|
207
|
|
|
class Maximum(Minimum): |
208
|
|
|
"""Aggregation scheme which remembers only the maximum value.""" |
209
|
|
|
def get_aggregator(self): |
210
|
|
|
self.storage = shared_like(self.variable) |
211
|
|
|
return self._build_aggregator(tensor.maximum(self.storage, |
212
|
|
|
self.variable)) |
213
|
|
|
|
214
|
|
|
maximum = partial(_simple_aggregation, Maximum) |
|
|
|
|
215
|
|
|
|
216
|
|
|
|
217
|
|
|
class Concatenate(Minimum): |
218
|
|
|
"""Aggregation scheme which remembers values from all batches. |
219
|
|
|
|
220
|
|
|
Parameters |
221
|
|
|
---------- |
222
|
|
|
variable: :class:`~tensor.TensorVariable` |
223
|
|
|
The variable that holds the desired value on a single batch. |
224
|
|
|
|
225
|
|
|
""" |
226
|
|
|
def __init__(self, variable): |
227
|
|
|
# Add an extra axis to concatenate along. Must be non-broadcastable |
228
|
|
|
# for concatenate to always work. |
229
|
|
|
variable = (tensor.unbroadcast(tensor.shape_padleft(variable, 1), 0) |
230
|
|
|
.copy(variable.name)) |
231
|
|
|
super(Concatenate, self).__init__(variable) |
232
|
|
|
|
233
|
|
|
def get_aggregator(self): |
234
|
|
|
self.storage = shared_like(self.variable) |
235
|
|
|
return self._build_aggregator(tensor.concatenate([self.storage, |
236
|
|
|
self.variable])) |
237
|
|
|
|
238
|
|
|
concatenate = partial(_simple_aggregation, Concatenate) |
|
|
|
|
239
|
|
|
|
240
|
|
|
|
241
|
|
|
@add_metaclass(ABCMeta) |
|
|
|
|
242
|
|
|
class MonitoredQuantity(object): |
243
|
|
|
"""The base class for monitored-quantities. |
244
|
|
|
|
245
|
|
|
To monitor a non-Theano quantity in Blocks you have to implement this |
246
|
|
|
interface for it. The initialize method initializes accumulators and |
247
|
|
|
the parameters needed to compute this quantity, aggregate method |
248
|
|
|
aggregates results for every batch, and finally readout is called |
249
|
|
|
to get the aggregated results. |
250
|
|
|
|
251
|
|
|
Attributes |
252
|
|
|
---------- |
253
|
|
|
requires : list |
254
|
|
|
List of Theano variables needed to calculate this quantity. |
255
|
|
|
name : str |
256
|
|
|
The name of monitored quantity which appears in the log. |
257
|
|
|
|
258
|
|
|
See Also |
259
|
|
|
-------- |
260
|
|
|
:class:`~blocks.monitoring.evaluators.DatasetEvaluator` |
261
|
|
|
:class:`~blocks.extensions.DataStreamMonitoring` |
262
|
|
|
|
263
|
|
|
""" |
264
|
|
|
def __init__(self, requires=None, name=None): |
265
|
|
|
if requires is None: |
266
|
|
|
requires = [] |
267
|
|
|
self.requires = requires |
268
|
|
|
self.name = name |
269
|
|
|
|
270
|
|
|
@abstractmethod |
271
|
|
|
def initialize(self): |
272
|
|
|
"""Initialize accumulators for this monitored quantity.""" |
273
|
|
|
pass |
274
|
|
|
|
275
|
|
|
@abstractmethod |
276
|
|
|
def aggregate(self, *args): |
277
|
|
|
r"""Aggregate results for every batch. |
278
|
|
|
|
279
|
|
|
\*args : list of :class:`~numpy.ndarray` |
280
|
|
|
The values of the variables required to aggregate the |
281
|
|
|
value of the quantity. |
282
|
|
|
|
283
|
|
|
""" |
284
|
|
|
pass |
285
|
|
|
|
286
|
|
|
@abstractmethod |
287
|
|
|
def get_aggregated_value(self): |
288
|
|
|
"""Obtain the result of aggregation.""" |
289
|
|
|
pass |
290
|
|
|
|
It is generally advisable to initialize the super-class by calling its
__init__
method: