|
1
|
|
|
#!/usr/bin/env python |
|
2
|
|
|
# -*- coding: utf-8 -*- |
|
3
|
|
|
|
|
4
|
|
|
from collections import OrderedDict, defaultdict |
|
5
|
|
|
|
|
6
|
|
|
class TrainingController(object): |
|
7
|
|
|
""" |
|
8
|
|
|
Abstract class of training controllers. |
|
9
|
|
|
""" |
|
10
|
|
|
|
|
11
|
|
|
def bind(self, trainer): |
|
12
|
|
|
""" |
|
13
|
|
|
:type trainer: deepy.trainers.base.NeuralTrainer |
|
14
|
|
|
""" |
|
15
|
|
|
self._trainer = trainer |
|
16
|
|
|
|
|
17
|
|
|
def invoke(self): |
|
18
|
|
|
""" |
|
19
|
|
|
Return True to exit training. |
|
20
|
|
|
""" |
|
21
|
|
|
return False |
|
22
|
|
|
|
|
23
|
|
|
class TrainingValidator(TrainingController): |
|
24
|
|
|
""" |
|
25
|
|
|
A validator that allows validating the model with another graph. |
|
26
|
|
|
""" |
|
27
|
|
|
|
|
28
|
|
|
def __init__(self, valid_model=None, data_split='valid', freq=1500, save_path=None, criteria='cost', |
|
29
|
|
|
smaller_is_better=True): |
|
30
|
|
|
""" |
|
31
|
|
|
Initialize the training validator. |
|
32
|
|
|
""" |
|
33
|
|
|
self._model = valid_model |
|
34
|
|
|
self._data_split = data_split |
|
35
|
|
|
self._freq = freq |
|
36
|
|
|
self._save_path = save_path |
|
37
|
|
|
self._criteria = criteria |
|
38
|
|
|
self._smaller_is_better = smaller_is_better |
|
39
|
|
|
self._best_criteria = None |
|
40
|
|
|
self._counter = 0 |
|
41
|
|
|
|
|
42
|
|
|
def compare(self, cost_map): |
|
43
|
|
|
""" |
|
44
|
|
|
Compare to previous records and return whether the given cost is a new best. |
|
45
|
|
|
:return: True if the given cost is a new best |
|
46
|
|
|
""" |
|
47
|
|
|
cri_val = cost_map[self._criteria] |
|
48
|
|
|
if self._best_criteria is None: |
|
49
|
|
|
self._best_criteria = cri_val |
|
50
|
|
|
return True |
|
51
|
|
|
else: |
|
52
|
|
|
if self._smaller_is_better and cri_val < self._best_criteria: |
|
53
|
|
|
self._best_criteria = cri_val |
|
54
|
|
|
return True |
|
55
|
|
|
elif not self._smaller_is_better and cri_val > self._best_criteria: |
|
56
|
|
|
self._best_criteria = cri_val |
|
57
|
|
|
return True |
|
58
|
|
|
else: |
|
59
|
|
|
return False |
|
60
|
|
|
|
|
61
|
|
|
def compute(self, *x): |
|
62
|
|
|
""" |
|
63
|
|
|
Compute with the validation model given data x. |
|
64
|
|
|
""" |
|
65
|
|
|
return self._model.compute(*x) |
|
66
|
|
|
|
|
67
|
|
|
def _extract_costs(self, vars): |
|
68
|
|
|
ret_map = OrderedDict() |
|
69
|
|
|
sub_costs = OrderedDict() |
|
70
|
|
|
for k, val in vars.items(): |
|
71
|
|
|
if val.ndim == 0: |
|
72
|
|
|
if k == self._criteria: |
|
73
|
|
|
ret_map[k] = val |
|
74
|
|
|
else: |
|
75
|
|
|
sub_costs[k] = val |
|
76
|
|
|
ret_map.update(sub_costs) |
|
77
|
|
|
return ret_map |
|
78
|
|
|
|
|
79
|
|
|
def run(self, data_x): |
|
80
|
|
|
""" |
|
81
|
|
|
Run the model with validation data and return costs. |
|
82
|
|
|
""" |
|
83
|
|
|
output_vars = self.compute(*data_x) |
|
84
|
|
|
return self._extract_costs(output_vars) |
|
85
|
|
|
|
|
86
|
|
|
def invoke(self): |
|
87
|
|
|
""" |
|
88
|
|
|
This function will be called after each iteration. |
|
89
|
|
|
""" |
|
90
|
|
|
self._counter += 1 |
|
91
|
|
|
if self._counter % self._freq == 0: |
|
92
|
|
|
cnt = 0. |
|
93
|
|
|
sum_map = defaultdict(float) |
|
94
|
|
|
for x in self._trainer.get_data(self._data_split): |
|
95
|
|
|
val_map = self.run(x) |
|
96
|
|
|
if not isinstance(val_map, dict): |
|
97
|
|
|
raise Exception("Monitor.run must return a dict.") |
|
98
|
|
|
for k, val in val_map.items(): |
|
99
|
|
|
sum_map[k] += val |
|
100
|
|
|
cnt += 1 |
|
101
|
|
|
for k in sum_map: |
|
102
|
|
|
sum_map[k] /= cnt |
|
103
|
|
|
new_best = self.compare(sum_map) |
|
104
|
|
|
self._trainer.report(sum_map, self._data_split, new_best=new_best) |
|
105
|
|
|
if new_best: |
|
106
|
|
|
self._trainer.save_checkpoint(self._save_path) |
zomux /
deepy
