|
1
|
|
|
#!/usr/bin/env python |
|
2
|
|
|
# -*- coding: utf-8 -*- |
|
3
|
|
|
|
|
4
|
|
|
import os, sys |
|
5
|
|
|
import subprocess, threading |
|
6
|
|
|
import logging as loggers |
|
7
|
|
|
import time |
|
8
|
|
|
loggers.basicConfig(level=loggers.INFO) |
|
9
|
|
|
logging = loggers.getLogger(__name__) |
|
10
|
|
|
|
|
11
|
|
|
|
|
12
|
|
|
class ParallelExecutor(object): |
|
13
|
|
|
def __init__(self, cmd): |
|
14
|
|
|
self.cmd = cmd |
|
15
|
|
|
self.process = None |
|
16
|
|
|
self.stdout_output = "" |
|
17
|
|
|
self.stderr_output = "" |
|
18
|
|
|
|
|
19
|
|
|
def run(self, timeout): |
|
20
|
|
|
def target(): |
|
21
|
|
|
logging.info(self.cmd) |
|
22
|
|
|
self.process = subprocess.Popen(self.cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
|
23
|
|
|
self.stdout_output, self.stderr_output = self.process.communicate() |
|
24
|
|
|
logging.info("Thread finished") |
|
25
|
|
|
|
|
26
|
|
|
thread = threading.Thread(target=target) |
|
27
|
|
|
thread.daemon = True |
|
28
|
|
|
thread.start() |
|
29
|
|
|
|
|
30
|
|
|
thread.join(timeout) |
|
31
|
|
|
if thread.is_alive(): |
|
32
|
|
|
logging.info("Time out, terminating process") |
|
33
|
|
|
self.process.terminate() |
|
34
|
|
|
time.sleep(1) |
|
35
|
|
|
try: |
|
36
|
|
|
self.process.kill() |
|
37
|
|
|
except OSError: |
|
38
|
|
|
pass |
|
39
|
|
|
# thread.join() |
|
40
|
|
|
try: |
|
41
|
|
|
thread._stop() |
|
42
|
|
|
except: |
|
43
|
|
|
pass |
|
44
|
|
|
return self.process.returncode |
|
45
|
|
|
|
|
46
|
|
|
ERROR_KEYWORD = "Traceback (most recent call last)" |
|
47
|
|
|
|
|
48
|
|
|
def test_experiment(path, timeout=60): |
|
49
|
|
|
path = os.path.join("experiments", path) |
|
50
|
|
|
executor = ParallelExecutor("python %s" % path) |
|
51
|
|
|
executor.run(timeout=timeout) |
|
52
|
|
|
if executor.stderr_output == None: |
|
53
|
|
|
logging.info("stderr is none") |
|
54
|
|
|
return |
|
55
|
|
|
if executor.stdout_output: |
|
56
|
|
|
logging.info(executor.stdout_output) |
|
57
|
|
|
if executor.stderr_output: |
|
58
|
|
|
logging.info(executor.stderr_output) |
|
59
|
|
|
if ERROR_KEYWORD in executor.stderr_output: |
|
60
|
|
|
logging.info("------ Error was found ------") |
|
61
|
|
|
logging.info(executor.stderr_output) |
|
62
|
|
|
logging.info("-----------------------------") |
|
63
|
|
|
else: |
|
64
|
|
|
logging.info("------ No error was found ------") |
|
65
|
|
|
assert ERROR_KEYWORD not in executor.stderr_output |
|
66
|
|
|
|
|
67
|
|
|
|
|
68
|
|
|
if __name__ == '__main__': |
|
69
|
|
|
# Mnist tasks |
|
70
|
|
|
test_experiment("mnist/mlp_dropout.py", timeout=30) |
|
71
|
|
|
test_experiment("mnist/deep_convolution.py", timeout=30) |
|
72
|
|
|
# Auto-encoders |
|
73
|
|
|
test_experiment("auto_encoders/mnist_auto_encoder.py", timeout=60) |
|
74
|
|
|
test_experiment("auto_encoders/rnn_auto_encoder.py", timeout=120) |
|
75
|
|
|
# Highway networks |
|
76
|
|
|
test_experiment("highway_networks/mnist_highway.py", timeout=120) |
|
77
|
|
|
# TODO: recurrent neural networks |
|
78
|
|
|
# test_experiment("lm/baseline_rnnlm.py", timeout=180) |
|
79
|
|
|
# Scipy trainers |
|
80
|
|
|
test_experiment("scipy_training/mnist_cg.py", timeout=180) |
|
81
|
|
|
# Tutorials |
|
82
|
|
|
test_experiment("tutorials/tutorial1.py", timeout=120) |
|
83
|
|
|
# test_experiment("tutorials/tutorial2.py", timeout=120) |
|
84
|
|
|
# test_experiment("tutorials/tutorial3.py", timeout=120) |
|
85
|
|
|
sys.exit(0) |
|
86
|
|
|
|
zomux /
deepy
