|
1
|
|
|
# sns.set(color_codes=True) |
|
2
|
|
|
# sns.set_palette(sns.color_palette("RdBu", n_colors=7)) |
|
3
|
|
|
# sns.set(rc={'figure.figsize':(12, 9)}) |
|
4
|
|
|
|
|
5
|
|
|
import numpy as np |
|
6
|
|
|
import pandas as pd |
|
7
|
|
|
import matplotlib.pyplot as plt |
|
8
|
|
|
|
|
9
|
|
|
from sklearn.datasets import load_breast_cancer |
|
10
|
|
|
|
|
11
|
|
|
from hyperactive import HillClimbingOptimizer |
|
12
|
|
|
from hyperactive import StochasticHillClimbingOptimizer |
|
13
|
|
|
from hyperactive import TabuOptimizer |
|
14
|
|
|
from hyperactive import RandomSearchOptimizer |
|
15
|
|
|
from hyperactive import RandomRestartHillClimbingOptimizer |
|
16
|
|
|
from hyperactive import RandomAnnealingOptimizer |
|
17
|
|
|
from hyperactive import SimulatedAnnealingOptimizer |
|
18
|
|
|
from hyperactive import StochasticTunnelingOptimizer |
|
19
|
|
|
from hyperactive import ParallelTemperingOptimizer |
|
20
|
|
|
from hyperactive import ParticleSwarmOptimizer |
|
21
|
|
|
from hyperactive import EvolutionStrategyOptimizer |
|
22
|
|
|
from hyperactive import BayesianOptimizer |
|
23
|
|
|
|
|
24
|
|
|
breast_cancer_data = load_breast_cancer() |
|
25
|
|
|
X = breast_cancer_data.data |
|
26
|
|
|
y = breast_cancer_data.target |
|
27
|
|
|
|
|
28
|
|
|
opt_list = { |
|
29
|
|
|
"Hill Climbing": HillClimbingOptimizer, |
|
30
|
|
|
"Stoch. Hill Climbing": StochasticHillClimbingOptimizer, |
|
31
|
|
|
"Tabu Search": TabuOptimizer, |
|
32
|
|
|
"Random Search": RandomSearchOptimizer, |
|
33
|
|
|
"Rand. Rest. Hill Climbing": RandomRestartHillClimbingOptimizer, |
|
34
|
|
|
"Random Annealing": RandomAnnealingOptimizer, |
|
35
|
|
|
"Simulated Annealing": SimulatedAnnealingOptimizer, |
|
36
|
|
|
"Stochastic Tunneling": StochasticTunnelingOptimizer, |
|
37
|
|
|
"Parallel Tempering": ParallelTemperingOptimizer, |
|
38
|
|
|
"Particle Swarm": ParticleSwarmOptimizer, |
|
39
|
|
|
"Evolution Strategy": EvolutionStrategyOptimizer, |
|
40
|
|
|
"Bayesian Optimization": BayesianOptimizer, |
|
41
|
|
|
} |
|
42
|
|
|
|
|
43
|
|
|
search_config = { |
|
44
|
|
|
"sklearn.ensemble.GradientBoostingClassifier": { |
|
45
|
|
|
"n_estimators": range(1, 102, 1), |
|
46
|
|
|
"max_depth": range(1, 32, 1), |
|
47
|
|
|
} |
|
48
|
|
|
} |
|
49
|
|
|
n_iter = 150 |
|
50
|
|
|
|
|
51
|
|
|
opt_dict = {"cv": 5, "n_jobs": 1, "memory": False, "verbosity": 0} |
|
52
|
|
|
|
|
53
|
|
|
|
|
54
|
|
|
def _plot(plt, pos, score): |
|
55
|
|
|
df = pd.DataFrame( |
|
56
|
|
|
{"n_estimators": pos[:, 0], "max_depth": pos[:, 1], "score": score} |
|
57
|
|
|
) |
|
58
|
|
|
|
|
59
|
|
|
# plot |
|
60
|
|
|
plt.plot( |
|
61
|
|
|
"n_estimators", |
|
62
|
|
|
"max_depth", |
|
63
|
|
|
data=df, |
|
64
|
|
|
linestyle="-", |
|
65
|
|
|
marker=",", |
|
66
|
|
|
color="gray", |
|
67
|
|
|
alpha=0.33, |
|
68
|
|
|
) |
|
69
|
|
|
plt.scatter( |
|
70
|
|
|
df["n_estimators"], |
|
71
|
|
|
df["max_depth"], |
|
72
|
|
|
c=df["score"], |
|
73
|
|
|
marker="H", |
|
74
|
|
|
s=50, |
|
75
|
|
|
vmin=0.88, |
|
76
|
|
|
vmax=0.98, |
|
77
|
|
|
) |
|
78
|
|
|
|
|
79
|
|
|
return plt |
|
80
|
|
|
|
|
81
|
|
|
|
|
82
|
|
|
for opt in opt_list: |
|
83
|
|
|
n_iter_temp = n_iter |
|
84
|
|
|
opt_dict_temp = opt_dict |
|
85
|
|
|
|
|
86
|
|
|
if opt == "Parallel Tempering": |
|
87
|
|
|
n_iter_temp = int(n_iter / 10) |
|
88
|
|
|
opt_dict_temp["system_temps"] = [0.001, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100] |
|
89
|
|
|
|
|
90
|
|
|
if opt == "Particle Swarm": |
|
91
|
|
|
n_iter_temp = int(n_iter / 10) |
|
92
|
|
|
opt_dict_temp["n_part"] = 10 |
|
93
|
|
|
|
|
94
|
|
|
if opt == "Evolution Strategy": |
|
95
|
|
|
n_iter_temp = int(n_iter / 10) |
|
96
|
|
|
opt_dict_temp["individuals"] = 10 |
|
97
|
|
|
|
|
98
|
|
|
opt_ = opt_list[opt] |
|
99
|
|
|
opt_ = opt_( |
|
100
|
|
|
search_config, n_iter=n_iter_temp, get_search_path=True, **opt_dict_temp |
|
101
|
|
|
) |
|
102
|
|
|
opt_.fit(X, y) |
|
103
|
|
|
|
|
104
|
|
|
pos_list = opt_.pos_list |
|
105
|
|
|
score_list = opt_.score_list |
|
106
|
|
|
|
|
107
|
|
|
pos_list = np.array(pos_list) |
|
108
|
|
|
score_list = np.array(score_list) |
|
109
|
|
|
|
|
110
|
|
|
plt.figure(figsize=(15, 5)) |
|
111
|
|
|
plt.set_cmap("jet") |
|
112
|
|
|
|
|
113
|
|
|
pos_list = np.swapaxes(pos_list, 0, 1) |
|
114
|
|
|
score_list = np.swapaxes(score_list, 0, 1) |
|
115
|
|
|
|
|
116
|
|
|
# print("\npos_list\n", pos_list, pos_list.shape) |
|
117
|
|
|
# print("score_list\n", score_list, score_list.shape) |
|
118
|
|
|
|
|
119
|
|
|
for pos, score in zip(pos_list, score_list): |
|
120
|
|
|
# print(pos[:, 0]) |
|
121
|
|
|
# print(pos[:, 1]) |
|
122
|
|
|
# print(score, "\n") |
|
123
|
|
|
plt = _plot(plt, pos, score) |
|
124
|
|
|
|
|
125
|
|
|
plt.title(opt) |
|
126
|
|
|
plt.xlabel("n_estimators") |
|
127
|
|
|
plt.ylabel("max_depth") |
|
128
|
|
|
|
|
129
|
|
|
plt.xlim((0, 100)) |
|
130
|
|
|
plt.ylim((0, 30)) |
|
131
|
|
|
plt.colorbar() |
|
132
|
|
|
|
|
133
|
|
|
plt.tight_layout() |
|
134
|
|
|
plt.savefig("search_path_" + opt + ".png") |
|
135
|
|
|
|
SimonBlanke /
Hyperactive
