|
1
|
|
|
""" |
|
2
|
|
|
Fabfile to setup and deploy gage-web |
|
3
|
|
|
""" |
|
4
|
|
|
from fabric.api import cd, run, local, env, sudo, put, prompt, lcd, prefix |
|
5
|
|
|
from fabric.contrib.console import confirm |
|
6
|
|
|
from fabtools import require |
|
7
|
|
|
import fabtools |
|
8
|
|
|
|
|
9
|
|
|
# uses a file called fabhosts where servers can be defined but gitignored |
|
10
|
|
|
# looks like |
|
11
|
|
|
# |
|
12
|
|
|
# from fabric.api import env |
|
13
|
|
|
# |
|
14
|
|
|
# def prod(): |
|
15
|
|
|
# env.user = 'username' |
|
16
|
|
|
# env.hosts = ['server1', 'server2'] |
|
17
|
|
|
# |
|
18
|
|
|
try: |
|
19
|
|
|
from fabhosts import (prod, # noqa |
|
20
|
|
|
register_deployment, |
|
21
|
|
|
WWW_DIR, |
|
22
|
|
|
ENV_DIR, |
|
23
|
|
|
USER, |
|
24
|
|
|
GROUP, |
|
25
|
|
|
DB, |
|
26
|
|
|
DB_USER, |
|
27
|
|
|
DB_PASSWORD, |
|
28
|
|
|
GIT_DIR) |
|
29
|
|
|
except ImportError: |
|
30
|
|
|
pass |
|
31
|
|
|
|
|
32
|
|
|
LOCAL_APP_DIR = '.' |
|
33
|
|
|
LOCAL_CONFIG_DIR = LOCAL_APP_DIR + '/server-config' |
|
34
|
|
|
|
|
35
|
|
|
|
|
36
|
|
|
def apt_upgrade(): |
|
37
|
|
|
""" |
|
38
|
|
|
Run apt-get upgrade |
|
39
|
|
|
""" |
|
40
|
|
|
require.deb.uptodate_index(max_age={'day': 1}) |
|
41
|
|
|
sudo('apt-get upgrade') |
|
42
|
|
|
|
|
43
|
|
|
|
|
44
|
|
|
def create_user(): |
|
45
|
|
|
""" |
|
46
|
|
|
Create a gage_www user for running gage-web |
|
47
|
|
|
""" |
|
48
|
|
|
require.groups.group(GROUP) |
|
49
|
|
|
require.users.user(USER, group=GROUP, system=True) |
|
50
|
|
|
|
|
51
|
|
|
|
|
52
|
|
|
def create_www_folder(): |
|
53
|
|
|
""" |
|
54
|
|
|
Folder for gage-web to run from |
|
55
|
|
|
""" |
|
56
|
|
|
require.directory(WWW_DIR, |
|
57
|
|
|
use_sudo=True, |
|
58
|
|
|
owner=USER) |
|
59
|
|
|
|
|
60
|
|
|
|
|
61
|
|
|
def create_venv(): |
|
62
|
|
|
""" |
|
63
|
|
|
Create virutalenv for gage-web |
|
64
|
|
|
""" |
|
65
|
|
|
require.python.virtualenv(ENV_DIR, |
|
66
|
|
|
use_sudo=True) |
|
67
|
|
|
|
|
68
|
|
|
|
|
69
|
|
|
def install_system_requirements(): |
|
70
|
|
|
""" |
|
71
|
|
|
Install required system packages |
|
72
|
|
|
""" |
|
73
|
|
|
require.deb.packages(['python', |
|
74
|
|
|
'python-dev', |
|
75
|
|
|
'python-pip', |
|
76
|
|
|
'python-virtualenv', |
|
77
|
|
|
'nginx', |
|
78
|
|
|
'postgresql-9.4', |
|
79
|
|
|
'postgresql-9.4-postgis-2.1', |
|
80
|
|
|
'postgresql-server-dev-9.4', |
|
81
|
|
|
'libpq-dev', |
|
82
|
|
|
'git', |
|
83
|
|
|
'gdal-bin', |
|
84
|
|
|
'gfortran', |
|
85
|
|
|
'python-gdal', |
|
86
|
|
|
'python-numpy', |
|
87
|
|
|
'python-scipy', |
|
88
|
|
|
'python-pandas', |
|
89
|
|
|
'libblas-dev', |
|
90
|
|
|
'liblapack-dev', |
|
91
|
|
|
'libgdal-dev', |
|
92
|
|
|
'supervisor', |
|
93
|
|
|
'redis-server', |
|
94
|
|
|
'libxslt1.dev', |
|
95
|
|
|
'libjpeg8-dev', |
|
96
|
|
|
'libjpeg-dev', |
|
97
|
|
|
'libfreetype6-dev', |
|
98
|
|
|
'zlib1g-dev', |
|
99
|
|
|
'libpng12-dev']) |
|
100
|
|
|
sudo('export CPLUS_INCLUDE_PATH=/usr/include/gdal') |
|
101
|
|
|
sudo('export C_INCLUDE_PATH=/usr/include/gdal') |
|
102
|
|
|
|
|
103
|
|
|
|
|
104
|
|
|
def create_database(): |
|
105
|
|
|
""" |
|
106
|
|
|
Create database and setup postgis |
|
107
|
|
|
""" |
|
108
|
|
|
output = sudo('psql -lqt | cut -d \| -f 1', user='postgres') |
|
109
|
|
|
if DB in output: |
|
110
|
|
|
print('{DB} database found!'.format(DB=DB)) |
|
111
|
|
|
else: |
|
112
|
|
|
print('{DB} not found! Creating DB'.format(DB=DB)) |
|
113
|
|
|
# require.postgres.user(DB_USER, password=DB_PASSWORD) |
|
114
|
|
|
sudo("""psql -c "CREATE USER '{DB_USER}' NOSUPERUSER NOCREATEDB NOCREATEROLE INHERIT LOGIN UNENCRYPTED PASSWORD '{DB_PASSWORD}';""".format(DB_USER=DB_USER, DB_PASSWORD=DB_PASSWORD), |
|
115
|
|
|
user='postgres') |
|
116
|
|
|
# require.postgres.database(DB, owner=DB_USER) |
|
117
|
|
|
sudo('createdb --owner {DB_USER} {DB}'.format(DB_USER=DB_USER, DB=DB), |
|
118
|
|
|
user='postgres') |
|
119
|
|
|
sudo('psql -d {DB} -c "CREATE EXTENSION postgis;"'.format(DB=DB), |
|
120
|
|
|
user='postgres') |
|
121
|
|
|
sudo('psql -d {DB} -c "CREATE EXTENSION postgis_topology;"'.format(DB=DB), |
|
122
|
|
|
user='postgres') |
|
123
|
|
|
|
|
124
|
|
|
|
|
125
|
|
|
def configure_git(): |
|
126
|
|
|
""" |
|
127
|
|
|
1. Setup bare Git Repo |
|
128
|
|
|
2. Create post-recieve hook |
|
129
|
|
|
""" |
|
130
|
|
|
require.directory(GIT_DIR, use_sudo=True) |
|
131
|
|
|
with cd(GIT_DIR): |
|
132
|
|
|
sudo('mkdir gage-web.git') |
|
133
|
|
|
with cd('gage-web.git'): |
|
134
|
|
|
sudo('git init --bare') |
|
135
|
|
|
with lcd(LOCAL_CONFIG_DIR): |
|
136
|
|
|
with cd('hooks'): |
|
137
|
|
|
put('./post-receive', './', use_sudo=True) |
|
138
|
|
|
sudo('chmod +x post-receive') |
|
139
|
|
|
with lcd(LOCAL_APP_DIR): |
|
140
|
|
|
local( |
|
141
|
|
|
'git remote add production {user}@{server}:{GIT_DIR}/gage-web.git' |
|
142
|
|
|
.format(user=env.user, server=env.host_string, GIT_DIR=GIT_DIR)) |
|
143
|
|
|
|
|
144
|
|
|
|
|
145
|
|
|
def deploy(): |
|
146
|
|
|
""" |
|
147
|
|
|
Push current master to production and restart gunicorn |
|
148
|
|
|
""" |
|
149
|
|
|
with lcd(LOCAL_APP_DIR): |
|
150
|
|
|
local('git push production master') |
|
151
|
|
|
sudo('supervisorctl restart gage:*') |
|
152
|
|
|
register_deployment(LOCAL_APP_DIR) |
|
153
|
|
|
|
|
154
|
|
|
|
|
155
|
|
|
def install_requirements(): |
|
156
|
|
|
""" |
|
157
|
|
|
Install requirements into virtualenv |
|
158
|
|
|
""" |
|
159
|
|
|
with fabtools.python.virtualenv(ENV_DIR): |
|
160
|
|
|
with cd(WWW_DIR): |
|
161
|
|
|
require.python.requirements('requirements.txt') |
|
162
|
|
|
|
|
163
|
|
|
|
|
164
|
|
|
def install_config_files(): |
|
165
|
|
|
""" |
|
166
|
|
|
Put config files for gunicorn, supervisord, nginx |
|
167
|
|
|
""" |
|
168
|
|
|
# host-export |
|
169
|
|
|
with cd(WWW_DIR + '/server-config'): |
|
170
|
|
|
require.file('host-export', source=LOCAL_CONFIG_DIR+'/host-export') |
|
171
|
|
|
sudo('chmod +x host-export') |
|
172
|
|
|
# gunicorn |
|
173
|
|
|
with cd('/home/www'): |
|
174
|
|
|
fabtools.files.upload_template( |
|
175
|
|
|
LOCAL_CONFIG_DIR + '/gunicorn-start-gage', |
|
176
|
|
|
'.', |
|
177
|
|
|
use_sudo=True, |
|
178
|
|
|
use_jinja=True, |
|
179
|
|
|
user=USER, |
|
180
|
|
|
context={ |
|
181
|
|
|
'WWW_DIR': WWW_DIR, |
|
182
|
|
|
'ENV_DIR': ENV_DIR, |
|
183
|
|
|
'USER': USER, |
|
184
|
|
|
'GROUP': GROUP |
|
185
|
|
|
}, |
|
186
|
|
|
chown=True), |
|
187
|
|
|
sudo('chmod +x gunicorn-start-gage') |
|
188
|
|
|
# supervisord |
|
189
|
|
|
with cd('/etc/supervisor/conf.d/'): |
|
190
|
|
|
require.file('gage-web.conf', |
|
191
|
|
|
source=LOCAL_CONFIG_DIR+'/gage-web.conf', |
|
192
|
|
|
use_sudo=True) |
|
193
|
|
|
require.directory('/home/www/logs/gage-web/') |
|
194
|
|
|
sudo('supervisorctl reread') |
|
195
|
|
|
sudo('supervisorctl update') |
|
196
|
|
|
# nginx |
|
197
|
|
|
with cd('/etc/nginx/sites-available'): |
|
198
|
|
|
require.file('gage-web', |
|
199
|
|
|
source=LOCAL_CONFIG_DIR+'/gage-web', |
|
200
|
|
|
use_sudo=True) |
|
201
|
|
|
sudo('ln -s /etc/nginx/sites-available/gage-web' + |
|
202
|
|
|
' /etc/nginx/sites-enabled/gage-web') |
|
203
|
|
|
sudo('service nginx configtest') |
|
204
|
|
|
sudo('service nginx restart') |
|
205
|
|
|
|
|
206
|
|
|
|
|
207
|
|
|
def upgrade_db_schema(): |
|
208
|
|
|
""" |
|
209
|
|
|
With manage.py setup database |
|
210
|
|
|
""" |
|
211
|
|
|
with prefix('source {ENV_DIR}/bin/activate'.format(ENV_DIR=ENV_DIR)): |
|
212
|
|
|
with prefix('source {WWW_DIR}/server-config/host-export'.format(WWW_DIR=WWW_DIR)): |
|
213
|
|
|
with cd(WWW_DIR): |
|
214
|
|
|
sudo('python manage.py db upgrade') |
|
215
|
|
|
|
|
216
|
|
|
|
|
217
|
|
|
def create_roles(): |
|
218
|
|
|
""" |
|
219
|
|
|
Create user and admin roles |
|
220
|
|
|
""" |
|
221
|
|
|
with prefix('source {ENV_DIR}/bin/activate'.format(ENV_DIR=ENV_DIR)): |
|
222
|
|
|
with prefix('source {WWW_DIR}/server-config/host-export'.format(WWW_DIR=WWW_DIR)): |
|
223
|
|
|
with cd(WWW_DIR): |
|
224
|
|
|
sudo('python manage.py user create_role -n admin -d "Site Administrators"') |
|
225
|
|
|
sudo('python manage.py user create_role -n user -d Users') |
|
226
|
|
|
|
|
227
|
|
|
def setup_celery(): |
|
228
|
|
|
""" |
|
229
|
|
|
Upload configs for celery, make sure redis is running |
|
230
|
|
|
""" |
|
231
|
|
|
with cd('/home/www'): |
|
232
|
|
|
fabtools.files.upload_template( |
|
233
|
|
|
LOCAL_CONFIG_DIR + '/celery-start-gage', |
|
234
|
|
|
'.', |
|
235
|
|
|
use_sudo=True, |
|
236
|
|
|
use_jinja=True, |
|
237
|
|
|
user=USER, |
|
238
|
|
|
context={ |
|
239
|
|
|
'WWW_DIR': WWW_DIR, |
|
240
|
|
|
'ENV_DIR': ENV_DIR, |
|
241
|
|
|
'USER': USER, |
|
242
|
|
|
'GROUP': GROUP |
|
243
|
|
|
}, |
|
244
|
|
|
chown=True), |
|
245
|
|
|
sudo('chmod +x celery-start-gage') |
|
246
|
|
|
fabtools.files.upload_template( |
|
247
|
|
|
LOCAL_CONFIG_DIR + '/celery-start-beat-gage', |
|
248
|
|
|
'.', |
|
249
|
|
|
use_sudo=True, |
|
250
|
|
|
use_jinja=True, |
|
251
|
|
|
user=USER, |
|
252
|
|
|
context={ |
|
253
|
|
|
'WWW_DIR': WWW_DIR, |
|
254
|
|
|
'ENV_DIR': ENV_DIR, |
|
255
|
|
|
'USER': USER, |
|
256
|
|
|
'GROUP': GROUP |
|
257
|
|
|
}, |
|
258
|
|
|
chown=True), |
|
259
|
|
|
sudo('chmod +x celery-start-beat-gage') |
|
260
|
|
|
with cd('/etc/supervisor/conf.d/'): |
|
261
|
|
|
require.file('gage-web.conf', |
|
262
|
|
|
source=LOCAL_CONFIG_DIR+'/gage-web.conf', |
|
263
|
|
|
use_sudo=True) |
|
264
|
|
|
require.directory('/home/www/logs/gage-web/') |
|
265
|
|
|
sudo('supervisorctl reread') |
|
266
|
|
|
sudo('supervisorctl update') |
|
267
|
|
|
sudo('supervisorctl restart gage:*') |
|
268
|
|
|
|
|
269
|
|
|
|
|
270
|
|
|
def gage_restart(): |
|
271
|
|
|
""" |
|
272
|
|
|
Cron script to restart the supervisorctl gage:gage-celery-beat hourly |
|
273
|
|
|
""" |
|
274
|
|
|
fabtools.cron.add_task('restart-gage-beat', '@hourly', 'root', WWW_DIR+'/server-config/restart-gage.sh') |
|
275
|
|
|
|
|
276
|
|
|
|
|
277
|
|
|
|
|
278
|
|
|
def bootstrap(): |
|
279
|
|
|
""" |
|
280
|
|
|
Setup all the things |
|
281
|
|
|
""" |
|
282
|
|
|
create_user() |
|
283
|
|
|
create_www_folder() |
|
284
|
|
|
create_venv() |
|
285
|
|
|
install_system_requirements() |
|
286
|
|
|
# create_database() |
|
287
|
|
|
# configure_git() |
|
288
|
|
|
deploy() |
|
289
|
|
|
install_requirements() |
|
290
|
|
|
# Install ssl scripts |
|
291
|
|
|
install_config_files() |
|
292
|
|
|
upgrade_db_schema() |
|
293
|
|
|
create_roles() |
|
294
|
|
|
|
abkfenris /
gage-web
