1
|
|
|
#!/usr/bin/env python |
2
|
|
|
# -*- coding: UTF-8 -*- |
3
|
|
|
|
4
|
|
|
# Isomer - The distributed application framework |
5
|
|
|
# ============================================== |
6
|
|
|
# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others. |
7
|
|
|
# |
8
|
|
|
# This program is free software: you can redistribute it and/or modify |
9
|
|
|
# it under the terms of the GNU Affero General Public License as published by |
10
|
|
|
# the Free Software Foundation, either version 3 of the License, or |
11
|
|
|
# (at your option) any later version. |
12
|
|
|
# |
13
|
|
|
# This program is distributed in the hope that it will be useful, |
14
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
15
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
16
|
|
|
# GNU Affero General Public License for more details. |
17
|
|
|
# |
18
|
|
|
# You should have received a copy of the GNU Affero General Public License |
19
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
20
|
|
|
|
21
|
|
|
""" |
22
|
|
|
|
23
|
|
|
Module: Instance |
24
|
|
|
================ |
25
|
|
|
|
26
|
|
|
Instance management functionality. |
27
|
|
|
|
28
|
|
|
instance info |
29
|
|
|
instance list |
30
|
|
|
instance set |
31
|
|
|
instance create |
32
|
|
|
instance install |
33
|
|
|
instance clear |
34
|
|
|
instance remove |
35
|
|
|
instance install-module |
36
|
|
|
instance turnover |
37
|
|
|
instance cert |
38
|
|
|
instance service |
39
|
|
|
instance update-nginx |
40
|
|
|
|
41
|
|
|
""" |
42
|
|
|
|
43
|
|
|
import os |
44
|
|
|
import webbrowser |
45
|
|
|
from socket import gethostname |
46
|
|
|
from shutil import rmtree |
47
|
|
|
|
48
|
|
|
import tomlkit |
49
|
|
|
import click |
50
|
|
|
from click_didyoumean import DYMGroup |
51
|
|
|
from isomer.error import ( |
52
|
|
|
abort, |
53
|
|
|
EXIT_INVALID_CONFIGURATION, |
54
|
|
|
EXIT_INSTALLATION_FAILED, |
55
|
|
|
EXIT_INSTANCE_EXISTS, |
56
|
|
|
EXIT_INSTANCE_UNKNOWN, |
57
|
|
|
EXIT_SERVICE_INVALID, |
58
|
|
|
EXIT_USER_BAILED_OUT, |
59
|
|
|
EXIT_INVALID_PARAMETER, |
60
|
|
|
) |
61
|
|
|
from isomer.migration import apply_migrations |
62
|
|
|
from isomer.misc import sorted_alphanumerical |
63
|
|
|
from isomer.misc.path import set_instance, get_etc_instance_path, get_path |
64
|
|
|
from isomer.logger import warn, critical |
65
|
|
|
from isomer.tool import ( |
66
|
|
|
_get_system_configuration, |
67
|
|
|
_get_configuration, |
68
|
|
|
get_next_environment, |
69
|
|
|
ask, |
70
|
|
|
finish, |
71
|
|
|
run_process, |
72
|
|
|
format_result, |
73
|
|
|
log, |
74
|
|
|
error, |
75
|
|
|
debug, |
76
|
|
|
) |
77
|
|
|
from isomer.tool.etc import ( |
78
|
|
|
write_instance, |
79
|
|
|
valid_configuration, |
80
|
|
|
remove_instance, |
81
|
|
|
instance_template, |
82
|
|
|
) |
83
|
|
|
from isomer.tool.templates import write_template |
84
|
|
|
from isomer.tool.defaults import ( |
85
|
|
|
service_template, |
86
|
|
|
nginx_template, |
87
|
|
|
distribution, |
88
|
|
|
) |
89
|
|
|
from isomer.tool.environment import ( |
90
|
|
|
_install_environment, |
91
|
|
|
install_environment_modules, |
92
|
|
|
_clear_environment, |
93
|
|
|
_check_environment, |
94
|
|
|
) |
95
|
|
|
from isomer.tool.database import copy_database |
96
|
|
|
from isomer.tool.version import _get_versions |
97
|
|
|
from isomer.ui.builder import copy_directory_tree |
98
|
|
|
from isomer.ui.store import DEFAULT_STORE_URL |
99
|
|
|
|
100
|
|
|
|
101
|
|
|
@click.group(cls=DYMGroup) |
102
|
|
|
@click.pass_context |
103
|
|
|
def instance(ctx): |
104
|
|
|
"""[GROUP] instance various aspects of Isomer""" |
105
|
|
|
|
106
|
|
|
if ctx.invoked_subcommand in ("info", "list", "create"): |
107
|
|
|
return |
108
|
|
|
|
109
|
|
|
_get_configuration(ctx) |
110
|
|
|
|
111
|
|
|
|
112
|
|
View Code Duplication |
@instance.command(name="info", short_help="show system configuration of instance") |
|
|
|
|
113
|
|
|
@click.pass_context |
114
|
|
|
def info_instance(ctx): |
115
|
|
|
"""Print information about the selected instance""" |
116
|
|
|
|
117
|
|
|
instance_name = ctx.obj["instance"] |
118
|
|
|
instances = ctx.obj["instances"] |
119
|
|
|
instance_configuration = instances[instance_name] |
120
|
|
|
|
121
|
|
|
environment_name = instance_configuration["environment"] |
122
|
|
|
environment_config = instance_configuration["environments"][environment_name] |
123
|
|
|
|
124
|
|
|
if instance_name not in instances: |
125
|
|
|
log("Instance %s unknown!" % instance_name, lvl=warn) |
126
|
|
|
abort(EXIT_INSTANCE_UNKNOWN) |
127
|
|
|
|
128
|
|
|
log("Instance configuration:", instance_configuration, pretty=True) |
129
|
|
|
log("Active environment (%s):" % environment_name, environment_config, pretty=True) |
130
|
|
|
|
131
|
|
|
finish(ctx) |
132
|
|
|
|
133
|
|
|
|
134
|
|
View Code Duplication |
@instance.command( |
|
|
|
|
135
|
|
|
name="check", short_help="check instance configuration and environments" |
136
|
|
|
) |
137
|
|
|
@click.pass_context |
138
|
|
|
def check_instance(ctx): |
139
|
|
|
"""Check health of the selected instance""" |
140
|
|
|
|
141
|
|
|
instance_name = ctx.obj["instance"] |
142
|
|
|
instances = ctx.obj["instances"] |
143
|
|
|
instance_configuration = instances[instance_name] |
144
|
|
|
|
145
|
|
|
environment_name = instance_configuration["environment"] |
146
|
|
|
environment_config = instance_configuration["environments"][environment_name] |
147
|
|
|
|
148
|
|
|
if instance_name not in instances: |
149
|
|
|
log("Instance %s unknown!" % instance_name, lvl=warn) |
150
|
|
|
abort(EXIT_INSTANCE_UNKNOWN) |
151
|
|
|
|
152
|
|
|
# TODO: Validate instance config |
153
|
|
|
|
154
|
|
|
_check_environment(ctx, "blue") |
155
|
|
|
_check_environment(ctx, "green") |
156
|
|
|
|
157
|
|
|
finish(ctx) |
158
|
|
|
|
159
|
|
|
|
160
|
|
|
@instance.command(name="browser", short_help="try to point a browser to this instance") |
161
|
|
|
@click.pass_context |
162
|
|
|
def browser(ctx): |
163
|
|
|
"""Tries to start or point a browser towards this instance's frontend""" |
164
|
|
|
instance_configuration = ctx.obj["instance_configuration"] |
165
|
|
|
|
166
|
|
|
server = instance_configuration.get("webserver", "internal") |
167
|
|
|
protocol = "http" |
168
|
|
|
|
169
|
|
|
if server != "internal": |
170
|
|
|
protocol += "s" |
171
|
|
|
|
172
|
|
|
host = instance_configuration.get( |
173
|
|
|
"web_hostname", instance_configuration.get("web_address", "127.0.0.1") |
174
|
|
|
) |
175
|
|
|
port = instance_configuration.get("web_port", None) |
176
|
|
|
|
177
|
|
|
if port is None: |
178
|
|
|
url = "%s://%s" % (protocol, host) |
179
|
|
|
else: |
180
|
|
|
url = "%s://%s:%i" % (protocol, host, port) |
181
|
|
|
|
182
|
|
|
log("Opening browser to:", url) |
183
|
|
|
log( |
184
|
|
|
"If this is empty or unreachable, check your instance with:\n" |
185
|
|
|
" iso instance info\n" |
186
|
|
|
"Also try the health checks:\n" |
187
|
|
|
" iso instance check" |
188
|
|
|
) |
189
|
|
|
|
190
|
|
|
webbrowser.open(url) |
191
|
|
|
|
192
|
|
|
|
193
|
|
|
@instance.command(name="list", short_help="List all instances") |
194
|
|
|
@click.pass_context |
195
|
|
|
def list_instances(ctx): |
196
|
|
|
"""List all known instances""" |
197
|
|
|
|
198
|
|
|
for instance_name in ctx.obj["instances"]: |
199
|
|
|
log(instance_name, pretty=True) |
200
|
|
|
finish(ctx) |
201
|
|
|
|
202
|
|
|
|
203
|
|
|
@instance.command(name="set", short_help="Set a parameter of an instance") |
204
|
|
|
@click.argument("parameter") |
205
|
|
|
@click.argument("value") |
206
|
|
|
@click.pass_context |
207
|
|
|
def set_parameter(ctx, parameter, value): |
208
|
|
|
"""Set a configuration parameter of an instance""" |
209
|
|
|
|
210
|
|
|
log("Setting %s to %s" % (parameter, value)) |
211
|
|
|
instance_configuration = ctx.obj["instance_configuration"] |
212
|
|
|
defaults = instance_template |
213
|
|
|
converted_value = None |
214
|
|
|
|
215
|
|
|
try: |
216
|
|
|
parameter_type = type(defaults[parameter]) |
217
|
|
|
log(parameter_type, pretty=True, lvl=debug) |
218
|
|
|
|
219
|
|
|
if parameter_type == tomlkit.items.Integer: |
220
|
|
|
converted_value = int(value) |
221
|
|
|
elif parameter_type == bool: |
222
|
|
|
converted_value = value.upper() == "TRUE" |
223
|
|
|
else: |
224
|
|
|
converted_value = value |
225
|
|
|
except KeyError: |
226
|
|
|
log("Available parameters:", sorted(list(defaults.keys()))) |
227
|
|
|
abort(EXIT_INVALID_PARAMETER) |
228
|
|
|
|
229
|
|
|
if converted_value is None: |
230
|
|
|
log("Converted value was None! Recheck the new config!", lvl=warn) |
231
|
|
|
|
232
|
|
|
instance_configuration[parameter] = converted_value |
233
|
|
|
log("New config:", instance_configuration, pretty=True, lvl=debug) |
234
|
|
|
|
235
|
|
|
ctx.obj["instances"][ctx.obj["instance"]] = instance_configuration |
236
|
|
|
|
237
|
|
|
if valid_configuration(ctx): |
238
|
|
|
write_instance(instance_configuration) |
239
|
|
|
finish(ctx) |
240
|
|
|
else: |
241
|
|
|
log("New configuration would not be valid", lvl=critical) |
242
|
|
|
abort(EXIT_INVALID_CONFIGURATION) |
243
|
|
|
|
244
|
|
|
|
245
|
|
|
@instance.command(short_help="Create a new instance") |
246
|
|
|
@click.argument("instance_name", default="") |
247
|
|
|
@click.pass_context |
248
|
|
|
def create(ctx, instance_name): |
249
|
|
|
"""Create a new instance""" |
250
|
|
|
|
251
|
|
|
if instance_name is "": |
252
|
|
|
instance_name = ctx.obj["instance"] |
253
|
|
|
|
254
|
|
|
if instance_name in ctx.obj["instances"]: |
255
|
|
|
abort(EXIT_INSTANCE_EXISTS) |
256
|
|
|
|
257
|
|
|
log("Creating instance:", instance_name) |
258
|
|
|
instance_configuration = instance_template |
259
|
|
|
instance_configuration["name"] = instance_name |
260
|
|
|
ctx.obj["instances"][instance_name] = instance_configuration |
261
|
|
|
|
262
|
|
|
write_instance(instance_configuration) |
263
|
|
|
finish(ctx) |
264
|
|
|
|
265
|
|
|
|
266
|
|
|
@instance.command(name="install", short_help="Install a fresh instance") |
267
|
|
|
@click.option("--force", "-f", is_flag=True, default=False) |
268
|
|
|
@click.option( |
269
|
|
|
"--source", "-s", default="git", |
270
|
|
|
type=click.Choice(["link", "copy", "git", "github"]) |
271
|
|
|
) |
272
|
|
|
@click.option("--url", "-u", default="", type=click.Path()) |
273
|
|
|
@click.option( |
274
|
|
|
"--import-file", "--import", default=None, help="Import the specified backup" |
275
|
|
|
) |
276
|
|
|
@click.option( |
277
|
|
|
"--no-sudo", |
278
|
|
|
is_flag=True, |
279
|
|
|
default=False, |
280
|
|
|
help="Do not use sudo to install (Mostly for tests)", |
281
|
|
|
) |
282
|
|
|
@click.option( |
283
|
|
|
"--release", "-r", default=None, help="Override installed release version" |
284
|
|
|
) |
285
|
|
|
@click.option("--skip-modules", is_flag=True, default=False) |
286
|
|
|
@click.option("--skip-data", is_flag=True, default=False) |
287
|
|
|
@click.option("--skip-frontend", is_flag=True, default=False) |
288
|
|
|
@click.option("--skip-test", is_flag=True, default=False) |
289
|
|
|
@click.option("--skip-provisions", is_flag=True, default=False) |
290
|
|
|
@click.pass_context |
291
|
|
|
def install(ctx, **kwargs): |
292
|
|
|
"""Install a new environment of an instance""" |
293
|
|
|
|
294
|
|
|
log("Installing instance") |
295
|
|
|
|
296
|
|
|
env = ctx.obj["instance_configuration"]["environments"] |
297
|
|
|
|
298
|
|
|
green = env["green"]["installed"] |
299
|
|
|
blue = env["blue"]["installed"] |
300
|
|
|
|
301
|
|
|
if green or blue: |
302
|
|
|
log( |
303
|
|
|
"At least one environment is installed or in a non-clear state.\n" |
304
|
|
|
"Please use 'iso instance upgrade' to upgrade an instance.", |
305
|
|
|
lvl=warn, |
306
|
|
|
) |
307
|
|
|
abort(50081) |
308
|
|
|
|
309
|
|
|
_clear_instance(ctx, force=kwargs["force"], clear=False, no_archive=True) |
310
|
|
|
|
311
|
|
|
_install_environment(ctx, **kwargs) |
312
|
|
|
|
313
|
|
|
ctx.obj["instance_configuration"]["source"] = kwargs["source"] |
314
|
|
|
ctx.obj["instance_configuration"]["url"] = kwargs["url"] |
315
|
|
|
|
316
|
|
|
write_instance(ctx.obj["instance_configuration"]) |
317
|
|
|
|
318
|
|
|
_turnover(ctx, force=kwargs["force"]) |
319
|
|
|
|
320
|
|
|
finish(ctx) |
321
|
|
|
|
322
|
|
|
|
323
|
|
|
@instance.command(name="clear", short_help="Clear the whole instance (CAUTION)") |
324
|
|
|
@click.option("--force", "-f", is_flag=True, default=False) |
325
|
|
|
@click.option("--no-archive", "-n", is_flag=True, default=False) |
326
|
|
|
@click.pass_context |
327
|
|
|
def clear_instance(ctx, force, no_archive): |
328
|
|
|
"""Irrevocably clear all environments of an instance""" |
329
|
|
|
|
330
|
|
|
_clear_instance(ctx, force, False, no_archive) |
331
|
|
|
|
332
|
|
|
|
333
|
|
|
def _clear_instance(ctx, force, clear, no_archive): |
334
|
|
|
log("Clearing instance:", ctx.obj["instance"]) |
335
|
|
|
log("Clearing blue environment.", lvl=debug) |
336
|
|
|
_clear_environment(ctx, force, "blue", clear, no_archive) |
337
|
|
|
log("Clearing green environment.", lvl=debug) |
338
|
|
|
_clear_environment(ctx, force, "green", clear, no_archive) |
339
|
|
|
finish(ctx) |
340
|
|
|
|
341
|
|
|
|
342
|
|
|
@instance.command(short_help="Remove a whole instance (CAUTION)") |
343
|
|
|
@click.option( |
344
|
|
|
"--clear", "-c", is_flag=True, help="Clear instance before removal", default=False |
345
|
|
|
) |
346
|
|
|
@click.option("--no-archive", "-n", is_flag=True, default=False) |
347
|
|
|
@click.pass_context |
348
|
|
|
def remove(ctx, clear, no_archive): |
349
|
|
|
"""Irrevocably remove a whole instance""" |
350
|
|
|
|
351
|
|
|
if clear: |
352
|
|
|
log("Destructively removing instance:", ctx.obj["instance"], lvl=warn) |
353
|
|
|
|
354
|
|
|
if not ask("Are you sure", default=False, data_type="bool"): |
355
|
|
|
abort(EXIT_USER_BAILED_OUT) |
356
|
|
|
|
357
|
|
|
if clear: |
358
|
|
|
_clear_instance(ctx, force=True, clear=clear, no_archive=no_archive) |
359
|
|
|
|
360
|
|
|
remove_instance(ctx.obj["instance"]) |
361
|
|
|
finish(ctx) |
362
|
|
|
|
363
|
|
|
|
364
|
|
|
@instance.command( |
365
|
|
|
"install-modules", short_help="Add (and install) modules to an instance" |
366
|
|
|
) |
367
|
|
|
@click.option( |
368
|
|
|
"--source", |
369
|
|
|
"-s", |
370
|
|
|
default="git", |
371
|
|
|
type=click.Choice(["link", "copy", "git", "develop", "store"]), |
372
|
|
|
help="Specify installation source/method", |
373
|
|
|
) |
374
|
|
|
@click.option( |
375
|
|
|
"--store-url", |
376
|
|
|
default=DEFAULT_STORE_URL, |
377
|
|
|
help="Specify alternative store url (Default: %s)" % DEFAULT_STORE_URL, |
378
|
|
|
) |
379
|
|
|
@click.option( |
380
|
|
|
"--install-env", |
381
|
|
|
"--install", |
382
|
|
|
"-i", |
383
|
|
|
is_flag=True, |
384
|
|
|
default=False, |
385
|
|
|
help="Install modules on active environment", |
386
|
|
|
) |
387
|
|
|
@click.option( |
388
|
|
|
"--force", |
389
|
|
|
"-f", |
390
|
|
|
default=False, |
391
|
|
|
is_flag=True, |
392
|
|
|
help="Force installation (overwrites old modules)", |
393
|
|
|
) |
394
|
|
|
@click.argument("urls", nargs=-1) |
395
|
|
|
@click.pass_context |
396
|
|
|
def install_instance_modules(ctx, source, urls, install_env, force, store_url): |
397
|
|
|
"""Add (and optionally immediately install) modules for an instance. |
398
|
|
|
|
399
|
|
|
This will add them to the instance's configuration, so they will be upgraded as well |
400
|
|
|
as reinstalled on other environment changes. |
401
|
|
|
|
402
|
|
|
If you're installing from a store, you can specify a custom store URL with the |
403
|
|
|
--store-url argument. |
404
|
|
|
""" |
405
|
|
|
|
406
|
|
|
instance_name = ctx.obj["instance"] |
407
|
|
|
instance_configuration = ctx.obj["instances"][instance_name] |
408
|
|
|
|
409
|
|
|
for url in urls: |
410
|
|
|
descriptor = [source, url] |
411
|
|
|
if store_url != DEFAULT_STORE_URL: |
412
|
|
|
descriptor.append(store_url) |
413
|
|
|
if descriptor not in instance_configuration["modules"]: |
414
|
|
|
instance_configuration["modules"].append(descriptor) |
415
|
|
|
elif not force: |
416
|
|
|
log("Module %s is already installed. Use --force to install anyway." % url) |
417
|
|
|
abort(50000) |
418
|
|
|
|
419
|
|
|
write_instance(instance_configuration) |
420
|
|
|
|
421
|
|
|
if install_env is True: |
422
|
|
|
next_environment = get_next_environment(ctx) |
423
|
|
|
environments = instance_configuration['environments'] |
424
|
|
|
|
425
|
|
|
if environments[next_environment]["installed"] is False: |
426
|
|
|
log("Environment %s is not installed, cannot install modules." |
427
|
|
|
% next_environment, lvl=warn) |
428
|
|
|
abort(50600) |
429
|
|
|
return |
430
|
|
|
del ctx.params["install_env"] |
431
|
|
|
ctx.forward(install_environment_modules) |
432
|
|
|
|
433
|
|
|
finish(ctx) |
434
|
|
|
|
435
|
|
|
|
436
|
|
|
@instance.command(short_help="Activates the other environment") |
437
|
|
|
@click.option("--force", "-f", is_flag=True, default=False, help="Force turnover") |
438
|
|
|
@click.pass_context |
439
|
|
|
def turnover(ctx, **kwargs): |
440
|
|
|
"""Activates the other environment """ |
441
|
|
|
|
442
|
|
|
_turnover(ctx, **kwargs) |
443
|
|
|
finish(ctx) |
444
|
|
|
|
445
|
|
|
|
446
|
|
|
def _turnover(ctx, force): |
447
|
|
|
"""Internal turnover operation""" |
448
|
|
|
|
449
|
|
|
# if ctx.obj['acting_environment'] is not None: |
450
|
|
|
# next_environment = ctx.obj['acting_environment'] |
451
|
|
|
# else: |
452
|
|
|
next_environment = get_next_environment(ctx) |
453
|
|
|
|
454
|
|
|
log("Activating environment:", next_environment) |
455
|
|
|
env = ctx.obj["instance_configuration"]["environments"][next_environment] |
456
|
|
|
|
457
|
|
|
log("Inspecting new environment") |
458
|
|
|
|
459
|
|
|
if not force: |
460
|
|
|
if env.get("database", "") == "": |
461
|
|
|
log("Database has not been set up correctly.", lvl=critical) |
462
|
|
|
abort(EXIT_INSTALLATION_FAILED) |
463
|
|
|
|
464
|
|
|
if ( |
465
|
|
|
not env.get("installed", False) |
466
|
|
|
or not env.get("tested", False) |
467
|
|
|
or not env.get("migrated", False) |
468
|
|
|
): |
469
|
|
|
log("Installation failed, cannot activate!", lvl=critical) |
470
|
|
|
abort(EXIT_INSTALLATION_FAILED) |
471
|
|
|
|
472
|
|
|
update_service(ctx, next_environment) |
473
|
|
|
|
474
|
|
|
ctx.obj["instance_configuration"]["environment"] = next_environment |
475
|
|
|
|
476
|
|
|
write_instance(ctx.obj["instance_configuration"]) |
477
|
|
|
|
478
|
|
|
# TODO: Effect reload of service |
479
|
|
|
# * Systemctl reload |
480
|
|
|
# * (Re)start service |
481
|
|
|
# * confirm correct operation |
482
|
|
|
# - if not, switch back to the other instance, maybe indicate a broken |
483
|
|
|
# state for next_environment |
484
|
|
|
# - if yes, Store instance configuration and terminate, we're done |
485
|
|
|
|
486
|
|
|
log("Turned instance over to", next_environment) |
487
|
|
|
|
488
|
|
|
|
489
|
|
|
@instance.command(short_help="Upgrades the other environment") |
490
|
|
|
@click.option("--release", "-r", default=None, help="Specify release to upgrade to") |
491
|
|
|
@click.option("--upgrade-modules", default=False, is_flag=True, |
492
|
|
|
help="Also, upgrade modules if possible") |
493
|
|
|
@click.option("--restart", default=False, is_flag=True, |
494
|
|
|
help="Restart systemd service via systemctl on success") |
495
|
|
|
@click.option("--handle-cache", "-c", |
496
|
|
|
type=click.Choice(["ignore", "move", "copy"], case_sensitive=False), |
497
|
|
|
default="ignore", help="Handle cached data as well (ignore, move, copy)") |
498
|
|
|
@click.option( |
499
|
|
|
"--source", |
500
|
|
|
"-s", |
501
|
|
|
default="git", |
502
|
|
|
type=click.Choice(["link", "copy", "git", "develop", "github", "pypi"]), |
503
|
|
|
help="Specify installation source/method", |
504
|
|
|
) |
505
|
|
|
@click.option("--url", "-u", default="", type=click.Path()) |
506
|
|
|
@click.pass_context |
507
|
|
|
def upgrade(ctx, release, upgrade_modules, restart, handle_cache, source, url): |
508
|
|
|
"""Upgrades an instance on its other environment and turns over on success. |
509
|
|
|
|
510
|
|
|
\b |
511
|
|
|
1. Test if other environment is empty |
512
|
|
|
1.1. No - archive and clear it |
513
|
|
|
2. Copy current environment to other environment |
514
|
|
|
3. Clear old bits (venv, frontend) |
515
|
|
|
4. Fetch updates in other environment repository |
516
|
|
|
5. Select a release |
517
|
|
|
6. Checkout that release and its submodules |
518
|
|
|
7. Install release |
519
|
|
|
8. Copy database |
520
|
|
|
9. Migrate data (WiP) |
521
|
|
|
10. Turnover |
522
|
|
|
|
523
|
|
|
""" |
524
|
|
|
|
525
|
|
|
instance_config = ctx.obj["instance_configuration"] |
526
|
|
|
repository = get_path("lib", "repository") |
527
|
|
|
|
528
|
|
|
installation_source = source if source is not None else instance_config['source'] |
529
|
|
|
installation_url = url if url is not None else instance_config['url'] |
530
|
|
|
|
531
|
|
|
environments = instance_config["environments"] |
532
|
|
|
|
533
|
|
|
active = instance_config["environment"] |
534
|
|
|
|
535
|
|
|
next_environment = get_next_environment(ctx) |
536
|
|
|
if environments[next_environment]["installed"] is True: |
537
|
|
|
_clear_environment(ctx, clear_env=next_environment) |
538
|
|
|
|
539
|
|
|
source_paths = [ |
540
|
|
|
get_path("lib", "", environment=active), |
541
|
|
|
get_path("local", "", environment=active) |
542
|
|
|
] |
543
|
|
|
|
544
|
|
|
destination_paths = [ |
545
|
|
|
get_path("lib", "", environment=next_environment), |
546
|
|
|
get_path("local", "", environment=next_environment) |
547
|
|
|
] |
548
|
|
|
|
549
|
|
|
log(source_paths, destination_paths, pretty=True) |
550
|
|
|
|
551
|
|
|
for source, destination in zip(source_paths, destination_paths): |
552
|
|
|
log("Copying to new environment:", source, destination) |
553
|
|
|
copy_directory_tree(source, destination) |
554
|
|
|
|
555
|
|
|
if handle_cache != "ignore": |
556
|
|
|
log("Handling cache") |
557
|
|
|
move = handle_cache == "move" |
558
|
|
|
copy_directory_tree( |
559
|
|
|
get_path("cache", "", environment=active), |
560
|
|
|
get_path("cache", "", environment=next_environment), |
561
|
|
|
move=move |
562
|
|
|
) |
563
|
|
|
|
564
|
|
|
rmtree(get_path("lib", "venv"), ignore_errors=True) |
565
|
|
|
# TODO: This potentially leaves frontend-dev: |
566
|
|
|
rmtree(get_path("lib", "frontend"), ignore_errors=True) |
567
|
|
|
|
568
|
|
|
releases = _get_versions( |
569
|
|
|
ctx, |
570
|
|
|
source=installation_source, |
571
|
|
|
url=installation_url, |
572
|
|
|
fetch=True |
573
|
|
|
) |
574
|
|
|
|
575
|
|
|
releases_keys = sorted_alphanumerical(releases.keys()) |
576
|
|
|
|
577
|
|
|
if release is None: |
578
|
|
|
release = releases_keys[-1] |
579
|
|
|
else: |
580
|
|
|
if release not in releases_keys: |
581
|
|
|
log("Unknown release. Maybe try a different release or source.") |
582
|
|
|
abort(50100) |
583
|
|
|
|
584
|
|
|
log("Choosing release", release) |
585
|
|
|
|
586
|
|
|
_install_environment( |
587
|
|
|
ctx, installation_source, installation_url, upgrade=True, release=release |
588
|
|
|
) |
589
|
|
|
|
590
|
|
|
new_database_name = instance_config["name"] + "_" + next_environment |
591
|
|
|
|
592
|
|
|
copy_database( |
593
|
|
|
ctx.obj["dbhost"], |
594
|
|
|
active['database'], |
595
|
|
|
new_database_name |
596
|
|
|
) |
597
|
|
|
|
598
|
|
|
apply_migrations(ctx) |
599
|
|
|
|
600
|
|
|
finish(ctx) |
601
|
|
|
|
602
|
|
|
|
603
|
|
|
def update_service(ctx, next_environment): |
604
|
|
|
"""Updates the specified service configuration""" |
605
|
|
|
|
606
|
|
|
validated, message = validate_services(ctx) |
607
|
|
|
|
608
|
|
|
if not validated: |
609
|
|
|
log("Service configuration validation failed:", message, lvl=error) |
610
|
|
|
abort(EXIT_SERVICE_INVALID) |
611
|
|
|
|
612
|
|
|
init = ctx.obj["config"]["meta"]["init"] |
613
|
|
|
environment_config = ctx.obj["instance_configuration"]["environments"][ |
614
|
|
|
next_environment |
615
|
|
|
] |
616
|
|
|
|
617
|
|
|
log( |
618
|
|
|
"Updating %s configuration of instance %s to %s" |
619
|
|
|
% (init, ctx.obj["instance"], next_environment) |
620
|
|
|
) |
621
|
|
|
log("New environment:", environment_config, pretty=True) |
622
|
|
|
|
623
|
|
|
# TODO: Add update for systemd |
624
|
|
|
# * Stop possibly running service (it should not be running, though!) |
625
|
|
|
# * Actually update service files |
626
|
|
|
|
627
|
|
|
instance_name = ctx.obj["instance"] |
628
|
|
|
config = ctx.obj["instance_configuration"] |
629
|
|
|
|
630
|
|
|
env_path = "/var/lib/isomer/" + instance_name + "/" + next_environment |
631
|
|
|
|
632
|
|
|
log("Updating systemd service for %s (%s)" % (instance_name, next_environment)) |
633
|
|
|
|
634
|
|
|
launcher = os.path.join(env_path, "repository/iso") |
635
|
|
|
executable = os.path.join(env_path, "venv/bin/python3") + " " + launcher |
636
|
|
|
executable += " --quiet --instance " + instance_name + " launch" |
637
|
|
|
|
638
|
|
|
definitions = { |
639
|
|
|
"instance": instance_name, |
640
|
|
|
"executable": executable, |
641
|
|
|
"environment": next_environment, |
642
|
|
|
"user_name": config["user"], |
643
|
|
|
"user_group": config["group"], |
644
|
|
|
} |
645
|
|
|
service_name = "isomer-" + instance_name + ".service" |
646
|
|
|
|
647
|
|
|
write_template( |
648
|
|
|
service_template, |
649
|
|
|
os.path.join("/etc/systemd/system/", service_name), |
650
|
|
|
definitions, |
651
|
|
|
) |
652
|
|
|
|
653
|
|
|
|
654
|
|
|
def _launch_service(ctx): |
655
|
|
|
"""Actually enable and launch newly set up environment""" |
656
|
|
|
instance_name = ctx.obj["instance"] |
657
|
|
|
|
658
|
|
|
service_name = "isomer-" + instance_name + ".service" |
659
|
|
|
|
660
|
|
|
success, result = run_process( |
661
|
|
|
"/", ["systemctl", "enable", service_name], sudo="root" |
662
|
|
|
) |
663
|
|
|
|
664
|
|
|
if not success: |
665
|
|
|
log("Error activating service:", format_result(result), pretty=True, lvl=error) |
666
|
|
|
abort(5000) |
667
|
|
|
|
668
|
|
|
log("Launching service") |
669
|
|
|
|
670
|
|
|
success, result = run_process( |
671
|
|
|
"/", ["systemctl", "start", service_name], sudo="root" |
672
|
|
|
) |
673
|
|
|
|
674
|
|
|
if not success: |
675
|
|
|
log("Error activating service:", format_result(result), pretty=True, lvl=error) |
676
|
|
|
abort(5000) |
677
|
|
|
|
678
|
|
|
return True |
679
|
|
|
|
680
|
|
|
|
681
|
|
|
def validate_services(ctx): |
682
|
|
|
"""Checks init configuration settings so nothing gets mis-configured""" |
683
|
|
|
|
684
|
|
|
# TODO: Service validation |
685
|
|
|
# * Check through all configurations that we're not messing with port numbers |
686
|
|
|
# * ??? |
687
|
|
|
|
688
|
|
|
_ = ctx |
689
|
|
|
|
690
|
|
|
return True, "VALIDATION_NOT_IMPLEMENTED" |
691
|
|
|
|
692
|
|
|
|
693
|
|
|
@instance.command(short_help="instance ssl certificate") |
694
|
|
|
@click.option( |
695
|
|
|
"--selfsigned", |
696
|
|
|
help="Use a self-signed certificate", |
697
|
|
|
default=False, |
698
|
|
|
is_flag=True, |
699
|
|
|
) |
700
|
|
|
@click.pass_context |
701
|
|
|
def cert(ctx, selfsigned): |
702
|
|
|
"""instance a local SSL certificate""" |
703
|
|
|
|
704
|
|
|
instance_configuration = ctx.obj["instance_configuration"] |
705
|
|
|
instance_name = ctx.obj["instance"] |
706
|
|
|
next_environment = get_next_environment(ctx) |
707
|
|
|
|
708
|
|
|
set_instance(instance_name, next_environment) |
709
|
|
|
|
710
|
|
|
if selfsigned: |
711
|
|
|
_instance_selfsigned(instance_configuration) |
712
|
|
|
else: |
713
|
|
|
log("This is work in progress") |
714
|
|
|
abort(55555) |
715
|
|
|
|
716
|
|
|
_instance_letsencrypt(instance_configuration) |
717
|
|
|
|
718
|
|
|
finish(ctx) |
719
|
|
|
|
720
|
|
|
|
721
|
|
|
def _instance_letsencrypt(instance_configuration): |
722
|
|
|
hostnames = instance_configuration.get("web_hostnames", False) |
723
|
|
|
hostnames = hostnames.replace(" ", "") |
724
|
|
|
|
725
|
|
|
if not hostnames or hostnames == "localhost": |
726
|
|
|
log( |
727
|
|
|
"Please configure the public fully qualified domain names of this instance.\n" |
728
|
|
|
"Use 'iso instance set web_hostnames your.hostname.tld' to do that.\n" |
729
|
|
|
"You can add multiple names by separating them with commas.", |
730
|
|
|
lvl=error, |
731
|
|
|
) |
732
|
|
|
abort(50031) |
733
|
|
|
|
734
|
|
|
contact = instance_configuration.get("contact", False) |
735
|
|
|
if not contact: |
736
|
|
|
log( |
737
|
|
|
"You need to specify a contact mail address for this instance to generate certificates.\n" |
738
|
|
|
"Use 'iso instance set contact [email protected]' to do that.", |
739
|
|
|
lvl=error, |
740
|
|
|
) |
741
|
|
|
abort(50032) |
742
|
|
|
|
743
|
|
|
success, result = run_process( |
744
|
|
|
"/", |
745
|
|
|
[ |
746
|
|
|
"certbot", |
747
|
|
|
"--nginx", |
748
|
|
|
"certonly", |
749
|
|
|
"-m", |
750
|
|
|
contact, |
751
|
|
|
"-d", |
752
|
|
|
hostnames, |
753
|
|
|
"--agree-tos", |
754
|
|
|
"-n", |
755
|
|
|
], |
756
|
|
|
) |
757
|
|
|
if not success: |
758
|
|
|
log( |
759
|
|
|
"Error getting certificate:", |
760
|
|
|
format_result(result), |
761
|
|
|
pretty=True, |
762
|
|
|
lvl=error, |
763
|
|
|
) |
764
|
|
|
abort(50033) |
765
|
|
|
|
766
|
|
|
|
767
|
|
|
def _instance_selfsigned(instance_configuration): |
768
|
|
|
"""Generates a snakeoil certificate that has only been self signed""" |
769
|
|
|
|
770
|
|
|
log("Generating self signed certificate/key combination") |
771
|
|
|
|
772
|
|
|
try: |
773
|
|
|
os.mkdir("/etc/ssl/certs/isomer") |
774
|
|
|
except FileExistsError: |
775
|
|
|
pass |
776
|
|
|
except PermissionError: |
777
|
|
|
log("Need root (e.g. via sudo) to generate ssl certificate") |
778
|
|
|
abort(1) |
779
|
|
|
|
780
|
|
|
try: |
781
|
|
|
from OpenSSL import crypto |
782
|
|
|
except ImportError: |
783
|
|
|
log("Need python3-openssl to do this.") |
784
|
|
|
abort(1) |
785
|
|
|
return |
786
|
|
|
|
787
|
|
|
def create_self_signed_cert(target): |
788
|
|
|
"""Create a simple self signed SSL certificate""" |
789
|
|
|
|
790
|
|
|
key_file = os.path.join(target, "selfsigned.key") |
791
|
|
|
cert_file = os.path.join(target, "selfsigned.crt") |
792
|
|
|
combined_file = os.path.join(target, "selfsigned.pem") |
793
|
|
|
|
794
|
|
|
cert_conf = {k: v for k, v in instance_configuration.items() if |
795
|
|
|
k.startswith("web_certificate")} |
796
|
|
|
|
797
|
|
|
log("Certificate data:", cert_conf, pretty=True) |
798
|
|
|
|
799
|
|
|
hostname = instance_configuration.get("web_hostnames", gethostname()) |
800
|
|
|
if isinstance(hostname, list): |
801
|
|
|
hostname = hostname[0] |
802
|
|
|
|
803
|
|
|
# create a key pair |
804
|
|
|
k = crypto.PKey() |
805
|
|
|
k.generate_key(crypto.TYPE_RSA, 2048) |
806
|
|
|
|
807
|
|
|
if os.path.exists(cert_file): |
808
|
|
|
try: |
809
|
|
|
certificate = open(cert_file, "rb").read() |
810
|
|
|
old_cert = crypto.load_certificate(crypto.FILETYPE_PEM, certificate) |
811
|
|
|
serial = old_cert.get_serial_number() + 1 |
812
|
|
|
except (crypto.Error, OSError) as e: |
813
|
|
|
log( |
814
|
|
|
"Could not read old certificate to increment serial:", |
815
|
|
|
type(e), |
816
|
|
|
e, |
817
|
|
|
exc=True, |
818
|
|
|
lvl=warn, |
819
|
|
|
) |
820
|
|
|
serial = 1 |
821
|
|
|
else: |
822
|
|
|
serial = 1 |
823
|
|
|
|
824
|
|
|
# create a self-signed certificate |
825
|
|
|
certificate = crypto.X509() |
826
|
|
|
certificate.get_subject().C = cert_conf.get("web_certificate_country", |
827
|
|
|
"EU") |
828
|
|
|
certificate.get_subject().ST = cert_conf.get("web_certificate_state", "Sol") |
829
|
|
|
certificate.get_subject().L = cert_conf.get("web_certificate_location", "Earth") |
830
|
|
|
# noinspection PyPep8 |
831
|
|
|
certificate.get_subject().O = cert_conf.get("web_certificate_issuer", "Unknown") |
832
|
|
|
certificate.get_subject().OU = cert_conf.get("web_certificate_unit", "Unknown") |
833
|
|
|
certificate.get_subject().CN = hostname |
834
|
|
|
certificate.set_serial_number(serial) |
835
|
|
|
certificate.gmtime_adj_notBefore(0) |
836
|
|
|
certificate.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) |
837
|
|
|
certificate.set_issuer(certificate.get_subject()) |
838
|
|
|
certificate.set_pubkey(k) |
839
|
|
|
certificate.sign(k, "sha512") |
840
|
|
|
|
841
|
|
|
open(key_file, "wt").write( |
842
|
|
|
str(crypto.dump_privatekey(crypto.FILETYPE_PEM, k), encoding="ASCII") |
843
|
|
|
) |
844
|
|
|
|
845
|
|
|
open(cert_file, "wt").write( |
846
|
|
|
str( |
847
|
|
|
crypto.dump_certificate(crypto.FILETYPE_PEM, certificate), |
848
|
|
|
encoding="ASCII", |
849
|
|
|
) |
850
|
|
|
) |
851
|
|
|
|
852
|
|
|
open(combined_file, "wt").write( |
853
|
|
|
str( |
854
|
|
|
crypto.dump_certificate(crypto.FILETYPE_PEM, certificate), |
855
|
|
|
encoding="ASCII", |
856
|
|
|
) |
857
|
|
|
+ str(crypto.dump_privatekey(crypto.FILETYPE_PEM, k), encoding="ASCII") |
858
|
|
|
) |
859
|
|
|
|
860
|
|
|
location = os.path.join( |
861
|
|
|
get_etc_instance_path(), |
862
|
|
|
instance_configuration.get("name") |
863
|
|
|
) |
864
|
|
|
|
865
|
|
|
create_self_signed_cert(location) |
866
|
|
|
|
867
|
|
|
instance_configuration["web_key"] = os.path.join(location, "selfsigned.key") |
868
|
|
|
instance_configuration["web_certificate"] = os.path.join(location, "selfsigned.crt") |
869
|
|
|
|
870
|
|
|
write_instance(instance_configuration) |
871
|
|
|
|
872
|
|
|
|
873
|
|
|
@instance.command(short_help="install systemd service") |
874
|
|
|
@click.pass_context |
875
|
|
|
def service(ctx): |
876
|
|
|
"""instance systemd service configuration""" |
877
|
|
|
|
878
|
|
|
update_service(ctx, ctx.obj["instance_configuration"]["environment"]) |
879
|
|
|
finish(ctx) |
880
|
|
|
|
881
|
|
|
|
882
|
|
|
@instance.command(short_help="instance nginx configuration") |
883
|
|
|
@click.option( |
884
|
|
|
"--hostname", |
885
|
|
|
default=None, |
886
|
|
|
help="Override public Hostname (FQDN) Default from active system " "configuration", |
887
|
|
|
) |
888
|
|
|
@click.pass_context |
889
|
|
|
def update_nginx(ctx, hostname): |
890
|
|
|
"""instance nginx configuration""" |
891
|
|
|
|
892
|
|
|
ctx.obj["hostname"] = hostname |
893
|
|
|
|
894
|
|
|
_create_nginx_config(ctx) |
895
|
|
|
finish(ctx) |
896
|
|
|
|
897
|
|
|
|
898
|
|
|
def _create_nginx_config(ctx): |
899
|
|
|
"""instance nginx configuration""" |
900
|
|
|
|
901
|
|
|
# TODO: Specify template url very precisely. Currently one needs to be in |
902
|
|
|
# the repository root |
903
|
|
|
|
904
|
|
|
instance_name = ctx.obj["instance"] |
905
|
|
|
config = ctx.obj["instance_configuration"] |
906
|
|
|
|
907
|
|
|
current_env = config["environment"] |
908
|
|
|
env = config["environments"][current_env] |
909
|
|
|
|
910
|
|
|
dbhost = config["database_host"] |
911
|
|
|
dbname = env["database"] |
912
|
|
|
|
913
|
|
|
hostnames = ctx.obj.get("web_hostnames", None) |
914
|
|
|
if hostnames is None: |
915
|
|
|
hostnames = config.get("web_hostnames", None) |
916
|
|
|
if hostnames is None: |
917
|
|
|
try: |
918
|
|
|
configuration = _get_system_configuration(dbhost, dbname) |
919
|
|
|
hostnames = configuration.hostname |
920
|
|
|
except Exception as e: |
921
|
|
|
log("Exception:", e, type(e), exc=True, lvl=error) |
922
|
|
|
log( |
923
|
|
|
"""Could not determine public fully qualified hostname! |
924
|
|
|
Check systemconfig (see db view and db modify commands) or specify |
925
|
|
|
manually with --hostname host.domain.tld |
926
|
|
|
|
927
|
|
|
Using 'localhost' for now""", |
928
|
|
|
lvl=warn, |
929
|
|
|
) |
930
|
|
|
hostnames = "localhost" |
931
|
|
|
port = config["web_port"] |
932
|
|
|
address = config["web_address"] |
933
|
|
|
|
934
|
|
|
log( |
935
|
|
|
"Creating nginx configuration for %s:%i using %s@%s" |
936
|
|
|
% (hostnames, port, dbname, dbhost) |
937
|
|
|
) |
938
|
|
|
|
939
|
|
|
definitions = { |
940
|
|
|
"server_public_name": hostnames.replace(",", " "), |
941
|
|
|
"ssl_certificate": config["web_certificate"], |
942
|
|
|
"ssl_key": config["web_key"], |
943
|
|
|
"host_url": "http://%s:%i/" % (address, port), |
944
|
|
|
"instance": instance_name, |
945
|
|
|
"environment": current_env, |
946
|
|
|
} |
947
|
|
|
|
948
|
|
|
if distribution == "DEBIAN": |
949
|
|
|
configuration_file = "/etc/nginx/sites-available/isomer.%s.conf" % instance_name |
950
|
|
|
configuration_link = "/etc/nginx/sites-enabled/isomer.%s.conf" % instance_name |
951
|
|
|
elif distribution == "ARCH": |
952
|
|
|
configuration_file = "/etc/nginx/nginx.conf" |
953
|
|
|
configuration_link = None |
954
|
|
|
else: |
955
|
|
|
log( |
956
|
|
|
"Unsure how to proceed, you may need to specify your " "distribution", |
957
|
|
|
lvl=error, |
958
|
|
|
) |
959
|
|
|
return |
960
|
|
|
|
961
|
|
|
log("Writing nginx Isomer site definition") |
962
|
|
|
write_template(nginx_template, configuration_file, definitions) |
963
|
|
|
|
964
|
|
|
if configuration_link is not None: |
965
|
|
|
log("Enabling nginx Isomer site (symlink)") |
966
|
|
|
if not os.path.exists(configuration_link): |
967
|
|
|
os.symlink(configuration_file, configuration_link) |
968
|
|
|
|
969
|
|
|
if os.path.exists("/bin/systemctl"): |
970
|
|
|
log("Restarting nginx service") |
971
|
|
|
run_process("/", ["systemctl", "restart", "nginx.service"], sudo="root") |
972
|
|
|
else: |
973
|
|
|
log("No systemctl found, not restarting nginx") |
974
|
|
|
|
975
|
|
|
# TODO: Add instance user |
976
|
|
|
|