1
|
|
|
#!/usr/bin/env python |
2
|
|
|
# -*- coding: UTF-8 -*- |
3
|
|
|
|
4
|
|
|
# Isomer - The distributed application framework |
5
|
|
|
# ============================================== |
6
|
|
|
# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others. |
7
|
|
|
# |
8
|
|
|
# This program is free software: you can redistribute it and/or modify |
9
|
|
|
# it under the terms of the GNU Affero General Public License as published by |
10
|
|
|
# the Free Software Foundation, either version 3 of the License, or |
11
|
|
|
# (at your option) any later version. |
12
|
|
|
# |
13
|
|
|
# This program is distributed in the hope that it will be useful, |
14
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
15
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
16
|
|
|
# GNU Affero General Public License for more details. |
17
|
|
|
# |
18
|
|
|
# You should have received a copy of the GNU Affero General Public License |
19
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
20
|
|
|
|
21
|
|
|
""" |
22
|
|
|
|
23
|
|
|
Module: Instance |
24
|
|
|
================ |
25
|
|
|
|
26
|
|
|
Instance management functionality. |
27
|
|
|
|
28
|
|
|
instance info |
29
|
|
|
instance list |
30
|
|
|
instance set |
31
|
|
|
instance create |
32
|
|
|
instance install |
33
|
|
|
instance clear |
34
|
|
|
instance remove |
35
|
|
|
instance install-module |
36
|
|
|
instance turnover |
37
|
|
|
instance cert |
38
|
|
|
instance service |
39
|
|
|
instance update-nginx |
40
|
|
|
|
41
|
|
|
""" |
42
|
|
|
|
43
|
|
|
import os |
44
|
|
|
import webbrowser |
45
|
|
|
from socket import gethostname |
46
|
|
|
from shutil import rmtree |
47
|
|
|
|
48
|
|
|
import tomlkit |
49
|
|
|
import click |
50
|
|
|
from click_didyoumean import DYMGroup |
51
|
|
|
from isomer.error import ( |
52
|
|
|
abort, |
53
|
|
|
EXIT_INVALID_CONFIGURATION, |
54
|
|
|
EXIT_INSTALLATION_FAILED, |
55
|
|
|
EXIT_INSTANCE_EXISTS, |
56
|
|
|
EXIT_INSTANCE_UNKNOWN, |
57
|
|
|
EXIT_SERVICE_INVALID, |
58
|
|
|
EXIT_USER_BAILED_OUT, |
59
|
|
|
EXIT_INVALID_PARAMETER, |
60
|
|
|
) |
61
|
|
|
from isomer.migration import apply_migrations |
62
|
|
|
from isomer.misc import sorted_alphanumerical |
63
|
|
|
from isomer.misc.path import set_instance, get_etc_instance_path, get_path |
64
|
|
|
from isomer.logger import warn, critical |
65
|
|
|
from isomer.tool import ( |
66
|
|
|
_get_system_configuration, |
67
|
|
|
_get_configuration, |
68
|
|
|
get_next_environment, |
69
|
|
|
ask, |
70
|
|
|
finish, |
71
|
|
|
run_process, |
72
|
|
|
format_result, |
73
|
|
|
log, |
74
|
|
|
error, |
75
|
|
|
debug, |
76
|
|
|
) |
77
|
|
|
from isomer.tool.etc import ( |
78
|
|
|
write_instance, |
79
|
|
|
valid_configuration, |
80
|
|
|
remove_instance, |
81
|
|
|
instance_template, |
82
|
|
|
) |
83
|
|
|
from isomer.tool.templates import write_template |
84
|
|
|
from isomer.tool.defaults import ( |
85
|
|
|
service_template, |
86
|
|
|
nginx_template, |
87
|
|
|
distribution, |
88
|
|
|
) |
89
|
|
|
from isomer.tool.environment import ( |
90
|
|
|
_install_environment, |
91
|
|
|
install_environment_modules, |
92
|
|
|
_clear_environment, |
93
|
|
|
_check_environment, |
94
|
|
|
) |
95
|
|
|
from isomer.tool.database import copy_database |
96
|
|
|
from isomer.tool.version import _get_versions |
97
|
|
|
from isomer.ui.builder import copy_directory_tree |
98
|
|
|
from isomer.ui.store import DEFAULT_STORE_URL |
99
|
|
|
|
100
|
|
|
|
101
|
|
|
@click.group( |
102
|
|
|
cls=DYMGroup, |
103
|
|
|
short_help="Instance handling" |
104
|
|
|
) |
105
|
|
|
@click.pass_context |
106
|
|
|
def instance(ctx): |
107
|
|
|
"""[GROUP] instance various aspects of Isomer""" |
108
|
|
|
|
109
|
|
|
if ctx.invoked_subcommand in ("info", "list", "create"): |
110
|
|
|
return |
111
|
|
|
|
112
|
|
|
_get_configuration(ctx) |
113
|
|
|
|
114
|
|
|
|
115
|
|
View Code Duplication |
@instance.command(name="info", short_help="show system configuration of instance") |
|
|
|
|
116
|
|
|
@click.pass_context |
117
|
|
|
def info_instance(ctx): |
118
|
|
|
"""Print information about the selected instance""" |
119
|
|
|
|
120
|
|
|
instance_name = ctx.obj["instance"] |
121
|
|
|
instances = ctx.obj["instances"] |
122
|
|
|
instance_configuration = instances[instance_name] |
123
|
|
|
|
124
|
|
|
environment_name = instance_configuration["environment"] |
125
|
|
|
environment_config = instance_configuration["environments"][environment_name] |
126
|
|
|
|
127
|
|
|
if instance_name not in instances: |
128
|
|
|
log("Instance %s unknown!" % instance_name, lvl=warn) |
129
|
|
|
abort(EXIT_INSTANCE_UNKNOWN) |
130
|
|
|
|
131
|
|
|
log("Instance configuration:", instance_configuration, pretty=True) |
132
|
|
|
log("Active environment (%s):" % environment_name, environment_config, pretty=True) |
133
|
|
|
|
134
|
|
|
finish(ctx) |
135
|
|
|
|
136
|
|
|
|
137
|
|
View Code Duplication |
@instance.command( |
|
|
|
|
138
|
|
|
name="check", short_help="check instance configuration and environments" |
139
|
|
|
) |
140
|
|
|
@click.pass_context |
141
|
|
|
def check_instance(ctx): |
142
|
|
|
"""Check health of the selected instance""" |
143
|
|
|
|
144
|
|
|
instance_name = ctx.obj["instance"] |
145
|
|
|
instances = ctx.obj["instances"] |
146
|
|
|
instance_configuration = instances[instance_name] |
147
|
|
|
|
148
|
|
|
environment_name = instance_configuration["environment"] |
149
|
|
|
environment_config = instance_configuration["environments"][environment_name] |
150
|
|
|
|
151
|
|
|
if instance_name not in instances: |
152
|
|
|
log("Instance %s unknown!" % instance_name, lvl=warn) |
153
|
|
|
abort(EXIT_INSTANCE_UNKNOWN) |
154
|
|
|
|
155
|
|
|
# TODO: Validate instance config |
156
|
|
|
|
157
|
|
|
_check_environment(ctx, "blue") |
158
|
|
|
_check_environment(ctx, "green") |
159
|
|
|
|
160
|
|
|
finish(ctx) |
161
|
|
|
|
162
|
|
|
|
163
|
|
|
@instance.command(name="browser", short_help="try to point a browser to this instance") |
164
|
|
|
@click.pass_context |
165
|
|
|
def browser(ctx): |
166
|
|
|
"""Tries to start or point a browser towards this instance's frontend""" |
167
|
|
|
instance_configuration = ctx.obj["instance_configuration"] |
168
|
|
|
|
169
|
|
|
server = instance_configuration.get("webserver", "internal") |
170
|
|
|
protocol = "http" |
171
|
|
|
|
172
|
|
|
if server != "internal": |
173
|
|
|
protocol += "s" |
174
|
|
|
|
175
|
|
|
host = instance_configuration.get( |
176
|
|
|
"web_hostname", instance_configuration.get("web_address", "127.0.0.1") |
177
|
|
|
) |
178
|
|
|
port = instance_configuration.get("web_port", None) |
179
|
|
|
|
180
|
|
|
if port is None: |
181
|
|
|
url = "%s://%s" % (protocol, host) |
182
|
|
|
else: |
183
|
|
|
url = "%s://%s:%i" % (protocol, host, port) |
184
|
|
|
|
185
|
|
|
log("Opening browser to:", url) |
186
|
|
|
log( |
187
|
|
|
"If this is empty or unreachable, check your instance with:\n" |
188
|
|
|
" iso instance info\n" |
189
|
|
|
"Also try the health checks:\n" |
190
|
|
|
" iso instance check" |
191
|
|
|
) |
192
|
|
|
|
193
|
|
|
webbrowser.open(url) |
194
|
|
|
|
195
|
|
|
|
196
|
|
|
@instance.command(name="list", short_help="List all instances") |
197
|
|
|
@click.pass_context |
198
|
|
|
def list_instances(ctx): |
199
|
|
|
"""List all known instances""" |
200
|
|
|
|
201
|
|
|
for instance_name in ctx.obj["instances"]: |
202
|
|
|
log(instance_name, pretty=True) |
203
|
|
|
finish(ctx) |
204
|
|
|
|
205
|
|
|
|
206
|
|
|
@instance.command(name="set", short_help="Set a parameter of an instance") |
207
|
|
|
@click.argument("parameter") |
208
|
|
|
@click.argument("value") |
209
|
|
|
@click.pass_context |
210
|
|
|
def set_parameter(ctx, parameter, value): |
211
|
|
|
"""Set a configuration parameter of an instance""" |
212
|
|
|
|
213
|
|
|
log("Setting %s to %s" % (parameter, value)) |
214
|
|
|
instance_configuration = ctx.obj["instance_configuration"] |
215
|
|
|
defaults = instance_template |
216
|
|
|
converted_value = None |
217
|
|
|
|
218
|
|
|
try: |
219
|
|
|
parameter_type = type(defaults[parameter]) |
220
|
|
|
log(parameter_type, pretty=True, lvl=debug) |
221
|
|
|
|
222
|
|
|
if parameter_type == tomlkit.items.Integer: |
223
|
|
|
converted_value = int(value) |
224
|
|
|
elif parameter_type == bool: |
225
|
|
|
converted_value = value.upper() == "TRUE" |
226
|
|
|
else: |
227
|
|
|
converted_value = value |
228
|
|
|
except KeyError: |
229
|
|
|
log("Available parameters:", sorted(list(defaults.keys()))) |
230
|
|
|
abort(EXIT_INVALID_PARAMETER) |
231
|
|
|
|
232
|
|
|
if converted_value is None: |
233
|
|
|
log("Converted value was None! Recheck the new config!", lvl=warn) |
234
|
|
|
|
235
|
|
|
instance_configuration[parameter] = converted_value |
236
|
|
|
log("New config:", instance_configuration, pretty=True, lvl=debug) |
237
|
|
|
|
238
|
|
|
ctx.obj["instances"][ctx.obj["instance"]] = instance_configuration |
239
|
|
|
|
240
|
|
|
if valid_configuration(ctx): |
241
|
|
|
write_instance(instance_configuration) |
242
|
|
|
finish(ctx) |
243
|
|
|
else: |
244
|
|
|
log("New configuration would not be valid", lvl=critical) |
245
|
|
|
abort(EXIT_INVALID_CONFIGURATION) |
246
|
|
|
|
247
|
|
|
|
248
|
|
|
@instance.command(short_help="Create a new instance") |
249
|
|
|
@click.argument("instance_name", default="") |
250
|
|
|
@click.pass_context |
251
|
|
|
def create(ctx, instance_name): |
252
|
|
|
"""Create a new instance""" |
253
|
|
|
|
254
|
|
|
if instance_name == "": |
255
|
|
|
instance_name = ctx.obj["instance"] |
256
|
|
|
|
257
|
|
|
if instance_name in ctx.obj["instances"]: |
258
|
|
|
abort(EXIT_INSTANCE_EXISTS) |
259
|
|
|
|
260
|
|
|
log("Creating instance:", instance_name) |
261
|
|
|
instance_configuration = instance_template |
262
|
|
|
instance_configuration["name"] = instance_name |
263
|
|
|
ctx.obj["instances"][instance_name] = instance_configuration |
264
|
|
|
|
265
|
|
|
write_instance(instance_configuration) |
266
|
|
|
finish(ctx) |
267
|
|
|
|
268
|
|
|
|
269
|
|
|
@instance.command(name="install", short_help="Install a fresh instance") |
270
|
|
|
@click.option("--force", "-f", is_flag=True, default=False) |
271
|
|
|
@click.option( |
272
|
|
|
"--source", "-s", default="git", |
273
|
|
|
type=click.Choice(["link", "copy", "git", "github"]) |
274
|
|
|
) |
275
|
|
|
@click.option("--url", "-u", default="", type=click.Path()) |
276
|
|
|
@click.option( |
277
|
|
|
"--import-file", "--import", default=None, help="Import the specified backup" |
278
|
|
|
) |
279
|
|
|
@click.option( |
280
|
|
|
"--no-sudo", |
281
|
|
|
is_flag=True, |
282
|
|
|
default=False, |
283
|
|
|
help="Do not use sudo to install (Mostly for tests)", |
284
|
|
|
) |
285
|
|
|
@click.option( |
286
|
|
|
"--release", "-r", default=None, help="Override installed release version" |
287
|
|
|
) |
288
|
|
|
@click.option("--skip-modules", is_flag=True, default=False) |
289
|
|
|
@click.option("--skip-data", is_flag=True, default=False) |
290
|
|
|
@click.option("--skip-frontend", is_flag=True, default=False) |
291
|
|
|
@click.option("--skip-test", is_flag=True, default=False) |
292
|
|
|
@click.option("--skip-provisions", is_flag=True, default=False) |
293
|
|
|
@click.pass_context |
294
|
|
|
def install(ctx, **kwargs): |
295
|
|
|
"""Install a new environment of an instance""" |
296
|
|
|
|
297
|
|
|
log("Installing instance") |
298
|
|
|
|
299
|
|
|
env = ctx.obj["instance_configuration"]["environments"] |
300
|
|
|
|
301
|
|
|
green = env["green"]["installed"] |
302
|
|
|
blue = env["blue"]["installed"] |
303
|
|
|
|
304
|
|
|
if green or blue: |
305
|
|
|
log( |
306
|
|
|
"At least one environment is installed or in a non-clear state.\n" |
307
|
|
|
"Please use 'iso instance upgrade' to upgrade an instance.", |
308
|
|
|
lvl=warn, |
309
|
|
|
) |
310
|
|
|
abort(50081) |
311
|
|
|
|
312
|
|
|
_clear_instance(ctx, force=kwargs["force"], clear=False, no_archive=True) |
313
|
|
|
|
314
|
|
|
_install_environment(ctx, **kwargs) |
315
|
|
|
|
316
|
|
|
ctx.obj["instance_configuration"]["source"] = kwargs["source"] |
317
|
|
|
ctx.obj["instance_configuration"]["url"] = kwargs["url"] |
318
|
|
|
|
319
|
|
|
write_instance(ctx.obj["instance_configuration"]) |
320
|
|
|
|
321
|
|
|
_turnover(ctx, force=kwargs["force"]) |
322
|
|
|
|
323
|
|
|
finish(ctx) |
324
|
|
|
|
325
|
|
|
|
326
|
|
|
@instance.command(name="clear", short_help="Clear the whole instance (CAUTION)") |
327
|
|
|
@click.option("--force", "-f", is_flag=True, default=False) |
328
|
|
|
@click.option("--no-archive", "-n", is_flag=True, default=False) |
329
|
|
|
@click.pass_context |
330
|
|
|
def clear_instance(ctx, force, no_archive): |
331
|
|
|
"""Irrevocably clear all environments of an instance""" |
332
|
|
|
|
333
|
|
|
_clear_instance(ctx, force, False, no_archive) |
334
|
|
|
|
335
|
|
|
|
336
|
|
|
def _clear_instance(ctx, force, clear, no_archive): |
337
|
|
|
log("Clearing instance:", ctx.obj["instance"]) |
338
|
|
|
log("Clearing blue environment.", lvl=debug) |
339
|
|
|
_clear_environment(ctx, force, "blue", clear, no_archive) |
340
|
|
|
log("Clearing green environment.", lvl=debug) |
341
|
|
|
_clear_environment(ctx, force, "green", clear, no_archive) |
342
|
|
|
finish(ctx) |
343
|
|
|
|
344
|
|
|
|
345
|
|
|
@instance.command(short_help="Remove a whole instance (CAUTION)") |
346
|
|
|
@click.option( |
347
|
|
|
"--clear", "-c", is_flag=True, help="Clear instance before removal", default=False |
348
|
|
|
) |
349
|
|
|
@click.option("--no-archive", "-n", is_flag=True, default=False) |
350
|
|
|
@click.pass_context |
351
|
|
|
def remove(ctx, clear, no_archive): |
352
|
|
|
"""Irrevocably remove a whole instance""" |
353
|
|
|
|
354
|
|
|
if clear: |
355
|
|
|
log("Destructively removing instance:", ctx.obj["instance"], lvl=warn) |
356
|
|
|
|
357
|
|
|
if not ask("Are you sure", default=False, data_type="bool"): |
358
|
|
|
abort(EXIT_USER_BAILED_OUT) |
359
|
|
|
|
360
|
|
|
if clear: |
361
|
|
|
_clear_instance(ctx, force=True, clear=clear, no_archive=no_archive) |
362
|
|
|
|
363
|
|
|
remove_instance(ctx.obj["instance"]) |
364
|
|
|
finish(ctx) |
365
|
|
|
|
366
|
|
|
|
367
|
|
|
@instance.command( |
368
|
|
|
"install-modules", short_help="Add (and install) modules to an instance" |
369
|
|
|
) |
370
|
|
|
@click.option( |
371
|
|
|
"--source", |
372
|
|
|
"-s", |
373
|
|
|
default="git", |
374
|
|
|
type=click.Choice(["link", "copy", "git", "develop", "store"]), |
375
|
|
|
help="Specify installation source/method", |
376
|
|
|
) |
377
|
|
|
@click.option( |
378
|
|
|
"--store-url", |
379
|
|
|
default=DEFAULT_STORE_URL, |
380
|
|
|
help="Specify alternative store url (Default: %s)" % DEFAULT_STORE_URL, |
381
|
|
|
) |
382
|
|
|
@click.option( |
383
|
|
|
"--install-env", |
384
|
|
|
"--install", |
385
|
|
|
"-i", |
386
|
|
|
is_flag=True, |
387
|
|
|
default=False, |
388
|
|
|
help="Install modules on active environment", |
389
|
|
|
) |
390
|
|
|
@click.option( |
391
|
|
|
"--force", |
392
|
|
|
"-f", |
393
|
|
|
default=False, |
394
|
|
|
is_flag=True, |
395
|
|
|
help="Force installation (overwrites old modules)", |
396
|
|
|
) |
397
|
|
|
@click.argument("urls", nargs=-1) |
398
|
|
|
@click.pass_context |
399
|
|
|
def install_instance_modules(ctx, source, urls, install_env, force, store_url): |
400
|
|
|
"""Add (and optionally immediately install) modules for an instance. |
401
|
|
|
|
402
|
|
|
This will add them to the instance's configuration, so they will be upgraded as well |
403
|
|
|
as reinstalled on other environment changes. |
404
|
|
|
|
405
|
|
|
If you're installing from a store, you can specify a custom store URL with the |
406
|
|
|
--store-url argument. |
407
|
|
|
""" |
408
|
|
|
|
409
|
|
|
instance_name = ctx.obj["instance"] |
410
|
|
|
instance_configuration = ctx.obj["instances"][instance_name] |
411
|
|
|
|
412
|
|
|
for url in urls: |
413
|
|
|
descriptor = [source, url] |
414
|
|
|
if store_url != DEFAULT_STORE_URL: |
415
|
|
|
descriptor.append(store_url) |
416
|
|
|
if descriptor not in instance_configuration["modules"]: |
417
|
|
|
instance_configuration["modules"].append(descriptor) |
418
|
|
|
elif not force: |
419
|
|
|
log("Module %s is already installed. Use --force to install anyway." % url) |
420
|
|
|
abort(50000) |
421
|
|
|
|
422
|
|
|
write_instance(instance_configuration) |
423
|
|
|
|
424
|
|
|
if install_env is True: |
425
|
|
|
next_environment = get_next_environment(ctx) |
426
|
|
|
environments = instance_configuration['environments'] |
427
|
|
|
|
428
|
|
|
if environments[next_environment]["installed"] is False: |
429
|
|
|
log("Environment %s is not installed, cannot install modules." |
430
|
|
|
% next_environment, lvl=warn) |
431
|
|
|
abort(50600) |
432
|
|
|
return |
433
|
|
|
del ctx.params["install_env"] |
434
|
|
|
ctx.forward(install_environment_modules) |
435
|
|
|
|
436
|
|
|
finish(ctx) |
437
|
|
|
|
438
|
|
|
|
439
|
|
|
@instance.command(short_help="Activates the other environment") |
440
|
|
|
@click.option("--force", "-f", is_flag=True, default=False, help="Force turnover") |
441
|
|
|
@click.pass_context |
442
|
|
|
def turnover(ctx, **kwargs): |
443
|
|
|
"""Activates the other environment """ |
444
|
|
|
|
445
|
|
|
_turnover(ctx, **kwargs) |
446
|
|
|
finish(ctx) |
447
|
|
|
|
448
|
|
|
|
449
|
|
|
def _turnover(ctx, force): |
450
|
|
|
"""Internal turnover operation""" |
451
|
|
|
|
452
|
|
|
# if ctx.obj['acting_environment'] is not None: |
453
|
|
|
# next_environment = ctx.obj['acting_environment'] |
454
|
|
|
# else: |
455
|
|
|
next_environment = get_next_environment(ctx) |
456
|
|
|
|
457
|
|
|
log("Activating environment:", next_environment) |
458
|
|
|
env = ctx.obj["instance_configuration"]["environments"][next_environment] |
459
|
|
|
|
460
|
|
|
log("Inspecting new environment") |
461
|
|
|
|
462
|
|
|
if not force: |
463
|
|
|
if env.get("database", "") == "": |
464
|
|
|
log("Database has not been set up correctly.", lvl=critical) |
465
|
|
|
abort(EXIT_INSTALLATION_FAILED) |
466
|
|
|
|
467
|
|
|
if ( |
468
|
|
|
not env.get("installed", False) |
469
|
|
|
or not env.get("tested", False) |
470
|
|
|
or not env.get("migrated", False) |
471
|
|
|
): |
472
|
|
|
log("Installation failed, cannot activate!", lvl=critical) |
473
|
|
|
abort(EXIT_INSTALLATION_FAILED) |
474
|
|
|
|
475
|
|
|
update_service(ctx, next_environment) |
476
|
|
|
|
477
|
|
|
ctx.obj["instance_configuration"]["environment"] = next_environment |
478
|
|
|
|
479
|
|
|
write_instance(ctx.obj["instance_configuration"]) |
480
|
|
|
|
481
|
|
|
# TODO: Effect reload of service |
482
|
|
|
# * Systemctl reload |
483
|
|
|
# * (Re)start service |
484
|
|
|
# * confirm correct operation |
485
|
|
|
# - if not, switch back to the other instance, maybe indicate a broken |
486
|
|
|
# state for next_environment |
487
|
|
|
# - if yes, Store instance configuration and terminate, we're done |
488
|
|
|
|
489
|
|
|
log("Turned instance over to", next_environment) |
490
|
|
|
|
491
|
|
|
|
492
|
|
|
@instance.command(short_help="Upgrades the other environment") |
493
|
|
|
@click.option("--release", "-r", default=None, help="Specify release to upgrade to") |
494
|
|
|
@click.option("--upgrade-modules", default=False, is_flag=True, |
495
|
|
|
help="Also, upgrade modules if possible") |
496
|
|
|
@click.option("--restart", default=False, is_flag=True, |
497
|
|
|
help="Restart systemd service via systemctl on success") |
498
|
|
|
@click.option("--handle-cache", "-c", |
499
|
|
|
type=click.Choice(["ignore", "move", "copy"], case_sensitive=False), |
500
|
|
|
default="ignore", help="Handle cached data as well (ignore, move, copy)") |
501
|
|
|
@click.option( |
502
|
|
|
"--source", |
503
|
|
|
"-s", |
504
|
|
|
default="git", |
505
|
|
|
type=click.Choice(["link", "copy", "git", "develop", "github", "pypi"]), |
506
|
|
|
help="Specify installation source/method", |
507
|
|
|
) |
508
|
|
|
@click.option("--url", "-u", default="", type=click.Path()) |
509
|
|
|
@click.pass_context |
510
|
|
|
def upgrade(ctx, release, upgrade_modules, restart, handle_cache, source, url): |
511
|
|
|
"""Upgrades an instance on its other environment and turns over on success. |
512
|
|
|
|
513
|
|
|
\b |
514
|
|
|
1. Test if other environment is empty |
515
|
|
|
1.1. No - archive and clear it |
516
|
|
|
2. Copy current environment to other environment |
517
|
|
|
3. Clear old bits (venv, frontend) |
518
|
|
|
4. Fetch updates in other environment repository |
519
|
|
|
5. Select a release |
520
|
|
|
6. Checkout that release and its submodules |
521
|
|
|
7. Install release |
522
|
|
|
8. Copy database |
523
|
|
|
9. Migrate data (WiP) |
524
|
|
|
10. Turnover |
525
|
|
|
|
526
|
|
|
""" |
527
|
|
|
|
528
|
|
|
instance_config = ctx.obj["instance_configuration"] |
529
|
|
|
repository = get_path("lib", "repository") |
530
|
|
|
|
531
|
|
|
installation_source = source if source is not None else instance_config['source'] |
532
|
|
|
installation_url = url if url is not None else instance_config['url'] |
533
|
|
|
|
534
|
|
|
environments = instance_config["environments"] |
535
|
|
|
|
536
|
|
|
active = instance_config["environment"] |
537
|
|
|
|
538
|
|
|
next_environment = get_next_environment(ctx) |
539
|
|
|
if environments[next_environment]["installed"] is True: |
540
|
|
|
_clear_environment(ctx, clear_env=next_environment) |
541
|
|
|
|
542
|
|
|
source_paths = [ |
543
|
|
|
get_path("lib", "", environment=active), |
544
|
|
|
get_path("local", "", environment=active) |
545
|
|
|
] |
546
|
|
|
|
547
|
|
|
destination_paths = [ |
548
|
|
|
get_path("lib", "", environment=next_environment), |
549
|
|
|
get_path("local", "", environment=next_environment) |
550
|
|
|
] |
551
|
|
|
|
552
|
|
|
log(source_paths, destination_paths, pretty=True) |
553
|
|
|
|
554
|
|
|
for source, destination in zip(source_paths, destination_paths): |
555
|
|
|
log("Copying to new environment:", source, destination) |
556
|
|
|
copy_directory_tree(source, destination) |
557
|
|
|
|
558
|
|
|
if handle_cache != "ignore": |
559
|
|
|
log("Handling cache") |
560
|
|
|
move = handle_cache == "move" |
561
|
|
|
copy_directory_tree( |
562
|
|
|
get_path("cache", "", environment=active), |
563
|
|
|
get_path("cache", "", environment=next_environment), |
564
|
|
|
move=move |
565
|
|
|
) |
566
|
|
|
|
567
|
|
|
rmtree(get_path("lib", "venv"), ignore_errors=True) |
568
|
|
|
# TODO: This potentially leaves frontend-dev: |
569
|
|
|
rmtree(get_path("lib", "frontend"), ignore_errors=True) |
570
|
|
|
|
571
|
|
|
releases = _get_versions( |
572
|
|
|
ctx, |
573
|
|
|
source=installation_source, |
574
|
|
|
url=installation_url, |
575
|
|
|
fetch=True |
576
|
|
|
) |
577
|
|
|
|
578
|
|
|
releases_keys = sorted_alphanumerical(releases.keys()) |
579
|
|
|
|
580
|
|
|
if release is None: |
581
|
|
|
release = releases_keys[-1] |
582
|
|
|
else: |
583
|
|
|
if release not in releases_keys: |
584
|
|
|
log("Unknown release. Maybe try a different release or source.") |
585
|
|
|
abort(50100) |
586
|
|
|
|
587
|
|
|
log("Choosing release", release) |
588
|
|
|
|
589
|
|
|
_install_environment( |
590
|
|
|
ctx, installation_source, installation_url, upgrade=True, release=release |
591
|
|
|
) |
592
|
|
|
|
593
|
|
|
new_database_name = instance_config["name"] + "_" + next_environment |
594
|
|
|
|
595
|
|
|
copy_database( |
596
|
|
|
ctx.obj["dbhost"], |
597
|
|
|
active['database'], |
598
|
|
|
new_database_name |
599
|
|
|
) |
600
|
|
|
|
601
|
|
|
apply_migrations(ctx) |
602
|
|
|
|
603
|
|
|
finish(ctx) |
604
|
|
|
|
605
|
|
|
|
606
|
|
|
def update_service(ctx, next_environment): |
607
|
|
|
"""Updates the specified service configuration""" |
608
|
|
|
|
609
|
|
|
validated, message = validate_services(ctx) |
610
|
|
|
|
611
|
|
|
if not validated: |
612
|
|
|
log("Service configuration validation failed:", message, lvl=error) |
613
|
|
|
abort(EXIT_SERVICE_INVALID) |
614
|
|
|
|
615
|
|
|
init = ctx.obj["config"]["meta"]["init"] |
616
|
|
|
environment_config = ctx.obj["instance_configuration"]["environments"][ |
617
|
|
|
next_environment |
618
|
|
|
] |
619
|
|
|
|
620
|
|
|
log( |
621
|
|
|
"Updating %s configuration of instance %s to %s" |
622
|
|
|
% (init, ctx.obj["instance"], next_environment) |
623
|
|
|
) |
624
|
|
|
log("New environment:", environment_config, pretty=True) |
625
|
|
|
|
626
|
|
|
# TODO: Add update for systemd |
627
|
|
|
# * Stop possibly running service (it should not be running, though!) |
628
|
|
|
# * Actually update service files |
629
|
|
|
|
630
|
|
|
instance_name = ctx.obj["instance"] |
631
|
|
|
config = ctx.obj["instance_configuration"] |
632
|
|
|
|
633
|
|
|
env_path = "/var/lib/isomer/" + instance_name + "/" + next_environment |
634
|
|
|
|
635
|
|
|
log("Updating systemd service for %s (%s)" % (instance_name, next_environment)) |
636
|
|
|
|
637
|
|
|
launcher = os.path.join(env_path, "repository/iso") |
638
|
|
|
executable = os.path.join(env_path, "venv/bin/python3") + " " + launcher |
639
|
|
|
executable += " --quiet --instance " + instance_name + " launch" |
640
|
|
|
|
641
|
|
|
definitions = { |
642
|
|
|
"instance": instance_name, |
643
|
|
|
"executable": executable, |
644
|
|
|
"environment": next_environment, |
645
|
|
|
"user_name": config["user"], |
646
|
|
|
"user_group": config["group"], |
647
|
|
|
} |
648
|
|
|
service_name = "isomer-" + instance_name + ".service" |
649
|
|
|
|
650
|
|
|
write_template( |
651
|
|
|
service_template, |
652
|
|
|
os.path.join("/etc/systemd/system/", service_name), |
653
|
|
|
definitions, |
654
|
|
|
) |
655
|
|
|
|
656
|
|
|
|
657
|
|
|
def _launch_service(ctx): |
658
|
|
|
"""Actually enable and launch newly set up environment""" |
659
|
|
|
instance_name = ctx.obj["instance"] |
660
|
|
|
|
661
|
|
|
service_name = "isomer-" + instance_name + ".service" |
662
|
|
|
|
663
|
|
|
success, result = run_process( |
664
|
|
|
"/", ["systemctl", "enable", service_name], sudo="root" |
665
|
|
|
) |
666
|
|
|
|
667
|
|
|
if not success: |
668
|
|
|
log("Error activating service:", format_result(result), pretty=True, lvl=error) |
669
|
|
|
abort(5000) |
670
|
|
|
|
671
|
|
|
log("Launching service") |
672
|
|
|
|
673
|
|
|
success, result = run_process( |
674
|
|
|
"/", ["systemctl", "start", service_name], sudo="root" |
675
|
|
|
) |
676
|
|
|
|
677
|
|
|
if not success: |
678
|
|
|
log("Error activating service:", format_result(result), pretty=True, lvl=error) |
679
|
|
|
abort(5000) |
680
|
|
|
|
681
|
|
|
return True |
682
|
|
|
|
683
|
|
|
|
684
|
|
|
def validate_services(ctx): |
685
|
|
|
"""Checks init configuration settings so nothing gets mis-configured""" |
686
|
|
|
|
687
|
|
|
# TODO: Service validation |
688
|
|
|
# * Check through all configurations that we're not messing with port numbers |
689
|
|
|
# * ??? |
690
|
|
|
|
691
|
|
|
_ = ctx |
692
|
|
|
|
693
|
|
|
return True, "VALIDATION_NOT_IMPLEMENTED" |
694
|
|
|
|
695
|
|
|
|
696
|
|
|
@instance.command(short_help="instance ssl certificate") |
697
|
|
|
@click.option( |
698
|
|
|
"--selfsigned", |
699
|
|
|
help="Use a self-signed certificate", |
700
|
|
|
default=False, |
701
|
|
|
is_flag=True, |
702
|
|
|
) |
703
|
|
|
@click.pass_context |
704
|
|
|
def cert(ctx, selfsigned): |
705
|
|
|
"""instance a local SSL certificate""" |
706
|
|
|
|
707
|
|
|
instance_configuration = ctx.obj["instance_configuration"] |
708
|
|
|
instance_name = ctx.obj["instance"] |
709
|
|
|
next_environment = get_next_environment(ctx) |
710
|
|
|
|
711
|
|
|
set_instance(instance_name, next_environment) |
712
|
|
|
|
713
|
|
|
if selfsigned: |
714
|
|
|
_instance_selfsigned(instance_configuration) |
715
|
|
|
else: |
716
|
|
|
log("This is work in progress") |
717
|
|
|
abort(55555) |
718
|
|
|
|
719
|
|
|
_instance_letsencrypt(instance_configuration) |
720
|
|
|
|
721
|
|
|
finish(ctx) |
722
|
|
|
|
723
|
|
|
|
724
|
|
|
def _instance_letsencrypt(instance_configuration): |
725
|
|
|
hostnames = instance_configuration.get("web_hostnames", False) |
726
|
|
|
hostnames = hostnames.replace(" ", "") |
727
|
|
|
|
728
|
|
|
if not hostnames or hostnames == "localhost": |
729
|
|
|
log( |
730
|
|
|
"Please configure the public fully qualified domain names of this instance.\n" |
731
|
|
|
"Use 'iso instance set web_hostnames your.hostname.tld' to do that.\n" |
732
|
|
|
"You can add multiple names by separating them with commas.", |
733
|
|
|
lvl=error, |
734
|
|
|
) |
735
|
|
|
abort(50031) |
736
|
|
|
|
737
|
|
|
contact = instance_configuration.get("contact", False) |
738
|
|
|
if not contact: |
739
|
|
|
log( |
740
|
|
|
"You need to specify a contact mail address for this instance to generate certificates.\n" |
741
|
|
|
"Use 'iso instance set contact [email protected]' to do that.", |
742
|
|
|
lvl=error, |
743
|
|
|
) |
744
|
|
|
abort(50032) |
745
|
|
|
|
746
|
|
|
success, result = run_process( |
747
|
|
|
"/", |
748
|
|
|
[ |
749
|
|
|
"certbot", |
750
|
|
|
"--nginx", |
751
|
|
|
"certonly", |
752
|
|
|
"-m", |
753
|
|
|
contact, |
754
|
|
|
"-d", |
755
|
|
|
hostnames, |
756
|
|
|
"--agree-tos", |
757
|
|
|
"-n", |
758
|
|
|
], |
759
|
|
|
) |
760
|
|
|
if not success: |
761
|
|
|
log( |
762
|
|
|
"Error getting certificate:", |
763
|
|
|
format_result(result), |
764
|
|
|
pretty=True, |
765
|
|
|
lvl=error, |
766
|
|
|
) |
767
|
|
|
abort(50033) |
768
|
|
|
|
769
|
|
|
|
770
|
|
|
def _instance_selfsigned(instance_configuration): |
771
|
|
|
"""Generates a snakeoil certificate that has only been self signed""" |
772
|
|
|
|
773
|
|
|
log("Generating self signed certificate/key combination") |
774
|
|
|
|
775
|
|
|
try: |
776
|
|
|
os.mkdir("/etc/ssl/certs/isomer") |
777
|
|
|
except FileExistsError: |
778
|
|
|
pass |
779
|
|
|
except PermissionError: |
780
|
|
|
log("Need root (e.g. via sudo) to generate ssl certificate") |
781
|
|
|
abort(1) |
782
|
|
|
|
783
|
|
|
try: |
784
|
|
|
from OpenSSL import crypto |
785
|
|
|
except ImportError: |
786
|
|
|
log("Need python3-openssl to do this.") |
787
|
|
|
abort(1) |
788
|
|
|
return |
789
|
|
|
|
790
|
|
|
def create_self_signed_cert(target): |
791
|
|
|
"""Create a simple self signed SSL certificate""" |
792
|
|
|
|
793
|
|
|
key_file = os.path.join(target, "selfsigned.key") |
794
|
|
|
cert_file = os.path.join(target, "selfsigned.crt") |
795
|
|
|
combined_file = os.path.join(target, "selfsigned.pem") |
796
|
|
|
|
797
|
|
|
cert_conf = {k: v for k, v in instance_configuration.items() if |
798
|
|
|
k.startswith("web_certificate")} |
799
|
|
|
|
800
|
|
|
log("Certificate data:", cert_conf, pretty=True) |
801
|
|
|
|
802
|
|
|
hostname = instance_configuration.get("web_hostnames", gethostname()) |
803
|
|
|
if isinstance(hostname, list): |
804
|
|
|
hostname = hostname[0] |
805
|
|
|
|
806
|
|
|
# create a key pair |
807
|
|
|
k = crypto.PKey() |
808
|
|
|
k.generate_key(crypto.TYPE_RSA, 2048) |
809
|
|
|
|
810
|
|
|
if os.path.exists(cert_file): |
811
|
|
|
try: |
812
|
|
|
certificate = open(cert_file, "rb").read() |
813
|
|
|
old_cert = crypto.load_certificate(crypto.FILETYPE_PEM, certificate) |
814
|
|
|
serial = old_cert.get_serial_number() + 1 |
815
|
|
|
except (crypto.Error, OSError) as e: |
816
|
|
|
log( |
817
|
|
|
"Could not read old certificate to increment serial:", |
818
|
|
|
type(e), |
819
|
|
|
e, |
820
|
|
|
exc=True, |
821
|
|
|
lvl=warn, |
822
|
|
|
) |
823
|
|
|
serial = 1 |
824
|
|
|
else: |
825
|
|
|
serial = 1 |
826
|
|
|
|
827
|
|
|
# create a self-signed certificate |
828
|
|
|
certificate = crypto.X509() |
829
|
|
|
certificate.get_subject().C = cert_conf.get("web_certificate_country", |
830
|
|
|
"EU") |
831
|
|
|
certificate.get_subject().ST = cert_conf.get("web_certificate_state", "Sol") |
832
|
|
|
certificate.get_subject().L = cert_conf.get("web_certificate_location", "Earth") |
833
|
|
|
# noinspection PyPep8 |
834
|
|
|
certificate.get_subject().O = cert_conf.get("web_certificate_issuer", "Unknown") |
835
|
|
|
certificate.get_subject().OU = cert_conf.get("web_certificate_unit", "Unknown") |
836
|
|
|
certificate.get_subject().CN = hostname |
837
|
|
|
certificate.set_serial_number(serial) |
838
|
|
|
certificate.gmtime_adj_notBefore(0) |
839
|
|
|
certificate.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) |
840
|
|
|
certificate.set_issuer(certificate.get_subject()) |
841
|
|
|
certificate.set_pubkey(k) |
842
|
|
|
certificate.sign(k, "sha512") |
843
|
|
|
|
844
|
|
|
open(key_file, "wt").write( |
845
|
|
|
str(crypto.dump_privatekey(crypto.FILETYPE_PEM, k), encoding="ASCII") |
846
|
|
|
) |
847
|
|
|
|
848
|
|
|
open(cert_file, "wt").write( |
849
|
|
|
str( |
850
|
|
|
crypto.dump_certificate(crypto.FILETYPE_PEM, certificate), |
851
|
|
|
encoding="ASCII", |
852
|
|
|
) |
853
|
|
|
) |
854
|
|
|
|
855
|
|
|
open(combined_file, "wt").write( |
856
|
|
|
str( |
857
|
|
|
crypto.dump_certificate(crypto.FILETYPE_PEM, certificate), |
858
|
|
|
encoding="ASCII", |
859
|
|
|
) |
860
|
|
|
+ str(crypto.dump_privatekey(crypto.FILETYPE_PEM, k), encoding="ASCII") |
861
|
|
|
) |
862
|
|
|
|
863
|
|
|
location = os.path.join( |
864
|
|
|
get_etc_instance_path(), |
865
|
|
|
instance_configuration.get("name") |
866
|
|
|
) |
867
|
|
|
|
868
|
|
|
create_self_signed_cert(location) |
869
|
|
|
|
870
|
|
|
instance_configuration["web_key"] = os.path.join(location, "selfsigned.key") |
871
|
|
|
instance_configuration["web_certificate"] = os.path.join(location, "selfsigned.crt") |
872
|
|
|
|
873
|
|
|
write_instance(instance_configuration) |
874
|
|
|
|
875
|
|
|
|
876
|
|
|
@instance.command(short_help="install systemd service") |
877
|
|
|
@click.pass_context |
878
|
|
|
def service(ctx): |
879
|
|
|
"""instance systemd service configuration""" |
880
|
|
|
|
881
|
|
|
update_service(ctx, ctx.obj["instance_configuration"]["environment"]) |
882
|
|
|
finish(ctx) |
883
|
|
|
|
884
|
|
|
|
885
|
|
|
@instance.command(short_help="instance nginx configuration") |
886
|
|
|
@click.option( |
887
|
|
|
"--hostname", |
888
|
|
|
default=None, |
889
|
|
|
help="Override public Hostname (FQDN) Default from active system " "configuration", |
890
|
|
|
) |
891
|
|
|
@click.pass_context |
892
|
|
|
def update_nginx(ctx, hostname): |
893
|
|
|
"""instance nginx configuration""" |
894
|
|
|
|
895
|
|
|
ctx.obj["hostname"] = hostname |
896
|
|
|
|
897
|
|
|
_create_nginx_config(ctx) |
898
|
|
|
finish(ctx) |
899
|
|
|
|
900
|
|
|
|
901
|
|
|
def _create_nginx_config(ctx): |
902
|
|
|
"""instance nginx configuration""" |
903
|
|
|
|
904
|
|
|
# TODO: Specify template url very precisely. Currently one needs to be in |
905
|
|
|
# the repository root |
906
|
|
|
|
907
|
|
|
instance_name = ctx.obj["instance"] |
908
|
|
|
config = ctx.obj["instance_configuration"] |
909
|
|
|
|
910
|
|
|
current_env = config["environment"] |
911
|
|
|
env = config["environments"][current_env] |
912
|
|
|
|
913
|
|
|
dbhost = config["database_host"] |
914
|
|
|
dbname = env["database"] |
915
|
|
|
|
916
|
|
|
hostnames = ctx.obj.get("web_hostnames", None) |
917
|
|
|
if hostnames is None: |
918
|
|
|
hostnames = config.get("web_hostnames", None) |
919
|
|
|
if hostnames is None: |
920
|
|
|
try: |
921
|
|
|
configuration = _get_system_configuration(dbhost, dbname) |
922
|
|
|
hostnames = configuration.hostname |
923
|
|
|
except Exception as e: |
924
|
|
|
log("Exception:", e, type(e), exc=True, lvl=error) |
925
|
|
|
log( |
926
|
|
|
"""Could not determine public fully qualified hostname! |
927
|
|
|
Check systemconfig (see db view and db modify commands) or specify |
928
|
|
|
manually with --hostname host.domain.tld |
929
|
|
|
|
930
|
|
|
Using 'localhost' for now""", |
931
|
|
|
lvl=warn, |
932
|
|
|
) |
933
|
|
|
hostnames = "localhost" |
934
|
|
|
port = config["web_port"] |
935
|
|
|
address = config["web_address"] |
936
|
|
|
|
937
|
|
|
log( |
938
|
|
|
"Creating nginx configuration for %s:%i using %s@%s" |
939
|
|
|
% (hostnames, port, dbname, dbhost) |
940
|
|
|
) |
941
|
|
|
|
942
|
|
|
definitions = { |
943
|
|
|
"server_public_name": hostnames.replace(",", " "), |
944
|
|
|
"ssl_certificate": config["web_certificate"], |
945
|
|
|
"ssl_key": config["web_key"], |
946
|
|
|
"host_url": "http://%s:%i/" % (address, port), |
947
|
|
|
"instance": instance_name, |
948
|
|
|
"environment": current_env, |
949
|
|
|
} |
950
|
|
|
|
951
|
|
|
if distribution == "DEBIAN": |
952
|
|
|
configuration_file = "/etc/nginx/sites-available/isomer.%s.conf" % instance_name |
953
|
|
|
configuration_link = "/etc/nginx/sites-enabled/isomer.%s.conf" % instance_name |
954
|
|
|
elif distribution == "ARCH": |
955
|
|
|
configuration_file = "/etc/nginx/nginx.conf" |
956
|
|
|
configuration_link = None |
957
|
|
|
else: |
958
|
|
|
log( |
959
|
|
|
"Unsure how to proceed, you may need to specify your " "distribution", |
960
|
|
|
lvl=error, |
961
|
|
|
) |
962
|
|
|
return |
963
|
|
|
|
964
|
|
|
log("Writing nginx Isomer site definition") |
965
|
|
|
write_template(nginx_template, configuration_file, definitions) |
966
|
|
|
|
967
|
|
|
if configuration_link is not None: |
968
|
|
|
log("Enabling nginx Isomer site (symlink)") |
969
|
|
|
if not os.path.exists(configuration_link): |
970
|
|
|
os.symlink(configuration_file, configuration_link) |
971
|
|
|
|
972
|
|
|
if os.path.exists("/bin/systemctl"): |
973
|
|
|
log("Restarting nginx service") |
974
|
|
|
run_process("/", ["systemctl", "restart", "nginx.service"], sudo="root") |
975
|
|
|
else: |
976
|
|
|
log("No systemctl found, not restarting nginx") |
977
|
|
|
|
978
|
|
|
# TODO: Add instance user |
979
|
|
|
|