| Total Complexity | 82 |
| Total Lines | 977 |
| Duplicated Lines | 4.5 % |
| Changes | 0 | ||
Duplicate code is one of the most pungent code smells. A rule that is often used is to re-structure code once it is duplicated in three or more places.
Common duplication problems, and corresponding solutions are:
Complex classes like isomer.tool.instance often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | #!/usr/bin/env python |
||
| 2 | # -*- coding: UTF-8 -*- |
||
| 3 | |||
| 4 | # Isomer - The distributed application framework |
||
| 5 | # ============================================== |
||
| 6 | # Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others. |
||
| 7 | # |
||
| 8 | # This program is free software: you can redistribute it and/or modify |
||
| 9 | # it under the terms of the GNU Affero General Public License as published by |
||
| 10 | # the Free Software Foundation, either version 3 of the License, or |
||
| 11 | # (at your option) any later version. |
||
| 12 | # |
||
| 13 | # This program is distributed in the hope that it will be useful, |
||
| 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
| 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
| 16 | # GNU Affero General Public License for more details. |
||
| 17 | # |
||
| 18 | # You should have received a copy of the GNU Affero General Public License |
||
| 19 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
||
| 20 | |||
| 21 | """ |
||
| 22 | |||
| 23 | Module: Instance |
||
| 24 | ================ |
||
| 25 | |||
| 26 | Instance management functionality. |
||
| 27 | |||
| 28 | instance info |
||
| 29 | instance list |
||
| 30 | instance set |
||
| 31 | instance create |
||
| 32 | instance install |
||
| 33 | instance clear |
||
| 34 | instance remove |
||
| 35 | instance install-module |
||
| 36 | instance turnover |
||
| 37 | instance cert |
||
| 38 | instance service |
||
| 39 | instance update-nginx |
||
| 40 | |||
| 41 | """ |
||
| 42 | |||
| 43 | import os |
||
| 44 | import webbrowser |
||
| 45 | from socket import gethostname |
||
| 46 | from shutil import rmtree |
||
| 47 | |||
| 48 | import tomlkit |
||
| 49 | import click |
||
| 50 | from click_didyoumean import DYMGroup |
||
| 51 | from isomer.error import ( |
||
| 52 | abort, |
||
| 53 | EXIT_INVALID_CONFIGURATION, |
||
| 54 | EXIT_INSTALLATION_FAILED, |
||
| 55 | EXIT_INSTANCE_EXISTS, |
||
| 56 | EXIT_INSTANCE_UNKNOWN, |
||
| 57 | EXIT_SERVICE_INVALID, |
||
| 58 | EXIT_USER_BAILED_OUT, |
||
| 59 | EXIT_INVALID_PARAMETER, |
||
| 60 | ) |
||
| 61 | from isomer.migration import apply_migrations |
||
| 62 | from isomer.misc import sorted_alphanumerical |
||
| 63 | from isomer.misc.path import set_instance, get_etc_instance_path, get_path |
||
| 64 | from isomer.logger import warn, critical |
||
| 65 | from isomer.tool import ( |
||
| 66 | _get_system_configuration, |
||
| 67 | _get_configuration, |
||
| 68 | get_next_environment, |
||
| 69 | ask, |
||
| 70 | finish, |
||
| 71 | run_process, |
||
| 72 | format_result, |
||
| 73 | log, |
||
| 74 | error, |
||
| 75 | debug, |
||
| 76 | ) |
||
| 77 | from isomer.tool.etc import ( |
||
| 78 | write_instance, |
||
| 79 | valid_configuration, |
||
| 80 | remove_instance, |
||
| 81 | instance_template, |
||
| 82 | ) |
||
| 83 | from isomer.tool.templates import write_template |
||
| 84 | from isomer.tool.defaults import ( |
||
| 85 | service_template, |
||
| 86 | nginx_template, |
||
| 87 | distribution, |
||
| 88 | ) |
||
| 89 | from isomer.tool.environment import ( |
||
| 90 | _install_environment, |
||
| 91 | install_environment_modules, |
||
| 92 | _clear_environment, |
||
| 93 | _check_environment, |
||
| 94 | ) |
||
| 95 | from isomer.tool.database import copy_database |
||
| 96 | from isomer.tool.version import _get_versions |
||
| 97 | from isomer.ui.builder import copy_directory_tree |
||
| 98 | from isomer.ui.store import DEFAULT_STORE_URL |
||
| 99 | |||
| 100 | |||
| 101 | @click.group( |
||
| 102 | cls=DYMGroup, |
||
| 103 | short_help="Instance handling" |
||
| 104 | ) |
||
| 105 | @click.pass_context |
||
| 106 | def instance(ctx): |
||
| 107 | """[GROUP] instance various aspects of Isomer""" |
||
| 108 | |||
| 109 | if ctx.invoked_subcommand in ("info", "list", "create"): |
||
| 110 | return |
||
| 111 | |||
| 112 | _get_configuration(ctx) |
||
| 113 | |||
| 114 | |||
| 115 | View Code Duplication | @instance.command(name="info", short_help="show system configuration of instance") |
|
|
|
|||
| 116 | @click.pass_context |
||
| 117 | def info_instance(ctx): |
||
| 118 | """Print information about the selected instance""" |
||
| 119 | |||
| 120 | instance_name = ctx.obj["instance"] |
||
| 121 | instances = ctx.obj["instances"] |
||
| 122 | instance_configuration = instances[instance_name] |
||
| 123 | |||
| 124 | environment_name = instance_configuration["environment"] |
||
| 125 | environment_config = instance_configuration["environments"][environment_name] |
||
| 126 | |||
| 127 | if instance_name not in instances: |
||
| 128 | log("Instance %s unknown!" % instance_name, lvl=warn) |
||
| 129 | abort(EXIT_INSTANCE_UNKNOWN) |
||
| 130 | |||
| 131 | log("Instance configuration:", instance_configuration, pretty=True) |
||
| 132 | log("Active environment (%s):" % environment_name, environment_config, pretty=True) |
||
| 133 | |||
| 134 | finish(ctx) |
||
| 135 | |||
| 136 | |||
| 137 | View Code Duplication | @instance.command( |
|
| 138 | name="check", short_help="check instance configuration and environments" |
||
| 139 | ) |
||
| 140 | @click.pass_context |
||
| 141 | def check_instance(ctx): |
||
| 142 | """Check health of the selected instance""" |
||
| 143 | |||
| 144 | instance_name = ctx.obj["instance"] |
||
| 145 | instances = ctx.obj["instances"] |
||
| 146 | instance_configuration = instances[instance_name] |
||
| 147 | |||
| 148 | environment_name = instance_configuration["environment"] |
||
| 149 | environment_config = instance_configuration["environments"][environment_name] |
||
| 150 | |||
| 151 | if instance_name not in instances: |
||
| 152 | log("Instance %s unknown!" % instance_name, lvl=warn) |
||
| 153 | abort(EXIT_INSTANCE_UNKNOWN) |
||
| 154 | |||
| 155 | # TODO: Validate instance config |
||
| 156 | |||
| 157 | _check_environment(ctx, "blue") |
||
| 158 | _check_environment(ctx, "green") |
||
| 159 | |||
| 160 | finish(ctx) |
||
| 161 | |||
| 162 | |||
| 163 | @instance.command(name="browser", short_help="try to point a browser to this instance") |
||
| 164 | @click.pass_context |
||
| 165 | def browser(ctx): |
||
| 166 | """Tries to start or point a browser towards this instance's frontend""" |
||
| 167 | instance_configuration = ctx.obj["instance_configuration"] |
||
| 168 | |||
| 169 | server = instance_configuration.get("webserver", "internal") |
||
| 170 | protocol = "http" |
||
| 171 | |||
| 172 | if server != "internal": |
||
| 173 | protocol += "s" |
||
| 174 | |||
| 175 | host = instance_configuration.get( |
||
| 176 | "web_hostname", instance_configuration.get("web_address", "127.0.0.1") |
||
| 177 | ) |
||
| 178 | port = instance_configuration.get("web_port", None) |
||
| 179 | |||
| 180 | if port is None: |
||
| 181 | url = "%s://%s" % (protocol, host) |
||
| 182 | else: |
||
| 183 | url = "%s://%s:%i" % (protocol, host, port) |
||
| 184 | |||
| 185 | log("Opening browser to:", url) |
||
| 186 | log( |
||
| 187 | "If this is empty or unreachable, check your instance with:\n" |
||
| 188 | " iso instance info\n" |
||
| 189 | "Also try the health checks:\n" |
||
| 190 | " iso instance check" |
||
| 191 | ) |
||
| 192 | |||
| 193 | webbrowser.open(url) |
||
| 194 | |||
| 195 | |||
| 196 | @instance.command(name="list", short_help="List all instances") |
||
| 197 | @click.pass_context |
||
| 198 | def list_instances(ctx): |
||
| 199 | """List all known instances""" |
||
| 200 | |||
| 201 | for instance_name in ctx.obj["instances"]: |
||
| 202 | log(instance_name, pretty=True) |
||
| 203 | finish(ctx) |
||
| 204 | |||
| 205 | |||
| 206 | @instance.command(name="set", short_help="Set a parameter of an instance") |
||
| 207 | @click.argument("parameter") |
||
| 208 | @click.argument("value") |
||
| 209 | @click.pass_context |
||
| 210 | def set_parameter(ctx, parameter, value): |
||
| 211 | """Set a configuration parameter of an instance""" |
||
| 212 | |||
| 213 | log("Setting %s to %s" % (parameter, value)) |
||
| 214 | instance_configuration = ctx.obj["instance_configuration"] |
||
| 215 | defaults = instance_template |
||
| 216 | converted_value = None |
||
| 217 | |||
| 218 | try: |
||
| 219 | parameter_type = type(defaults[parameter]) |
||
| 220 | log(parameter_type, pretty=True, lvl=debug) |
||
| 221 | |||
| 222 | if parameter_type == tomlkit.items.Integer: |
||
| 223 | converted_value = int(value) |
||
| 224 | elif parameter_type == bool: |
||
| 225 | converted_value = value.upper() == "TRUE" |
||
| 226 | else: |
||
| 227 | converted_value = value |
||
| 228 | except KeyError: |
||
| 229 | log("Available parameters:", sorted(list(defaults.keys()))) |
||
| 230 | abort(EXIT_INVALID_PARAMETER) |
||
| 231 | |||
| 232 | if converted_value is None: |
||
| 233 | log("Converted value was None! Recheck the new config!", lvl=warn) |
||
| 234 | |||
| 235 | instance_configuration[parameter] = converted_value |
||
| 236 | log("New config:", instance_configuration, pretty=True, lvl=debug) |
||
| 237 | |||
| 238 | ctx.obj["instances"][ctx.obj["instance"]] = instance_configuration |
||
| 239 | |||
| 240 | if valid_configuration(ctx): |
||
| 241 | write_instance(instance_configuration) |
||
| 242 | finish(ctx) |
||
| 243 | else: |
||
| 244 | log("New configuration would not be valid", lvl=critical) |
||
| 245 | abort(EXIT_INVALID_CONFIGURATION) |
||
| 246 | |||
| 247 | |||
| 248 | @instance.command(short_help="Create a new instance") |
||
| 249 | @click.argument("instance_name", default="") |
||
| 250 | @click.pass_context |
||
| 251 | def create(ctx, instance_name): |
||
| 252 | """Create a new instance""" |
||
| 253 | |||
| 254 | if instance_name == "": |
||
| 255 | instance_name = ctx.obj["instance"] |
||
| 256 | |||
| 257 | if instance_name in ctx.obj["instances"]: |
||
| 258 | abort(EXIT_INSTANCE_EXISTS) |
||
| 259 | |||
| 260 | log("Creating instance:", instance_name) |
||
| 261 | instance_configuration = instance_template |
||
| 262 | instance_configuration["name"] = instance_name |
||
| 263 | ctx.obj["instances"][instance_name] = instance_configuration |
||
| 264 | |||
| 265 | write_instance(instance_configuration) |
||
| 266 | finish(ctx) |
||
| 267 | |||
| 268 | |||
| 269 | @instance.command(name="install", short_help="Install a fresh instance") |
||
| 270 | @click.option("--force", "-f", is_flag=True, default=False) |
||
| 271 | @click.option( |
||
| 272 | "--source", "-s", default="git", |
||
| 273 | type=click.Choice(["link", "copy", "git", "github"]) |
||
| 274 | ) |
||
| 275 | @click.option("--url", "-u", default="", type=click.Path()) |
||
| 276 | @click.option( |
||
| 277 | "--import-file", "--import", default=None, help="Import the specified backup" |
||
| 278 | ) |
||
| 279 | @click.option( |
||
| 280 | "--no-sudo", |
||
| 281 | is_flag=True, |
||
| 282 | default=False, |
||
| 283 | help="Do not use sudo to install (Mostly for tests)", |
||
| 284 | ) |
||
| 285 | @click.option( |
||
| 286 | "--release", "-r", default=None, help="Override installed release version" |
||
| 287 | ) |
||
| 288 | @click.option("--skip-modules", is_flag=True, default=False) |
||
| 289 | @click.option("--skip-data", is_flag=True, default=False) |
||
| 290 | @click.option("--skip-frontend", is_flag=True, default=False) |
||
| 291 | @click.option("--skip-test", is_flag=True, default=False) |
||
| 292 | @click.option("--skip-provisions", is_flag=True, default=False) |
||
| 293 | @click.pass_context |
||
| 294 | def install(ctx, **kwargs): |
||
| 295 | """Install a new environment of an instance""" |
||
| 296 | |||
| 297 | log("Installing instance") |
||
| 298 | |||
| 299 | env = ctx.obj["instance_configuration"]["environments"] |
||
| 300 | |||
| 301 | green = env["green"]["installed"] |
||
| 302 | blue = env["blue"]["installed"] |
||
| 303 | |||
| 304 | if green or blue: |
||
| 305 | log( |
||
| 306 | "At least one environment is installed or in a non-clear state.\n" |
||
| 307 | "Please use 'iso instance upgrade' to upgrade an instance.", |
||
| 308 | lvl=warn, |
||
| 309 | ) |
||
| 310 | abort(50081) |
||
| 311 | |||
| 312 | _clear_instance(ctx, force=kwargs["force"], clear=False, no_archive=True) |
||
| 313 | |||
| 314 | _install_environment(ctx, **kwargs) |
||
| 315 | |||
| 316 | ctx.obj["instance_configuration"]["source"] = kwargs["source"] |
||
| 317 | ctx.obj["instance_configuration"]["url"] = kwargs["url"] |
||
| 318 | |||
| 319 | write_instance(ctx.obj["instance_configuration"]) |
||
| 320 | |||
| 321 | _turnover(ctx, force=kwargs["force"]) |
||
| 322 | |||
| 323 | finish(ctx) |
||
| 324 | |||
| 325 | |||
| 326 | @instance.command(name="clear", short_help="Clear the whole instance (CAUTION)") |
||
| 327 | @click.option("--force", "-f", is_flag=True, default=False) |
||
| 328 | @click.option("--no-archive", "-n", is_flag=True, default=False) |
||
| 329 | @click.pass_context |
||
| 330 | def clear_instance(ctx, force, no_archive): |
||
| 331 | """Irrevocably clear all environments of an instance""" |
||
| 332 | |||
| 333 | _clear_instance(ctx, force, False, no_archive) |
||
| 334 | |||
| 335 | |||
| 336 | def _clear_instance(ctx, force, clear, no_archive): |
||
| 337 | log("Clearing instance:", ctx.obj["instance"]) |
||
| 338 | log("Clearing blue environment.", lvl=debug) |
||
| 339 | _clear_environment(ctx, force, "blue", clear, no_archive) |
||
| 340 | log("Clearing green environment.", lvl=debug) |
||
| 341 | _clear_environment(ctx, force, "green", clear, no_archive) |
||
| 342 | finish(ctx) |
||
| 343 | |||
| 344 | |||
| 345 | @instance.command(short_help="Remove a whole instance (CAUTION)") |
||
| 346 | @click.option( |
||
| 347 | "--clear", "-c", is_flag=True, help="Clear instance before removal", default=False |
||
| 348 | ) |
||
| 349 | @click.option("--no-archive", "-n", is_flag=True, default=False) |
||
| 350 | @click.pass_context |
||
| 351 | def remove(ctx, clear, no_archive): |
||
| 352 | """Irrevocably remove a whole instance""" |
||
| 353 | |||
| 354 | if clear: |
||
| 355 | log("Destructively removing instance:", ctx.obj["instance"], lvl=warn) |
||
| 356 | |||
| 357 | if not ask("Are you sure", default=False, data_type="bool"): |
||
| 358 | abort(EXIT_USER_BAILED_OUT) |
||
| 359 | |||
| 360 | if clear: |
||
| 361 | _clear_instance(ctx, force=True, clear=clear, no_archive=no_archive) |
||
| 362 | |||
| 363 | remove_instance(ctx.obj["instance"]) |
||
| 364 | finish(ctx) |
||
| 365 | |||
| 366 | |||
| 367 | @instance.command( |
||
| 368 | "install-modules", short_help="Add (and install) modules to an instance" |
||
| 369 | ) |
||
| 370 | @click.option( |
||
| 371 | "--source", |
||
| 372 | "-s", |
||
| 373 | default="git", |
||
| 374 | type=click.Choice(["link", "copy", "git", "develop", "store"]), |
||
| 375 | help="Specify installation source/method", |
||
| 376 | ) |
||
| 377 | @click.option( |
||
| 378 | "--store-url", |
||
| 379 | default=DEFAULT_STORE_URL, |
||
| 380 | help="Specify alternative store url (Default: %s)" % DEFAULT_STORE_URL, |
||
| 381 | ) |
||
| 382 | @click.option( |
||
| 383 | "--install-env", |
||
| 384 | "--install", |
||
| 385 | "-i", |
||
| 386 | is_flag=True, |
||
| 387 | default=False, |
||
| 388 | help="Install modules on active environment", |
||
| 389 | ) |
||
| 390 | @click.option( |
||
| 391 | "--force", |
||
| 392 | "-f", |
||
| 393 | default=False, |
||
| 394 | is_flag=True, |
||
| 395 | help="Force installation (overwrites old modules)", |
||
| 396 | ) |
||
| 397 | @click.argument("urls", nargs=-1) |
||
| 398 | @click.pass_context |
||
| 399 | def install_instance_modules(ctx, source, urls, install_env, force, store_url): |
||
| 400 | """Add (and optionally immediately install) modules for an instance. |
||
| 401 | |||
| 402 | This will add them to the instance's configuration, so they will be upgraded as well |
||
| 403 | as reinstalled on other environment changes. |
||
| 404 | |||
| 405 | If you're installing from a store, you can specify a custom store URL with the |
||
| 406 | --store-url argument. |
||
| 407 | """ |
||
| 408 | |||
| 409 | instance_name = ctx.obj["instance"] |
||
| 410 | instance_configuration = ctx.obj["instances"][instance_name] |
||
| 411 | |||
| 412 | for url in urls: |
||
| 413 | descriptor = [source, url] |
||
| 414 | if store_url != DEFAULT_STORE_URL: |
||
| 415 | descriptor.append(store_url) |
||
| 416 | if descriptor not in instance_configuration["modules"]: |
||
| 417 | instance_configuration["modules"].append(descriptor) |
||
| 418 | elif not force: |
||
| 419 | log("Module %s is already installed. Use --force to install anyway." % url) |
||
| 420 | abort(50000) |
||
| 421 | |||
| 422 | write_instance(instance_configuration) |
||
| 423 | |||
| 424 | if install_env is True: |
||
| 425 | next_environment = get_next_environment(ctx) |
||
| 426 | environments = instance_configuration['environments'] |
||
| 427 | |||
| 428 | if environments[next_environment]["installed"] is False: |
||
| 429 | log("Environment %s is not installed, cannot install modules." |
||
| 430 | % next_environment, lvl=warn) |
||
| 431 | abort(50600) |
||
| 432 | return |
||
| 433 | del ctx.params["install_env"] |
||
| 434 | ctx.forward(install_environment_modules) |
||
| 435 | |||
| 436 | finish(ctx) |
||
| 437 | |||
| 438 | |||
| 439 | @instance.command(short_help="Activates the other environment") |
||
| 440 | @click.option("--force", "-f", is_flag=True, default=False, help="Force turnover") |
||
| 441 | @click.pass_context |
||
| 442 | def turnover(ctx, **kwargs): |
||
| 443 | """Activates the other environment """ |
||
| 444 | |||
| 445 | _turnover(ctx, **kwargs) |
||
| 446 | finish(ctx) |
||
| 447 | |||
| 448 | |||
| 449 | def _turnover(ctx, force): |
||
| 450 | """Internal turnover operation""" |
||
| 451 | |||
| 452 | # if ctx.obj['acting_environment'] is not None: |
||
| 453 | # next_environment = ctx.obj['acting_environment'] |
||
| 454 | # else: |
||
| 455 | next_environment = get_next_environment(ctx) |
||
| 456 | |||
| 457 | log("Activating environment:", next_environment) |
||
| 458 | env = ctx.obj["instance_configuration"]["environments"][next_environment] |
||
| 459 | |||
| 460 | log("Inspecting new environment") |
||
| 461 | |||
| 462 | if not force: |
||
| 463 | if env.get("database", "") == "": |
||
| 464 | log("Database has not been set up correctly.", lvl=critical) |
||
| 465 | abort(EXIT_INSTALLATION_FAILED) |
||
| 466 | |||
| 467 | if ( |
||
| 468 | not env.get("installed", False) |
||
| 469 | or not env.get("tested", False) |
||
| 470 | or not env.get("migrated", False) |
||
| 471 | ): |
||
| 472 | log("Installation failed, cannot activate!", lvl=critical) |
||
| 473 | abort(EXIT_INSTALLATION_FAILED) |
||
| 474 | |||
| 475 | update_service(ctx, next_environment) |
||
| 476 | |||
| 477 | ctx.obj["instance_configuration"]["environment"] = next_environment |
||
| 478 | |||
| 479 | write_instance(ctx.obj["instance_configuration"]) |
||
| 480 | |||
| 481 | # TODO: Effect reload of service |
||
| 482 | # * Systemctl reload |
||
| 483 | # * (Re)start service |
||
| 484 | # * confirm correct operation |
||
| 485 | # - if not, switch back to the other instance, maybe indicate a broken |
||
| 486 | # state for next_environment |
||
| 487 | # - if yes, Store instance configuration and terminate, we're done |
||
| 488 | |||
| 489 | log("Turned instance over to", next_environment) |
||
| 490 | |||
| 491 | |||
| 492 | @instance.command(short_help="Upgrades the other environment") |
||
| 493 | @click.option("--release", "-r", default=None, help="Specify release to upgrade to") |
||
| 494 | @click.option("--upgrade-modules", default=False, is_flag=True, |
||
| 495 | help="Also, upgrade modules if possible") |
||
| 496 | @click.option("--restart", default=False, is_flag=True, |
||
| 497 | help="Restart systemd service via systemctl on success") |
||
| 498 | @click.option("--handle-cache", "-c", |
||
| 499 | type=click.Choice(["ignore", "move", "copy"], case_sensitive=False), |
||
| 500 | default="ignore", help="Handle cached data as well (ignore, move, copy)") |
||
| 501 | @click.option( |
||
| 502 | "--source", |
||
| 503 | "-s", |
||
| 504 | default="git", |
||
| 505 | type=click.Choice(["link", "copy", "git", "develop", "github", "pypi"]), |
||
| 506 | help="Specify installation source/method", |
||
| 507 | ) |
||
| 508 | @click.option("--url", "-u", default="", type=click.Path()) |
||
| 509 | @click.pass_context |
||
| 510 | def upgrade(ctx, release, upgrade_modules, restart, handle_cache, source, url): |
||
| 511 | """Upgrades an instance on its other environment and turns over on success. |
||
| 512 | |||
| 513 | \b |
||
| 514 | 1. Test if other environment is empty |
||
| 515 | 1.1. No - archive and clear it |
||
| 516 | 2. Copy current environment to other environment |
||
| 517 | 3. Clear old bits (venv, frontend) |
||
| 518 | 4. Fetch updates in other environment repository |
||
| 519 | 5. Select a release |
||
| 520 | 6. Checkout that release and its submodules |
||
| 521 | 7. Install release |
||
| 522 | 8. Copy database |
||
| 523 | 9. Migrate data (WiP) |
||
| 524 | 10. Turnover |
||
| 525 | |||
| 526 | """ |
||
| 527 | |||
| 528 | instance_config = ctx.obj["instance_configuration"] |
||
| 529 | repository = get_path("lib", "repository") |
||
| 530 | |||
| 531 | installation_source = source if source is not None else instance_config['source'] |
||
| 532 | installation_url = url if url is not None else instance_config['url'] |
||
| 533 | |||
| 534 | environments = instance_config["environments"] |
||
| 535 | |||
| 536 | active = instance_config["environment"] |
||
| 537 | |||
| 538 | next_environment = get_next_environment(ctx) |
||
| 539 | if environments[next_environment]["installed"] is True: |
||
| 540 | _clear_environment(ctx, clear_env=next_environment) |
||
| 541 | |||
| 542 | source_paths = [ |
||
| 543 | get_path("lib", "", environment=active), |
||
| 544 | get_path("local", "", environment=active) |
||
| 545 | ] |
||
| 546 | |||
| 547 | destination_paths = [ |
||
| 548 | get_path("lib", "", environment=next_environment), |
||
| 549 | get_path("local", "", environment=next_environment) |
||
| 550 | ] |
||
| 551 | |||
| 552 | log(source_paths, destination_paths, pretty=True) |
||
| 553 | |||
| 554 | for source, destination in zip(source_paths, destination_paths): |
||
| 555 | log("Copying to new environment:", source, destination) |
||
| 556 | copy_directory_tree(source, destination) |
||
| 557 | |||
| 558 | if handle_cache != "ignore": |
||
| 559 | log("Handling cache") |
||
| 560 | move = handle_cache == "move" |
||
| 561 | copy_directory_tree( |
||
| 562 | get_path("cache", "", environment=active), |
||
| 563 | get_path("cache", "", environment=next_environment), |
||
| 564 | move=move |
||
| 565 | ) |
||
| 566 | |||
| 567 | rmtree(get_path("lib", "venv"), ignore_errors=True) |
||
| 568 | # TODO: This potentially leaves frontend-dev: |
||
| 569 | rmtree(get_path("lib", "frontend"), ignore_errors=True) |
||
| 570 | |||
| 571 | releases = _get_versions( |
||
| 572 | ctx, |
||
| 573 | source=installation_source, |
||
| 574 | url=installation_url, |
||
| 575 | fetch=True |
||
| 576 | ) |
||
| 577 | |||
| 578 | releases_keys = sorted_alphanumerical(releases.keys()) |
||
| 579 | |||
| 580 | if release is None: |
||
| 581 | release = releases_keys[-1] |
||
| 582 | else: |
||
| 583 | if release not in releases_keys: |
||
| 584 | log("Unknown release. Maybe try a different release or source.") |
||
| 585 | abort(50100) |
||
| 586 | |||
| 587 | log("Choosing release", release) |
||
| 588 | |||
| 589 | _install_environment( |
||
| 590 | ctx, installation_source, installation_url, upgrade=True, release=release |
||
| 591 | ) |
||
| 592 | |||
| 593 | new_database_name = instance_config["name"] + "_" + next_environment |
||
| 594 | |||
| 595 | copy_database( |
||
| 596 | ctx.obj["dbhost"], |
||
| 597 | active['database'], |
||
| 598 | new_database_name |
||
| 599 | ) |
||
| 600 | |||
| 601 | apply_migrations(ctx) |
||
| 602 | |||
| 603 | finish(ctx) |
||
| 604 | |||
| 605 | |||
| 606 | def update_service(ctx, next_environment): |
||
| 607 | """Updates the specified service configuration""" |
||
| 608 | |||
| 609 | validated, message = validate_services(ctx) |
||
| 610 | |||
| 611 | if not validated: |
||
| 612 | log("Service configuration validation failed:", message, lvl=error) |
||
| 613 | abort(EXIT_SERVICE_INVALID) |
||
| 614 | |||
| 615 | init = ctx.obj["config"]["meta"]["init"] |
||
| 616 | environment_config = ctx.obj["instance_configuration"]["environments"][ |
||
| 617 | next_environment |
||
| 618 | ] |
||
| 619 | |||
| 620 | log( |
||
| 621 | "Updating %s configuration of instance %s to %s" |
||
| 622 | % (init, ctx.obj["instance"], next_environment) |
||
| 623 | ) |
||
| 624 | log("New environment:", environment_config, pretty=True) |
||
| 625 | |||
| 626 | # TODO: Add update for systemd |
||
| 627 | # * Stop possibly running service (it should not be running, though!) |
||
| 628 | # * Actually update service files |
||
| 629 | |||
| 630 | instance_name = ctx.obj["instance"] |
||
| 631 | config = ctx.obj["instance_configuration"] |
||
| 632 | |||
| 633 | env_path = "/var/lib/isomer/" + instance_name + "/" + next_environment |
||
| 634 | |||
| 635 | log("Updating systemd service for %s (%s)" % (instance_name, next_environment)) |
||
| 636 | |||
| 637 | launcher = os.path.join(env_path, "repository/iso") |
||
| 638 | executable = os.path.join(env_path, "venv/bin/python3") + " " + launcher |
||
| 639 | executable += " --quiet --instance " + instance_name + " launch" |
||
| 640 | |||
| 641 | definitions = { |
||
| 642 | "instance": instance_name, |
||
| 643 | "executable": executable, |
||
| 644 | "environment": next_environment, |
||
| 645 | "user_name": config["user"], |
||
| 646 | "user_group": config["group"], |
||
| 647 | } |
||
| 648 | service_name = "isomer-" + instance_name + ".service" |
||
| 649 | |||
| 650 | write_template( |
||
| 651 | service_template, |
||
| 652 | os.path.join("/etc/systemd/system/", service_name), |
||
| 653 | definitions, |
||
| 654 | ) |
||
| 655 | |||
| 656 | |||
| 657 | def _launch_service(ctx): |
||
| 658 | """Actually enable and launch newly set up environment""" |
||
| 659 | instance_name = ctx.obj["instance"] |
||
| 660 | |||
| 661 | service_name = "isomer-" + instance_name + ".service" |
||
| 662 | |||
| 663 | success, result = run_process( |
||
| 664 | "/", ["systemctl", "enable", service_name], sudo="root" |
||
| 665 | ) |
||
| 666 | |||
| 667 | if not success: |
||
| 668 | log("Error activating service:", format_result(result), pretty=True, lvl=error) |
||
| 669 | abort(5000) |
||
| 670 | |||
| 671 | log("Launching service") |
||
| 672 | |||
| 673 | success, result = run_process( |
||
| 674 | "/", ["systemctl", "start", service_name], sudo="root" |
||
| 675 | ) |
||
| 676 | |||
| 677 | if not success: |
||
| 678 | log("Error activating service:", format_result(result), pretty=True, lvl=error) |
||
| 679 | abort(5000) |
||
| 680 | |||
| 681 | return True |
||
| 682 | |||
| 683 | |||
| 684 | def validate_services(ctx): |
||
| 685 | """Checks init configuration settings so nothing gets mis-configured""" |
||
| 686 | |||
| 687 | # TODO: Service validation |
||
| 688 | # * Check through all configurations that we're not messing with port numbers |
||
| 689 | # * ??? |
||
| 690 | |||
| 691 | _ = ctx |
||
| 692 | |||
| 693 | return True, "VALIDATION_NOT_IMPLEMENTED" |
||
| 694 | |||
| 695 | |||
| 696 | @instance.command(short_help="instance ssl certificate") |
||
| 697 | @click.option( |
||
| 698 | "--selfsigned", |
||
| 699 | help="Use a self-signed certificate", |
||
| 700 | default=False, |
||
| 701 | is_flag=True, |
||
| 702 | ) |
||
| 703 | @click.pass_context |
||
| 704 | def cert(ctx, selfsigned): |
||
| 705 | """instance a local SSL certificate""" |
||
| 706 | |||
| 707 | instance_configuration = ctx.obj["instance_configuration"] |
||
| 708 | instance_name = ctx.obj["instance"] |
||
| 709 | next_environment = get_next_environment(ctx) |
||
| 710 | |||
| 711 | set_instance(instance_name, next_environment) |
||
| 712 | |||
| 713 | if selfsigned: |
||
| 714 | _instance_selfsigned(instance_configuration) |
||
| 715 | else: |
||
| 716 | log("This is work in progress") |
||
| 717 | abort(55555) |
||
| 718 | |||
| 719 | _instance_letsencrypt(instance_configuration) |
||
| 720 | |||
| 721 | finish(ctx) |
||
| 722 | |||
| 723 | |||
| 724 | def _instance_letsencrypt(instance_configuration): |
||
| 725 | hostnames = instance_configuration.get("web_hostnames", False) |
||
| 726 | hostnames = hostnames.replace(" ", "") |
||
| 727 | |||
| 728 | if not hostnames or hostnames == "localhost": |
||
| 729 | log( |
||
| 730 | "Please configure the public fully qualified domain names of this instance.\n" |
||
| 731 | "Use 'iso instance set web_hostnames your.hostname.tld' to do that.\n" |
||
| 732 | "You can add multiple names by separating them with commas.", |
||
| 733 | lvl=error, |
||
| 734 | ) |
||
| 735 | abort(50031) |
||
| 736 | |||
| 737 | contact = instance_configuration.get("contact", False) |
||
| 738 | if not contact: |
||
| 739 | log( |
||
| 740 | "You need to specify a contact mail address for this instance to generate certificates.\n" |
||
| 741 | "Use 'iso instance set contact [email protected]' to do that.", |
||
| 742 | lvl=error, |
||
| 743 | ) |
||
| 744 | abort(50032) |
||
| 745 | |||
| 746 | success, result = run_process( |
||
| 747 | "/", |
||
| 748 | [ |
||
| 749 | "certbot", |
||
| 750 | "--nginx", |
||
| 751 | "certonly", |
||
| 752 | "-m", |
||
| 753 | contact, |
||
| 754 | "-d", |
||
| 755 | hostnames, |
||
| 756 | "--agree-tos", |
||
| 757 | "-n", |
||
| 758 | ], |
||
| 759 | ) |
||
| 760 | if not success: |
||
| 761 | log( |
||
| 762 | "Error getting certificate:", |
||
| 763 | format_result(result), |
||
| 764 | pretty=True, |
||
| 765 | lvl=error, |
||
| 766 | ) |
||
| 767 | abort(50033) |
||
| 768 | |||
| 769 | |||
| 770 | def _instance_selfsigned(instance_configuration): |
||
| 771 | """Generates a snakeoil certificate that has only been self signed""" |
||
| 772 | |||
| 773 | log("Generating self signed certificate/key combination") |
||
| 774 | |||
| 775 | try: |
||
| 776 | os.mkdir("/etc/ssl/certs/isomer") |
||
| 777 | except FileExistsError: |
||
| 778 | pass |
||
| 779 | except PermissionError: |
||
| 780 | log("Need root (e.g. via sudo) to generate ssl certificate") |
||
| 781 | abort(1) |
||
| 782 | |||
| 783 | try: |
||
| 784 | from OpenSSL import crypto |
||
| 785 | except ImportError: |
||
| 786 | log("Need python3-openssl to do this.") |
||
| 787 | abort(1) |
||
| 788 | return |
||
| 789 | |||
| 790 | def create_self_signed_cert(target): |
||
| 791 | """Create a simple self signed SSL certificate""" |
||
| 792 | |||
| 793 | key_file = os.path.join(target, "selfsigned.key") |
||
| 794 | cert_file = os.path.join(target, "selfsigned.crt") |
||
| 795 | combined_file = os.path.join(target, "selfsigned.pem") |
||
| 796 | |||
| 797 | cert_conf = {k: v for k, v in instance_configuration.items() if |
||
| 798 | k.startswith("web_certificate")} |
||
| 799 | |||
| 800 | log("Certificate data:", cert_conf, pretty=True) |
||
| 801 | |||
| 802 | hostname = instance_configuration.get("web_hostnames", gethostname()) |
||
| 803 | if isinstance(hostname, list): |
||
| 804 | hostname = hostname[0] |
||
| 805 | |||
| 806 | # create a key pair |
||
| 807 | k = crypto.PKey() |
||
| 808 | k.generate_key(crypto.TYPE_RSA, 2048) |
||
| 809 | |||
| 810 | if os.path.exists(cert_file): |
||
| 811 | try: |
||
| 812 | certificate = open(cert_file, "rb").read() |
||
| 813 | old_cert = crypto.load_certificate(crypto.FILETYPE_PEM, certificate) |
||
| 814 | serial = old_cert.get_serial_number() + 1 |
||
| 815 | except (crypto.Error, OSError) as e: |
||
| 816 | log( |
||
| 817 | "Could not read old certificate to increment serial:", |
||
| 818 | type(e), |
||
| 819 | e, |
||
| 820 | exc=True, |
||
| 821 | lvl=warn, |
||
| 822 | ) |
||
| 823 | serial = 1 |
||
| 824 | else: |
||
| 825 | serial = 1 |
||
| 826 | |||
| 827 | # create a self-signed certificate |
||
| 828 | certificate = crypto.X509() |
||
| 829 | certificate.get_subject().C = cert_conf.get("web_certificate_country", |
||
| 830 | "EU") |
||
| 831 | certificate.get_subject().ST = cert_conf.get("web_certificate_state", "Sol") |
||
| 832 | certificate.get_subject().L = cert_conf.get("web_certificate_location", "Earth") |
||
| 833 | # noinspection PyPep8 |
||
| 834 | certificate.get_subject().O = cert_conf.get("web_certificate_issuer", "Unknown") |
||
| 835 | certificate.get_subject().OU = cert_conf.get("web_certificate_unit", "Unknown") |
||
| 836 | certificate.get_subject().CN = hostname |
||
| 837 | certificate.set_serial_number(serial) |
||
| 838 | certificate.gmtime_adj_notBefore(0) |
||
| 839 | certificate.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) |
||
| 840 | certificate.set_issuer(certificate.get_subject()) |
||
| 841 | certificate.set_pubkey(k) |
||
| 842 | certificate.sign(k, "sha512") |
||
| 843 | |||
| 844 | open(key_file, "wt").write( |
||
| 845 | str(crypto.dump_privatekey(crypto.FILETYPE_PEM, k), encoding="ASCII") |
||
| 846 | ) |
||
| 847 | |||
| 848 | open(cert_file, "wt").write( |
||
| 849 | str( |
||
| 850 | crypto.dump_certificate(crypto.FILETYPE_PEM, certificate), |
||
| 851 | encoding="ASCII", |
||
| 852 | ) |
||
| 853 | ) |
||
| 854 | |||
| 855 | open(combined_file, "wt").write( |
||
| 856 | str( |
||
| 857 | crypto.dump_certificate(crypto.FILETYPE_PEM, certificate), |
||
| 858 | encoding="ASCII", |
||
| 859 | ) |
||
| 860 | + str(crypto.dump_privatekey(crypto.FILETYPE_PEM, k), encoding="ASCII") |
||
| 861 | ) |
||
| 862 | |||
| 863 | location = os.path.join( |
||
| 864 | get_etc_instance_path(), |
||
| 865 | instance_configuration.get("name") |
||
| 866 | ) |
||
| 867 | |||
| 868 | create_self_signed_cert(location) |
||
| 869 | |||
| 870 | instance_configuration["web_key"] = os.path.join(location, "selfsigned.key") |
||
| 871 | instance_configuration["web_certificate"] = os.path.join(location, "selfsigned.crt") |
||
| 872 | |||
| 873 | write_instance(instance_configuration) |
||
| 874 | |||
| 875 | |||
| 876 | @instance.command(short_help="install systemd service") |
||
| 877 | @click.pass_context |
||
| 878 | def service(ctx): |
||
| 879 | """instance systemd service configuration""" |
||
| 880 | |||
| 881 | update_service(ctx, ctx.obj["instance_configuration"]["environment"]) |
||
| 882 | finish(ctx) |
||
| 883 | |||
| 884 | |||
| 885 | @instance.command(short_help="instance nginx configuration") |
||
| 886 | @click.option( |
||
| 887 | "--hostname", |
||
| 888 | default=None, |
||
| 889 | help="Override public Hostname (FQDN) Default from active system " "configuration", |
||
| 890 | ) |
||
| 891 | @click.pass_context |
||
| 892 | def update_nginx(ctx, hostname): |
||
| 893 | """instance nginx configuration""" |
||
| 894 | |||
| 895 | ctx.obj["hostname"] = hostname |
||
| 896 | |||
| 897 | _create_nginx_config(ctx) |
||
| 898 | finish(ctx) |
||
| 899 | |||
| 900 | |||
| 901 | def _create_nginx_config(ctx): |
||
| 902 | """instance nginx configuration""" |
||
| 903 | |||
| 904 | # TODO: Specify template url very precisely. Currently one needs to be in |
||
| 905 | # the repository root |
||
| 906 | |||
| 907 | instance_name = ctx.obj["instance"] |
||
| 908 | config = ctx.obj["instance_configuration"] |
||
| 909 | |||
| 910 | current_env = config["environment"] |
||
| 911 | env = config["environments"][current_env] |
||
| 912 | |||
| 913 | dbhost = config["database_host"] |
||
| 914 | dbname = env["database"] |
||
| 915 | |||
| 916 | hostnames = ctx.obj.get("web_hostnames", None) |
||
| 917 | if hostnames is None: |
||
| 918 | hostnames = config.get("web_hostnames", None) |
||
| 919 | if hostnames is None: |
||
| 920 | try: |
||
| 921 | configuration = _get_system_configuration(dbhost, dbname) |
||
| 922 | hostnames = configuration.hostname |
||
| 923 | except Exception as e: |
||
| 924 | log("Exception:", e, type(e), exc=True, lvl=error) |
||
| 925 | log( |
||
| 926 | """Could not determine public fully qualified hostname! |
||
| 927 | Check systemconfig (see db view and db modify commands) or specify |
||
| 928 | manually with --hostname host.domain.tld |
||
| 929 | |||
| 930 | Using 'localhost' for now""", |
||
| 931 | lvl=warn, |
||
| 932 | ) |
||
| 933 | hostnames = "localhost" |
||
| 934 | port = config["web_port"] |
||
| 935 | address = config["web_address"] |
||
| 936 | |||
| 937 | log( |
||
| 938 | "Creating nginx configuration for %s:%i using %s@%s" |
||
| 939 | % (hostnames, port, dbname, dbhost) |
||
| 940 | ) |
||
| 941 | |||
| 942 | definitions = { |
||
| 943 | "server_public_name": hostnames.replace(",", " "), |
||
| 944 | "ssl_certificate": config["web_certificate"], |
||
| 945 | "ssl_key": config["web_key"], |
||
| 946 | "host_url": "http://%s:%i/" % (address, port), |
||
| 947 | "instance": instance_name, |
||
| 948 | "environment": current_env, |
||
| 949 | } |
||
| 950 | |||
| 951 | if distribution == "DEBIAN": |
||
| 952 | configuration_file = "/etc/nginx/sites-available/isomer.%s.conf" % instance_name |
||
| 953 | configuration_link = "/etc/nginx/sites-enabled/isomer.%s.conf" % instance_name |
||
| 954 | elif distribution == "ARCH": |
||
| 955 | configuration_file = "/etc/nginx/nginx.conf" |
||
| 956 | configuration_link = None |
||
| 957 | else: |
||
| 958 | log( |
||
| 959 | "Unsure how to proceed, you may need to specify your " "distribution", |
||
| 960 | lvl=error, |
||
| 961 | ) |
||
| 962 | return |
||
| 963 | |||
| 964 | log("Writing nginx Isomer site definition") |
||
| 965 | write_template(nginx_template, configuration_file, definitions) |
||
| 966 | |||
| 967 | if configuration_link is not None: |
||
| 968 | log("Enabling nginx Isomer site (symlink)") |
||
| 969 | if not os.path.exists(configuration_link): |
||
| 970 | os.symlink(configuration_file, configuration_link) |
||
| 971 | |||
| 972 | if os.path.exists("/bin/systemctl"): |
||
| 973 | log("Restarting nginx service") |
||
| 974 | run_process("/", ["systemctl", "restart", "nginx.service"], sudo="root") |
||
| 975 | else: |
||
| 976 | log("No systemctl found, not restarting nginx") |
||
| 977 | |||
| 979 |