data.datasets.osmtgmod.import_osm_data()

Complexity

Conditions

10

Size

Total Lines	154
Code Lines	101

Duplication

Lines	0
Ratio	0 %

Importance

Changes

0

from pathlib import Path
def import_osm_data():

    osmtgmod_repos = Path(".") / "osmTGmod"

    # Delete repository if it already exists
    if osmtgmod_repos.exists() and osmtgmod_repos.is_dir():
        try:
            status = subprocess.check_output(
                ["git", "status"], cwd=(osmtgmod_repos).absolute()
            )
            if status.startswith(
                b"Auf Branch features/egon"
            ) or status.startswith(b"On branch features/egon"):
                logger.info("OsmTGmod cloned and right branch checked out.")

            else:
                subproc.run(
                    [
                        "git",
                        "checkout",
                        "features/egon",
                    ]
                )
        except subprocess.CalledProcessError:
            shutil.rmtree(osmtgmod_repos)
            subproc.run(
                [
                    "git",
                    "clone",
                    "--single-branch",
                    "--branch",
                    "features/egon",
                    "https://github.com/openego/osmTGmod.git",
                ]
            )
    else:

        subproc.run(
            [
                "git",
                "clone",
                "--single-branch",
                "--branch",
                "features/egon",
                "https://github.com/openego/osmTGmod.git",
            ]
        )

    data_config = egon.data.config.datasets()
    osm_config = data_config["openstreetmap"]["original_data"]

    if settings()["egon-data"]["--dataset-boundary"] == "Everything":
        target_path = osm_config["target"]["file"]
    else:
        target_path = osm_config["target"]["file_testmode"]

    filtered_osm_pbf_path_to_file = Path(".") / "openstreetmap" / target_path

    docker_db_config = db.credentials()
    config_database = docker_db_config["POSTGRES_DB"]
    config_basepath = "osmTGmod/egon-data"

    config = configparser.ConfigParser()
    config.read(config_basepath + ".cfg")
    config["postgres_server"]["host"] = docker_db_config["HOST"]
    config["postgres_server"]["port"] = docker_db_config["PORT"]
    config["postgres_server"]["user"] = docker_db_config["POSTGRES_USER"]
    config["postgres_server"]["password"] = docker_db_config[
        "POSTGRES_PASSWORD"
    ]

    logging.info("Creating status table ...")
    db.execute_sql(
        """
        DROP TABLE IF EXISTS _db_status;
        CREATE TABLE _db_status (module TEXT, status BOOLEAN);
        INSERT INTO _db_status (module, status) VALUES ('grid_model', FALSE);
        """
    )

    logging.info("Status table created.")

    # egon-specific, in order to not fill up the results schema,
    # it is dropped before creation
    logging.info("Dropping osmtgmod_results schema if exists")
    db.execute_sql("DROP SCHEMA IF EXISTS osmtgmod_results CASCADE;")

    logging.info("Loading functions and result schema ...")
    scripts = [
        "sql-scripts/extensions.sql",
        "sql-scripts/functions.sql",
        "sql-scripts/admin_boundaries.sql",
        "sql-scripts/electrical_properties.sql",
        "sql-scripts/build_up_db.sql",
    ]
    for script in scripts:
        logging.info("Running script {0} ...".format(script))
        with codecs.open(
            os.path.join("osmTGmod", script), "r", "utf-8-sig"
        ) as fd:
            sqlfile = fd.read()
        db.execute_sql(sqlfile)
        logging.info("Done.")

    db.execute_sql(
        """UPDATE _db_status SET status = TRUE
            WHERE module = 'grid_model'; """
    )

    logging.info("osmTGmod-database successfully built up!")

    logging.info("Importing OSM-data to database.")

    logging.info("Using pdf file: {}".format(filtered_osm_pbf_path_to_file))
    logging.info(
        f"""Assuming osmosis is avaliable at
        {config['osm_data']['osmosis_path_to_binary']}"""
    )

    # create directory to store osmosis' temp files
    osmosis_temp_dir = Path("osmTGmod") / "osmosis_temp/"
    if not os.path.exists(osmosis_temp_dir):
        os.mkdir(osmosis_temp_dir)

    subproc.run(
        "JAVACMD_OPTIONS='%s' %s --read-pbf %s --write-pgsql \
                database=%s host=%s user=%s password=%s"
        % (
            f"-Djava.io.tmpdir={osmosis_temp_dir}",
            os.path.join(
                "osmTGmod", config["osm_data"]["osmosis_path_to_binary"]
            ),
            filtered_osm_pbf_path_to_file,
            config_database,
            config["postgres_server"]["host"]
            + ":"
            + config["postgres_server"]["port"],
            config["postgres_server"]["user"],
            config["postgres_server"]["password"],
        ),
        shell=True,
    )
    logging.info("Importing OSM-Data...")

    # After updating OSM-Data, power_tables (for editing)
    # have to be updated as well
    logging.info("Creating power-tables...")
    db.execute_sql("SELECT otg_create_power_tables ();")

    # Update OSM Metadata
    logging.info("Updating OSM metadata")
    v_date = datetime.datetime.now().strftime("%Y-%m-%d")
    db.execute_sql(f"UPDATE osm_metadata SET imported = '{v_date}'")
    logging.info("OSM data imported to database successfully.")