openstreetmap.to_postgres() - Code Metrics - Inspection of "Place files created during the workflow below the..." - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#208)

by Stephan

created 2021-04-09 19:20 UTC

openstreetmap.to_postgres() B

↳ Parent: openstreetmap

Complexity

Conditions

Size

Total Lines	59
Code Lines	37

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	3
eloc	37
nop	2
dl	0
loc	59
rs	8.9919
c	0
b	0
f	0

How to fix Long Method

"""The central module containing all code dealing with importing OSM data.

This module either directly contains the code dealing with importing OSM
data, or it re-exports everything needed to handle it. Please refrain
from importing code from any modules below this one, because it might
lead to unwanted behaviour.

If you have to import code from a module below this one because the code
isn't exported from this module, please file a bug, so we can fix this.
"""

from pathlib import Path
from urllib.request import urlretrieve
import json
import shutil
import time

import importlib_resources as resources

from egon.data import db
from egon.data.config import settings
import egon.data.config
import egon.data.subprocess as subprocess


def download_pbf_file():
    """Download OpenStreetMap `.pbf` file."""
    data_config = egon.data.config.datasets()
    osm_config = data_config["openstreetmap"]["original_data"]

    if settings()["egon-data"]["--dataset-boundary"] == "Everything":
        source_url = osm_config["source"]["url"]
        target_path = osm_config["target"]["path"]
    else:
        source_url = osm_config["source"]["url_testmode"]
        target_path = osm_config["target"]["path_testmode"]

    target_file = Path(".") / "openstreetmap" / target_path

    if not target_file.exists():
        target_file.parent.mkdir(parents=True, exist_ok=True)
        urlretrieve(source_url, target_file)


def to_postgres(num_processes=1, cache_size=4096):
    """Import OSM data from a Geofabrik `.pbf` file into a PostgreSQL database.

    Parameters
    ----------
    num_processes : int, optional
        Number of parallel processes used for processing during data import
    cache_size: int, optional
        Memory used during data import

    """
    # Read database configuration from docker-compose.yml
    docker_db_config = db.credentials()

    # Get dataset config
    data_config = egon.data.config.datasets()
    osm_config = data_config["openstreetmap"]["original_data"]

    if settings()["egon-data"]["--dataset-boundary"] == "Everything":
        target_path = osm_config["target"]["path"]
    else:
        target_path = osm_config["target"]["path_testmode"]

    input_file = Path(".") / "openstreetmap" / target_path

    with resources.path(
        "egon.data.importing.openstreetmap", osm_config["source"]["stylefile"]
    ) as p:
        stylefile = shutil.copy(p, Path(".") / "openstreetmap")

    # Prepare osm2pgsql command
    cmd = [
        "osm2pgsql",
        "--create",
        "--slim",
        "--hstore-all",
        "--number-processes",
        f"{num_processes}",
        "--cache",
        f"{cache_size}",
        "-H",
        f"{docker_db_config['HOST']}",
        "-P",
        f"{docker_db_config['PORT']}",
        "-d",
        f"{docker_db_config['POSTGRES_DB']}",
        "-U",
        f"{docker_db_config['POSTGRES_USER']}",
        "-p",
        f"{osm_config['target']['table_prefix']}",
        "-S",
        stylefile,
        f"{input_file.absolute()}",
    ]

    # Execute osm2pgsql for import OSM data
    subprocess.run(
        cmd,
        env={"PGPASSWORD": docker_db_config["POSTGRES_PASSWORD"]},
    )


def add_metadata():
    """Writes metadata JSON string into table comment."""
    # Prepare variables
    osm_config = egon.data.config.datasets()["openstreetmap"]

    if settings()["egon-data"]["--dataset-boundary"] == "Everything":
        osm_url = osm_config["original_data"]["source"]["url"]
        target_path = osm_config["original_data"]["target"]["path"]
    else:
        osm_url = osm_config["original_data"]["source"]["url_testmode"]
        target_path = osm_config["original_data"]["target"]["path_testmode"]
    spatial_and_date = Path(target_path).name.split("-")
    spatial_extend = spatial_and_date[0]
    osm_data_date = (
        "20"
        + spatial_and_date[1][0:2]
        + "-"
        + spatial_and_date[1][2:4]
        + "-"
        + spatial_and_date[1][4:6]
    )

    # Insert metadata for each table
    licenses = [
        {
            "name": "Open Data Commons Open Database License 1.0",
            "title": "",
            "path": "https://opendatacommons.org/licenses/odbl/1.0/",
            "instruction": (
                "You are free: To Share, To Create, To Adapt;"
                " As long as you: Attribute, Share-Alike, Keep open!"
            ),
            "attribution": "© Reiner Lemoine Institut",
        }
    ]
    for table in osm_config["processed"]["tables"]:
        table_suffix = table.split("_")[1]
        meta = {
            "title": f"OpenStreetMap (OSM) - Germany - {table_suffix}",
            "description": (
                "OpenStreetMap is a free, editable map of the"
                " whole world that is being built by volunteers"
                " largely from scratch and released with"
                " an open-content license."
            ),
            "language": ["EN", "DE"],
            "spatial": {
                "location": "",
                "extent": f"{spatial_extend}",
                "resolution": "",
            },
            "temporal": {
                "referenceDate": f"{osm_data_date}",
                "timeseries": {
                    "start": "",
                    "end": "",
                    "resolution": "",
                    "alignment": "",
                    "aggregationType": "",
                },
            },
            "sources": [
                {
                    "title": (
                        "Geofabrik - Download - OpenStreetMap Data Extracts"
                    ),
                    "description": (
                        'Data dump taken on "referenceDate",'
                        f" i.e. {osm_data_date}."
                        " A subset of this is selected using osm2pgsql"
                        ' using the style file "oedb.style".'
                    ),
                    "path": f"{osm_url}",
                    "licenses": licenses,
                }
            ],
            "licenses": licenses,
            "contributors": [
                {
                    "title": "Guido Pleßmann",
                    "email": "http://github.com/gplssm",
                    "date": time.strftime("%Y-%m-%d"),
                    "object": "",
                    "comment": "Imported data",
                }
            ],
            "metaMetadata": {
                "metadataVersion": "OEP-1.4.0",
                "metadataLicense": {
                    "name": "CC0-1.0",
                    "title": "Creative Commons Zero v1.0 Universal",
                    "path": (
                        "https://creativecommons.org/publicdomain/zero/1.0/"
                    ),
                },
            },
        }

        meta_json = "'" + json.dumps(meta) + "'"

        db.submit_comment(meta_json, "openstreetmap", table)


1			"""The central module containing all code dealing with importing OSM data.
2
3			This module either directly contains the code dealing with importing OSM
4			data, or it re-exports everything needed to handle it. Please refrain
5			from importing code from any modules below this one, because it might
6			lead to unwanted behaviour.
7
8			If you have to import code from a module below this one because the code
9			isn't exported from this module, please file a bug, so we can fix this.
10			"""
11
12			from pathlib import Path
13			from urllib.request import urlretrieve
14			import json
15			import shutil
16			import time
17
18			import importlib_resources as resources
19
20			from egon.data import db
21			from egon.data.config import settings
22			import egon.data.config
23			import egon.data.subprocess as subprocess
24
25
26			def download_pbf_file():
27			"""Download OpenStreetMap `.pbf` file."""
28			data_config = egon.data.config.datasets()
29			osm_config = data_config["openstreetmap"]["original_data"]
30
31			if settings()["egon-data"]["--dataset-boundary"] == "Everything":
32			source_url = osm_config["source"]["url"]
33			target_path = osm_config["target"]["path"]
34			else:
35			source_url = osm_config["source"]["url_testmode"]
36			target_path = osm_config["target"]["path_testmode"]
37
38			target_file = Path(".") / "openstreetmap" / target_path
39
40			if not target_file.exists():
41			target_file.parent.mkdir(parents=True, exist_ok=True)
42			urlretrieve(source_url, target_file)
43
44
45			def to_postgres(num_processes=1, cache_size=4096):
46			"""Import OSM data from a Geofabrik `.pbf` file into a PostgreSQL database.
47
48			Parameters
49			----------
50			num_processes : int, optional
51			Number of parallel processes used for processing during data import
52			cache_size: int, optional
53			Memory used during data import
54
55			"""
56			# Read database configuration from docker-compose.yml
57			docker_db_config = db.credentials()
58
59			# Get dataset config
60			data_config = egon.data.config.datasets()
61			osm_config = data_config["openstreetmap"]["original_data"]
62
63			if settings()["egon-data"]["--dataset-boundary"] == "Everything":
64			target_path = osm_config["target"]["path"]
65			else:
66			target_path = osm_config["target"]["path_testmode"]
67
68			input_file = Path(".") / "openstreetmap" / target_path
69
70			with resources.path(
71			"egon.data.importing.openstreetmap", osm_config["source"]["stylefile"]
72			) as p:
73			stylefile = shutil.copy(p, Path(".") / "openstreetmap")
74
75			# Prepare osm2pgsql command
76			cmd = [
77			"osm2pgsql",
78			"--create",
79			"--slim",
80			"--hstore-all",
81			"--number-processes",
82			f"{num_processes}",
83			"--cache",
84			f"{cache_size}",
85			"-H",
86			f"{docker_db_config['HOST']}",
87			"-P",
88			f"{docker_db_config['PORT']}",
89			"-d",
90			f"{docker_db_config['POSTGRES_DB']}",
91			"-U",
92			f"{docker_db_config['POSTGRES_USER']}",
93			"-p",
94			f"{osm_config['target']['table_prefix']}",
95			"-S",
96			stylefile,
97			f"{input_file.absolute()}",
98			]
99
100			# Execute osm2pgsql for import OSM data
101			subprocess.run(
102			cmd,
103			env={"PGPASSWORD": docker_db_config["POSTGRES_PASSWORD"]},
104			)
105
106
107			def add_metadata():
108			"""Writes metadata JSON string into table comment."""
109			# Prepare variables
110			osm_config = egon.data.config.datasets()["openstreetmap"]
111
112			if settings()["egon-data"]["--dataset-boundary"] == "Everything":
113			osm_url = osm_config["original_data"]["source"]["url"]
114			target_path = osm_config["original_data"]["target"]["path"]
115			else:
116			osm_url = osm_config["original_data"]["source"]["url_testmode"]
117			target_path = osm_config["original_data"]["target"]["path_testmode"]
118			spatial_and_date = Path(target_path).name.split("-")
119			spatial_extend = spatial_and_date[0]
120			osm_data_date = (
121			"20"
122			+ spatial_and_date[1][0:2]
123			+ "-"
124			+ spatial_and_date[1][2:4]
125			+ "-"
126			+ spatial_and_date[1][4:6]
127			)
128
129			# Insert metadata for each table
130			licenses = [
131			{
132			"name": "Open Data Commons Open Database License 1.0",
133			"title": "",
134			"path": "https://opendatacommons.org/licenses/odbl/1.0/",
135			"instruction": (
136			"You are free: To Share, To Create, To Adapt;"
137			" As long as you: Attribute, Share-Alike, Keep open!"
138			),
139			"attribution": "© Reiner Lemoine Institut",
140			}
141			]
142			for table in osm_config["processed"]["tables"]:
143			table_suffix = table.split("_")[1]
144			meta = {
145			"title": f"OpenStreetMap (OSM) - Germany - {table_suffix}",
146			"description": (
147			"OpenStreetMap is a free, editable map of the"
148			" whole world that is being built by volunteers"
149			" largely from scratch and released with"
150			" an open-content license."
151			),
152			"language": ["EN", "DE"],
153			"spatial": {
154			"location": "",
155			"extent": f"{spatial_extend}",
156			"resolution": "",
157			},
158			"temporal": {
159			"referenceDate": f"{osm_data_date}",
160			"timeseries": {
161			"start": "",
162			"end": "",
163			"resolution": "",
164			"alignment": "",
165			"aggregationType": "",
166			},
167			},
168			"sources": [
169			{
170			"title": (
171			"Geofabrik - Download - OpenStreetMap Data Extracts"
172			),
173			"description": (
174			'Data dump taken on "referenceDate",'
175			f" i.e. {osm_data_date}."
176			" A subset of this is selected using osm2pgsql"
177			' using the style file "oedb.style".'
178			),
179			"path": f"{osm_url}",
180			"licenses": licenses,
181			}
182			],
183			"licenses": licenses,
184			"contributors": [
185			{
186			"title": "Guido Pleßmann",
187			"email": "http://github.com/gplssm",
188			"date": time.strftime("%Y-%m-%d"),
189			"object": "",
190			"comment": "Imported data",
191			}
192			],
193			"metaMetadata": {
194			"metadataVersion": "OEP-1.4.0",
195			"metadataLicense": {
196			"name": "CC0-1.0",
197			"title": "Creative Commons Zero v1.0 Universal",
198			"path": (
199			"https://creativecommons.org/publicdomain/zero/1.0/"
200			),
201			},
202			},
203			}
204
205			meta_json = "'" + json.dumps(meta) + "'"
206
207			db.submit_comment(meta_json, "openstreetmap", table)
208

openego / eGon-data

Pull Request — dev (#208)

openstreetmap.to_postgres() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like