openstreetmap.add_metadata() - Code Metrics - Inspection of "Integrating OSM data import" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#31)

by Stephan

created 2020-11-17 05:34 UTC

openstreetmap.add_metadata() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines	97
Code Lines	62

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	2
eloc	62
nop	0
dl	0
loc	97
rs	8.2436
c	0
b	0
f	0

How to fix Long Method

"""The central module containing all code dealing with importing OSM data.

This module either directly contains the code dealing with importing OSM
data, or it re-exports everything needed to handle it. Please refrain
from importing code from any modules below this one, because it might
lead to unwanted behaviour.

If you have to import code from a module below this one because the code
isn't exported from this module, please file a bug, so we can fix this.
"""

from urllib.request import urlretrieve
import json
import os
import subprocess
import time

from egon.data import db
import egon.data.config


def download_pbf_file():
    """Download OpenStreetMap `.pbf` file."""
    data_config = egon.data.config.datasets()
    osm_config = data_config["openstreetmap"]["original_data"]["osm"]

    if not os.path.isfile(osm_config["file"]):
        urlretrieve(osm_config["url"] + osm_config["file"], osm_config["file"])


def to_postgres(num_processes=4, cache_size=4096):
    """Import OSM data from a Geofabrik `.pbf` file into a PostgreSQL database.

    Parameters
    ----------
    num_processes : int, optional
        Number of parallel processes used for processing during data import
    cache_size: int, optional
        Memory used during data import

    """
    # Read database configuration from docker-compose.yml
    docker_db_config = db.credentials()

    # Get dataset config
    data_config = egon.data.config.datasets()
    osm_config = data_config["openstreetmap"]["original_data"]["osm"]

    # Prepare osm2pgsql command
    cmd = [
        "osm2pgsql",
        "--create",
        "--slim",
        "--hstore-all",
        f"--number-processes {num_processes}",
        f"--cache {cache_size}",
        f"-H {docker_db_config['HOST']} -P {docker_db_config['PORT']} "
        f"-d {docker_db_config['POSTGRES_DB']} "
        f"-U {docker_db_config['POSTGRES_USER']}",
        f"-p {osm_config['table_prefix']}",
        f"-S {osm_config['stylefile']}",
        f"{osm_config['file']}",
    ]

    # Execute osm2pgsql for import OSM data
    subprocess.run(
        " ".join(cmd),
        shell=True,
        env={"PGPASSWORD": docker_db_config["POSTGRES_PASSWORD"]},
        cwd=os.path.dirname(__file__),
    )


def add_metadata():
    """Writes metadata JSON string into table comment."""
    # Prepare variables
    osm_config = egon.data.config.datasets()["openstreetmap"]["original_data"][
        "osm"
    ]
    spatial_and_date = os.path.basename(osm_config["file"]).split("-")
    spatial_extend = spatial_and_date[0]
    osm_data_date = (
        "20"
        + spatial_and_date[1][0:2]
        + "-"
        + spatial_and_date[1][2:4]
        + "-"
        + spatial_and_date[1][4:6]
    )
    osm_url = osm_config["url"]

    # Insert metadata for each table
    licenses = [
        {
            "name": "Open Data Commons Open Database License 1.0",
            "title": "",
            "path": "https://opendatacommons.org/licenses/odbl/1.0/",
            "instruction": (
                "You are free: To Share, To Create, To Adapt;"
                " As long as you: Attribute, Share-Alike, Keep open!"
            ),
            "attribution": "© Reiner Lemoine Institut",
        }
    ]
    for table in osm_config["output_tables"]:
        table_suffix = table.split("_")[1]
        meta = {
            "title": f"OpenStreetMap (OSM) - Germany - {table_suffix}",
            "description": (
                "OpenStreetMap is a free, editable map of the"
                " whole world that is being built by volunteers"
                " largely from scratch and released with"
                " an open-content license."
            ),
            "language": ["EN", "DE"],
            "spatial": {
                "location": "",
                "extent": f"{spatial_extend}",
                "resolution": "",
            },
            "temporal": {
                "referenceDate": f"{osm_data_date}",
                "timeseries": {
                    "start": "",
                    "end": "",
                    "resolution": "",
                    "alignment": "",
                    "aggregationType": "",
                },
            },
            "sources": [
                {
                    "title": (
                        "Geofabrik - Download - OpenStreetMap Data Extracts"
                    ),
                    "description": (
                        'Data dump taken on "referenceDate",'
                        f" i.e. {osm_data_date}."
                        " A subset of this is selected using osm2pgsql"
                        ' using the style file "oedb.style".'
                    ),
                    "path": f"{osm_url}",
                    "licenses": licenses,
                }
            ],
            "licenses": licenses,
            "contributors": [
                {
                    "title": "Guido Pleßmann",
                    "email": "http://github.com/gplssm",
                    "date": time.strftime("%Y-%m-%d"),
                    "object": "",
                    "comment": "Imported data",
                }
            ],
            "metaMetadata": {
                "metadataVersion": "OEP-1.4.0",
                "metadataLicense": {
                    "name": "CC0-1.0",
                    "title": "Creative Commons Zero v1.0 Universal",
                    "path": (
                        "https://creativecommons.org/publicdomain/zero/1.0/"
                    ),
                },
            },
        }

        meta_json = "'" + json.dumps(meta) + "'"

        db.submit_comment(meta_json, "openstreetmap", table)


1			"""The central module containing all code dealing with importing OSM data.
2
3			This module either directly contains the code dealing with importing OSM
4			data, or it re-exports everything needed to handle it. Please refrain
5			from importing code from any modules below this one, because it might
6			lead to unwanted behaviour.
7
8			If you have to import code from a module below this one because the code
9			isn't exported from this module, please file a bug, so we can fix this.
10			"""
11
12			from urllib.request import urlretrieve
13			import json
14			import os
15			import subprocess
16			import time
17
18			from egon.data import db
19			import egon.data.config
20
21
22			def download_pbf_file():
23			"""Download OpenStreetMap `.pbf` file."""
24			data_config = egon.data.config.datasets()
25			osm_config = data_config["openstreetmap"]["original_data"]["osm"]
26
27			if not os.path.isfile(osm_config["file"]):
28			urlretrieve(osm_config["url"] + osm_config["file"], osm_config["file"])
29
30
31			def to_postgres(num_processes=4, cache_size=4096):
32			"""Import OSM data from a Geofabrik `.pbf` file into a PostgreSQL database.
33
34			Parameters
35			----------
36			num_processes : int, optional
37			Number of parallel processes used for processing during data import
38			cache_size: int, optional
39			Memory used during data import
40
41			"""
42			# Read database configuration from docker-compose.yml
43			docker_db_config = db.credentials()
44
45			# Get dataset config
46			data_config = egon.data.config.datasets()
47			osm_config = data_config["openstreetmap"]["original_data"]["osm"]
48
49			# Prepare osm2pgsql command
50			cmd = [
51			"osm2pgsql",
52			"--create",
53			"--slim",
54			"--hstore-all",
55			f"--number-processes {num_processes}",
56			f"--cache {cache_size}",
57			f"-H {docker_db_config['HOST']} -P {docker_db_config['PORT']} "
58			f"-d {docker_db_config['POSTGRES_DB']} "
59			f"-U {docker_db_config['POSTGRES_USER']}",
60			f"-p {osm_config['table_prefix']}",
61			f"-S {osm_config['stylefile']}",
62			f"{osm_config['file']}",
63			]
64
65			# Execute osm2pgsql for import OSM data
66			subprocess.run(
67			" ".join(cmd),
68			shell=True,
69			env={"PGPASSWORD": docker_db_config["POSTGRES_PASSWORD"]},
70			cwd=os.path.dirname(__file__),
71			)
72
73
74			def add_metadata():
75			"""Writes metadata JSON string into table comment."""
76			# Prepare variables
77			osm_config = egon.data.config.datasets()["openstreetmap"]["original_data"][
78			"osm"
79			]
80			spatial_and_date = os.path.basename(osm_config["file"]).split("-")
81			spatial_extend = spatial_and_date[0]
82			osm_data_date = (
83			"20"
84			+ spatial_and_date[1][0:2]
85			+ "-"
86			+ spatial_and_date[1][2:4]
87			+ "-"
88			+ spatial_and_date[1][4:6]
89			)
90			osm_url = osm_config["url"]
91
92			# Insert metadata for each table
93			licenses = [
94			{
95			"name": "Open Data Commons Open Database License 1.0",
96			"title": "",
97			"path": "https://opendatacommons.org/licenses/odbl/1.0/",
98			"instruction": (
99			"You are free: To Share, To Create, To Adapt;"
100			" As long as you: Attribute, Share-Alike, Keep open!"
101			),
102			"attribution": "© Reiner Lemoine Institut",
103			}
104			]
105			for table in osm_config["output_tables"]:
106			table_suffix = table.split("_")[1]
107			meta = {
108			"title": f"OpenStreetMap (OSM) - Germany - {table_suffix}",
109			"description": (
110			"OpenStreetMap is a free, editable map of the"
111			" whole world that is being built by volunteers"
112			" largely from scratch and released with"
113			" an open-content license."
114			),
115			"language": ["EN", "DE"],
116			"spatial": {
117			"location": "",
118			"extent": f"{spatial_extend}",
119			"resolution": "",
120			},
121			"temporal": {
122			"referenceDate": f"{osm_data_date}",
123			"timeseries": {
124			"start": "",
125			"end": "",
126			"resolution": "",
127			"alignment": "",
128			"aggregationType": "",
129			},
130			},
131			"sources": [
132			{
133			"title": (
134			"Geofabrik - Download - OpenStreetMap Data Extracts"
135			),
136			"description": (
137			'Data dump taken on "referenceDate",'
138			f" i.e. {osm_data_date}."
139			" A subset of this is selected using osm2pgsql"
140			' using the style file "oedb.style".'
141			),
142			"path": f"{osm_url}",
143			"licenses": licenses,
144			}
145			],
146			"licenses": licenses,
147			"contributors": [
148			{
149			"title": "Guido Pleßmann",
150			"email": "http://github.com/gplssm",
151			"date": time.strftime("%Y-%m-%d"),
152			"object": "",
153			"comment": "Imported data",
154			}
155			],
156			"metaMetadata": {
157			"metadataVersion": "OEP-1.4.0",
158			"metadataLicense": {
159			"name": "CC0-1.0",
160			"title": "Creative Commons Zero v1.0 Universal",
161			"path": (
162			"https://creativecommons.org/publicdomain/zero/1.0/"
163			),
164			},
165			},
166			}
167
168			meta_json = "'" + json.dumps(meta) + "'"
169
170			db.submit_comment(meta_json, "openstreetmap", table)
171

openego / eGon-data

Pull Request — dev (#31)

openstreetmap.add_metadata() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like