openstreetmap.add_metadata() - Code Metrics - Inspection of "Integrating OSM data import" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#31)

by Stephan

created 2020-11-19 15:18 UTC

openstreetmap.add_metadata() B

↳ Parent: openstreetmap

Complexity

Conditions

Size

Total Lines	97
Code Lines	63

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	2
eloc	63
nop	0
dl	0
loc	97
rs	8.2109
c	0
b	0
f	0

How to fix Long Method

"""The central module containing all code dealing with importing OSM data.

This module either directly contains the code dealing with importing OSM
data, or it re-exports everything needed to handle it. Please refrain
from importing code from any modules below this one, because it might
lead to unwanted behaviour.

If you have to import code from a module below this one because the code
isn't exported from this module, please file a bug, so we can fix this.
"""

from urllib.request import urlretrieve
import json
import os
import subprocess
import time

from egon.data import db
import egon.data.config


def download_pbf_file():
    """Download OpenStreetMap `.pbf` file."""
    data_config = egon.data.config.datasets()
    osm_config = data_config["openstreetmap"]["original_data"]

    target_file = os.path.join(
        os.path.dirname(__file__), osm_config["target"]["path"]
    )

    if not os.path.isfile(target_file):
        urlretrieve(osm_config["source"]["url"], target_file)


def to_postgres(num_processes=4, cache_size=4096):
    """Import OSM data from a Geofabrik `.pbf` file into a PostgreSQL database.

    Parameters
    ----------
    num_processes : int, optional
        Number of parallel processes used for processing during data import
    cache_size: int, optional
        Memory used during data import

    """
    # Read database configuration from docker-compose.yml
    docker_db_config = db.credentials()

    # Get dataset config
    data_config = egon.data.config.datasets()
    osm_config = data_config["openstreetmap"]["original_data"]
    input_file = os.path.join(
        os.path.dirname(__file__), osm_config["target"]["path"]
    )

    # Prepare osm2pgsql command
    cmd = [
        "osm2pgsql",
        "--create",
        "--slim",
        "--hstore-all",
        f"--number-processes {num_processes}",
        f"--cache {cache_size}",
        f"-H {docker_db_config['HOST']} -P {docker_db_config['PORT']} "
        f"-d {docker_db_config['POSTGRES_DB']} "
        f"-U {docker_db_config['POSTGRES_USER']}",
        f"-p {osm_config['target']['table_prefix']}",
        f"-S {osm_config['source']['stylefile']}",
        f"{input_file}",
    ]

    # Execute osm2pgsql for import OSM data
    subprocess.run(
        " ".join(cmd),
        shell=True,
        env={"PGPASSWORD": docker_db_config["POSTGRES_PASSWORD"]},
        cwd=os.path.dirname(__file__),
    )


def add_metadata():
    """Writes metadata JSON string into table comment."""
    # Prepare variables
    osm_config = egon.data.config.datasets()["openstreetmap"]
    spatial_and_date = os.path.basename(
        osm_config["original_data"]["target"]["path"]
    ).split("-")
    spatial_extend = spatial_and_date[0]
    osm_data_date = (
        "20"
        + spatial_and_date[1][0:2]
        + "-"
        + spatial_and_date[1][2:4]
        + "-"
        + spatial_and_date[1][4:6]
    )
    osm_url = osm_config["original_data"]["source"]["url"]

    # Insert metadata for each table
    licenses = [
        {
            "name": "Open Data Commons Open Database License 1.0",
            "title": "",
            "path": "https://opendatacommons.org/licenses/odbl/1.0/",
            "instruction": (
                "You are free: To Share, To Create, To Adapt;"
                " As long as you: Attribute, Share-Alike, Keep open!"
            ),
            "attribution": "© Reiner Lemoine Institut",
        }
    ]
    for table in osm_config["processed"]["tables"]:
        table_suffix = table.split("_")[1]
        meta = {
            "title": f"OpenStreetMap (OSM) - Germany - {table_suffix}",
            "description": (
                "OpenStreetMap is a free, editable map of the"
                " whole world that is being built by volunteers"
                " largely from scratch and released with"
                " an open-content license."
            ),
            "language": ["EN", "DE"],
            "spatial": {
                "location": "",
                "extent": f"{spatial_extend}",
                "resolution": "",
            },
            "temporal": {
                "referenceDate": f"{osm_data_date}",
                "timeseries": {
                    "start": "",
                    "end": "",
                    "resolution": "",
                    "alignment": "",
                    "aggregationType": "",
                },
            },
            "sources": [
                {
                    "title": (
                        "Geofabrik - Download - OpenStreetMap Data Extracts"
                    ),
                    "description": (
                        'Data dump taken on "referenceDate",'
                        f" i.e. {osm_data_date}."
                        " A subset of this is selected using osm2pgsql"
                        ' using the style file "oedb.style".'
                    ),
                    "path": f"{osm_url}",
                    "licenses": licenses,
                }
            ],
            "licenses": licenses,
            "contributors": [
                {
                    "title": "Guido Pleßmann",
                    "email": "http://github.com/gplssm",
                    "date": time.strftime("%Y-%m-%d"),
                    "object": "",
                    "comment": "Imported data",
                }
            ],
            "metaMetadata": {
                "metadataVersion": "OEP-1.4.0",
                "metadataLicense": {
                    "name": "CC0-1.0",
                    "title": "Creative Commons Zero v1.0 Universal",
                    "path": (
                        "https://creativecommons.org/publicdomain/zero/1.0/"
                    ),
                },
            },
        }

        meta_json = "'" + json.dumps(meta) + "'"

        db.submit_comment(meta_json, "openstreetmap", table)


1			"""The central module containing all code dealing with importing OSM data.
2
3			This module either directly contains the code dealing with importing OSM
4			data, or it re-exports everything needed to handle it. Please refrain
5			from importing code from any modules below this one, because it might
6			lead to unwanted behaviour.
7
8			If you have to import code from a module below this one because the code
9			isn't exported from this module, please file a bug, so we can fix this.
10			"""
11
12			from urllib.request import urlretrieve
13			import json
14			import os
15			import subprocess
16			import time
17
18			from egon.data import db
19			import egon.data.config
20
21
22			def download_pbf_file():
23			"""Download OpenStreetMap `.pbf` file."""
24			data_config = egon.data.config.datasets()
25			osm_config = data_config["openstreetmap"]["original_data"]
26
27			target_file = os.path.join(
28			os.path.dirname(__file__), osm_config["target"]["path"]
29			)
30
31			if not os.path.isfile(target_file):
32			urlretrieve(osm_config["source"]["url"], target_file)
33
34
35			def to_postgres(num_processes=4, cache_size=4096):
36			"""Import OSM data from a Geofabrik `.pbf` file into a PostgreSQL database.
37
38			Parameters
39			----------
40			num_processes : int, optional
41			Number of parallel processes used for processing during data import
42			cache_size: int, optional
43			Memory used during data import
44
45			"""
46			# Read database configuration from docker-compose.yml
47			docker_db_config = db.credentials()
48
49			# Get dataset config
50			data_config = egon.data.config.datasets()
51			osm_config = data_config["openstreetmap"]["original_data"]
52			input_file = os.path.join(
53			os.path.dirname(__file__), osm_config["target"]["path"]
54			)
55
56			# Prepare osm2pgsql command
57			cmd = [
58			"osm2pgsql",
59			"--create",
60			"--slim",
61			"--hstore-all",
62			f"--number-processes {num_processes}",
63			f"--cache {cache_size}",
64			f"-H {docker_db_config['HOST']} -P {docker_db_config['PORT']} "
65			f"-d {docker_db_config['POSTGRES_DB']} "
66			f"-U {docker_db_config['POSTGRES_USER']}",
67			f"-p {osm_config['target']['table_prefix']}",
68			f"-S {osm_config['source']['stylefile']}",
69			f"{input_file}",
70			]
71
72			# Execute osm2pgsql for import OSM data
73			subprocess.run(
74			" ".join(cmd),
75			shell=True,
76			env={"PGPASSWORD": docker_db_config["POSTGRES_PASSWORD"]},
77			cwd=os.path.dirname(__file__),
78			)
79
80
81			def add_metadata():
82			"""Writes metadata JSON string into table comment."""
83			# Prepare variables
84			osm_config = egon.data.config.datasets()["openstreetmap"]
85			spatial_and_date = os.path.basename(
86			osm_config["original_data"]["target"]["path"]
87			).split("-")
88			spatial_extend = spatial_and_date[0]
89			osm_data_date = (
90			"20"
91			+ spatial_and_date[1][0:2]
92			+ "-"
93			+ spatial_and_date[1][2:4]
94			+ "-"
95			+ spatial_and_date[1][4:6]
96			)
97			osm_url = osm_config["original_data"]["source"]["url"]
98
99			# Insert metadata for each table
100			licenses = [
101			{
102			"name": "Open Data Commons Open Database License 1.0",
103			"title": "",
104			"path": "https://opendatacommons.org/licenses/odbl/1.0/",
105			"instruction": (
106			"You are free: To Share, To Create, To Adapt;"
107			" As long as you: Attribute, Share-Alike, Keep open!"
108			),
109			"attribution": "© Reiner Lemoine Institut",
110			}
111			]
112			for table in osm_config["processed"]["tables"]:
113			table_suffix = table.split("_")[1]
114			meta = {
115			"title": f"OpenStreetMap (OSM) - Germany - {table_suffix}",
116			"description": (
117			"OpenStreetMap is a free, editable map of the"
118			" whole world that is being built by volunteers"
119			" largely from scratch and released with"
120			" an open-content license."
121			),
122			"language": ["EN", "DE"],
123			"spatial": {
124			"location": "",
125			"extent": f"{spatial_extend}",
126			"resolution": "",
127			},
128			"temporal": {
129			"referenceDate": f"{osm_data_date}",
130			"timeseries": {
131			"start": "",
132			"end": "",
133			"resolution": "",
134			"alignment": "",
135			"aggregationType": "",
136			},
137			},
138			"sources": [
139			{
140			"title": (
141			"Geofabrik - Download - OpenStreetMap Data Extracts"
142			),
143			"description": (
144			'Data dump taken on "referenceDate",'
145			f" i.e. {osm_data_date}."
146			" A subset of this is selected using osm2pgsql"
147			' using the style file "oedb.style".'
148			),
149			"path": f"{osm_url}",
150			"licenses": licenses,
151			}
152			],
153			"licenses": licenses,
154			"contributors": [
155			{
156			"title": "Guido Pleßmann",
157			"email": "http://github.com/gplssm",
158			"date": time.strftime("%Y-%m-%d"),
159			"object": "",
160			"comment": "Imported data",
161			}
162			],
163			"metaMetadata": {
164			"metadataVersion": "OEP-1.4.0",
165			"metadataLicense": {
166			"name": "CC0-1.0",
167			"title": "Creative Commons Zero v1.0 Universal",
168			"path": (
169			"https://creativecommons.org/publicdomain/zero/1.0/"
170			),
171			},
172			},
173			}
174
175			meta_json = "'" + json.dumps(meta) + "'"
176
177			db.submit_comment(meta_json, "openstreetmap", table)
178

openego / eGon-data

Pull Request — dev (#31)

openstreetmap.add_metadata() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like