|
1
|
|
|
"""The central module containing all code dealing with importing OSM data. |
|
2
|
|
|
|
|
3
|
|
|
This module either directly contains the code dealing with importing OSM |
|
4
|
|
|
data, or it re-exports everything needed to handle it. Please refrain |
|
5
|
|
|
from importing code from any modules below this one, because it might |
|
6
|
|
|
lead to unwanted behaviour. |
|
7
|
|
|
|
|
8
|
|
|
If you have to import code from a module below this one because the code |
|
9
|
|
|
isn't exported from this module, please file a bug, so we can fix this. |
|
10
|
|
|
""" |
|
11
|
|
|
|
|
12
|
|
|
from pathlib import Path |
|
13
|
|
|
from urllib.request import urlretrieve |
|
14
|
|
|
import json |
|
15
|
|
|
import shutil |
|
16
|
|
|
import time |
|
17
|
|
|
|
|
18
|
|
|
import importlib_resources as resources |
|
19
|
|
|
|
|
20
|
|
|
from egon.data import db |
|
21
|
|
|
from egon.data.config import settings |
|
22
|
|
|
import egon.data.config |
|
23
|
|
|
import egon.data.subprocess as subprocess |
|
24
|
|
|
|
|
25
|
|
|
|
|
26
|
|
|
def download_pbf_file(): |
|
27
|
|
|
"""Download OpenStreetMap `.pbf` file.""" |
|
28
|
|
|
data_config = egon.data.config.datasets() |
|
29
|
|
|
osm_config = data_config["openstreetmap"]["original_data"] |
|
30
|
|
|
|
|
31
|
|
|
if settings()["egon-data"]["--dataset-boundary"] == "Everything": |
|
32
|
|
|
source_url = osm_config["source"]["url"] |
|
33
|
|
|
target_path = osm_config["target"]["path"] |
|
34
|
|
|
else: |
|
35
|
|
|
source_url = osm_config["source"]["url_testmode"] |
|
36
|
|
|
target_path = osm_config["target"]["path_testmode"] |
|
37
|
|
|
|
|
38
|
|
|
target_file = Path(".") / "openstreetmap" / target_path |
|
39
|
|
|
|
|
40
|
|
|
if not target_file.exists(): |
|
41
|
|
|
target_file.parent.mkdir(parents=True, exist_ok=True) |
|
42
|
|
|
urlretrieve(source_url, target_file) |
|
43
|
|
|
|
|
44
|
|
|
|
|
45
|
|
|
def to_postgres(num_processes=1, cache_size=4096): |
|
46
|
|
|
"""Import OSM data from a Geofabrik `.pbf` file into a PostgreSQL database. |
|
47
|
|
|
|
|
48
|
|
|
Parameters |
|
49
|
|
|
---------- |
|
50
|
|
|
num_processes : int, optional |
|
51
|
|
|
Number of parallel processes used for processing during data import |
|
52
|
|
|
cache_size: int, optional |
|
53
|
|
|
Memory used during data import |
|
54
|
|
|
|
|
55
|
|
|
""" |
|
56
|
|
|
# Read database configuration from docker-compose.yml |
|
57
|
|
|
docker_db_config = db.credentials() |
|
58
|
|
|
|
|
59
|
|
|
# Get dataset config |
|
60
|
|
|
data_config = egon.data.config.datasets() |
|
61
|
|
|
osm_config = data_config["openstreetmap"]["original_data"] |
|
62
|
|
|
|
|
63
|
|
|
if settings()["egon-data"]["--dataset-boundary"] == "Everything": |
|
64
|
|
|
target_path = osm_config["target"]["path"] |
|
65
|
|
|
else: |
|
66
|
|
|
target_path = osm_config["target"]["path_testmode"] |
|
67
|
|
|
|
|
68
|
|
|
input_file = Path(".") / "openstreetmap" / target_path |
|
69
|
|
|
|
|
70
|
|
|
with resources.path( |
|
71
|
|
|
"egon.data.importing.openstreetmap", osm_config["source"]["stylefile"] |
|
72
|
|
|
) as p: |
|
73
|
|
|
stylefile = shutil.copy(p, Path(".") / "openstreetmap") |
|
74
|
|
|
|
|
75
|
|
|
# Prepare osm2pgsql command |
|
76
|
|
|
cmd = [ |
|
77
|
|
|
"osm2pgsql", |
|
78
|
|
|
"--create", |
|
79
|
|
|
"--slim", |
|
80
|
|
|
"--hstore-all", |
|
81
|
|
|
"--number-processes", |
|
82
|
|
|
f"{num_processes}", |
|
83
|
|
|
"--cache", |
|
84
|
|
|
f"{cache_size}", |
|
85
|
|
|
"-H", |
|
86
|
|
|
f"{docker_db_config['HOST']}", |
|
87
|
|
|
"-P", |
|
88
|
|
|
f"{docker_db_config['PORT']}", |
|
89
|
|
|
"-d", |
|
90
|
|
|
f"{docker_db_config['POSTGRES_DB']}", |
|
91
|
|
|
"-U", |
|
92
|
|
|
f"{docker_db_config['POSTGRES_USER']}", |
|
93
|
|
|
"-p", |
|
94
|
|
|
f"{osm_config['target']['table_prefix']}", |
|
95
|
|
|
"-S", |
|
96
|
|
|
stylefile, |
|
97
|
|
|
f"{input_file.absolute()}", |
|
98
|
|
|
] |
|
99
|
|
|
|
|
100
|
|
|
# Execute osm2pgsql for import OSM data |
|
101
|
|
|
subprocess.run( |
|
102
|
|
|
cmd, |
|
103
|
|
|
env={"PGPASSWORD": docker_db_config["POSTGRES_PASSWORD"]}, |
|
104
|
|
|
) |
|
105
|
|
|
|
|
106
|
|
|
|
|
107
|
|
|
def add_metadata(): |
|
108
|
|
|
"""Writes metadata JSON string into table comment.""" |
|
109
|
|
|
# Prepare variables |
|
110
|
|
|
osm_config = egon.data.config.datasets()["openstreetmap"] |
|
111
|
|
|
|
|
112
|
|
|
if settings()["egon-data"]["--dataset-boundary"] == "Everything": |
|
113
|
|
|
osm_url = osm_config["original_data"]["source"]["url"] |
|
114
|
|
|
target_path = osm_config["original_data"]["target"]["path"] |
|
115
|
|
|
else: |
|
116
|
|
|
osm_url = osm_config["original_data"]["source"]["url_testmode"] |
|
117
|
|
|
target_path = osm_config["original_data"]["target"]["path_testmode"] |
|
118
|
|
|
spatial_and_date = Path(target_path).name.split("-") |
|
119
|
|
|
spatial_extend = spatial_and_date[0] |
|
120
|
|
|
osm_data_date = ( |
|
121
|
|
|
"20" |
|
122
|
|
|
+ spatial_and_date[1][0:2] |
|
123
|
|
|
+ "-" |
|
124
|
|
|
+ spatial_and_date[1][2:4] |
|
125
|
|
|
+ "-" |
|
126
|
|
|
+ spatial_and_date[1][4:6] |
|
127
|
|
|
) |
|
128
|
|
|
|
|
129
|
|
|
# Insert metadata for each table |
|
130
|
|
|
licenses = [ |
|
131
|
|
|
{ |
|
132
|
|
|
"name": "Open Data Commons Open Database License 1.0", |
|
133
|
|
|
"title": "", |
|
134
|
|
|
"path": "https://opendatacommons.org/licenses/odbl/1.0/", |
|
135
|
|
|
"instruction": ( |
|
136
|
|
|
"You are free: To Share, To Create, To Adapt;" |
|
137
|
|
|
" As long as you: Attribute, Share-Alike, Keep open!" |
|
138
|
|
|
), |
|
139
|
|
|
"attribution": "© Reiner Lemoine Institut", |
|
140
|
|
|
} |
|
141
|
|
|
] |
|
142
|
|
|
for table in osm_config["processed"]["tables"]: |
|
143
|
|
|
table_suffix = table.split("_")[1] |
|
144
|
|
|
meta = { |
|
145
|
|
|
"title": f"OpenStreetMap (OSM) - Germany - {table_suffix}", |
|
146
|
|
|
"description": ( |
|
147
|
|
|
"OpenStreetMap is a free, editable map of the" |
|
148
|
|
|
" whole world that is being built by volunteers" |
|
149
|
|
|
" largely from scratch and released with" |
|
150
|
|
|
" an open-content license." |
|
151
|
|
|
), |
|
152
|
|
|
"language": ["EN", "DE"], |
|
153
|
|
|
"spatial": { |
|
154
|
|
|
"location": "", |
|
155
|
|
|
"extent": f"{spatial_extend}", |
|
156
|
|
|
"resolution": "", |
|
157
|
|
|
}, |
|
158
|
|
|
"temporal": { |
|
159
|
|
|
"referenceDate": f"{osm_data_date}", |
|
160
|
|
|
"timeseries": { |
|
161
|
|
|
"start": "", |
|
162
|
|
|
"end": "", |
|
163
|
|
|
"resolution": "", |
|
164
|
|
|
"alignment": "", |
|
165
|
|
|
"aggregationType": "", |
|
166
|
|
|
}, |
|
167
|
|
|
}, |
|
168
|
|
|
"sources": [ |
|
169
|
|
|
{ |
|
170
|
|
|
"title": ( |
|
171
|
|
|
"Geofabrik - Download - OpenStreetMap Data Extracts" |
|
172
|
|
|
), |
|
173
|
|
|
"description": ( |
|
174
|
|
|
'Data dump taken on "referenceDate",' |
|
175
|
|
|
f" i.e. {osm_data_date}." |
|
176
|
|
|
" A subset of this is selected using osm2pgsql" |
|
177
|
|
|
' using the style file "oedb.style".' |
|
178
|
|
|
), |
|
179
|
|
|
"path": f"{osm_url}", |
|
180
|
|
|
"licenses": licenses, |
|
181
|
|
|
} |
|
182
|
|
|
], |
|
183
|
|
|
"licenses": licenses, |
|
184
|
|
|
"contributors": [ |
|
185
|
|
|
{ |
|
186
|
|
|
"title": "Guido Pleßmann", |
|
187
|
|
|
"email": "http://github.com/gplssm", |
|
188
|
|
|
"date": time.strftime("%Y-%m-%d"), |
|
189
|
|
|
"object": "", |
|
190
|
|
|
"comment": "Imported data", |
|
191
|
|
|
} |
|
192
|
|
|
], |
|
193
|
|
|
"metaMetadata": { |
|
194
|
|
|
"metadataVersion": "OEP-1.4.0", |
|
195
|
|
|
"metadataLicense": { |
|
196
|
|
|
"name": "CC0-1.0", |
|
197
|
|
|
"title": "Creative Commons Zero v1.0 Universal", |
|
198
|
|
|
"path": ( |
|
199
|
|
|
"https://creativecommons.org/publicdomain/zero/1.0/" |
|
200
|
|
|
), |
|
201
|
|
|
}, |
|
202
|
|
|
}, |
|
203
|
|
|
} |
|
204
|
|
|
|
|
205
|
|
|
meta_json = "'" + json.dumps(meta) + "'" |
|
206
|
|
|
|
|
207
|
|
|
db.submit_comment(meta_json, "openstreetmap", table) |
|
208
|
|
|
|
openego /
eGon-data
