|
1
|
|
|
"""The central module containing all code dealing with small scale input-data |
|
2
|
|
|
""" |
|
3
|
|
|
|
|
4
|
|
|
|
|
5
|
|
|
from pathlib import Path |
|
6
|
|
|
from urllib.request import urlretrieve |
|
7
|
|
|
import shutil |
|
8
|
|
|
import zipfile |
|
9
|
|
|
|
|
10
|
|
|
from egon.data import config |
|
11
|
|
|
from egon.data.datasets import Dataset |
|
12
|
|
|
|
|
13
|
|
|
|
|
14
|
|
|
def download(): |
|
15
|
|
|
""" |
|
16
|
|
|
Download small scale input data from Zenodo |
|
17
|
|
|
Parameters |
|
18
|
|
|
---------- |
|
19
|
|
|
|
|
20
|
|
|
""" |
|
21
|
|
|
data_bundle_path = Path(".") / "data_bundle_egon_data" |
|
22
|
|
|
# Delete folder if it already exists |
|
23
|
|
|
if data_bundle_path.exists() and data_bundle_path.is_dir(): |
|
24
|
|
|
shutil.rmtree(data_bundle_path) |
|
25
|
|
|
# Get parameters from config and set download URL |
|
26
|
|
|
sources = config.datasets()["data-bundle"]["sources"]["zenodo"] |
|
27
|
|
|
url = ( |
|
28
|
|
|
f"https://zenodo.org/record/{sources['deposit_id']}/files/" |
|
29
|
|
|
"data_bundle_egon_data.zip" |
|
30
|
|
|
) |
|
31
|
|
|
target_file = config.datasets()["data-bundle"]["targets"]["file"] |
|
32
|
|
|
|
|
33
|
|
|
# check if file exists |
|
34
|
|
|
if not Path(target_file).exists(): |
|
35
|
|
|
# Retrieve files |
|
36
|
|
|
urlretrieve(url, target_file) |
|
37
|
|
|
|
|
38
|
|
|
with zipfile.ZipFile(target_file, "r") as zip_ref: |
|
39
|
|
|
zip_ref.extractall(".") |
|
40
|
|
|
|
|
41
|
|
|
powerd_data_bundle_path = Path(".") / "data_bundle_powerd_data" |
|
42
|
|
|
# Delete folder if it already exists |
|
43
|
|
|
if powerd_data_bundle_path.exists() and powerd_data_bundle_path.is_dir(): |
|
44
|
|
|
shutil.rmtree(powerd_data_bundle_path) |
|
45
|
|
|
|
|
46
|
|
|
url = f"""https://zenodo.org/record/{sources['deposit_id_powerd']}/files/data_bundle_powerd_data.zip""" |
|
47
|
|
|
target_file = config.datasets()["data-bundle"]["targets"]["file_powerd"] |
|
48
|
|
|
|
|
49
|
|
|
# check if file exists |
|
50
|
|
|
if not Path(target_file).exists(): |
|
51
|
|
|
# Retrieve files |
|
52
|
|
|
urlretrieve(url, target_file) |
|
53
|
|
|
|
|
54
|
|
|
with zipfile.ZipFile(target_file, "r") as zip_ref: |
|
55
|
|
|
zip_ref.extractall(".") |
|
56
|
|
|
|
|
57
|
|
|
|
|
58
|
|
|
class DataBundle(Dataset): |
|
59
|
|
|
def __init__(self, dependencies): |
|
60
|
|
|
deposit_id = config.datasets()["data-bundle"]["sources"][ |
|
61
|
|
|
"zenodo" |
|
62
|
|
|
]["deposit_id"] |
|
63
|
|
|
deposit_id_powerd = config.datasets()["data-bundle"]["sources"][ |
|
64
|
|
|
"zenodo" |
|
65
|
|
|
]["deposit_id"] |
|
66
|
|
|
super().__init__( |
|
67
|
|
|
name="DataBundle", |
|
68
|
|
|
version=f"{deposit_id}-{deposit_id_powerd}-0.0.3", |
|
69
|
|
|
dependencies=dependencies, |
|
70
|
|
|
tasks=(download,), |
|
71
|
|
|
) |
|
72
|
|
|
|
openego /
eGon-data
