1
|
|
|
"""The central module containing all code dealing with electrical neighbours |
2
|
|
|
""" |
3
|
|
|
|
4
|
|
|
from os import path |
5
|
|
|
from pathlib import Path |
6
|
|
|
import datetime |
7
|
|
|
import logging |
8
|
|
|
import os.path |
9
|
|
|
import zipfile |
10
|
|
|
|
11
|
|
|
from shapely.geometry import LineString |
12
|
|
|
from sqlalchemy.orm import sessionmaker |
13
|
|
|
import entsoe |
14
|
|
|
import requests |
15
|
|
|
|
16
|
|
|
import geopandas as gpd |
17
|
|
|
import pandas as pd |
18
|
|
|
|
19
|
|
|
from egon.data import config, db, logger |
20
|
|
|
from egon.data.db import session_scope |
21
|
|
|
from egon.data.datasets import Dataset, wrapped_partial |
22
|
|
|
from egon.data.datasets.fill_etrago_gen import add_marginal_costs |
23
|
|
|
from egon.data.datasets.fix_ehv_subnetworks import select_bus_id |
24
|
|
|
from egon.data.datasets.scenario_parameters import get_sector_parameters |
25
|
|
|
import egon.data.datasets.etrago_setup as etrago |
26
|
|
|
import egon.data.datasets.scenario_parameters.parameters as scenario_parameters |
27
|
|
|
from egon.data.datasets.pypsaeur import prepared_network |
28
|
|
|
|
29
|
|
|
|
30
|
|
|
def get_cross_border_buses(scenario, sources): |
31
|
|
|
"""Returns buses from osmTGmod which are outside of Germany. |
32
|
|
|
|
33
|
|
|
Parameters |
34
|
|
|
---------- |
35
|
|
|
sources : dict |
36
|
|
|
List of sources |
37
|
|
|
|
38
|
|
|
Returns |
39
|
|
|
------- |
40
|
|
|
geopandas.GeoDataFrame |
41
|
|
|
Electricity buses outside of Germany |
42
|
|
|
|
43
|
|
|
""" |
44
|
|
|
return db.select_geodataframe( |
45
|
|
|
f""" |
46
|
|
|
SELECT * |
47
|
|
|
FROM {sources['electricity_buses']['schema']}. |
48
|
|
|
{sources['electricity_buses']['table']} |
49
|
|
|
WHERE |
50
|
|
|
NOT ST_INTERSECTS ( |
51
|
|
|
geom, |
52
|
|
|
(SELECT ST_Transform(ST_Buffer(geometry, 5), 4326) FROM |
53
|
|
|
{sources['german_borders']['schema']}. |
54
|
|
|
{sources['german_borders']['table']})) |
55
|
|
|
AND (bus_id IN ( |
56
|
|
|
SELECT bus0 FROM |
57
|
|
|
{sources['lines']['schema']}.{sources['lines']['table']}) |
58
|
|
|
OR bus_id IN ( |
59
|
|
|
SELECT bus1 FROM |
60
|
|
|
{sources['lines']['schema']}.{sources['lines']['table']})) |
61
|
|
|
AND scn_name = '{scenario}'; |
62
|
|
|
""", |
63
|
|
|
epsg=4326, |
64
|
|
|
) |
65
|
|
|
|
66
|
|
|
|
67
|
|
|
def get_cross_border_lines(scenario, sources): |
68
|
|
|
"""Returns lines from osmTGmod which end or start outside of Germany. |
69
|
|
|
|
70
|
|
|
Parameters |
71
|
|
|
---------- |
72
|
|
|
sources : dict |
73
|
|
|
List of sources |
74
|
|
|
|
75
|
|
|
Returns |
76
|
|
|
------- |
77
|
|
|
geopandas.GeoDataFrame |
78
|
|
|
AC-lines outside of Germany |
79
|
|
|
|
80
|
|
|
""" |
81
|
|
|
return db.select_geodataframe( |
82
|
|
|
f""" |
83
|
|
|
SELECT * |
84
|
|
|
FROM {sources['lines']['schema']}.{sources['lines']['table']} a |
85
|
|
|
WHERE |
86
|
|
|
ST_INTERSECTS ( |
87
|
|
|
a.topo, |
88
|
|
|
(SELECT ST_Transform(ST_boundary(geometry), 4326) |
89
|
|
|
FROM {sources['german_borders']['schema']}. |
90
|
|
|
{sources['german_borders']['table']})) |
91
|
|
|
AND scn_name = '{scenario}'; |
92
|
|
|
""", |
93
|
|
|
epsg=4326, |
94
|
|
|
) |
95
|
|
|
|
96
|
|
|
|
97
|
|
|
def central_buses_pypsaeur(sources, scenario): |
98
|
|
|
"""Returns buses in the middle of foreign countries based on prepared pypsa-eur network |
99
|
|
|
|
100
|
|
|
Parameters |
101
|
|
|
---------- |
102
|
|
|
sources : dict |
103
|
|
|
List of sources |
104
|
|
|
|
105
|
|
|
Returns |
106
|
|
|
------- |
107
|
|
|
pandas.DataFrame |
108
|
|
|
Buses in the center of foreign countries |
109
|
|
|
|
110
|
|
|
""" |
111
|
|
|
|
112
|
|
|
wanted_countries = [ |
113
|
|
|
"AT", |
114
|
|
|
"CH", |
115
|
|
|
"CZ", |
116
|
|
|
"PL", |
117
|
|
|
"SE", |
118
|
|
|
"NO", |
119
|
|
|
"DK", |
120
|
|
|
"GB", |
121
|
|
|
"NL", |
122
|
|
|
"BE", |
123
|
|
|
"FR", |
124
|
|
|
"LU", |
125
|
|
|
] |
126
|
|
|
network = prepared_network() |
127
|
|
|
|
128
|
|
|
df = network.buses[ |
129
|
|
|
(network.buses.carrier == "AC") |
130
|
|
|
& (network.buses.country.isin(wanted_countries)) |
131
|
|
|
] |
132
|
|
|
|
133
|
|
|
return df |
134
|
|
|
|
135
|
|
|
|
136
|
|
|
def buses(scenario, sources, targets): |
137
|
|
|
"""Insert central buses in foreign countries per scenario |
138
|
|
|
|
139
|
|
|
Parameters |
140
|
|
|
---------- |
141
|
|
|
sources : dict |
142
|
|
|
List of dataset sources |
143
|
|
|
targets : dict |
144
|
|
|
List of dataset targets |
145
|
|
|
|
146
|
|
|
Returns |
147
|
|
|
------- |
148
|
|
|
central_buses : geoapndas.GeoDataFrame |
149
|
|
|
Buses in the center of foreign countries |
150
|
|
|
|
151
|
|
|
""" |
152
|
|
|
sql_delete = f""" |
153
|
|
|
DELETE FROM {sources['electricity_buses']['schema']}. |
154
|
|
|
{sources['electricity_buses']['table']} |
155
|
|
|
WHERE country != 'DE' AND scn_name = '{scenario}' |
156
|
|
|
AND carrier = 'AC' |
157
|
|
|
AND bus_id NOT IN ( |
158
|
|
|
SELECT bus_i |
159
|
|
|
FROM {sources['osmtgmod_bus']['schema']}. |
160
|
|
|
{sources['osmtgmod_bus']['table']}) |
161
|
|
|
""" |
162
|
|
|
|
163
|
|
|
# Delete existing buses |
164
|
|
|
db.execute_sql(sql_delete) |
165
|
|
|
|
166
|
|
|
central_buses = central_buses_pypsaeur(sources, scenario) |
167
|
|
|
|
168
|
|
|
next_bus_id = db.next_etrago_id("bus") + 1 |
169
|
|
|
|
170
|
|
|
central_buses["bus_id"] = central_buses.reset_index().index + next_bus_id |
171
|
|
|
|
172
|
|
|
next_bus_id += len(central_buses) |
173
|
|
|
|
174
|
|
|
# if in test mode, add bus in center of Germany |
175
|
|
|
if config.settings()["egon-data"]["--dataset-boundary"] != "Everything": |
176
|
|
|
central_buses = pd.concat( |
177
|
|
|
[ |
178
|
|
|
central_buses, |
179
|
|
|
pd.DataFrame( |
180
|
|
|
index=[central_buses.bus_id.max() + 1], |
181
|
|
|
data={ |
182
|
|
|
"scn_name": scenario, |
183
|
|
|
"bus_id": next_bus_id, |
184
|
|
|
"x": 10.4234469, |
185
|
|
|
"y": 51.0834196, |
186
|
|
|
"country": "DE", |
187
|
|
|
"carrier": "AC", |
188
|
|
|
"v_nom": 380.0, |
189
|
|
|
}, |
190
|
|
|
), |
191
|
|
|
], |
192
|
|
|
ignore_index=True, |
193
|
|
|
) |
194
|
|
|
next_bus_id += 1 |
195
|
|
|
|
196
|
|
|
# Add buses for other voltage levels |
197
|
|
|
foreign_buses = get_cross_border_buses(scenario, sources) |
198
|
|
|
if config.settings()["egon-data"]["--dataset-boundary"] == "Everything": |
199
|
|
|
foreign_buses = foreign_buses[foreign_buses.country != "DE"] |
200
|
|
|
vnom_per_country = foreign_buses.groupby("country").v_nom.unique().copy() |
201
|
|
|
for cntr in vnom_per_country.index: |
202
|
|
|
print(cntr) |
203
|
|
View Code Duplication |
if 110.0 in vnom_per_country[cntr]: |
|
|
|
|
204
|
|
|
central_buses = pd.concat( |
205
|
|
|
[ |
206
|
|
|
central_buses, |
207
|
|
|
pd.DataFrame( |
208
|
|
|
index=[next_bus_id], |
209
|
|
|
data={ |
210
|
|
|
"scn_name": scenario, |
211
|
|
|
"bus_id": next_bus_id, |
212
|
|
|
"x": central_buses[ |
213
|
|
|
central_buses.country == cntr |
214
|
|
|
].x.unique()[0], |
215
|
|
|
"y": central_buses[ |
216
|
|
|
central_buses.country == cntr |
217
|
|
|
].y.unique()[0], |
218
|
|
|
"country": cntr, |
219
|
|
|
"carrier": "AC", |
220
|
|
|
"v_nom": 110.0, |
221
|
|
|
}, |
222
|
|
|
), |
223
|
|
|
], |
224
|
|
|
ignore_index=True, |
225
|
|
|
) |
226
|
|
|
next_bus_id += 1 |
227
|
|
|
|
228
|
|
View Code Duplication |
if 220.0 in vnom_per_country[cntr]: |
|
|
|
|
229
|
|
|
central_buses = pd.concat( |
230
|
|
|
[ |
231
|
|
|
central_buses, |
232
|
|
|
pd.DataFrame( |
233
|
|
|
index=[next_bus_id], |
234
|
|
|
data={ |
235
|
|
|
"scn_name": scenario, |
236
|
|
|
"bus_id": next_bus_id, |
237
|
|
|
"x": central_buses[ |
238
|
|
|
central_buses.country == cntr |
239
|
|
|
].x.unique()[0], |
240
|
|
|
"y": central_buses[ |
241
|
|
|
central_buses.country == cntr |
242
|
|
|
].y.unique()[0], |
243
|
|
|
"country": cntr, |
244
|
|
|
"carrier": "AC", |
245
|
|
|
"v_nom": 220.0, |
246
|
|
|
}, |
247
|
|
|
), |
248
|
|
|
], |
249
|
|
|
ignore_index=True, |
250
|
|
|
) |
251
|
|
|
next_bus_id += 1 |
252
|
|
|
|
253
|
|
|
# Add geometry column |
254
|
|
|
central_buses = gpd.GeoDataFrame( |
255
|
|
|
central_buses, |
256
|
|
|
geometry=gpd.points_from_xy(central_buses.x, central_buses.y), |
257
|
|
|
crs="EPSG:4326", |
258
|
|
|
) |
259
|
|
|
central_buses["geom"] = central_buses.geometry.copy() |
260
|
|
|
central_buses = central_buses.set_geometry("geom").drop( |
261
|
|
|
"geometry", axis="columns" |
262
|
|
|
) |
263
|
|
|
central_buses.scn_name = scenario |
264
|
|
|
|
265
|
|
|
central_buses.drop( |
266
|
|
|
["control", "generator", "location", "unit", "sub_network", "substation_off", "substation_lv"], |
267
|
|
|
axis="columns", |
268
|
|
|
inplace=True, |
269
|
|
|
errors="ignore" |
270
|
|
|
) |
271
|
|
|
|
272
|
|
|
# Insert all central buses for eGon2035 |
273
|
|
|
if scenario in [ |
274
|
|
|
"eGon2035", |
275
|
|
|
"status2019", |
276
|
|
|
"status2023", |
277
|
|
|
]: # TODO: status2023 this is hardcoded shit |
278
|
|
|
central_buses.to_postgis( |
279
|
|
|
targets["buses"]["table"], |
280
|
|
|
schema=targets["buses"]["schema"], |
281
|
|
|
if_exists="append", |
282
|
|
|
con=db.engine(), |
283
|
|
|
index=False, |
284
|
|
|
) |
285
|
|
|
# Insert only buses for eGon100RE that are not coming from pypsa-eur-sec |
286
|
|
|
# (buses with another voltage_level or inside Germany in test mode) |
287
|
|
|
else: |
288
|
|
|
central_buses[central_buses.carrier=="AC"].to_postgis( |
289
|
|
|
targets["buses"]["table"], |
290
|
|
|
schema=targets["buses"]["schema"], |
291
|
|
|
if_exists="append", |
292
|
|
|
con=db.engine(), |
293
|
|
|
index=False, |
294
|
|
|
) |
295
|
|
|
|
296
|
|
|
return central_buses |
297
|
|
|
|
298
|
|
|
|
299
|
|
|
def lines_between_foreign_countries(scenario, sorces, targets, central_buses): |
300
|
|
|
# import network from pypsa-eur |
301
|
|
|
network = prepared_network() |
302
|
|
|
|
303
|
|
|
gdf_buses = gpd.GeoDataFrame( |
304
|
|
|
network.buses, |
305
|
|
|
geometry=gpd.points_from_xy(network.buses.x, network.buses.y), |
306
|
|
|
) |
307
|
|
|
|
308
|
|
|
central_buses_pypsaeur = gpd.sjoin( |
309
|
|
|
gdf_buses[gdf_buses.carrier == "AC"], central_buses |
310
|
|
|
) |
311
|
|
|
|
312
|
|
|
central_buses_pypsaeur = central_buses_pypsaeur[ |
313
|
|
|
central_buses_pypsaeur.v_nom_right == 380 |
314
|
|
|
] |
315
|
|
|
|
316
|
|
|
lines_to_add = network.lines[ |
317
|
|
|
(network.lines.bus0.isin(central_buses_pypsaeur.index)) |
318
|
|
|
& (network.lines.bus1.isin(central_buses_pypsaeur.index)) |
319
|
|
|
] |
320
|
|
|
|
321
|
|
|
lines_to_add.loc[:, "lifetime"] = get_sector_parameters( |
322
|
|
|
"electricity", scenario |
323
|
|
|
)["lifetime"]["ac_ehv_overhead_line"] |
324
|
|
|
lines_to_add.loc[:, "line_id"] = ( |
325
|
|
|
lines_to_add.reset_index().index.astype(int) |
326
|
|
|
+ db.next_etrago_id("line") |
327
|
|
|
+ 1 |
328
|
|
|
) |
329
|
|
|
|
330
|
|
|
links_to_add = network.links[ |
331
|
|
|
(network.links.bus0.isin(central_buses_pypsaeur.index)) |
332
|
|
|
& (network.links.bus1.isin(central_buses_pypsaeur.index)) |
333
|
|
|
] |
334
|
|
|
|
335
|
|
|
links_to_add.loc[:, "lifetime"] = get_sector_parameters( |
336
|
|
|
"electricity", scenario |
337
|
|
|
)["lifetime"]["dc_overhead_line"] |
338
|
|
|
links_to_add.loc[:, "link_id"] = ( |
339
|
|
|
links_to_add.reset_index().index.astype(int) |
340
|
|
|
+ db.next_etrago_id("link") |
341
|
|
|
+ 1 |
342
|
|
|
) |
343
|
|
|
|
344
|
|
|
for df in [lines_to_add, links_to_add]: |
345
|
|
|
df.loc[:, "scn_name"] = scenario |
346
|
|
|
gdf = gpd.GeoDataFrame(df) |
347
|
|
|
gdf["geom_bus0"] = gdf_buses.geometry[df.bus0].values |
348
|
|
|
gdf["geom_bus1"] = gdf_buses.geometry[df.bus1].values |
349
|
|
|
gdf["geometry"] = gdf.apply( |
350
|
|
|
lambda x: LineString([x["geom_bus0"], x["geom_bus1"]]), |
351
|
|
|
axis=1, |
352
|
|
|
) |
353
|
|
|
|
354
|
|
|
gdf = gdf.set_geometry("geometry") |
355
|
|
|
gdf = gdf.set_crs(4326) |
356
|
|
|
|
357
|
|
|
gdf = gdf.rename_geometry("topo") |
358
|
|
|
|
359
|
|
|
gdf.loc[:, "bus0"] = central_buses_pypsaeur.bus_id.loc[df.bus0].values |
360
|
|
|
gdf.loc[:, "bus1"] = central_buses_pypsaeur.bus_id.loc[df.bus1].values |
361
|
|
|
|
362
|
|
|
gdf.drop(["geom_bus0", "geom_bus1"], inplace=True, axis="columns") |
363
|
|
|
if "link_id" in df.columns: |
364
|
|
|
table_name = "link" |
365
|
|
|
gdf.drop( |
366
|
|
|
[ |
367
|
|
|
"tags", |
368
|
|
|
"under_construction", |
369
|
|
|
"underground", |
370
|
|
|
"underwater_fraction", |
371
|
|
|
"bus2", |
372
|
|
|
"efficiency2", |
373
|
|
|
"length_original", |
374
|
|
|
"bus4", |
375
|
|
|
"efficiency4", |
376
|
|
|
"reversed", |
377
|
|
|
"ramp_limit_up", |
378
|
|
|
"ramp_limit_down", |
379
|
|
|
"p_nom_opt", |
380
|
|
|
"bus3", |
381
|
|
|
"efficiency3", |
382
|
|
|
"location", |
383
|
|
|
"project_status", |
384
|
|
|
"dc", |
385
|
|
|
"voltage", |
386
|
|
|
], |
387
|
|
|
axis="columns", |
388
|
|
|
inplace=True, |
389
|
|
|
) |
390
|
|
|
else: |
391
|
|
|
table_name = "line" |
392
|
|
|
gdf.drop( |
393
|
|
|
[ |
394
|
|
|
"i_nom", |
395
|
|
|
"sub_network", |
396
|
|
|
"x_pu", |
397
|
|
|
"r_pu", |
398
|
|
|
"g_pu", |
399
|
|
|
"b_pu", |
400
|
|
|
"x_pu_eff", |
401
|
|
|
"r_pu_eff", |
402
|
|
|
"s_nom_opt", |
403
|
|
|
"dc", |
404
|
|
|
], |
405
|
|
|
axis="columns", |
406
|
|
|
inplace=True, |
407
|
|
|
) |
408
|
|
|
|
409
|
|
|
gdf = gdf.set_index(f"{table_name}_id") |
410
|
|
|
gdf.to_postgis( |
411
|
|
|
f"egon_etrago_{table_name}", |
412
|
|
|
db.engine(), |
413
|
|
|
schema="grid", |
414
|
|
|
if_exists="append", |
415
|
|
|
index=True, |
416
|
|
|
index_label=f"{table_name}_id", |
417
|
|
|
) |
418
|
|
|
|
419
|
|
|
|
420
|
|
|
def cross_border_lines(scenario, sources, targets, central_buses): |
421
|
|
|
"""Adds lines which connect border-crossing lines from osmtgmod |
422
|
|
|
to the central buses in the corresponding neigbouring country |
423
|
|
|
|
424
|
|
|
Parameters |
425
|
|
|
---------- |
426
|
|
|
sources : dict |
427
|
|
|
List of dataset sources |
428
|
|
|
targets : dict |
429
|
|
|
List of dataset targets |
430
|
|
|
central_buses : geopandas.GeoDataFrame |
431
|
|
|
Buses in the center of foreign countries |
432
|
|
|
|
433
|
|
|
Returns |
434
|
|
|
------- |
435
|
|
|
new_lines : geopandas.GeoDataFrame |
436
|
|
|
Lines that connect cross-border lines to central bus per country |
437
|
|
|
|
438
|
|
|
""" |
439
|
|
|
# Delete existing data |
440
|
|
|
db.execute_sql( |
441
|
|
|
f""" |
442
|
|
|
DELETE FROM {targets['lines']['schema']}. |
443
|
|
|
{targets['lines']['table']} |
444
|
|
|
WHERE scn_name = '{scenario}' |
445
|
|
|
AND line_id NOT IN ( |
446
|
|
|
SELECT branch_id |
447
|
|
|
FROM {sources['osmtgmod_branch']['schema']}. |
448
|
|
|
{sources['osmtgmod_branch']['table']} |
449
|
|
|
WHERE result_id = 1 and (link_type = 'line' or |
450
|
|
|
link_type = 'cable')) |
451
|
|
|
AND bus0 IN ( |
452
|
|
|
SELECT bus_i |
453
|
|
|
FROM {sources['osmtgmod_bus']['schema']}. |
454
|
|
|
{sources['osmtgmod_bus']['table']}) |
455
|
|
|
AND bus1 NOT IN ( |
456
|
|
|
SELECT bus_i |
457
|
|
|
FROM {sources['osmtgmod_bus']['schema']}. |
458
|
|
|
{sources['osmtgmod_bus']['table']}) |
459
|
|
|
""" |
460
|
|
|
) |
461
|
|
|
|
462
|
|
|
# Calculate cross-border busses and lines from osmtgmod |
463
|
|
|
foreign_buses = get_cross_border_buses(scenario, sources) |
464
|
|
|
foreign_buses.dropna(subset="country", inplace=True) |
465
|
|
|
|
466
|
|
|
if config.settings()["egon-data"]["--dataset-boundary"] == "Everything": |
467
|
|
|
foreign_buses = foreign_buses[foreign_buses.country != "DE"] |
468
|
|
|
lines = get_cross_border_lines(scenario, sources) |
469
|
|
|
|
470
|
|
|
# Select bus outside of Germany from border-crossing lines |
471
|
|
|
lines.loc[ |
472
|
|
|
lines[lines.bus0.isin(foreign_buses.bus_id)].index, "foreign_bus" |
473
|
|
|
] = lines.loc[lines[lines.bus0.isin(foreign_buses.bus_id)].index, "bus0"] |
474
|
|
|
lines.loc[ |
475
|
|
|
lines[lines.bus1.isin(foreign_buses.bus_id)].index, "foreign_bus" |
476
|
|
|
] = lines.loc[lines[lines.bus1.isin(foreign_buses.bus_id)].index, "bus1"] |
477
|
|
|
|
478
|
|
|
# Drop lines with start and endpoint in Germany |
479
|
|
|
lines = lines[lines.foreign_bus.notnull()] |
480
|
|
|
lines.loc[:, "foreign_bus"] = lines.loc[:, "foreign_bus"].astype(int) |
481
|
|
|
|
482
|
|
|
# Copy all parameters from border-crossing lines |
483
|
|
|
new_lines = lines.copy().set_crs(4326) |
484
|
|
|
|
485
|
|
|
# Set bus0 as foreign_bus from osmtgmod |
486
|
|
|
new_lines.bus0 = new_lines.foreign_bus.copy() |
487
|
|
|
new_lines.bus0 = new_lines.bus0.astype(int) |
488
|
|
|
|
489
|
|
|
# Add country tag and set index |
490
|
|
|
new_lines["country"] = ( |
491
|
|
|
foreign_buses.set_index("bus_id") |
492
|
|
|
.loc[lines.foreign_bus, "country"] |
493
|
|
|
.values |
494
|
|
|
) |
495
|
|
|
|
496
|
|
|
if config.settings()["egon-data"]["--dataset-boundary"] == "Everything": |
497
|
|
|
new_lines = new_lines[~new_lines.country.isnull()] |
498
|
|
|
new_lines.line_id = range( |
499
|
|
|
db.next_etrago_id("line"), db.next_etrago_id("line") + len(new_lines) |
500
|
|
|
) |
501
|
|
|
|
502
|
|
|
# Set bus in center of foreign countries as bus1 |
503
|
|
|
for i, row in new_lines.iterrows(): |
504
|
|
|
print(row) |
505
|
|
|
new_lines.loc[i, "bus1"] = central_buses.bus_id[ |
506
|
|
|
(central_buses.country == row.country) |
507
|
|
|
& (central_buses.v_nom == row.v_nom) |
508
|
|
|
].values[0] |
509
|
|
|
|
510
|
|
|
# Create geometry for new lines |
511
|
|
|
new_lines["geom_bus0"] = ( |
512
|
|
|
foreign_buses.set_index("bus_id").geom[new_lines.bus0].values |
513
|
|
|
) |
514
|
|
|
new_lines["geom_bus1"] = ( |
515
|
|
|
central_buses.set_index("bus_id").geom[new_lines.bus1].values |
516
|
|
|
) |
517
|
|
|
new_lines["topo"] = new_lines.apply( |
518
|
|
|
lambda x: LineString([x["geom_bus0"], x["geom_bus1"]]), axis=1 |
519
|
|
|
) |
520
|
|
|
|
521
|
|
|
# Set topo as geometry column |
522
|
|
|
new_lines = new_lines.set_geometry("topo").set_crs(4326) |
523
|
|
|
# Calcultae length of lines based on topology |
524
|
|
|
old_length = new_lines["length"].copy() |
525
|
|
|
new_lines["length"] = new_lines.to_crs(3035).length / 1000 |
526
|
|
|
|
527
|
|
|
if (new_lines["length"] == 0).any(): |
528
|
|
|
print("WARNING! THERE ARE LINES WITH LENGTH = 0") |
529
|
|
|
condition = new_lines["length"] != 0 |
530
|
|
|
new_lines["length"] = new_lines["length"].where(condition, 1) |
531
|
|
|
|
532
|
|
|
|
533
|
|
|
# Set electrical parameters based on lines from osmtgmod |
534
|
|
|
for parameter in ["x", "r"]: |
535
|
|
|
new_lines[parameter] = ( |
536
|
|
|
new_lines[parameter] / old_length * new_lines["length"] |
537
|
|
|
) |
538
|
|
|
for parameter in ["b", "g"]: |
539
|
|
|
new_lines[parameter] = ( |
540
|
|
|
new_lines[parameter] * old_length / new_lines["length"] |
541
|
|
|
) |
542
|
|
|
|
543
|
|
|
|
544
|
|
|
# Drop intermediate columns |
545
|
|
|
new_lines.drop( |
546
|
|
|
["foreign_bus", "country", "geom_bus0", "geom_bus1", "geom"], |
547
|
|
|
axis="columns", |
548
|
|
|
inplace=True, |
549
|
|
|
) |
550
|
|
|
|
551
|
|
|
new_lines = new_lines[new_lines.bus0 != new_lines.bus1] |
552
|
|
|
|
553
|
|
|
new_lines["cables"] = new_lines["cables"].apply(int) |
554
|
|
|
|
555
|
|
|
# Insert lines to the database |
556
|
|
|
new_lines.to_postgis( |
557
|
|
|
targets["lines"]["table"], |
558
|
|
|
schema=targets["lines"]["schema"], |
559
|
|
|
if_exists="append", |
560
|
|
|
con=db.engine(), |
561
|
|
|
index=False, |
562
|
|
|
) |
563
|
|
|
|
564
|
|
|
return new_lines |
565
|
|
|
|
566
|
|
|
|
567
|
|
|
def choose_transformer(s_nom): |
568
|
|
|
"""Select transformer and parameters from existing data in the grid model |
569
|
|
|
|
570
|
|
|
It is assumed that transformers in the foreign countries are not limiting |
571
|
|
|
the electricity flow, so the capacitiy s_nom is set to the minimum sum |
572
|
|
|
of attached AC-lines. |
573
|
|
|
The electrical parameters are set according to already inserted |
574
|
|
|
transformers in the grid model for Germany. |
575
|
|
|
|
576
|
|
|
Parameters |
577
|
|
|
---------- |
578
|
|
|
s_nom : float |
579
|
|
|
Minimal sum of nominal power of lines at one side |
580
|
|
|
|
581
|
|
|
Returns |
582
|
|
|
------- |
583
|
|
|
int |
584
|
|
|
Selected transformer nominal power |
585
|
|
|
float |
586
|
|
|
Selected transformer nominal impedance |
587
|
|
|
|
588
|
|
|
""" |
589
|
|
|
|
590
|
|
|
if s_nom <= 600: |
591
|
|
|
return 600, 0.0002 |
592
|
|
|
elif (s_nom > 600) & (s_nom <= 1200): |
593
|
|
|
return 1200, 0.0001 |
594
|
|
|
elif (s_nom > 1200) & (s_nom <= 1600): |
595
|
|
|
return 1600, 0.000075 |
596
|
|
|
elif (s_nom > 1600) & (s_nom <= 2100): |
597
|
|
|
return 2100, 0.00006667 |
598
|
|
|
elif (s_nom > 2100) & (s_nom <= 2600): |
599
|
|
|
return 2600, 0.0000461538 |
600
|
|
|
elif (s_nom > 2600) & (s_nom <= 4800): |
601
|
|
|
return 4800, 0.000025 |
602
|
|
|
elif (s_nom > 4800) & (s_nom <= 6000): |
603
|
|
|
return 6000, 0.0000225 |
604
|
|
|
elif (s_nom > 6000) & (s_nom <= 7200): |
605
|
|
|
return 7200, 0.0000194444 |
606
|
|
|
elif (s_nom > 7200) & (s_nom <= 8000): |
607
|
|
|
return 8000, 0.000016875 |
608
|
|
|
elif (s_nom > 8000) & (s_nom <= 9000): |
609
|
|
|
return 9000, 0.000015 |
610
|
|
|
elif (s_nom > 9000) & (s_nom <= 13000): |
611
|
|
|
return 13000, 0.0000103846 |
612
|
|
|
elif (s_nom > 13000) & (s_nom <= 20000): |
613
|
|
|
return 20000, 0.00000675 |
614
|
|
|
elif (s_nom > 20000) & (s_nom <= 33000): |
615
|
|
|
return 33000, 0.00000409091 |
616
|
|
|
|
617
|
|
|
|
618
|
|
|
def central_transformer(scenario, sources, targets, central_buses, new_lines): |
619
|
|
|
"""Connect central foreign buses with different voltage levels |
620
|
|
|
|
621
|
|
|
Parameters |
622
|
|
|
---------- |
623
|
|
|
sources : dict |
624
|
|
|
List of dataset sources |
625
|
|
|
targets : dict |
626
|
|
|
List of dataset targets |
627
|
|
|
central_buses : geopandas.GeoDataFrame |
628
|
|
|
Buses in the center of foreign countries |
629
|
|
|
new_lines : geopandas.GeoDataFrame |
630
|
|
|
Lines that connect cross-border lines to central bus per country |
631
|
|
|
|
632
|
|
|
Returns |
633
|
|
|
------- |
634
|
|
|
None. |
635
|
|
|
|
636
|
|
|
""" |
637
|
|
|
# Delete existing transformers in foreign countries |
638
|
|
|
db.execute_sql( |
639
|
|
|
f""" |
640
|
|
|
DELETE FROM {targets['transformers']['schema']}. |
641
|
|
|
{targets['transformers']['table']} |
642
|
|
|
WHERE scn_name = '{scenario}' |
643
|
|
|
AND trafo_id NOT IN ( |
644
|
|
|
SELECT branch_id |
645
|
|
|
FROM {sources['osmtgmod_branch']['schema']}. |
646
|
|
|
{sources['osmtgmod_branch']['table']} |
647
|
|
|
WHERE result_id = 1 and link_type = 'transformer') |
648
|
|
|
""" |
649
|
|
|
) |
650
|
|
|
|
651
|
|
|
# Initalize the dataframe for transformers |
652
|
|
|
trafo = gpd.GeoDataFrame( |
653
|
|
|
columns=["trafo_id", "bus0", "bus1", "s_nom"], dtype=int |
654
|
|
|
) |
655
|
|
|
trafo_id = db.next_etrago_id("transformer") |
656
|
|
|
|
657
|
|
|
# Add one transformer per central foreign bus with v_nom != 380 |
658
|
|
|
for i, row in central_buses[central_buses.v_nom != 380].iterrows(): |
659
|
|
|
s_nom_0 = new_lines[new_lines.bus0 == row.bus_id].s_nom.sum() |
660
|
|
|
s_nom_1 = new_lines[new_lines.bus1 == row.bus_id].s_nom.sum() |
661
|
|
|
if s_nom_0 == 0.0: |
662
|
|
|
s_nom = s_nom_1 |
663
|
|
|
elif s_nom_1 == 0.0: |
664
|
|
|
s_nom = s_nom_0 |
665
|
|
|
else: |
666
|
|
|
s_nom = min([s_nom_0, s_nom_1]) |
667
|
|
|
|
668
|
|
|
s_nom, x = choose_transformer(s_nom) |
669
|
|
|
|
670
|
|
|
trafo = pd.concat( |
671
|
|
|
[ |
672
|
|
|
trafo, |
673
|
|
|
pd.DataFrame( |
674
|
|
|
index=[trafo.index.max() + 1], |
675
|
|
|
data={ |
676
|
|
|
"trafo_id": trafo_id, |
677
|
|
|
"bus0": row.bus_id, |
678
|
|
|
"bus1": central_buses[ |
679
|
|
|
(central_buses.v_nom == 380) |
680
|
|
|
& (central_buses.country == row.country) |
681
|
|
|
].bus_id.values[0], |
682
|
|
|
"s_nom": s_nom, |
683
|
|
|
"x": x, |
684
|
|
|
}, |
685
|
|
|
), |
686
|
|
|
], |
687
|
|
|
ignore_index=True, |
688
|
|
|
) |
689
|
|
|
trafo_id += 1 |
690
|
|
|
|
691
|
|
|
# Set data type |
692
|
|
|
trafo = trafo.astype({"trafo_id": "int", "bus0": "int", "bus1": "int"}) |
693
|
|
|
trafo["scn_name"] = scenario |
694
|
|
|
|
695
|
|
|
# Insert transformers to the database |
696
|
|
|
trafo.to_sql( |
697
|
|
|
targets["transformers"]["table"], |
698
|
|
|
schema=targets["transformers"]["schema"], |
699
|
|
|
if_exists="append", |
700
|
|
|
con=db.engine(), |
701
|
|
|
index=False, |
702
|
|
|
) |
703
|
|
|
|
704
|
|
|
|
705
|
|
|
def foreign_dc_lines(scenario, sources, targets, central_buses): |
706
|
|
|
"""Insert DC lines to foreign countries manually |
707
|
|
|
|
708
|
|
|
Parameters |
709
|
|
|
---------- |
710
|
|
|
sources : dict |
711
|
|
|
List of dataset sources |
712
|
|
|
targets : dict |
713
|
|
|
List of dataset targets |
714
|
|
|
central_buses : geopandas.GeoDataFrame |
715
|
|
|
Buses in the center of foreign countries |
716
|
|
|
|
717
|
|
|
Returns |
718
|
|
|
------- |
719
|
|
|
None. |
720
|
|
|
|
721
|
|
|
""" |
722
|
|
|
# Delete existing dc lines to foreign countries |
723
|
|
|
db.execute_sql( |
724
|
|
|
f""" |
725
|
|
|
DELETE FROM {targets['links']['schema']}. |
726
|
|
|
{targets['links']['table']} |
727
|
|
|
WHERE scn_name = '{scenario}' |
728
|
|
|
AND carrier = 'DC' |
729
|
|
|
AND bus0 IN ( |
730
|
|
|
SELECT bus_id |
731
|
|
|
FROM {sources['electricity_buses']['schema']}. |
732
|
|
|
{sources['electricity_buses']['table']} |
733
|
|
|
WHERE scn_name = '{scenario}' |
734
|
|
|
AND carrier = 'AC' |
735
|
|
|
AND country = 'DE') |
736
|
|
|
AND bus1 IN ( |
737
|
|
|
SELECT bus_id |
738
|
|
|
FROM {sources['electricity_buses']['schema']}. |
739
|
|
|
{sources['electricity_buses']['table']} |
740
|
|
|
WHERE scn_name = '{scenario}' |
741
|
|
|
AND carrier = 'AC' |
742
|
|
|
AND country != 'DE') |
743
|
|
|
""" |
744
|
|
|
) |
745
|
|
|
capital_cost = get_sector_parameters("electricity", scenario)[ |
746
|
|
|
"capital_cost" |
747
|
|
|
] |
748
|
|
|
|
749
|
|
|
# Add DC line from Lübeck to Sweden |
750
|
|
|
converter_luebeck = select_bus_id( |
751
|
|
|
10.802358024202768, |
752
|
|
|
53.897547401787, |
753
|
|
|
380, |
754
|
|
|
scenario, |
755
|
|
|
"AC", |
756
|
|
|
find_closest=True, |
757
|
|
|
) |
758
|
|
|
|
759
|
|
|
foreign_links = pd.DataFrame( |
760
|
|
|
index=[0], |
761
|
|
|
data={ |
762
|
|
|
"link_id": db.next_etrago_id("link"), |
763
|
|
|
"bus0": converter_luebeck, |
764
|
|
|
"bus1": central_buses[ |
765
|
|
|
(central_buses.country == "SE") & (central_buses.v_nom == 380) |
766
|
|
|
].iloc[0] |
767
|
|
|
.squeeze() |
768
|
|
|
.bus_id, |
769
|
|
|
"p_nom": 600, |
770
|
|
|
"length": 262, |
771
|
|
|
}, |
772
|
|
|
) |
773
|
|
|
|
774
|
|
|
# When not in test-mode, add DC line from Bentwisch to Denmark |
775
|
|
|
if config.settings()["egon-data"]["--dataset-boundary"] == "Everything": |
776
|
|
|
converter_bentwisch = select_bus_id( |
777
|
|
|
12.213671694775988, |
778
|
|
|
54.09974494662279, |
779
|
|
|
380, |
780
|
|
|
scenario, |
781
|
|
|
"AC", |
782
|
|
|
find_closest=True, |
783
|
|
|
) |
784
|
|
|
|
785
|
|
|
foreign_links = pd.concat( |
786
|
|
|
[ |
787
|
|
|
foreign_links, |
788
|
|
|
pd.DataFrame( |
789
|
|
|
index=[1], |
790
|
|
|
data={ |
791
|
|
|
"link_id": db.next_etrago_id("link") + 1, |
792
|
|
|
"bus0": converter_bentwisch, |
793
|
|
|
"bus1": central_buses[ |
794
|
|
|
(central_buses.country == "DK") |
795
|
|
|
& (central_buses.v_nom == 380) |
796
|
|
|
& (central_buses.x > 10) |
797
|
|
|
].iloc[0] |
798
|
|
|
.squeeze() |
799
|
|
|
.bus_id, |
800
|
|
|
"p_nom": 600, |
801
|
|
|
"length": 170, |
802
|
|
|
}, |
803
|
|
|
), |
804
|
|
|
] |
805
|
|
|
) |
806
|
|
|
|
807
|
|
|
# Set parameters for all DC lines |
808
|
|
|
foreign_links["capital_cost"] = ( |
809
|
|
|
capital_cost["dc_cable"] * foreign_links.length |
810
|
|
|
+ 2 * capital_cost["dc_inverter"] |
811
|
|
|
) |
812
|
|
|
foreign_links["p_min_pu"] = -1 |
813
|
|
|
foreign_links["p_nom_extendable"] = True |
814
|
|
|
foreign_links["p_nom_min"] = foreign_links["p_nom"] |
815
|
|
|
foreign_links["scn_name"] = scenario |
816
|
|
|
foreign_links["carrier"] = "DC" |
817
|
|
|
foreign_links["efficiency"] = 1 |
818
|
|
|
|
819
|
|
|
# Add topology |
820
|
|
|
foreign_links = etrago.link_geom_from_buses(foreign_links, scenario) |
821
|
|
|
|
822
|
|
|
# Insert DC lines to the database |
823
|
|
|
foreign_links.to_postgis( |
824
|
|
|
targets["links"]["table"], |
825
|
|
|
schema=targets["links"]["schema"], |
826
|
|
|
if_exists="append", |
827
|
|
|
con=db.engine(), |
828
|
|
|
index=False, |
829
|
|
|
) |
830
|
|
|
|
831
|
|
|
|
832
|
|
|
def grid(): |
833
|
|
|
"""Insert electrical grid compoenents for neighbouring countries |
834
|
|
|
|
835
|
|
|
Returns |
836
|
|
|
------- |
837
|
|
|
None. |
838
|
|
|
|
839
|
|
|
""" |
840
|
|
|
# Select sources and targets from dataset configuration |
841
|
|
|
sources = config.datasets()["electrical_neighbours"]["sources"] |
842
|
|
|
targets = config.datasets()["electrical_neighbours"]["targets"] |
843
|
|
|
|
844
|
|
|
for scenario in config.settings()["egon-data"]["--scenarios"]: |
845
|
|
|
central_buses = buses(scenario, sources, targets) |
846
|
|
|
|
847
|
|
|
foreign_lines = cross_border_lines( |
848
|
|
|
scenario, sources, targets, central_buses |
849
|
|
|
) |
850
|
|
|
|
851
|
|
|
central_transformer( |
852
|
|
|
scenario, sources, targets, central_buses, foreign_lines |
853
|
|
|
) |
854
|
|
|
|
855
|
|
|
foreign_dc_lines(scenario, sources, targets, central_buses) |
856
|
|
|
|
857
|
|
|
if scenario != "eGon100RE": |
858
|
|
|
lines_between_foreign_countries( |
859
|
|
|
scenario, sources, targets, central_buses |
860
|
|
|
) |
861
|
|
|
|
862
|
|
|
|
863
|
|
|
def map_carriers_tyndp(): |
864
|
|
|
"""Map carriers from TYNDP-data to carriers used in eGon |
865
|
|
|
Returns |
866
|
|
|
------- |
867
|
|
|
dict |
868
|
|
|
Carrier from TYNDP and eGon |
869
|
|
|
""" |
870
|
|
|
return { |
871
|
|
|
"Battery": "battery", |
872
|
|
|
"DSR": "demand_side_response", |
873
|
|
|
"Gas CCGT new": "gas", |
874
|
|
|
"Gas CCGT old 2": "gas", |
875
|
|
|
"Gas CCGT present 1": "gas", |
876
|
|
|
"Gas CCGT present 2": "gas", |
877
|
|
|
"Gas conventional old 1": "gas", |
878
|
|
|
"Gas conventional old 2": "gas", |
879
|
|
|
"Gas OCGT new": "gas", |
880
|
|
|
"Gas OCGT old": "gas", |
881
|
|
|
"Gas CCGT old 1": "gas", |
882
|
|
|
"Gas CCGT old 2 Bio": "biogas", |
883
|
|
|
"Gas conventional old 2 Bio": "biogas", |
884
|
|
|
"Hard coal new": "coal", |
885
|
|
|
"Hard coal old 1": "coal", |
886
|
|
|
"Hard coal old 2": "coal", |
887
|
|
|
"Hard coal old 2 Bio": "coal", |
888
|
|
|
"Heavy oil old 1": "oil", |
889
|
|
|
"Heavy oil old 1 Bio": "oil", |
890
|
|
|
"Heavy oil old 2": "oil", |
891
|
|
|
"Light oil": "oil", |
892
|
|
|
"Lignite new": "lignite", |
893
|
|
|
"Lignite old 1": "lignite", |
894
|
|
|
"Lignite old 2": "lignite", |
895
|
|
|
"Lignite old 1 Bio": "lignite", |
896
|
|
|
"Lignite old 2 Bio": "lignite", |
897
|
|
|
"Nuclear": "nuclear", |
898
|
|
|
"Offshore Wind": "wind_offshore", |
899
|
|
|
"Onshore Wind": "wind_onshore", |
900
|
|
|
"Other non-RES": "others", |
901
|
|
|
"Other RES": "others", |
902
|
|
|
"P2G": "power_to_gas", |
903
|
|
|
"PS Closed": "pumped_hydro", |
904
|
|
|
"PS Open": "reservoir", |
905
|
|
|
"Reservoir": "reservoir", |
906
|
|
|
"Run-of-River": "run_of_river", |
907
|
|
|
"Solar PV": "solar", |
908
|
|
|
"Solar Thermal": "others", |
909
|
|
|
"Waste": "Other RES", |
910
|
|
|
} |
911
|
|
|
|
912
|
|
|
|
913
|
|
View Code Duplication |
def get_foreign_bus_id(scenario): |
|
|
|
|
914
|
|
|
"""Calculte the etrago bus id from Nodes of TYNDP based on the geometry |
915
|
|
|
|
916
|
|
|
Returns |
917
|
|
|
------- |
918
|
|
|
pandas.Series |
919
|
|
|
List of mapped node_ids from TYNDP and etragos bus_id |
920
|
|
|
|
921
|
|
|
""" |
922
|
|
|
|
923
|
|
|
sources = config.datasets()["electrical_neighbours"]["sources"] |
924
|
|
|
|
925
|
|
|
bus_id = db.select_geodataframe( |
926
|
|
|
f"""SELECT bus_id, ST_Buffer(geom, 1) as geom, country |
927
|
|
|
FROM grid.egon_etrago_bus |
928
|
|
|
WHERE scn_name = '{scenario}' |
929
|
|
|
AND carrier = 'AC' |
930
|
|
|
AND v_nom = 380. |
931
|
|
|
AND country != 'DE' |
932
|
|
|
AND bus_id NOT IN ( |
933
|
|
|
SELECT bus_i |
934
|
|
|
FROM osmtgmod_results.bus_data) |
935
|
|
|
""", |
936
|
|
|
epsg=3035, |
937
|
|
|
) |
938
|
|
|
|
939
|
|
|
# insert installed capacities |
940
|
|
|
file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}") |
941
|
|
|
|
942
|
|
|
# Select buses in neighbouring countries as geodataframe |
943
|
|
|
buses = pd.read_excel( |
944
|
|
|
file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(), |
945
|
|
|
sheet_name="Nodes - Dict", |
946
|
|
|
).query("longitude==longitude") |
947
|
|
|
buses = gpd.GeoDataFrame( |
948
|
|
|
buses, |
949
|
|
|
crs=4326, |
950
|
|
|
geometry=gpd.points_from_xy(buses.longitude, buses.latitude), |
951
|
|
|
).to_crs(3035) |
952
|
|
|
|
953
|
|
|
buses["bus_id"] = 0 |
954
|
|
|
|
955
|
|
|
# Select bus_id from etrago with shortest distance to TYNDP node |
956
|
|
|
for i, row in buses.iterrows(): |
957
|
|
|
distance = bus_id.set_index("bus_id").geom.distance(row.geometry) |
958
|
|
|
buses.loc[i, "bus_id"] = distance[ |
959
|
|
|
distance == distance.min() |
960
|
|
|
].index.values[0] |
961
|
|
|
|
962
|
|
|
return buses.set_index("node_id").bus_id |
963
|
|
|
|
964
|
|
|
|
965
|
|
|
def calc_capacities(): |
966
|
|
|
"""Calculates installed capacities from TYNDP data |
967
|
|
|
|
968
|
|
|
Returns |
969
|
|
|
------- |
970
|
|
|
pandas.DataFrame |
971
|
|
|
Installed capacities per foreign node and energy carrier |
972
|
|
|
|
973
|
|
|
""" |
974
|
|
|
|
975
|
|
|
sources = config.datasets()["electrical_neighbours"]["sources"] |
976
|
|
|
|
977
|
|
|
countries = [ |
978
|
|
|
"AT", |
979
|
|
|
"BE", |
980
|
|
|
"CH", |
981
|
|
|
"CZ", |
982
|
|
|
"DK", |
983
|
|
|
"FR", |
984
|
|
|
"NL", |
985
|
|
|
"NO", |
986
|
|
|
"SE", |
987
|
|
|
"PL", |
988
|
|
|
"UK", |
989
|
|
|
] |
990
|
|
|
|
991
|
|
|
# insert installed capacities |
992
|
|
|
file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}") |
993
|
|
|
df = pd.read_excel( |
994
|
|
|
file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(), |
995
|
|
|
sheet_name="Capacity", |
996
|
|
|
) |
997
|
|
|
|
998
|
|
|
# differneces between different climate years are very small (<1MW) |
999
|
|
|
# choose 1984 because it is the mean value |
1000
|
|
|
df_2030 = ( |
1001
|
|
|
df.rename({"Climate Year": "Climate_Year"}, axis="columns") |
1002
|
|
|
.query( |
1003
|
|
|
'Scenario == "Distributed Energy" & Year == 2030 & ' |
1004
|
|
|
"Climate_Year == 1984" |
1005
|
|
|
) |
1006
|
|
|
.set_index(["Node/Line", "Generator_ID"]) |
1007
|
|
|
) |
1008
|
|
|
|
1009
|
|
|
df_2040 = ( |
1010
|
|
|
df.rename({"Climate Year": "Climate_Year"}, axis="columns") |
1011
|
|
|
.query( |
1012
|
|
|
'Scenario == "Distributed Energy" & Year == 2040 & ' |
1013
|
|
|
"Climate_Year == 1984" |
1014
|
|
|
) |
1015
|
|
|
.set_index(["Node/Line", "Generator_ID"]) |
1016
|
|
|
) |
1017
|
|
|
|
1018
|
|
|
# interpolate linear between 2030 and 2040 for 2035 accordning to |
1019
|
|
|
# scenario report of TSO's and the approval by BNetzA |
1020
|
|
|
df_2035 = pd.DataFrame(index=df_2030.index) |
1021
|
|
|
df_2035["cap_2030"] = df_2030.Value |
1022
|
|
|
df_2035["cap_2040"] = df_2040.Value |
1023
|
|
|
df_2035.fillna(0.0, inplace=True) |
1024
|
|
|
df_2035["cap_2035"] = ( |
1025
|
|
|
df_2035["cap_2030"] + (df_2035["cap_2040"] - df_2035["cap_2030"]) / 2 |
1026
|
|
|
) |
1027
|
|
|
df_2035 = df_2035.reset_index() |
1028
|
|
|
df_2035["carrier"] = df_2035.Generator_ID.map(map_carriers_tyndp()) |
1029
|
|
|
|
1030
|
|
|
# group capacities by new carriers |
1031
|
|
|
grouped_capacities = ( |
1032
|
|
|
df_2035.groupby(["carrier", "Node/Line"]).cap_2035.sum().reset_index() |
1033
|
|
|
) |
1034
|
|
|
|
1035
|
|
|
# choose capacities for considered countries |
1036
|
|
|
return grouped_capacities[ |
1037
|
|
|
grouped_capacities["Node/Line"].str[:2].isin(countries) |
1038
|
|
|
] |
1039
|
|
|
|
1040
|
|
|
|
1041
|
|
|
def insert_generators_tyndp(capacities): |
1042
|
|
|
"""Insert generators for foreign countries based on TYNDP-data |
1043
|
|
|
|
1044
|
|
|
Parameters |
1045
|
|
|
---------- |
1046
|
|
|
capacities : pandas.DataFrame |
1047
|
|
|
Installed capacities per foreign node and energy carrier |
1048
|
|
|
|
1049
|
|
|
Returns |
1050
|
|
|
------- |
1051
|
|
|
None. |
1052
|
|
|
|
1053
|
|
|
""" |
1054
|
|
|
targets = config.datasets()["electrical_neighbours"]["targets"] |
1055
|
|
|
map_buses = get_map_buses() |
1056
|
|
|
|
1057
|
|
|
# Delete existing data |
1058
|
|
|
db.execute_sql( |
1059
|
|
|
f""" |
1060
|
|
|
DELETE FROM |
1061
|
|
|
{targets['generators']['schema']}.{targets['generators']['table']} |
1062
|
|
|
WHERE bus IN ( |
1063
|
|
|
SELECT bus_id FROM |
1064
|
|
|
{targets['buses']['schema']}.{targets['buses']['table']} |
1065
|
|
|
WHERE country != 'DE' |
1066
|
|
|
AND scn_name = 'eGon2035') |
1067
|
|
|
AND scn_name = 'eGon2035' |
1068
|
|
|
AND carrier != 'CH4' |
1069
|
|
|
""" |
1070
|
|
|
) |
1071
|
|
|
|
1072
|
|
|
db.execute_sql( |
1073
|
|
|
f""" |
1074
|
|
|
DELETE FROM |
1075
|
|
|
{targets['generators_timeseries']['schema']}. |
1076
|
|
|
{targets['generators_timeseries']['table']} |
1077
|
|
|
WHERE generator_id NOT IN ( |
1078
|
|
|
SELECT generator_id FROM |
1079
|
|
|
{targets['generators']['schema']}.{targets['generators']['table']} |
1080
|
|
|
) |
1081
|
|
|
AND scn_name = 'eGon2035' |
1082
|
|
|
""" |
1083
|
|
|
) |
1084
|
|
|
|
1085
|
|
|
# Select generators from TYNDP capacities |
1086
|
|
|
gen = capacities[ |
1087
|
|
|
capacities.carrier.isin( |
1088
|
|
|
[ |
1089
|
|
|
"others", |
1090
|
|
|
"wind_offshore", |
1091
|
|
|
"wind_onshore", |
1092
|
|
|
"solar", |
1093
|
|
|
"reservoir", |
1094
|
|
|
"run_of_river", |
1095
|
|
|
"lignite", |
1096
|
|
|
"coal", |
1097
|
|
|
"oil", |
1098
|
|
|
"nuclear", |
1099
|
|
|
] |
1100
|
|
|
) |
1101
|
|
|
] |
1102
|
|
|
|
1103
|
|
|
# Set bus_id |
1104
|
|
|
gen.loc[ |
1105
|
|
|
gen[gen["Node/Line"].isin(map_buses.keys())].index, "Node/Line" |
1106
|
|
|
] = gen.loc[ |
1107
|
|
|
gen[gen["Node/Line"].isin(map_buses.keys())].index, "Node/Line" |
1108
|
|
|
].map( |
1109
|
|
|
map_buses |
1110
|
|
|
) |
1111
|
|
|
|
1112
|
|
|
gen.loc[:, "bus"] = ( |
1113
|
|
|
get_foreign_bus_id(scenario="eGon2035") |
1114
|
|
|
.loc[gen.loc[:, "Node/Line"]] |
1115
|
|
|
.values |
1116
|
|
|
) |
1117
|
|
|
|
1118
|
|
|
# Add scenario column |
1119
|
|
|
gen["scenario"] = "eGon2035" |
1120
|
|
|
|
1121
|
|
|
# Add marginal costs |
1122
|
|
|
gen = add_marginal_costs(gen) |
1123
|
|
|
|
1124
|
|
|
# insert generators data |
1125
|
|
|
session = sessionmaker(bind=db.engine())() |
1126
|
|
|
for i, row in gen.iterrows(): |
1127
|
|
|
entry = etrago.EgonPfHvGenerator( |
1128
|
|
|
scn_name=row.scenario, |
1129
|
|
|
generator_id=int(db.next_etrago_id("generator")), |
1130
|
|
|
bus=row.bus, |
1131
|
|
|
carrier=row.carrier, |
1132
|
|
|
p_nom=row.cap_2035, |
1133
|
|
|
marginal_cost=row.marginal_cost, |
1134
|
|
|
) |
1135
|
|
|
|
1136
|
|
|
session.add(entry) |
1137
|
|
|
session.commit() |
1138
|
|
|
|
1139
|
|
|
# assign generators time-series data |
1140
|
|
|
|
1141
|
|
|
renewable_timeseries_pypsaeur("eGon2035") |
1142
|
|
|
|
1143
|
|
|
|
1144
|
|
|
def insert_storage_tyndp(capacities): |
1145
|
|
|
"""Insert storage units for foreign countries based on TYNDP-data |
1146
|
|
|
|
1147
|
|
|
Parameters |
1148
|
|
|
---------- |
1149
|
|
|
capacities : pandas.DataFrame |
1150
|
|
|
Installed capacities per foreign node and energy carrier |
1151
|
|
|
|
1152
|
|
|
|
1153
|
|
|
Returns |
1154
|
|
|
------- |
1155
|
|
|
None. |
1156
|
|
|
|
1157
|
|
|
""" |
1158
|
|
|
targets = config.datasets()["electrical_neighbours"]["targets"] |
1159
|
|
|
map_buses = get_map_buses() |
1160
|
|
|
|
1161
|
|
|
# Delete existing data |
1162
|
|
|
db.execute_sql( |
1163
|
|
|
f""" |
1164
|
|
|
DELETE FROM {targets['storage']['schema']}.{targets['storage']['table']} |
1165
|
|
|
WHERE bus IN ( |
1166
|
|
|
SELECT bus_id FROM |
1167
|
|
|
{targets['buses']['schema']}.{targets['buses']['table']} |
1168
|
|
|
WHERE country != 'DE' |
1169
|
|
|
AND scn_name = 'eGon2035') |
1170
|
|
|
AND scn_name = 'eGon2035' |
1171
|
|
|
""" |
1172
|
|
|
) |
1173
|
|
|
|
1174
|
|
|
# Add missing information suitable for eTraGo selected from scenario_parameter table |
1175
|
|
|
parameters_pumped_hydro = scenario_parameters.electricity("eGon2035")[ |
1176
|
|
|
"efficiency" |
1177
|
|
|
]["pumped_hydro"] |
1178
|
|
|
|
1179
|
|
|
parameters_battery = scenario_parameters.electricity("eGon2035")[ |
1180
|
|
|
"efficiency" |
1181
|
|
|
]["battery"] |
1182
|
|
|
|
1183
|
|
|
# Select storage capacities from TYNDP-data |
1184
|
|
|
store = capacities[capacities.carrier.isin(["battery", "pumped_hydro"])] |
1185
|
|
|
|
1186
|
|
|
# Set bus_id |
1187
|
|
|
store.loc[ |
1188
|
|
|
store[store["Node/Line"].isin(map_buses.keys())].index, "Node/Line" |
1189
|
|
|
] = store.loc[ |
1190
|
|
|
store[store["Node/Line"].isin(map_buses.keys())].index, "Node/Line" |
1191
|
|
|
].map( |
1192
|
|
|
map_buses |
1193
|
|
|
) |
1194
|
|
|
|
1195
|
|
|
store.loc[:, "bus"] = ( |
1196
|
|
|
get_foreign_bus_id(scenario="eGon2035") |
1197
|
|
|
.loc[store.loc[:, "Node/Line"]] |
1198
|
|
|
.values |
1199
|
|
|
) |
1200
|
|
|
|
1201
|
|
|
# Add columns for additional parameters to df |
1202
|
|
|
( |
1203
|
|
|
store["dispatch"], |
1204
|
|
|
store["store"], |
1205
|
|
|
store["standing_loss"], |
1206
|
|
|
store["max_hours"], |
1207
|
|
|
) = (None, None, None, None) |
1208
|
|
|
|
1209
|
|
|
# Insert carrier specific parameters |
1210
|
|
|
|
1211
|
|
|
parameters = ["dispatch", "store", "standing_loss", "max_hours"] |
1212
|
|
|
|
1213
|
|
|
for x in parameters: |
1214
|
|
|
store.loc[store["carrier"] == "battery", x] = parameters_battery[x] |
1215
|
|
|
store.loc[store["carrier"] == "pumped_hydro", x] = ( |
1216
|
|
|
parameters_pumped_hydro[x] |
1217
|
|
|
) |
1218
|
|
|
|
1219
|
|
|
# insert data |
1220
|
|
|
session = sessionmaker(bind=db.engine())() |
1221
|
|
|
for i, row in store.iterrows(): |
1222
|
|
|
entry = etrago.EgonPfHvStorage( |
1223
|
|
|
scn_name="eGon2035", |
1224
|
|
|
storage_id=int(db.next_etrago_id("storage")), |
1225
|
|
|
bus=row.bus, |
1226
|
|
|
max_hours=row.max_hours, |
1227
|
|
|
efficiency_store=row.store, |
1228
|
|
|
efficiency_dispatch=row.dispatch, |
1229
|
|
|
standing_loss=row.standing_loss, |
1230
|
|
|
carrier=row.carrier, |
1231
|
|
|
p_nom=row.cap_2035, |
1232
|
|
|
) |
1233
|
|
|
|
1234
|
|
|
session.add(entry) |
1235
|
|
|
session.commit() |
1236
|
|
|
|
1237
|
|
|
|
1238
|
|
|
def get_map_buses(): |
1239
|
|
|
"""Returns a dictonary of foreign regions which are aggregated to another |
1240
|
|
|
|
1241
|
|
|
Returns |
1242
|
|
|
------- |
1243
|
|
|
Combination of aggregated regions |
1244
|
|
|
|
1245
|
|
|
|
1246
|
|
|
""" |
1247
|
|
|
return { |
1248
|
|
|
"DK00": "DKW1", |
1249
|
|
|
"DKKF": "DKE1", |
1250
|
|
|
"FR15": "FR00", |
1251
|
|
|
"NON1": "NOM1", |
1252
|
|
|
"NOS0": "NOM1", |
1253
|
|
|
"NOS1": "NOM1", |
1254
|
|
|
"PLE0": "PL00", |
1255
|
|
|
"PLI0": "PL00", |
1256
|
|
|
"SE00": "SE02", |
1257
|
|
|
"SE01": "SE02", |
1258
|
|
|
"SE03": "SE02", |
1259
|
|
|
"SE04": "SE02", |
1260
|
|
|
"RU": "RU00", |
1261
|
|
|
} |
1262
|
|
|
|
1263
|
|
|
|
1264
|
|
|
def tyndp_generation(): |
1265
|
|
|
"""Insert data from TYNDP 2020 accordning to NEP 2021 |
1266
|
|
|
Scenario 'Distributed Energy', linear interpolate between 2030 and 2040 |
1267
|
|
|
|
1268
|
|
|
Returns |
1269
|
|
|
------- |
1270
|
|
|
None. |
1271
|
|
|
""" |
1272
|
|
|
|
1273
|
|
|
capacities = calc_capacities() |
1274
|
|
|
|
1275
|
|
|
insert_generators_tyndp(capacities) |
1276
|
|
|
|
1277
|
|
|
insert_storage_tyndp(capacities) |
1278
|
|
|
|
1279
|
|
|
|
1280
|
|
|
def tyndp_demand(): |
1281
|
|
|
"""Copy load timeseries data from TYNDP 2020. |
1282
|
|
|
According to NEP 2021, the data for 2030 and 2040 is interpolated linearly. |
1283
|
|
|
|
1284
|
|
|
Returns |
1285
|
|
|
------- |
1286
|
|
|
None. |
1287
|
|
|
|
1288
|
|
|
""" |
1289
|
|
|
map_buses = get_map_buses() |
1290
|
|
|
|
1291
|
|
|
sources = config.datasets()["electrical_neighbours"]["sources"] |
1292
|
|
|
targets = config.datasets()["electrical_neighbours"]["targets"] |
1293
|
|
|
|
1294
|
|
|
# Delete existing data |
1295
|
|
|
db.execute_sql( |
1296
|
|
|
f""" |
1297
|
|
|
DELETE FROM {targets['loads']['schema']}. |
1298
|
|
|
{targets['loads']['table']} |
1299
|
|
|
WHERE |
1300
|
|
|
scn_name = 'eGon2035' |
1301
|
|
|
AND carrier = 'AC' |
1302
|
|
|
AND bus NOT IN ( |
1303
|
|
|
SELECT bus_i |
1304
|
|
|
FROM {sources['osmtgmod_bus']['schema']}. |
1305
|
|
|
{sources['osmtgmod_bus']['table']}) |
1306
|
|
|
""" |
1307
|
|
|
) |
1308
|
|
|
|
1309
|
|
|
# Connect to database |
1310
|
|
|
engine = db.engine() |
1311
|
|
|
session = sessionmaker(bind=engine)() |
1312
|
|
|
|
1313
|
|
|
nodes = [ |
1314
|
|
|
"AT00", |
1315
|
|
|
"BE00", |
1316
|
|
|
"CH00", |
1317
|
|
|
"CZ00", |
1318
|
|
|
"DKE1", |
1319
|
|
|
"DKW1", |
1320
|
|
|
"FR00", |
1321
|
|
|
"NL00", |
1322
|
|
|
"LUB1", |
1323
|
|
|
"LUF1", |
1324
|
|
|
"LUG1", |
1325
|
|
|
"NOM1", |
1326
|
|
|
"NON1", |
1327
|
|
|
"NOS0", |
1328
|
|
|
"SE01", |
1329
|
|
|
"SE02", |
1330
|
|
|
"SE03", |
1331
|
|
|
"SE04", |
1332
|
|
|
"PL00", |
1333
|
|
|
"UK00", |
1334
|
|
|
"UKNI", |
1335
|
|
|
] |
1336
|
|
|
# Assign etrago bus_id to TYNDP nodes |
1337
|
|
|
buses = pd.DataFrame({"nodes": nodes}) |
1338
|
|
|
buses.loc[buses[buses.nodes.isin(map_buses.keys())].index, "nodes"] = ( |
1339
|
|
|
buses[buses.nodes.isin(map_buses.keys())].nodes.map(map_buses) |
1340
|
|
|
) |
1341
|
|
|
buses.loc[:, "bus"] = ( |
1342
|
|
|
get_foreign_bus_id(scenario="eGon2035") |
1343
|
|
|
.loc[buses.loc[:, "nodes"]] |
1344
|
|
|
.values |
1345
|
|
|
) |
1346
|
|
|
buses.set_index("nodes", inplace=True) |
1347
|
|
|
buses = buses[~buses.index.duplicated(keep="first")] |
1348
|
|
|
|
1349
|
|
|
# Read in data from TYNDP for 2030 and 2040 |
1350
|
|
|
dataset_2030 = pd.read_excel( |
1351
|
|
|
f"tyndp/{sources['tyndp_demand_2030']}", sheet_name=nodes, skiprows=10 |
1352
|
|
|
) |
1353
|
|
|
|
1354
|
|
|
dataset_2040 = pd.read_excel( |
1355
|
|
|
f"tyndp/{sources['tyndp_demand_2040']}", sheet_name=None, skiprows=10 |
1356
|
|
|
) |
1357
|
|
|
|
1358
|
|
|
# Transform map_buses to pandas.Series and select only used values |
1359
|
|
|
map_series = pd.Series(map_buses) |
1360
|
|
|
map_series = map_series[map_series.index.isin(nodes)] |
1361
|
|
|
|
1362
|
|
|
# Calculate and insert demand timeseries per etrago bus_id |
1363
|
|
|
for bus in buses.index: |
1364
|
|
|
nodes = [bus] |
1365
|
|
|
|
1366
|
|
|
if bus in map_series.values: |
1367
|
|
|
nodes.extend(list(map_series[map_series == bus].index.values)) |
1368
|
|
|
|
1369
|
|
|
load_id = db.next_etrago_id("load") |
1370
|
|
|
|
1371
|
|
|
# Some etrago bus_ids represent multiple TYNDP nodes, |
1372
|
|
|
# in this cases the loads are summed |
1373
|
|
|
data_2030 = pd.Series(index=range(8760), data=0.0) |
1374
|
|
|
for node in nodes: |
1375
|
|
|
data_2030 = dataset_2030[node][2011] + data_2030 |
1376
|
|
|
|
1377
|
|
|
try: |
1378
|
|
|
data_2040 = pd.Series(index=range(8760), data=0.0) |
1379
|
|
|
|
1380
|
|
|
for node in nodes: |
1381
|
|
|
data_2040 = dataset_2040[node][2011] + data_2040 |
1382
|
|
|
except: |
1383
|
|
|
data_2040 = data_2030 |
1384
|
|
|
|
1385
|
|
|
# According to the NEP, data for 2030 and 2040 is linear interpolated |
1386
|
|
|
data_2035 = ((data_2030 + data_2040) / 2)[:8760] |
1387
|
|
|
|
1388
|
|
|
entry = etrago.EgonPfHvLoad( |
1389
|
|
|
scn_name="eGon2035", |
1390
|
|
|
load_id=int(load_id), |
1391
|
|
|
carrier="AC", |
1392
|
|
|
bus=int(buses.bus[bus]), |
1393
|
|
|
) |
1394
|
|
|
|
1395
|
|
|
entry_ts = etrago.EgonPfHvLoadTimeseries( |
1396
|
|
|
scn_name="eGon2035", |
1397
|
|
|
load_id=int(load_id), |
1398
|
|
|
temp_id=1, |
1399
|
|
|
p_set=list(data_2035.values), |
1400
|
|
|
) |
1401
|
|
|
|
1402
|
|
|
session.add(entry) |
1403
|
|
|
session.add(entry_ts) |
1404
|
|
|
session.commit() |
1405
|
|
|
|
1406
|
|
|
|
1407
|
|
|
def get_entsoe_token(): |
1408
|
|
|
"""Check for token in home dir. If not exists, check in working dir""" |
1409
|
|
|
token_path = path.join(path.expanduser("~"), ".entsoe-token") |
1410
|
|
|
if not os.path.isfile(token_path): |
1411
|
|
|
logger.info( |
1412
|
|
|
f"Token file not found at {token_path}. Will check in working directory." |
1413
|
|
|
) |
1414
|
|
|
token_path = Path(".entsoe-token") |
1415
|
|
|
if os.path.isfile(token_path): |
1416
|
|
|
logger.info(f"Token found at {token_path}") |
1417
|
|
|
entsoe_token = open(token_path, "r").read(36) |
1418
|
|
|
if entsoe_token is None: |
1419
|
|
|
raise FileNotFoundError("No entsoe-token found.") |
1420
|
|
|
return entsoe_token |
1421
|
|
|
|
1422
|
|
|
|
1423
|
|
|
def entsoe_historic_generation_capacities( |
1424
|
|
|
year_start="20190101", year_end="20200101" |
1425
|
|
|
): |
1426
|
|
|
entsoe_token = get_entsoe_token() |
1427
|
|
|
client = entsoe.EntsoePandasClient(api_key=entsoe_token) |
1428
|
|
|
|
1429
|
|
|
start = pd.Timestamp(year_start, tz="Europe/Brussels") |
1430
|
|
|
end = pd.Timestamp(year_end, tz="Europe/Brussels") |
1431
|
|
|
start_gb = pd.Timestamp(year_start, tz="Europe/London") |
1432
|
|
|
end_gb = pd.Timestamp(year_end, tz="Europe/London") |
1433
|
|
|
countries = [ |
1434
|
|
|
"LU", |
1435
|
|
|
"AT", |
1436
|
|
|
"FR", |
1437
|
|
|
"NL", |
1438
|
|
|
"CZ", |
1439
|
|
|
"DK_1", |
1440
|
|
|
"DK_2", |
1441
|
|
|
"PL", |
1442
|
|
|
"CH", |
1443
|
|
|
"NO", |
1444
|
|
|
"BE", |
1445
|
|
|
"SE", |
1446
|
|
|
"GB", |
1447
|
|
|
] |
1448
|
|
|
# No GB data after Brexit |
1449
|
|
|
if int(year_start[:4]) > 2021: |
1450
|
|
|
logger.warning( |
1451
|
|
|
"No GB data after Brexit. GB is dropped from entsoe query!" |
1452
|
|
|
) |
1453
|
|
|
countries = [c for c in countries if c != "GB"] |
1454
|
|
|
# todo: define wanted countries |
1455
|
|
|
|
1456
|
|
|
not_retrieved = [] |
1457
|
|
|
dfs = [] |
1458
|
|
|
for country in countries: |
1459
|
|
|
if country == "GB": |
1460
|
|
|
kwargs = dict(start=start_gb, end=end_gb) |
1461
|
|
|
else: |
1462
|
|
|
kwargs = dict(start=start, end=end) |
1463
|
|
|
try: |
1464
|
|
|
country_data = client.query_installed_generation_capacity( |
1465
|
|
|
country, **kwargs |
1466
|
|
|
) |
1467
|
|
|
dfs.append(country_data) |
1468
|
|
|
except (entsoe.exceptions.NoMatchingDataError, requests.HTTPError): |
1469
|
|
|
logger.warning( |
1470
|
|
|
f"Data for country: {country} could not be retrieved." |
1471
|
|
|
) |
1472
|
|
|
not_retrieved.append(country) |
1473
|
|
|
pass |
1474
|
|
|
|
1475
|
|
|
if dfs: |
1476
|
|
|
df = pd.concat(dfs) |
1477
|
|
|
df["country"] = [c for c in countries if c not in not_retrieved] |
1478
|
|
|
df.set_index("country", inplace=True) |
1479
|
|
|
if int(year_start[:4]) == 2023: |
1480
|
|
|
# https://www.bmreports.com/bmrs/?q=foregeneration/capacityaggregated |
1481
|
|
|
# could probably somehow be automised |
1482
|
|
|
# https://www.elexonportal.co.uk/category/view/178 |
1483
|
|
|
# in MW |
1484
|
|
|
installed_capacity_gb = pd.Series( |
1485
|
|
|
{ |
1486
|
|
|
"Biomass": 4438, |
1487
|
|
|
"Fossil Gas": 37047, |
1488
|
|
|
"Fossil Hard coal": 1491, |
1489
|
|
|
"Hydro Pumped Storage": 5603, |
1490
|
|
|
"Hydro Run-of-river and poundage": 2063, |
1491
|
|
|
"Nuclear": 4950, |
1492
|
|
|
"Other": 3313, |
1493
|
|
|
"Other renewable": 1462, |
1494
|
|
|
"Solar": 14518, |
1495
|
|
|
"Wind Offshore": 13038, |
1496
|
|
|
"Wind Onshore": 13907, |
1497
|
|
|
}, |
1498
|
|
|
name="GB", |
1499
|
|
|
) |
1500
|
|
|
df = pd.concat([df.T, installed_capacity_gb], axis=1).T |
1501
|
|
|
logger.info("Manually added generation capacities for GB 2023.") |
1502
|
|
|
not_retrieved = [c for c in not_retrieved if c != "GB"] |
1503
|
|
|
df.fillna(0, inplace=True) |
1504
|
|
|
else: |
1505
|
|
|
df = pd.DataFrame() |
1506
|
|
|
return df, not_retrieved |
1507
|
|
|
|
1508
|
|
|
|
1509
|
|
|
def entsoe_historic_demand(year_start="20190101", year_end="20200101"): |
1510
|
|
|
entsoe_token = get_entsoe_token() |
1511
|
|
|
client = entsoe.EntsoePandasClient(api_key=entsoe_token) |
1512
|
|
|
|
1513
|
|
|
start = pd.Timestamp(year_start, tz="Europe/Brussels") |
1514
|
|
|
end = pd.Timestamp(year_end, tz="Europe/Brussels") |
1515
|
|
|
start_gb = start.tz_convert("Europe/London") |
1516
|
|
|
end_gb = end.tz_convert("Europe/London") |
1517
|
|
|
|
1518
|
|
|
countries = [ |
1519
|
|
|
"LU", |
1520
|
|
|
"AT", |
1521
|
|
|
"FR", |
1522
|
|
|
"NL", |
1523
|
|
|
"CZ", |
1524
|
|
|
"DK_1", |
1525
|
|
|
"DK_2", |
1526
|
|
|
"PL", |
1527
|
|
|
"CH", |
1528
|
|
|
"NO", |
1529
|
|
|
"BE", |
1530
|
|
|
"SE", |
1531
|
|
|
"GB", |
1532
|
|
|
] |
1533
|
|
|
|
1534
|
|
|
# todo: define wanted countries |
1535
|
|
|
|
1536
|
|
|
not_retrieved = [] |
1537
|
|
|
dfs = [] |
1538
|
|
|
|
1539
|
|
|
for country in countries: |
1540
|
|
|
if country == "GB": |
1541
|
|
|
kwargs = dict(start=start_gb, end=end_gb) |
1542
|
|
|
else: |
1543
|
|
|
kwargs = dict(start=start, end=end) |
1544
|
|
|
try: |
1545
|
|
|
country_data = ( |
1546
|
|
|
client.query_load(country, **kwargs) |
1547
|
|
|
.resample("H")["Actual Load"] |
1548
|
|
|
.mean() |
1549
|
|
|
) |
1550
|
|
|
if country == "GB": |
1551
|
|
|
country_data.index = country_data.index.tz_convert( |
1552
|
|
|
"Europe/Brussels" |
1553
|
|
|
) |
1554
|
|
|
dfs.append(country_data) |
1555
|
|
|
except (entsoe.exceptions.NoMatchingDataError, requests.HTTPError): |
1556
|
|
|
not_retrieved.append(country) |
1557
|
|
|
logger.warning( |
1558
|
|
|
f"Data for country: {country} could not be retrieved." |
1559
|
|
|
) |
1560
|
|
|
pass |
1561
|
|
|
|
1562
|
|
|
if dfs: |
1563
|
|
|
df = pd.concat(dfs, axis=1) |
1564
|
|
|
df.columns = [c for c in countries if c not in not_retrieved] |
1565
|
|
|
df.index = pd.date_range(year_start, periods=8760, freq="H") |
1566
|
|
|
else: |
1567
|
|
|
df = pd.DataFrame() |
1568
|
|
|
return df, not_retrieved |
1569
|
|
|
|
1570
|
|
|
|
1571
|
|
|
def map_carriers_entsoe(): |
1572
|
|
|
"""Map carriers from entsoe-data to carriers used in eGon |
1573
|
|
|
Returns |
1574
|
|
|
------- |
1575
|
|
|
dict |
1576
|
|
|
Carrier from entsoe to eGon |
1577
|
|
|
""" |
1578
|
|
|
return { |
1579
|
|
|
"Biomass": "biomass", |
1580
|
|
|
"Fossil Brown coal/Lignite": "lignite", |
1581
|
|
|
"Fossil Coal-derived gas": "coal", |
1582
|
|
|
"Fossil Gas": "OCGT", |
1583
|
|
|
"Fossil Hard coal": "coal", |
1584
|
|
|
"Fossil Oil": "oil", |
1585
|
|
|
"Fossil Oil shale": "oil", |
1586
|
|
|
"Fossil Peat": "others", |
1587
|
|
|
"Geothermal": "geo_thermal", |
1588
|
|
|
"Hydro Pumped Storage": "Hydro Pumped Storage", |
1589
|
|
|
"Hydro Run-of-river and poundage": "run_of_river", |
1590
|
|
|
"Hydro Water Reservoir": "reservoir", |
1591
|
|
|
"Marine": "others", |
1592
|
|
|
"Nuclear": "nuclear", |
1593
|
|
|
"Other": "others", |
1594
|
|
|
"Other renewable": "others", |
1595
|
|
|
"Solar": "solar", |
1596
|
|
|
"Waste": "others", |
1597
|
|
|
"Wind Offshore": "wind_offshore", |
1598
|
|
|
"Wind Onshore": "wind_onshore", |
1599
|
|
|
} |
1600
|
|
|
|
1601
|
|
|
|
1602
|
|
|
def entsoe_to_bus_etrago(scenario="status2019"): |
1603
|
|
|
map_entsoe = pd.Series( |
1604
|
|
|
{ |
1605
|
|
|
"LU": "LU00", |
1606
|
|
|
"AT": "AT00", |
1607
|
|
|
"FR": "FR00", |
1608
|
|
|
"NL": "NL00", |
1609
|
|
|
"DK_1": "DK00", |
1610
|
|
|
"DK_2": "DKE1", |
1611
|
|
|
"PL": "PL00", |
1612
|
|
|
"CH": "CH00", |
1613
|
|
|
"NO": "NO00", |
1614
|
|
|
"BE": "BE00", |
1615
|
|
|
"SE": "SE00", |
1616
|
|
|
"GB": "UK00", |
1617
|
|
|
"CZ": "CZ00", |
1618
|
|
|
} |
1619
|
|
|
) |
1620
|
|
|
|
1621
|
|
|
for_bus = get_foreign_bus_id(scenario=scenario) |
1622
|
|
|
|
1623
|
|
|
return map_entsoe.map(for_bus) |
1624
|
|
|
|
1625
|
|
|
|
1626
|
|
|
def save_entsoe_data(df: pd.DataFrame, file_path: Path): |
1627
|
|
|
os.makedirs(file_path.parent, exist_ok=True) |
1628
|
|
|
if not df.empty: |
1629
|
|
|
df.to_csv(file_path, index_label="Index") |
1630
|
|
|
logger.info( |
1631
|
|
|
f"Saved entsoe data for {file_path.stem} " |
1632
|
|
|
f"to {file_path.parent} for countries: {df.index}" |
1633
|
|
|
) |
1634
|
|
|
|
1635
|
|
|
|
1636
|
|
|
def fill_by_backup_data_from_former_runs(df_sq, file_path, not_retrieved): |
1637
|
|
|
""" |
1638
|
|
|
Fills missing data from former runs |
1639
|
|
|
Parameters |
1640
|
|
|
---------- |
1641
|
|
|
df_sq: pd.DataFrame |
1642
|
|
|
file_path: str, Path |
1643
|
|
|
not_retrieved: list |
1644
|
|
|
|
1645
|
|
|
Returns |
1646
|
|
|
------- |
1647
|
|
|
df_sq, not_retrieved |
1648
|
|
|
|
1649
|
|
|
""" |
1650
|
|
|
sq_backup = pd.read_csv(file_path, index_col="Index") |
1651
|
|
|
# check for missing columns in backup (former runs) |
1652
|
|
|
c_backup = [c for c in sq_backup.columns if c in not_retrieved] |
1653
|
|
|
# remove columns, if found in backup |
1654
|
|
|
not_retrieved = [c for c in not_retrieved if c not in c_backup] |
1655
|
|
|
if c_backup: |
1656
|
|
|
df_sq = pd.concat([df_sq, sq_backup.loc[:, c_backup]], axis=1) |
1657
|
|
|
logger.info(f"Appended data from former runs for {c_backup}") |
1658
|
|
|
return df_sq, not_retrieved |
1659
|
|
|
|
1660
|
|
|
|
1661
|
|
|
def insert_storage_units_sq(scn_name="status2019"): |
1662
|
|
|
""" |
1663
|
|
|
Insert storage_units for foreign countries based on ENTSO-E data |
1664
|
|
|
|
1665
|
|
|
Parameters |
1666
|
|
|
---------- |
1667
|
|
|
scn_name : str |
1668
|
|
|
Scenario to which the foreign storage units will be assigned. |
1669
|
|
|
The default is "status2019". |
1670
|
|
|
|
1671
|
|
|
Returns |
1672
|
|
|
------- |
1673
|
|
|
None. |
1674
|
|
|
|
1675
|
|
|
""" |
1676
|
|
|
if "status" in scn_name: |
1677
|
|
|
year = int(scn_name.split("status")[-1]) |
1678
|
|
|
year_start_end = { |
1679
|
|
|
"year_start": f"{year}0101", |
1680
|
|
|
"year_end": f"{year+1}0101", |
1681
|
|
|
} |
1682
|
|
|
else: |
1683
|
|
|
raise ValueError("No valid scenario name!") |
1684
|
|
|
|
1685
|
|
|
df_gen_sq, not_retrieved = entsoe_historic_generation_capacities( |
1686
|
|
|
**year_start_end |
1687
|
|
|
) |
1688
|
|
|
|
1689
|
|
View Code Duplication |
if not_retrieved: |
|
|
|
|
1690
|
|
|
logger.warning("Generation data from entsoe could not be retrieved.") |
1691
|
|
|
# check for generation backup from former runs |
1692
|
|
|
file_path = Path( |
1693
|
|
|
"./", "entsoe_data", f"gen_entsoe_{scn_name}.csv" |
1694
|
|
|
).resolve() |
1695
|
|
|
if os.path.isfile(file_path): |
1696
|
|
|
df_gen_sq, not_retrieved = fill_by_backup_data_from_former_runs( |
1697
|
|
|
df_gen_sq, file_path, not_retrieved |
1698
|
|
|
) |
1699
|
|
|
save_entsoe_data(df_gen_sq, file_path=file_path) |
1700
|
|
|
|
1701
|
|
|
if not_retrieved: |
1702
|
|
|
logger.warning( |
1703
|
|
|
f"Backup data of 2019 is used instead for {not_retrieved}" |
1704
|
|
|
) |
1705
|
|
|
df_gen_sq_backup = pd.read_csv( |
1706
|
|
|
"data_bundle_egon_data/entsoe/gen_entsoe.csv", index_col="Index" |
1707
|
|
|
) |
1708
|
|
|
df_gen_sq = pd.concat( |
1709
|
|
|
[df_gen_sq, df_gen_sq_backup.loc[not_retrieved]], axis=1 |
1710
|
|
|
) |
1711
|
|
|
|
1712
|
|
|
sto_sq = df_gen_sq.loc[:, df_gen_sq.columns == "Hydro Pumped Storage"] |
1713
|
|
|
sto_sq.rename(columns={"Hydro Pumped Storage": "p_nom"}, inplace=True) |
1714
|
|
|
|
1715
|
|
|
targets = config.datasets()["electrical_neighbours"]["targets"] |
1716
|
|
|
|
1717
|
|
|
# Delete existing data |
1718
|
|
|
db.execute_sql( |
1719
|
|
|
f""" |
1720
|
|
|
DELETE FROM {targets['storage']['schema']}.{targets['storage']['table']} |
1721
|
|
|
WHERE bus IN ( |
1722
|
|
|
SELECT bus_id FROM |
1723
|
|
|
{targets['buses']['schema']}.{targets['buses']['table']} |
1724
|
|
|
WHERE country != 'DE' |
1725
|
|
|
AND scn_name = '{scn_name}') |
1726
|
|
|
AND scn_name = '{scn_name}' |
1727
|
|
|
""" |
1728
|
|
|
) |
1729
|
|
|
|
1730
|
|
|
# Add missing information suitable for eTraGo selected from scenario_parameter table |
1731
|
|
|
parameters_pumped_hydro = get_sector_parameters( |
1732
|
|
|
sector="electricity", scenario=scn_name |
1733
|
|
|
)["efficiency"]["pumped_hydro"] |
1734
|
|
|
|
1735
|
|
|
# Set bus_id |
1736
|
|
|
entsoe_to_bus = entsoe_to_bus_etrago(scenario=scn_name) |
1737
|
|
|
sto_sq["bus"] = sto_sq.index.map(entsoe_to_bus) |
1738
|
|
|
|
1739
|
|
|
# Insert carrier specific parameters |
1740
|
|
|
sto_sq["carrier"] = "pumped_hydro" |
1741
|
|
|
sto_sq["scn_name"] = scn_name |
1742
|
|
|
sto_sq["dispatch"] = parameters_pumped_hydro["dispatch"] |
1743
|
|
|
sto_sq["store"] = parameters_pumped_hydro["store"] |
1744
|
|
|
sto_sq["standing_loss"] = parameters_pumped_hydro["standing_loss"] |
1745
|
|
|
sto_sq["max_hours"] = parameters_pumped_hydro["max_hours"] |
1746
|
|
|
sto_sq["cyclic_state_of_charge"] = parameters_pumped_hydro[ |
1747
|
|
|
"cyclic_state_of_charge" |
1748
|
|
|
] |
1749
|
|
|
|
1750
|
|
|
next_id = int(db.next_etrago_id("storage")) |
1751
|
|
|
sto_sq["storage_id"] = range(next_id, next_id + len(sto_sq)) |
1752
|
|
|
|
1753
|
|
|
# Delete entrances without any installed capacity |
1754
|
|
|
sto_sq = sto_sq[sto_sq["p_nom"] > 0] |
1755
|
|
|
|
1756
|
|
|
# insert data pumped_hydro storage |
1757
|
|
|
|
1758
|
|
|
with session_scope() as session: |
1759
|
|
|
for i, row in sto_sq.iterrows(): |
1760
|
|
|
entry = etrago.EgonPfHvStorage( |
1761
|
|
|
scn_name=scn_name, |
1762
|
|
|
storage_id=row.storage_id, |
1763
|
|
|
bus=row.bus, |
1764
|
|
|
max_hours=row.max_hours, |
1765
|
|
|
efficiency_store=row.store, |
1766
|
|
|
efficiency_dispatch=row.dispatch, |
1767
|
|
|
standing_loss=row.standing_loss, |
1768
|
|
|
carrier=row.carrier, |
1769
|
|
|
p_nom=row.p_nom, |
1770
|
|
|
cyclic_state_of_charge=row.cyclic_state_of_charge, |
1771
|
|
|
) |
1772
|
|
|
session.add(entry) |
1773
|
|
|
session.commit() |
1774
|
|
|
|
1775
|
|
|
# big scale batteries |
1776
|
|
|
# info based on EASE data. https://ease-storage.eu/publication/emmes-7-0-march-2023/ |
1777
|
|
|
# batteries smaller than 100MW are neglected |
1778
|
|
|
|
1779
|
|
|
# TODO: include capacities between 2020 and 2023 |
1780
|
|
|
bat_per_country = { |
1781
|
|
|
"LU": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1782
|
|
|
"AT": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1783
|
|
|
"FR": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1784
|
|
|
"NL": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1785
|
|
|
"DK_1": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1786
|
|
|
"DK_2": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1787
|
|
|
"PL": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1788
|
|
|
"CH": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1789
|
|
|
"NO": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1790
|
|
|
"BE": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1791
|
|
|
"SE": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1792
|
|
|
"GB": [723.8, 952.3, 1380.9, 2333.3, 3928.5], |
1793
|
|
|
"CZ": [0, pd.NA, pd.NA, pd.NA, pd.NA], |
1794
|
|
|
} |
1795
|
|
|
bat_sq = pd.DataFrame(bat_per_country).T.set_axis( |
1796
|
|
|
["2019", "2020", "2021", "2022", "2023"], axis=1 |
1797
|
|
|
) |
1798
|
|
|
|
1799
|
|
|
# Select year of interest |
1800
|
|
|
bat_sq = bat_sq[[str(year)]] |
1801
|
|
|
bat_sq.rename(columns={str(year): "p_nom"}, inplace= True) |
1802
|
|
|
|
1803
|
|
|
# Add missing information suitable for eTraGo selected from scenario_parameter table |
1804
|
|
|
parameters_batteries = get_sector_parameters( |
1805
|
|
|
sector="electricity", scenario=scn_name |
1806
|
|
|
)["efficiency"]["battery"] |
1807
|
|
|
|
1808
|
|
|
# Set bus_id |
1809
|
|
|
entsoe_to_bus = entsoe_to_bus_etrago() |
1810
|
|
|
bat_sq["bus"] = bat_sq.index.map(entsoe_to_bus) |
1811
|
|
|
|
1812
|
|
|
# Insert carrier specific parameters |
1813
|
|
|
bat_sq["carrier"] = "battery" |
1814
|
|
|
bat_sq["scn_name"] = scn_name |
1815
|
|
|
bat_sq["dispatch"] = parameters_batteries["dispatch"] |
1816
|
|
|
bat_sq["store"] = parameters_batteries["store"] |
1817
|
|
|
bat_sq["standing_loss"] = parameters_batteries["standing_loss"] |
1818
|
|
|
bat_sq["max_hours"] = parameters_batteries["max_hours"] |
1819
|
|
|
bat_sq["cyclic_state_of_charge"] = parameters_batteries[ |
1820
|
|
|
"cyclic_state_of_charge" |
1821
|
|
|
] |
1822
|
|
|
|
1823
|
|
|
next_id = int(db.next_etrago_id("storage")) |
1824
|
|
|
bat_sq["storage_id"] = range(next_id, next_id + len(bat_sq)) |
1825
|
|
|
|
1826
|
|
|
# Delete entrances without any installed capacity |
1827
|
|
|
bat_sq = bat_sq[bat_sq["p_nom"] > 0] |
1828
|
|
|
|
1829
|
|
|
# insert data pumped_hydro storage |
1830
|
|
|
with db.session_scope() as session: |
1831
|
|
|
for i, row in bat_sq.iterrows(): |
1832
|
|
|
entry = etrago.EgonPfHvStorage( |
1833
|
|
|
scn_name=scn_name, |
1834
|
|
|
storage_id=row.storage_id, |
1835
|
|
|
bus=row.bus, |
1836
|
|
|
max_hours=row.max_hours, |
1837
|
|
|
efficiency_store=row.store, |
1838
|
|
|
efficiency_dispatch=row.dispatch, |
1839
|
|
|
standing_loss=row.standing_loss, |
1840
|
|
|
carrier=row.carrier, |
1841
|
|
|
p_nom=row.p_nom, |
1842
|
|
|
cyclic_state_of_charge=row.cyclic_state_of_charge, |
1843
|
|
|
) |
1844
|
|
|
session.add(entry) |
1845
|
|
|
session.commit() |
1846
|
|
|
|
1847
|
|
|
|
1848
|
|
|
def insert_generators_sq(scn_name="status2019"): |
1849
|
|
|
""" |
1850
|
|
|
Insert generators for foreign countries based on ENTSO-E data |
1851
|
|
|
|
1852
|
|
|
Parameters |
1853
|
|
|
---------- |
1854
|
|
|
gen_sq : pandas dataframe |
1855
|
|
|
df with all the foreign generators produced by the function |
1856
|
|
|
entsoe_historic_generation_capacities |
1857
|
|
|
scn_name : str |
1858
|
|
|
The default is "status2019". |
1859
|
|
|
|
1860
|
|
|
Returns |
1861
|
|
|
------- |
1862
|
|
|
None. |
1863
|
|
|
|
1864
|
|
|
""" |
1865
|
|
|
if "status" in scn_name: |
1866
|
|
|
year = int(scn_name.split("status")[-1]) |
1867
|
|
|
year_start_end = { |
1868
|
|
|
"year_start": f"{year}0101", |
1869
|
|
|
"year_end": f"{year+1}0101", |
1870
|
|
|
} |
1871
|
|
|
else: |
1872
|
|
|
raise ValueError("No valid scenario name!") |
1873
|
|
|
|
1874
|
|
|
df_gen_sq, not_retrieved = entsoe_historic_generation_capacities( |
1875
|
|
|
**year_start_end |
1876
|
|
|
) |
1877
|
|
|
|
1878
|
|
View Code Duplication |
if not_retrieved: |
|
|
|
|
1879
|
|
|
logger.warning("Generation data from entsoe could not be retrieved.") |
1880
|
|
|
# check for generation backup from former runs |
1881
|
|
|
file_path = Path( |
1882
|
|
|
"./", "entsoe_data", f"gen_entsoe_{scn_name}.csv" |
1883
|
|
|
).resolve() |
1884
|
|
|
if os.path.isfile(file_path): |
1885
|
|
|
df_gen_sq, not_retrieved = fill_by_backup_data_from_former_runs( |
1886
|
|
|
df_gen_sq, file_path, not_retrieved |
1887
|
|
|
) |
1888
|
|
|
save_entsoe_data(df_gen_sq, file_path=file_path) |
1889
|
|
|
|
1890
|
|
|
if not_retrieved: |
1891
|
|
|
logger.warning( |
1892
|
|
|
f"Backup data of 2019 is used instead for {not_retrieved}" |
1893
|
|
|
) |
1894
|
|
|
df_gen_sq_backup = pd.read_csv( |
1895
|
|
|
"data_bundle_egon_data/entsoe/gen_entsoe.csv", index_col="Index" |
1896
|
|
|
) |
1897
|
|
|
df_gen_sq = pd.concat( |
1898
|
|
|
[df_gen_sq, df_gen_sq_backup.loc[not_retrieved]], axis=1 |
1899
|
|
|
) |
1900
|
|
|
|
1901
|
|
|
targets = config.datasets()["electrical_neighbours"]["targets"] |
1902
|
|
|
# Delete existing data |
1903
|
|
|
db.execute_sql( |
1904
|
|
|
f""" |
1905
|
|
|
DELETE FROM |
1906
|
|
|
{targets['generators']['schema']}.{targets['generators']['table']} |
1907
|
|
|
WHERE bus IN ( |
1908
|
|
|
SELECT bus_id FROM |
1909
|
|
|
{targets['buses']['schema']}.{targets['buses']['table']} |
1910
|
|
|
WHERE country != 'DE' |
1911
|
|
|
AND scn_name = '{scn_name}') |
1912
|
|
|
AND scn_name = '{scn_name}' |
1913
|
|
|
AND carrier != 'CH4' |
1914
|
|
|
""" |
1915
|
|
|
) |
1916
|
|
|
|
1917
|
|
|
db.execute_sql( |
1918
|
|
|
f""" |
1919
|
|
|
DELETE FROM |
1920
|
|
|
{targets['generators_timeseries']['schema']}. |
1921
|
|
|
{targets['generators_timeseries']['table']} |
1922
|
|
|
WHERE generator_id NOT IN ( |
1923
|
|
|
SELECT generator_id FROM |
1924
|
|
|
{targets['generators']['schema']}.{targets['generators']['table']} |
1925
|
|
|
) |
1926
|
|
|
AND scn_name = '{scn_name}' |
1927
|
|
|
""" |
1928
|
|
|
) |
1929
|
|
|
entsoe_to_bus = entsoe_to_bus_etrago(scn_name) |
1930
|
|
|
carrier_entsoe = map_carriers_entsoe() |
1931
|
|
|
df_gen_sq = df_gen_sq.groupby(axis=1, by=carrier_entsoe).sum() |
1932
|
|
|
|
1933
|
|
|
# Filter generators modeled as storage and geothermal |
1934
|
|
|
df_gen_sq = df_gen_sq.loc[ |
1935
|
|
|
:, ~df_gen_sq.columns.isin(["Hydro Pumped Storage", "geo_thermal"]) |
1936
|
|
|
] |
1937
|
|
|
|
1938
|
|
|
list_gen_sq = pd.DataFrame( |
1939
|
|
|
dtype=int, columns=["carrier", "country", "capacity"] |
1940
|
|
|
) |
1941
|
|
|
for carrier in df_gen_sq.columns: |
1942
|
|
|
gen_carry = df_gen_sq[carrier] |
1943
|
|
|
for country, cap in gen_carry.items(): |
1944
|
|
|
gen = pd.DataFrame( |
1945
|
|
|
{"carrier": carrier, "country": country, "capacity": cap}, |
1946
|
|
|
index=[1], |
1947
|
|
|
) |
1948
|
|
|
# print(gen) |
1949
|
|
|
list_gen_sq = pd.concat([list_gen_sq, gen], ignore_index=True) |
1950
|
|
|
|
1951
|
|
|
list_gen_sq = list_gen_sq[list_gen_sq.capacity > 0] |
1952
|
|
|
list_gen_sq["scenario"] = scn_name |
1953
|
|
|
|
1954
|
|
|
# Add marginal costs |
1955
|
|
|
list_gen_sq = add_marginal_costs(list_gen_sq) |
1956
|
|
|
|
1957
|
|
|
# Find foreign bus to assign the generator |
1958
|
|
|
list_gen_sq["bus"] = list_gen_sq.country.map(entsoe_to_bus) |
1959
|
|
|
|
1960
|
|
|
# insert generators data |
1961
|
|
|
session = sessionmaker(bind=db.engine())() |
1962
|
|
|
for i, row in list_gen_sq.iterrows(): |
1963
|
|
|
entry = etrago.EgonPfHvGenerator( |
1964
|
|
|
scn_name=row.scenario, |
1965
|
|
|
generator_id=int(db.next_etrago_id("generator")), |
1966
|
|
|
bus=row.bus, |
1967
|
|
|
carrier=row.carrier, |
1968
|
|
|
p_nom=row.capacity, |
1969
|
|
|
marginal_cost=row.marginal_cost, |
1970
|
|
|
) |
1971
|
|
|
|
1972
|
|
|
session.add(entry) |
1973
|
|
|
session.commit() |
1974
|
|
|
|
1975
|
|
|
renewable_timeseries_pypsaeur(scn_name) |
1976
|
|
|
|
1977
|
|
|
|
1978
|
|
|
def renewable_timeseries_pypsaeur(scn_name): |
1979
|
|
|
# select generators from database to get index values |
1980
|
|
|
foreign_re_generators = db.select_dataframe( |
1981
|
|
|
f""" |
1982
|
|
|
SELECT generator_id, a.carrier, country, x, y |
1983
|
|
|
FROM grid.egon_etrago_generator a |
1984
|
|
|
JOIN grid.egon_etrago_bus b |
1985
|
|
|
ON a.bus = b.bus_id |
1986
|
|
|
WHERE a.scn_name = '{scn_name}' |
1987
|
|
|
AND b.scn_name = '{scn_name}' |
1988
|
|
|
AND b.carrier = 'AC' |
1989
|
|
|
AND b.country != 'DE' |
1990
|
|
|
AND a.carrier IN ('wind_onshore', 'wind_offshore', 'solar') |
1991
|
|
|
""" |
1992
|
|
|
) |
1993
|
|
|
|
1994
|
|
|
# Import prepared network from pypsa-eur |
1995
|
|
|
network = prepared_network() |
1996
|
|
|
|
1997
|
|
|
# Select fluctuating renewable generators |
1998
|
|
|
generators_pypsa_eur = network.generators.loc[ |
1999
|
|
|
network.generators[ |
2000
|
|
|
network.generators.carrier.isin(["onwind", "offwind-ac", "solar"]) |
2001
|
|
|
].index, |
2002
|
|
|
["bus", "carrier"], |
2003
|
|
|
] |
2004
|
|
|
|
2005
|
|
|
# Align carrier names for wind turbines |
2006
|
|
|
generators_pypsa_eur.loc[ |
2007
|
|
|
generators_pypsa_eur[generators_pypsa_eur.carrier == "onwind"].index, |
2008
|
|
|
"carrier", |
2009
|
|
|
] = "wind_onshore" |
2010
|
|
|
generators_pypsa_eur.loc[ |
2011
|
|
|
generators_pypsa_eur[ |
2012
|
|
|
generators_pypsa_eur.carrier == "offwind-ac" |
2013
|
|
|
].index, |
2014
|
|
|
"carrier", |
2015
|
|
|
] = "wind_offshore" |
2016
|
|
|
|
2017
|
|
|
# Set coordinates from bus table |
2018
|
|
|
generators_pypsa_eur["x"] = network.buses.loc[ |
2019
|
|
|
generators_pypsa_eur.bus.values, "x" |
2020
|
|
|
].values |
2021
|
|
|
generators_pypsa_eur["y"] = network.buses.loc[ |
2022
|
|
|
generators_pypsa_eur.bus.values, "y" |
2023
|
|
|
].values |
2024
|
|
|
|
2025
|
|
|
# Get p_max_pu time series from pypsa-eur |
2026
|
|
|
generators_pypsa_eur["p_max_pu"] = network.generators_t.p_max_pu[ |
2027
|
|
|
generators_pypsa_eur.index |
2028
|
|
|
].T.values.tolist() |
2029
|
|
|
|
2030
|
|
|
session = sessionmaker(bind=db.engine())() |
2031
|
|
|
|
2032
|
|
|
# Insert p_max_pu timeseries based on geometry and carrier |
2033
|
|
|
for gen in foreign_re_generators.index: |
2034
|
|
|
entry = etrago.EgonPfHvGeneratorTimeseries( |
2035
|
|
|
scn_name=scn_name, |
2036
|
|
|
generator_id=foreign_re_generators.loc[gen, "generator_id"], |
2037
|
|
|
temp_id=1, |
2038
|
|
|
p_max_pu=generators_pypsa_eur[ |
2039
|
|
|
( |
2040
|
|
|
( |
2041
|
|
|
generators_pypsa_eur.x |
2042
|
|
|
- foreign_re_generators.loc[gen, "x"] |
2043
|
|
|
).abs() |
2044
|
|
|
< 0.01 |
2045
|
|
|
) |
2046
|
|
|
& ( |
2047
|
|
|
( |
2048
|
|
|
generators_pypsa_eur.y |
2049
|
|
|
- foreign_re_generators.loc[gen, "y"] |
2050
|
|
|
).abs() |
2051
|
|
|
< 0.01 |
2052
|
|
|
) |
2053
|
|
|
& ( |
2054
|
|
|
generators_pypsa_eur.carrier |
2055
|
|
|
== foreign_re_generators.loc[gen, "carrier"] |
2056
|
|
|
) |
2057
|
|
|
].p_max_pu.iloc[0], |
2058
|
|
|
) |
2059
|
|
|
|
2060
|
|
|
session.add(entry) |
2061
|
|
|
session.commit() |
2062
|
|
|
|
2063
|
|
|
|
2064
|
|
|
def insert_loads_sq(scn_name="status2019"): |
2065
|
|
|
""" |
2066
|
|
|
Copy load timeseries data from entso-e. |
2067
|
|
|
|
2068
|
|
|
Returns |
2069
|
|
|
------- |
2070
|
|
|
None. |
2071
|
|
|
|
2072
|
|
|
""" |
2073
|
|
|
sources = config.datasets()["electrical_neighbours"]["sources"] |
2074
|
|
|
targets = config.datasets()["electrical_neighbours"]["targets"] |
2075
|
|
|
|
2076
|
|
|
if scn_name == "status2019": |
2077
|
|
|
year_start_end = {"year_start": "20190101", "year_end": "20200101"} |
2078
|
|
|
elif scn_name == "status2023": |
2079
|
|
|
year_start_end = {"year_start": "20230101", "year_end": "20240101"} |
2080
|
|
|
else: |
2081
|
|
|
raise ValueError("No valid scenario name!") |
2082
|
|
|
|
2083
|
|
|
df_load_sq, not_retrieved = entsoe_historic_demand(**year_start_end) |
2084
|
|
|
|
2085
|
|
|
if not_retrieved: |
2086
|
|
|
logger.warning("Demand data from entsoe could not be retrieved.") |
2087
|
|
|
# check for generation backup from former runs |
2088
|
|
|
file_path = Path( |
2089
|
|
|
"./", "entsoe_data", f"load_entsoe_{scn_name}.csv" |
2090
|
|
|
).resolve() |
2091
|
|
|
if os.path.isfile(file_path): |
2092
|
|
|
df_load_sq, not_retrieved = fill_by_backup_data_from_former_runs( |
2093
|
|
|
df_load_sq, file_path, not_retrieved |
2094
|
|
|
) |
2095
|
|
|
save_entsoe_data(df_load_sq, file_path=file_path) |
2096
|
|
|
|
2097
|
|
|
if not_retrieved: |
2098
|
|
|
logger.warning( |
2099
|
|
|
f"Backup data of 2019 is used instead for {not_retrieved}" |
2100
|
|
|
) |
2101
|
|
|
df_load_sq_backup = pd.read_csv( |
2102
|
|
|
"data_bundle_egon_data/entsoe/load_entsoe.csv", index_col="Index" |
2103
|
|
|
) |
2104
|
|
|
df_load_sq_backup.index = df_load_sq.index |
2105
|
|
|
df_load_sq = pd.concat( |
2106
|
|
|
[df_load_sq, df_load_sq_backup.loc[:, not_retrieved]], axis=1 |
2107
|
|
|
) |
2108
|
|
|
|
2109
|
|
|
# Delete existing data |
2110
|
|
|
db.execute_sql( |
2111
|
|
|
f""" |
2112
|
|
|
DELETE FROM {targets['load_timeseries']['schema']}. |
2113
|
|
|
{targets['load_timeseries']['table']} |
2114
|
|
|
WHERE |
2115
|
|
|
scn_name = '{scn_name}' |
2116
|
|
|
AND load_id IN ( |
2117
|
|
|
SELECT load_id FROM {targets['loads']['schema']}. |
2118
|
|
|
{targets['loads']['table']} |
2119
|
|
|
WHERE |
2120
|
|
|
scn_name = '{scn_name}' |
2121
|
|
|
AND carrier = 'AC' |
2122
|
|
|
AND bus NOT IN ( |
2123
|
|
|
SELECT bus_i |
2124
|
|
|
FROM {sources['osmtgmod_bus']['schema']}. |
2125
|
|
|
{sources['osmtgmod_bus']['table']})) |
2126
|
|
|
""" |
2127
|
|
|
) |
2128
|
|
|
|
2129
|
|
|
db.execute_sql( |
2130
|
|
|
f""" |
2131
|
|
|
DELETE FROM {targets['loads']['schema']}. |
2132
|
|
|
{targets['loads']['table']} |
2133
|
|
|
WHERE |
2134
|
|
|
scn_name = '{scn_name}' |
2135
|
|
|
AND carrier = 'AC' |
2136
|
|
|
AND bus NOT IN ( |
2137
|
|
|
SELECT bus_i |
2138
|
|
|
FROM {sources['osmtgmod_bus']['schema']}. |
2139
|
|
|
{sources['osmtgmod_bus']['table']}) |
2140
|
|
|
""" |
2141
|
|
|
) |
2142
|
|
|
|
2143
|
|
|
# get the corresponding bus per foreign country |
2144
|
|
|
entsoe_to_bus = entsoe_to_bus_etrago(scn_name) |
2145
|
|
|
|
2146
|
|
|
# Calculate and insert demand timeseries per etrago bus_id |
2147
|
|
|
with session_scope() as session: |
2148
|
|
|
for country in df_load_sq.columns: |
2149
|
|
|
load_id = db.next_etrago_id("load") |
2150
|
|
|
|
2151
|
|
|
entry = etrago.EgonPfHvLoad( |
2152
|
|
|
scn_name=scn_name, |
2153
|
|
|
load_id=int(load_id), |
2154
|
|
|
carrier="AC", |
2155
|
|
|
bus=int(entsoe_to_bus[country]), |
2156
|
|
|
) |
2157
|
|
|
|
2158
|
|
|
entry_ts = etrago.EgonPfHvLoadTimeseries( |
2159
|
|
|
scn_name=scn_name, |
2160
|
|
|
load_id=int(load_id), |
2161
|
|
|
temp_id=1, |
2162
|
|
|
p_set=list(df_load_sq[country]), |
2163
|
|
|
) |
2164
|
|
|
|
2165
|
|
|
session.add(entry) |
2166
|
|
|
session.add(entry_ts) |
2167
|
|
|
session.commit() |
2168
|
|
|
|
2169
|
|
|
|
2170
|
|
|
tasks = (grid,) |
2171
|
|
|
|
2172
|
|
|
insert_per_scenario = set() |
2173
|
|
|
|
2174
|
|
|
for scn_name in config.settings()["egon-data"]["--scenarios"]: |
2175
|
|
|
|
2176
|
|
|
if scn_name == "eGon2035": |
2177
|
|
|
insert_per_scenario.update([tyndp_generation, tyndp_demand]) |
2178
|
|
|
|
2179
|
|
|
if "status" in scn_name: |
2180
|
|
|
postfix = f"_{scn_name.split('status')[-1]}" |
2181
|
|
|
insert_per_scenario.update( |
2182
|
|
|
[ |
2183
|
|
|
wrapped_partial( |
2184
|
|
|
insert_generators_sq, scn_name=scn_name, postfix=postfix |
2185
|
|
|
), |
2186
|
|
|
wrapped_partial( |
2187
|
|
|
insert_loads_sq, scn_name=scn_name, postfix=postfix |
2188
|
|
|
), |
2189
|
|
|
wrapped_partial( |
2190
|
|
|
insert_storage_units_sq, scn_name=scn_name, postfix=postfix |
2191
|
|
|
), |
2192
|
|
|
] |
2193
|
|
|
) |
2194
|
|
|
|
2195
|
|
|
tasks = tasks + (insert_per_scenario,) |
2196
|
|
|
|
2197
|
|
|
|
2198
|
|
|
class ElectricalNeighbours(Dataset): |
2199
|
|
|
""" |
2200
|
|
|
Add lines, loads, generation and storage for electrical neighbours |
2201
|
|
|
|
2202
|
|
|
This dataset creates data for modelling the considered foreign countries and writes |
2203
|
|
|
that data into the database tables that can be read by the eTraGo tool. |
2204
|
|
|
Neighbouring countries are modelled in a lower spatial resolution, in general one node per |
2205
|
|
|
country is considered. |
2206
|
|
|
Defined load timeseries as well as generatrion and storage capacities are connected to these nodes. |
2207
|
|
|
The nodes are connected by AC and DC transmission lines with the German grid and other neighbouring countries |
2208
|
|
|
considering the grid topology from ENTSO-E. |
2209
|
|
|
|
2210
|
|
|
|
2211
|
|
|
*Dependencies* |
2212
|
|
|
* :py:class:`Tyndp <egon.data.datasets.tyndp.Tyndp>` |
2213
|
|
|
* :py:class:`PypsaEurSec <egon.data.datasets.pypsaeursec.PypsaEurSec>` |
2214
|
|
|
|
2215
|
|
|
|
2216
|
|
|
*Resulting tables* |
2217
|
|
|
* :py:class:`grid.egon_etrago_bus <egon.data.datasets.etrago_setup.EgonPfHvBus>` is extended |
2218
|
|
|
* :py:class:`grid.egon_etrago_link <egon.data.datasets.etrago_setup.EgonPfHvLink>` is extended |
2219
|
|
|
* :py:class:`grid.egon_etrago_line <egon.data.datasets.etrago_setup.EgonPfHvLine>` is extended |
2220
|
|
|
* :py:class:`grid.egon_etrago_load <egon.data.datasets.etrago_setup.EgonPfHvLoad>` is extended |
2221
|
|
|
* :py:class:`grid.egon_etrago_load_timeseries <egon.data.datasets.etrago_setup.EgonPfHvLoadTimeseries>` is extended |
2222
|
|
|
* :py:class:`grid.egon_etrago_storage <egon.data.datasets.etrago_setup.EgonPfHvStorageUnit>` is extended |
2223
|
|
|
* :py:class:`grid.egon_etrago_generator <egon.data.datasets.etrago_setup.EgonPfHvGenerator>` is extended |
2224
|
|
|
* :py:class:`grid.egon_etrago_generator_timeseries <egon.data.datasets.etrago_setup.EgonPfHvGeneratorTimeseries>` is extended |
2225
|
|
|
* :py:class:`grid.egon_etrago_transformer <egon.data.datasets.etrago_setup.EgonPfHvTransformer>` is extended |
2226
|
|
|
|
2227
|
|
|
""" |
2228
|
|
|
|
2229
|
|
|
#: |
2230
|
|
|
name: str = "ElectricalNeighbours" |
2231
|
|
|
#: |
2232
|
|
|
version: str = "0.0.11" |
2233
|
|
|
|
2234
|
|
|
def __init__(self, dependencies): |
2235
|
|
|
super().__init__( |
2236
|
|
|
name=self.name, |
2237
|
|
|
version=self.version, |
2238
|
|
|
dependencies=dependencies, |
2239
|
|
|
tasks=tasks, |
2240
|
|
|
) |
2241
|
|
|
|