cross_border_lines()   C
last analyzed

Complexity

Conditions 8

Size

Total Lines 143
Code Lines 65

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
eloc 65
dl 0
loc 143
rs 6.2787
c 0
b 0
f 0
cc 8
nop 4

How to fix   Long Method   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

1
"""The central module containing all code dealing with electrical neighbours"""
2
3
from os import path
4
from pathlib import Path
5
import datetime
6
import logging
7
import os.path
8
import zipfile
9
10
from shapely.geometry import LineString
11
from sqlalchemy.orm import sessionmaker
12
import entsoe
13
import requests
14
15
import geopandas as gpd
16
import pandas as pd
17
18
from egon.data import config, db, logger
19
from egon.data.db import session_scope
20
from egon.data.datasets import Dataset, wrapped_partial
21
from egon.data.datasets.fill_etrago_gen import add_marginal_costs
22
from egon.data.datasets.fix_ehv_subnetworks import select_bus_id
23
from egon.data.datasets.scenario_parameters import get_sector_parameters
24
import egon.data.datasets.etrago_setup as etrago
25
import egon.data.datasets.scenario_parameters.parameters as scenario_parameters
26
from egon.data.datasets.pypsaeur import prepared_network
27
28
29
def get_cross_border_buses(scenario, sources):
30
    """Returns buses from osmTGmod which are outside of Germany.
31
32
    Parameters
33
    ----------
34
    sources : dict
35
        List of sources
36
37
    Returns
38
    -------
39
    geopandas.GeoDataFrame
40
        Electricity buses outside of Germany
41
42
    """
43
    return db.select_geodataframe(
44
        f"""
45
        SELECT *
46
        FROM {sources['electricity_buses']['schema']}.
47
            {sources['electricity_buses']['table']}
48
        WHERE
49
        NOT ST_INTERSECTS (
50
            geom,
51
            (SELECT ST_Transform(ST_Buffer(geometry, 5), 4326) FROM
52
             {sources['german_borders']['schema']}.
53
            {sources['german_borders']['table']}))
54
        AND (bus_id IN (
55
            SELECT bus0 FROM
56
            {sources['lines']['schema']}.{sources['lines']['table']})
57
            OR bus_id IN (
58
            SELECT bus1 FROM
59
            {sources['lines']['schema']}.{sources['lines']['table']}))
60
        AND scn_name = '{scenario}';
61
        """,
62
        epsg=4326,
63
    )
64
65
66
def get_cross_border_lines(scenario, sources):
67
    """Returns lines from osmTGmod which end or start outside of Germany.
68
69
    Parameters
70
    ----------
71
    sources : dict
72
        List of sources
73
74
    Returns
75
    -------
76
    geopandas.GeoDataFrame
77
        AC-lines outside of Germany
78
79
    """
80
    return db.select_geodataframe(
81
        f"""
82
    SELECT *
83
    FROM {sources['lines']['schema']}.{sources['lines']['table']} a
84
    WHERE
85
    ST_INTERSECTS (
86
        a.topo,
87
        (SELECT ST_Transform(ST_boundary(geometry), 4326)
88
         FROM {sources['german_borders']['schema']}.
89
        {sources['german_borders']['table']}))
90
    AND scn_name = '{scenario}';
91
    """,
92
        epsg=4326,
93
    )
94
95
96
def central_buses_pypsaeur(sources, scenario):
97
    """Returns buses in the middle of foreign countries based on prepared pypsa-eur network
98
99
    Parameters
100
    ----------
101
    sources : dict
102
        List of sources
103
104
    Returns
105
    -------
106
    pandas.DataFrame
107
        Buses in the center of foreign countries
108
109
    """
110
111
    wanted_countries = [
112
        "AT",
113
        "CH",
114
        "CZ",
115
        "PL",
116
        "SE",
117
        "NO",
118
        "DK",
119
        "GB",
120
        "NL",
121
        "BE",
122
        "FR",
123
        "LU",
124
    ]
125
    network = prepared_network()
126
127
    df = network.buses[
128
        (network.buses.carrier == "AC")
129
        & (network.buses.country.isin(wanted_countries))
130
    ]
131
132
    return df
133
134
135
def buses(scenario, sources, targets):
136
    """Insert central buses in foreign countries per scenario
137
138
    Parameters
139
    ----------
140
    sources : dict
141
        List of dataset sources
142
    targets : dict
143
        List of dataset targets
144
145
    Returns
146
    -------
147
    central_buses : geoapndas.GeoDataFrame
148
        Buses in the center of foreign countries
149
150
    """
151
    sql_delete = f"""
152
        DELETE FROM {sources['electricity_buses']['schema']}.
153
            {sources['electricity_buses']['table']}
154
        WHERE country != 'DE' AND scn_name = '{scenario}'
155
        AND carrier = 'AC'
156
        AND bus_id NOT IN (
157
            SELECT bus_i
158
            FROM  {sources['osmtgmod_bus']['schema']}.
159
            {sources['osmtgmod_bus']['table']})
160
        """
161
162
    # Delete existing buses
163
    db.execute_sql(sql_delete)
164
165
    central_buses = central_buses_pypsaeur(sources, scenario)
166
167
    next_bus_id = db.next_etrago_id("bus") + 1
168
169
    central_buses["bus_id"] = central_buses.reset_index().index + next_bus_id
170
171
    next_bus_id += len(central_buses)
172
173
    # if in test mode, add bus in center of Germany
174
    if config.settings()["egon-data"]["--dataset-boundary"] != "Everything":
175
        central_buses = pd.concat(
176
            [
177
                central_buses,
178
                pd.DataFrame(
179
                    index=[central_buses.bus_id.max() + 1],
180
                    data={
181
                        "scn_name": scenario,
182
                        "bus_id": next_bus_id,
183
                        "x": 10.4234469,
184
                        "y": 51.0834196,
185
                        "country": "DE",
186
                        "carrier": "AC",
187
                        "v_nom": 380.0,
188
                    },
189
                ),
190
            ],
191
            ignore_index=True,
192
        )
193
        next_bus_id += 1
194
195
    # Add buses for other voltage levels
196
    foreign_buses = get_cross_border_buses(scenario, sources)
197
    if config.settings()["egon-data"]["--dataset-boundary"] == "Everything":
198
        foreign_buses = foreign_buses[foreign_buses.country != "DE"]
199
    vnom_per_country = foreign_buses.groupby("country").v_nom.unique().copy()
200
    for cntr in vnom_per_country.index:
201
        print(cntr)
202 View Code Duplication
        if 110.0 in vnom_per_country[cntr]:
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
203
            central_buses = pd.concat(
204
                [
205
                    central_buses,
206
                    pd.DataFrame(
207
                        index=[next_bus_id],
208
                        data={
209
                            "scn_name": scenario,
210
                            "bus_id": next_bus_id,
211
                            "x": central_buses[
212
                                central_buses.country == cntr
213
                            ].x.unique()[0],
214
                            "y": central_buses[
215
                                central_buses.country == cntr
216
                            ].y.unique()[0],
217
                            "country": cntr,
218
                            "carrier": "AC",
219
                            "v_nom": 110.0,
220
                        },
221
                    ),
222
                ],
223
                ignore_index=True,
224
            )
225
            next_bus_id += 1
226
227 View Code Duplication
        if 220.0 in vnom_per_country[cntr]:
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
228
            central_buses = pd.concat(
229
                [
230
                    central_buses,
231
                    pd.DataFrame(
232
                        index=[next_bus_id],
233
                        data={
234
                            "scn_name": scenario,
235
                            "bus_id": next_bus_id,
236
                            "x": central_buses[
237
                                central_buses.country == cntr
238
                            ].x.unique()[0],
239
                            "y": central_buses[
240
                                central_buses.country == cntr
241
                            ].y.unique()[0],
242
                            "country": cntr,
243
                            "carrier": "AC",
244
                            "v_nom": 220.0,
245
                        },
246
                    ),
247
                ],
248
                ignore_index=True,
249
            )
250
            next_bus_id += 1
251
252
    # Add geometry column
253
    central_buses = gpd.GeoDataFrame(
254
        central_buses,
255
        geometry=gpd.points_from_xy(central_buses.x, central_buses.y),
256
        crs="EPSG:4326",
257
    )
258
    central_buses["geom"] = central_buses.geometry.copy()
259
    central_buses = central_buses.set_geometry("geom").drop(
260
        "geometry", axis="columns"
261
    )
262
    central_buses.scn_name = scenario
263
264
    central_buses.drop(
265
        [
266
            "control",
267
            "generator",
268
            "location",
269
            "unit",
270
            "sub_network",
271
            "substation_off",
272
            "substation_lv",
273
        ],
274
        axis="columns",
275
        inplace=True,
276
        errors="ignore",
277
    )
278
279
    # Insert all central buses for eGon2035
280
    if scenario in [
281
        "eGon2035",
282
        "status2019",
283
        "status2023",
284
    ]:  # TODO: status2023 this is hardcoded shit
285
        central_buses.to_postgis(
286
            targets["buses"]["table"],
287
            schema=targets["buses"]["schema"],
288
            if_exists="append",
289
            con=db.engine(),
290
            index=False,
291
        )
292
    # Insert only buses for eGon100RE that are not coming from pypsa-eur-sec
293
    # (buses with another voltage_level or inside Germany in test mode)
294
    else:
295
        central_buses[central_buses.carrier == "AC"].to_postgis(
296
            targets["buses"]["table"],
297
            schema=targets["buses"]["schema"],
298
            if_exists="append",
299
            con=db.engine(),
300
            index=False,
301
        )
302
303
    return central_buses
304
305
306
def lines_between_foreign_countries(scenario, sorces, targets, central_buses):
307
    # import network from pypsa-eur
308
    network = prepared_network()
309
310
    gdf_buses = gpd.GeoDataFrame(
311
        network.buses,
312
        geometry=gpd.points_from_xy(network.buses.x, network.buses.y),
313
    )
314
315
    central_buses_pypsaeur = gpd.sjoin(
316
        gdf_buses[gdf_buses.carrier == "AC"], central_buses
317
    )
318
319
    central_buses_pypsaeur = central_buses_pypsaeur[
320
        central_buses_pypsaeur.v_nom_right == 380
321
    ]
322
323
    lines_to_add = network.lines[
324
        (network.lines.bus0.isin(central_buses_pypsaeur.index))
325
        & (network.lines.bus1.isin(central_buses_pypsaeur.index))
326
    ]
327
328
    lines_to_add.loc[:, "lifetime"] = get_sector_parameters(
329
        "electricity", scenario
330
    )["lifetime"]["ac_ehv_overhead_line"]
331
    lines_to_add.loc[:, "line_id"] = (
332
        lines_to_add.reset_index().index.astype(int)
333
        + db.next_etrago_id("line")
334
        + 1
335
    )
336
337
    links_to_add = network.links[
338
        (network.links.bus0.isin(central_buses_pypsaeur.index))
339
        & (network.links.bus1.isin(central_buses_pypsaeur.index))
340
    ]
341
342
    links_to_add.loc[:, "lifetime"] = get_sector_parameters(
343
        "electricity", scenario
344
    )["lifetime"]["dc_overhead_line"]
345
    links_to_add.loc[:, "link_id"] = (
346
        links_to_add.reset_index().index.astype(int)
347
        + db.next_etrago_id("link")
348
        + 1
349
    )
350
351
    for df in [lines_to_add, links_to_add]:
352
        df.loc[:, "scn_name"] = scenario
353
        gdf = gpd.GeoDataFrame(df)
354
        gdf["geom_bus0"] = gdf_buses.geometry[df.bus0].values
355
        gdf["geom_bus1"] = gdf_buses.geometry[df.bus1].values
356
        gdf["geometry"] = gdf.apply(
357
            lambda x: LineString([x["geom_bus0"], x["geom_bus1"]]),
358
            axis=1,
359
        )
360
361
        gdf = gdf.set_geometry("geometry")
362
        gdf = gdf.set_crs(4326)
363
364
        gdf = gdf.rename_geometry("topo")
365
366
        gdf.loc[:, "bus0"] = central_buses_pypsaeur.bus_id.loc[df.bus0].values
367
        gdf.loc[:, "bus1"] = central_buses_pypsaeur.bus_id.loc[df.bus1].values
368
369
        gdf.drop(["geom_bus0", "geom_bus1"], inplace=True, axis="columns")
370
        if "link_id" in df.columns:
371
            table_name = "link"
372
            gdf.drop(
373
                [
374
                    "tags",
375
                    "under_construction",
376
                    "underground",
377
                    "underwater_fraction",
378
                    "bus2",
379
                    "efficiency2",
380
                    "length_original",
381
                    "bus4",
382
                    "efficiency4",
383
                    "reversed",
384
                    "ramp_limit_up",
385
                    "ramp_limit_down",
386
                    "p_nom_opt",
387
                    "bus3",
388
                    "efficiency3",
389
                    "location",
390
                    "project_status",
391
                    "dc",
392
                    "voltage",
393
                ],
394
                axis="columns",
395
                inplace=True,
396
            )
397
        else:
398
            table_name = "line"
399
            gdf.drop(
400
                [
401
                    "i_nom",
402
                    "sub_network",
403
                    "x_pu",
404
                    "r_pu",
405
                    "g_pu",
406
                    "b_pu",
407
                    "x_pu_eff",
408
                    "r_pu_eff",
409
                    "s_nom_opt",
410
                    "dc",
411
                ],
412
                axis="columns",
413
                inplace=True,
414
            )
415
416
        gdf = gdf.set_index(f"{table_name}_id")
417
        gdf.to_postgis(
418
            f"egon_etrago_{table_name}",
419
            db.engine(),
420
            schema="grid",
421
            if_exists="append",
422
            index=True,
423
            index_label=f"{table_name}_id",
424
        )
425
426
427
def cross_border_lines(scenario, sources, targets, central_buses):
428
    """Adds lines which connect border-crossing lines from osmtgmod
429
    to the central buses in the corresponding neigbouring country
430
431
    Parameters
432
    ----------
433
    sources : dict
434
        List of dataset sources
435
    targets : dict
436
        List of dataset targets
437
    central_buses : geopandas.GeoDataFrame
438
        Buses in the center of foreign countries
439
440
    Returns
441
    -------
442
    new_lines : geopandas.GeoDataFrame
443
        Lines that connect cross-border lines to central bus per country
444
445
    """
446
    # Delete existing data
447
    db.execute_sql(
448
        f"""
449
        DELETE FROM {targets['lines']['schema']}.
450
        {targets['lines']['table']}
451
        WHERE scn_name = '{scenario}'
452
        AND line_id NOT IN (
453
            SELECT branch_id
454
            FROM  {sources['osmtgmod_branch']['schema']}.
455
            {sources['osmtgmod_branch']['table']}
456
              WHERE result_id = 1 and (link_type = 'line' or
457
                                       link_type = 'cable'))
458
        AND bus0 IN (
459
            SELECT bus_i
460
            FROM  {sources['osmtgmod_bus']['schema']}.
461
            {sources['osmtgmod_bus']['table']})
462
        AND bus1 NOT IN (
463
            SELECT bus_i
464
            FROM  {sources['osmtgmod_bus']['schema']}.
465
            {sources['osmtgmod_bus']['table']})
466
        """
467
    )
468
469
    # Calculate cross-border busses and lines from osmtgmod
470
    foreign_buses = get_cross_border_buses(scenario, sources)
471
    foreign_buses.dropna(subset="country", inplace=True)
472
473
    if config.settings()["egon-data"]["--dataset-boundary"] == "Everything":
474
        foreign_buses = foreign_buses[foreign_buses.country != "DE"]
475
    lines = get_cross_border_lines(scenario, sources)
476
477
    # Select bus outside of Germany from border-crossing lines
478
    lines.loc[
479
        lines[lines.bus0.isin(foreign_buses.bus_id)].index, "foreign_bus"
480
    ] = lines.loc[lines[lines.bus0.isin(foreign_buses.bus_id)].index, "bus0"]
481
    lines.loc[
482
        lines[lines.bus1.isin(foreign_buses.bus_id)].index, "foreign_bus"
483
    ] = lines.loc[lines[lines.bus1.isin(foreign_buses.bus_id)].index, "bus1"]
484
485
    # Drop lines with start and endpoint in Germany
486
    lines = lines[lines.foreign_bus.notnull()]
487
    lines.loc[:, "foreign_bus"] = lines.loc[:, "foreign_bus"].astype(int)
488
489
    # Copy all parameters from border-crossing lines
490
    new_lines = lines.copy().set_crs(4326)
491
492
    # Set bus0 as foreign_bus from osmtgmod
493
    new_lines.bus0 = new_lines.foreign_bus.copy()
494
    new_lines.bus0 = new_lines.bus0.astype(int)
495
496
    # Add country tag and set index
497
    new_lines["country"] = (
498
        foreign_buses.set_index("bus_id")
499
        .loc[lines.foreign_bus, "country"]
500
        .values
501
    )
502
503
    if config.settings()["egon-data"]["--dataset-boundary"] == "Everything":
504
        new_lines = new_lines[~new_lines.country.isnull()]
505
    new_lines.line_id = range(
506
        db.next_etrago_id("line"), db.next_etrago_id("line") + len(new_lines)
507
    )
508
509
    # Set bus in center of foreign countries as bus1
510
    for i, row in new_lines.iterrows():
511
        print(row)
512
        new_lines.loc[i, "bus1"] = central_buses.bus_id[
513
            (central_buses.country == row.country)
514
            & (central_buses.v_nom == row.v_nom)
515
        ].values[0]
516
517
    # Create geometry for new lines
518
    new_lines["geom_bus0"] = (
519
        foreign_buses.set_index("bus_id").geom[new_lines.bus0].values
520
    )
521
    new_lines["geom_bus1"] = (
522
        central_buses.set_index("bus_id").geom[new_lines.bus1].values
523
    )
524
    new_lines["topo"] = new_lines.apply(
525
        lambda x: LineString([x["geom_bus0"], x["geom_bus1"]]), axis=1
526
    )
527
528
    # Set topo as geometry column
529
    new_lines = new_lines.set_geometry("topo").set_crs(4326)
530
    # Calcultae length of lines based on topology
531
    old_length = new_lines["length"].copy()
532
    new_lines["length"] = new_lines.to_crs(3035).length / 1000
533
534
    if (new_lines["length"] == 0).any():
535
        print("WARNING! THERE ARE LINES WITH LENGTH = 0")
536
        condition = new_lines["length"] != 0
537
        new_lines["length"] = new_lines["length"].where(condition, 1)
538
539
    # Set electrical parameters based on lines from osmtgmod
540
    for parameter in ["x", "r"]:
541
        new_lines[parameter] = (
542
            new_lines[parameter] / old_length * new_lines["length"]
543
        )
544
    for parameter in ["b", "g"]:
545
        new_lines[parameter] = (
546
            new_lines[parameter] * old_length / new_lines["length"]
547
        )
548
549
    # Drop intermediate columns
550
    new_lines.drop(
551
        ["foreign_bus", "country", "geom_bus0", "geom_bus1", "geom"],
552
        axis="columns",
553
        inplace=True,
554
    )
555
556
    new_lines = new_lines[new_lines.bus0 != new_lines.bus1]
557
558
    new_lines["cables"] = new_lines["cables"].apply(int)
559
560
    # Insert lines to the database
561
    new_lines.to_postgis(
562
        targets["lines"]["table"],
563
        schema=targets["lines"]["schema"],
564
        if_exists="append",
565
        con=db.engine(),
566
        index=False,
567
    )
568
569
    return new_lines
570
571
572
def choose_transformer(s_nom):
573
    """Select transformer and parameters from existing data in the grid model
574
575
    It is assumed that transformers in the foreign countries are not limiting
576
    the electricity flow, so the capacitiy s_nom is set to the minimum sum
577
    of attached AC-lines.
578
    The electrical parameters are set according to already inserted
579
    transformers in the grid model for Germany.
580
581
    Parameters
582
    ----------
583
    s_nom : float
584
        Minimal sum of nominal power of lines at one side
585
586
    Returns
587
    -------
588
    int
589
        Selected transformer nominal power
590
    float
591
        Selected transformer nominal impedance
592
593
    """
594
595
    if s_nom <= 600:
596
        return 600, 0.0002
597
    elif (s_nom > 600) & (s_nom <= 1200):
598
        return 1200, 0.0001
599
    elif (s_nom > 1200) & (s_nom <= 1600):
600
        return 1600, 0.000075
601
    elif (s_nom > 1600) & (s_nom <= 2100):
602
        return 2100, 0.00006667
603
    elif (s_nom > 2100) & (s_nom <= 2600):
604
        return 2600, 0.0000461538
605
    elif (s_nom > 2600) & (s_nom <= 4800):
606
        return 4800, 0.000025
607
    elif (s_nom > 4800) & (s_nom <= 6000):
608
        return 6000, 0.0000225
609
    elif (s_nom > 6000) & (s_nom <= 7200):
610
        return 7200, 0.0000194444
611
    elif (s_nom > 7200) & (s_nom <= 8000):
612
        return 8000, 0.000016875
613
    elif (s_nom > 8000) & (s_nom <= 9000):
614
        return 9000, 0.000015
615
    elif (s_nom > 9000) & (s_nom <= 13000):
616
        return 13000, 0.0000103846
617
    elif (s_nom > 13000) & (s_nom <= 20000):
618
        return 20000, 0.00000675
619
    elif (s_nom > 20000) & (s_nom <= 33000):
620
        return 33000, 0.00000409091
621
622
623
def central_transformer(scenario, sources, targets, central_buses, new_lines):
624
    """Connect central foreign buses with different voltage levels
625
626
    Parameters
627
    ----------
628
    sources : dict
629
        List of dataset sources
630
    targets : dict
631
        List of dataset targets
632
    central_buses : geopandas.GeoDataFrame
633
        Buses in the center of foreign countries
634
    new_lines : geopandas.GeoDataFrame
635
        Lines that connect cross-border lines to central bus per country
636
637
    Returns
638
    -------
639
    None.
640
641
    """
642
    # Delete existing transformers in foreign countries
643
    db.execute_sql(
644
        f"""
645
        DELETE FROM {targets['transformers']['schema']}.
646
        {targets['transformers']['table']}
647
        WHERE scn_name = '{scenario}'
648
        AND trafo_id NOT IN (
649
            SELECT branch_id
650
            FROM {sources['osmtgmod_branch']['schema']}.
651
            {sources['osmtgmod_branch']['table']}
652
              WHERE result_id = 1 and link_type = 'transformer')
653
        """
654
    )
655
656
    # Initalize the dataframe for transformers
657
    trafo = gpd.GeoDataFrame(
658
        columns=["trafo_id", "bus0", "bus1", "s_nom"], dtype=int
659
    )
660
    trafo_id = db.next_etrago_id("transformer")
661
662
    # Add one transformer per central foreign bus with v_nom != 380
663
    for i, row in central_buses[central_buses.v_nom != 380].iterrows():
664
        s_nom_0 = new_lines[new_lines.bus0 == row.bus_id].s_nom.sum()
665
        s_nom_1 = new_lines[new_lines.bus1 == row.bus_id].s_nom.sum()
666
        if s_nom_0 == 0.0:
667
            s_nom = s_nom_1
668
        elif s_nom_1 == 0.0:
669
            s_nom = s_nom_0
670
        else:
671
            s_nom = min([s_nom_0, s_nom_1])
672
673
        s_nom, x = choose_transformer(s_nom)
674
675
        trafo = pd.concat(
676
            [
677
                trafo,
678
                pd.DataFrame(
679
                    index=[trafo.index.max() + 1],
680
                    data={
681
                        "trafo_id": trafo_id,
682
                        "bus0": row.bus_id,
683
                        "bus1": central_buses[
684
                            (central_buses.v_nom == 380)
685
                            & (central_buses.country == row.country)
686
                        ].bus_id.values[0],
687
                        "s_nom": s_nom,
688
                        "x": x,
689
                    },
690
                ),
691
            ],
692
            ignore_index=True,
693
        )
694
        trafo_id += 1
695
696
    # Set data type
697
    trafo = trafo.astype({"trafo_id": "int", "bus0": "int", "bus1": "int"})
698
    trafo["scn_name"] = scenario
699
700
    # Insert transformers to the database
701
    trafo.to_sql(
702
        targets["transformers"]["table"],
703
        schema=targets["transformers"]["schema"],
704
        if_exists="append",
705
        con=db.engine(),
706
        index=False,
707
    )
708
709
710
def foreign_dc_lines(scenario, sources, targets, central_buses):
711
    """Insert DC lines to foreign countries manually
712
713
    Parameters
714
    ----------
715
    sources : dict
716
        List of dataset sources
717
    targets : dict
718
        List of dataset targets
719
    central_buses : geopandas.GeoDataFrame
720
        Buses in the center of foreign countries
721
722
    Returns
723
    -------
724
    None.
725
726
    """
727
    # Delete existing dc lines to foreign countries
728
    db.execute_sql(
729
        f"""
730
        DELETE FROM {targets['links']['schema']}.
731
        {targets['links']['table']}
732
        WHERE scn_name = '{scenario}'
733
        AND carrier = 'DC'
734
        AND bus0 IN (
735
            SELECT bus_id
736
            FROM {sources['electricity_buses']['schema']}.
737
            {sources['electricity_buses']['table']}
738
              WHERE scn_name = '{scenario}'
739
              AND carrier = 'AC'
740
              AND country = 'DE')
741
        AND bus1 IN (
742
            SELECT bus_id
743
            FROM {sources['electricity_buses']['schema']}.
744
            {sources['electricity_buses']['table']}
745
              WHERE scn_name = '{scenario}'
746
              AND carrier = 'AC'
747
              AND country != 'DE')
748
        """
749
    )
750
    capital_cost = get_sector_parameters("electricity", scenario)[
751
        "capital_cost"
752
    ]
753
754
    # Add DC line from Lübeck to Sweden
755
    converter_luebeck = select_bus_id(
756
        10.802358024202768,
757
        53.897547401787,
758
        380,
759
        scenario,
760
        "AC",
761
        find_closest=True,
762
    )
763
764
    foreign_links = pd.DataFrame(
765
        index=[0],
766
        data={
767
            "link_id": db.next_etrago_id("link"),
768
            "bus0": converter_luebeck,
769
            "bus1": central_buses[
770
                (central_buses.country == "SE") & (central_buses.v_nom == 380)
771
            ]
772
            .iloc[0]
773
            .squeeze()
774
            .bus_id,
775
            "p_nom": 600,
776
            "length": 262,
777
        },
778
    )
779
780
    # When not in test-mode, add DC line from Bentwisch to Denmark
781
    if config.settings()["egon-data"]["--dataset-boundary"] == "Everything":
782
        converter_bentwisch = select_bus_id(
783
            12.213671694775988,
784
            54.09974494662279,
785
            380,
786
            scenario,
787
            "AC",
788
            find_closest=True,
789
        )
790
791
        foreign_links = pd.concat(
792
            [
793
                foreign_links,
794
                pd.DataFrame(
795
                    index=[1],
796
                    data={
797
                        "link_id": db.next_etrago_id("link") + 1,
798
                        "bus0": converter_bentwisch,
799
                        "bus1": central_buses[
800
                            (central_buses.country == "DK")
801
                            & (central_buses.v_nom == 380)
802
                            & (central_buses.x > 10)
803
                        ]
804
                        .iloc[0]
805
                        .squeeze()
806
                        .bus_id,
807
                        "p_nom": 600,
808
                        "length": 170,
809
                    },
810
                ),
811
            ]
812
        )
813
814
    # Set parameters for all DC lines
815
    foreign_links["capital_cost"] = (
816
        capital_cost["dc_cable"] * foreign_links.length
817
        + 2 * capital_cost["dc_inverter"]
818
    )
819
    foreign_links["p_min_pu"] = -1
820
    foreign_links["p_nom_extendable"] = True
821
    foreign_links["p_nom_min"] = foreign_links["p_nom"]
822
    foreign_links["scn_name"] = scenario
823
    foreign_links["carrier"] = "DC"
824
    foreign_links["efficiency"] = 1
825
826
    # Add topology
827
    foreign_links = etrago.link_geom_from_buses(foreign_links, scenario)
828
829
    # Insert DC lines to the database
830
    foreign_links.to_postgis(
831
        targets["links"]["table"],
832
        schema=targets["links"]["schema"],
833
        if_exists="append",
834
        con=db.engine(),
835
        index=False,
836
    )
837
838
839
def grid():
840
    """Insert electrical grid compoenents for neighbouring countries
841
842
    Returns
843
    -------
844
    None.
845
846
    """
847
    # Select sources and targets from dataset configuration
848
    sources = config.datasets()["electrical_neighbours"]["sources"]
849
    targets = config.datasets()["electrical_neighbours"]["targets"]
850
851
    for scenario in config.settings()["egon-data"]["--scenarios"]:
852
        central_buses = buses(scenario, sources, targets)
853
854
        foreign_lines = cross_border_lines(
855
            scenario, sources, targets, central_buses
856
        )
857
858
        central_transformer(
859
            scenario, sources, targets, central_buses, foreign_lines
860
        )
861
862
        foreign_dc_lines(scenario, sources, targets, central_buses)
863
864
        if scenario != "eGon100RE":
865
            lines_between_foreign_countries(
866
                scenario, sources, targets, central_buses
867
            )
868
869
870
def map_carriers_tyndp():
871
    """Map carriers from TYNDP-data to carriers used in eGon
872
    Returns
873
    -------
874
    dict
875
        Carrier from TYNDP and eGon
876
    """
877
    return {
878
        "Battery": "battery",
879
        "DSR": "demand_side_response",
880
        "Gas CCGT new": "gas",
881
        "Gas CCGT old 2": "gas",
882
        "Gas CCGT present 1": "gas",
883
        "Gas CCGT present 2": "gas",
884
        "Gas conventional old 1": "gas",
885
        "Gas conventional old 2": "gas",
886
        "Gas OCGT new": "gas",
887
        "Gas OCGT old": "gas",
888
        "Gas CCGT old 1": "gas",
889
        "Gas CCGT old 2 Bio": "biogas",
890
        "Gas conventional old 2 Bio": "biogas",
891
        "Hard coal new": "coal",
892
        "Hard coal old 1": "coal",
893
        "Hard coal old 2": "coal",
894
        "Hard coal old 2 Bio": "coal",
895
        "Heavy oil old 1": "oil",
896
        "Heavy oil old 1 Bio": "oil",
897
        "Heavy oil old 2": "oil",
898
        "Light oil": "oil",
899
        "Lignite new": "lignite",
900
        "Lignite old 1": "lignite",
901
        "Lignite old 2": "lignite",
902
        "Lignite old 1 Bio": "lignite",
903
        "Lignite old 2 Bio": "lignite",
904
        "Nuclear": "nuclear",
905
        "Offshore Wind": "wind_offshore",
906
        "Onshore Wind": "wind_onshore",
907
        "Other non-RES": "others",
908
        "Other RES": "others",
909
        "P2G": "power_to_gas",
910
        "PS Closed": "pumped_hydro",
911
        "PS Open": "reservoir",
912
        "Reservoir": "reservoir",
913
        "Run-of-River": "run_of_river",
914
        "Solar PV": "solar",
915
        "Solar Thermal": "others",
916
        "Waste": "Other RES",
917
    }
918
919
920 View Code Duplication
def get_foreign_bus_id(scenario):
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
921
    """Calculte the etrago bus id from Nodes of TYNDP based on the geometry
922
923
    Returns
924
    -------
925
    pandas.Series
926
        List of mapped node_ids from TYNDP and etragos bus_id
927
928
    """
929
930
    sources = config.datasets()["electrical_neighbours"]["sources"]
931
932
    bus_id = db.select_geodataframe(
933
        f"""SELECT bus_id, ST_Buffer(geom, 1) as geom, country
934
        FROM grid.egon_etrago_bus
935
        WHERE scn_name = '{scenario}'
936
        AND carrier = 'AC'
937
        AND v_nom = 380.
938
        AND country != 'DE'
939
        AND bus_id NOT IN (
940
            SELECT bus_i
941
            FROM osmtgmod_results.bus_data)
942
        """,
943
        epsg=3035,
944
    )
945
946
    # insert installed capacities
947
    file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}")
948
949
    # Select buses in neighbouring countries as geodataframe
950
    buses = pd.read_excel(
951
        file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(),
952
        sheet_name="Nodes - Dict",
953
    ).query("longitude==longitude")
954
    buses = gpd.GeoDataFrame(
955
        buses,
956
        crs=4326,
957
        geometry=gpd.points_from_xy(buses.longitude, buses.latitude),
958
    ).to_crs(3035)
959
960
    buses["bus_id"] = 0
961
962
    # Select bus_id from etrago with shortest distance to TYNDP node
963
    for i, row in buses.iterrows():
964
        distance = bus_id.set_index("bus_id").geom.distance(row.geometry)
965
        buses.loc[i, "bus_id"] = distance[
966
            distance == distance.min()
967
        ].index.values[0]
968
969
    return buses.set_index("node_id").bus_id
970
971
972
def calc_capacities():
973
    """Calculates installed capacities from TYNDP data
974
975
    Returns
976
    -------
977
    pandas.DataFrame
978
        Installed capacities per foreign node and energy carrier
979
980
    """
981
982
    sources = config.datasets()["electrical_neighbours"]["sources"]
983
984
    countries = [
985
        "AT",
986
        "BE",
987
        "CH",
988
        "CZ",
989
        "DK",
990
        "FR",
991
        "NL",
992
        "NO",
993
        "SE",
994
        "PL",
995
        "UK",
996
    ]
997
998
    # insert installed capacities
999
    file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}")
1000
    df = pd.read_excel(
1001
        file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(),
1002
        sheet_name="Capacity",
1003
    )
1004
1005
    # differneces between different climate years are very small (<1MW)
1006
    # choose 1984 because it is the mean value
1007
    df_2030 = (
1008
        df.rename({"Climate Year": "Climate_Year"}, axis="columns")
1009
        .query(
1010
            'Scenario == "Distributed Energy" & Year == 2030 & '
1011
            "Climate_Year == 1984"
1012
        )
1013
        .set_index(["Node/Line", "Generator_ID"])
1014
    )
1015
1016
    df_2040 = (
1017
        df.rename({"Climate Year": "Climate_Year"}, axis="columns")
1018
        .query(
1019
            'Scenario == "Distributed Energy" & Year == 2040 & '
1020
            "Climate_Year == 1984"
1021
        )
1022
        .set_index(["Node/Line", "Generator_ID"])
1023
    )
1024
1025
    # interpolate linear between 2030 and 2040 for 2035 accordning to
1026
    # scenario report of TSO's and the approval by BNetzA
1027
    df_2035 = pd.DataFrame(index=df_2030.index)
1028
    df_2035["cap_2030"] = df_2030.Value
1029
    df_2035["cap_2040"] = df_2040.Value
1030
    df_2035.fillna(0.0, inplace=True)
1031
    df_2035["cap_2035"] = (
1032
        df_2035["cap_2030"] + (df_2035["cap_2040"] - df_2035["cap_2030"]) / 2
1033
    )
1034
    df_2035 = df_2035.reset_index()
1035
    df_2035["carrier"] = df_2035.Generator_ID.map(map_carriers_tyndp())
1036
1037
    # group capacities by new carriers
1038
    grouped_capacities = (
1039
        df_2035.groupby(["carrier", "Node/Line"]).cap_2035.sum().reset_index()
1040
    )
1041
1042
    # choose capacities for considered countries
1043
    return grouped_capacities[
1044
        grouped_capacities["Node/Line"].str[:2].isin(countries)
1045
    ]
1046
1047
1048
def insert_generators_tyndp(capacities):
1049
    """Insert generators for foreign countries based on TYNDP-data
1050
1051
    Parameters
1052
    ----------
1053
    capacities : pandas.DataFrame
1054
        Installed capacities per foreign node and energy carrier
1055
1056
    Returns
1057
    -------
1058
    None.
1059
1060
    """
1061
    targets = config.datasets()["electrical_neighbours"]["targets"]
1062
    map_buses = get_map_buses()
1063
1064
    # Delete existing data
1065
    db.execute_sql(
1066
        f"""
1067
        DELETE FROM
1068
        {targets['generators']['schema']}.{targets['generators']['table']}
1069
        WHERE bus IN (
1070
            SELECT bus_id FROM
1071
            {targets['buses']['schema']}.{targets['buses']['table']}
1072
            WHERE country != 'DE'
1073
            AND scn_name = 'eGon2035')
1074
        AND scn_name = 'eGon2035'
1075
        AND carrier != 'CH4'
1076
        """
1077
    )
1078
1079
    db.execute_sql(
1080
        f"""
1081
        DELETE FROM
1082
        {targets['generators_timeseries']['schema']}.
1083
        {targets['generators_timeseries']['table']}
1084
        WHERE generator_id NOT IN (
1085
            SELECT generator_id FROM
1086
            {targets['generators']['schema']}.{targets['generators']['table']}
1087
        )
1088
        AND scn_name = 'eGon2035'
1089
        """
1090
    )
1091
1092
    # Select generators from TYNDP capacities
1093
    gen = capacities[
1094
        capacities.carrier.isin(
1095
            [
1096
                "others",
1097
                "wind_offshore",
1098
                "wind_onshore",
1099
                "solar",
1100
                "reservoir",
1101
                "run_of_river",
1102
                "lignite",
1103
                "coal",
1104
                "oil",
1105
                "nuclear",
1106
            ]
1107
        )
1108
    ]
1109
1110
    # Set bus_id
1111
    gen.loc[
1112
        gen[gen["Node/Line"].isin(map_buses.keys())].index, "Node/Line"
1113
    ] = gen.loc[
1114
        gen[gen["Node/Line"].isin(map_buses.keys())].index, "Node/Line"
1115
    ].map(
1116
        map_buses
1117
    )
1118
1119
    gen.loc[:, "bus"] = (
1120
        get_foreign_bus_id(scenario="eGon2035")
1121
        .loc[gen.loc[:, "Node/Line"]]
1122
        .values
1123
    )
1124
1125
    # Add scenario column
1126
    gen["scenario"] = "eGon2035"
1127
1128
    # Add marginal costs
1129
    gen = add_marginal_costs(gen)
1130
1131
    # insert generators data
1132
    session = sessionmaker(bind=db.engine())()
1133
    for i, row in gen.iterrows():
1134
        entry = etrago.EgonPfHvGenerator(
1135
            scn_name=row.scenario,
1136
            generator_id=int(db.next_etrago_id("generator")),
1137
            bus=row.bus,
1138
            carrier=row.carrier,
1139
            p_nom=row.cap_2035,
1140
            marginal_cost=row.marginal_cost,
1141
        )
1142
1143
        session.add(entry)
1144
        session.commit()
1145
1146
    # assign generators time-series data
1147
1148
    renewable_timeseries_pypsaeur("eGon2035")
1149
1150
1151
def insert_storage_tyndp(capacities):
1152
    """Insert storage units for foreign countries based on TYNDP-data
1153
1154
    Parameters
1155
    ----------
1156
    capacities : pandas.DataFrame
1157
        Installed capacities per foreign node and energy carrier
1158
1159
1160
    Returns
1161
    -------
1162
    None.
1163
1164
    """
1165
    targets = config.datasets()["electrical_neighbours"]["targets"]
1166
    map_buses = get_map_buses()
1167
1168
    # Delete existing data
1169
    db.execute_sql(
1170
        f"""
1171
        DELETE FROM {targets['storage']['schema']}.{targets['storage']['table']}
1172
        WHERE bus IN (
1173
            SELECT bus_id FROM
1174
            {targets['buses']['schema']}.{targets['buses']['table']}
1175
            WHERE country != 'DE'
1176
            AND scn_name = 'eGon2035')
1177
        AND scn_name = 'eGon2035'
1178
        """
1179
    )
1180
1181
    # Add missing information suitable for eTraGo selected from scenario_parameter table
1182
    parameters_pumped_hydro = scenario_parameters.electricity("eGon2035")[
1183
        "efficiency"
1184
    ]["pumped_hydro"]
1185
1186
    parameters_battery = scenario_parameters.electricity("eGon2035")[
1187
        "efficiency"
1188
    ]["battery"]
1189
1190
    # Select storage capacities from TYNDP-data
1191
    store = capacities[capacities.carrier.isin(["battery", "pumped_hydro"])]
1192
1193
    # Set bus_id
1194
    store.loc[
1195
        store[store["Node/Line"].isin(map_buses.keys())].index, "Node/Line"
1196
    ] = store.loc[
1197
        store[store["Node/Line"].isin(map_buses.keys())].index, "Node/Line"
1198
    ].map(
1199
        map_buses
1200
    )
1201
1202
    store.loc[:, "bus"] = (
1203
        get_foreign_bus_id(scenario="eGon2035")
1204
        .loc[store.loc[:, "Node/Line"]]
1205
        .values
1206
    )
1207
1208
    # Add columns for additional parameters to df
1209
    (
1210
        store["dispatch"],
1211
        store["store"],
1212
        store["standing_loss"],
1213
        store["max_hours"],
1214
    ) = (None, None, None, None)
1215
1216
    # Insert carrier specific parameters
1217
1218
    parameters = ["dispatch", "store", "standing_loss", "max_hours"]
1219
1220
    for x in parameters:
1221
        store.loc[store["carrier"] == "battery", x] = parameters_battery[x]
1222
        store.loc[store["carrier"] == "pumped_hydro", x] = (
1223
            parameters_pumped_hydro[x]
1224
        )
1225
1226
    # insert data
1227
    session = sessionmaker(bind=db.engine())()
1228
    for i, row in store.iterrows():
1229
        entry = etrago.EgonPfHvStorage(
1230
            scn_name="eGon2035",
1231
            storage_id=int(db.next_etrago_id("storage")),
1232
            bus=row.bus,
1233
            max_hours=row.max_hours,
1234
            efficiency_store=row.store,
1235
            efficiency_dispatch=row.dispatch,
1236
            standing_loss=row.standing_loss,
1237
            carrier=row.carrier,
1238
            p_nom=row.cap_2035,
1239
        )
1240
1241
        session.add(entry)
1242
        session.commit()
1243
1244
1245
def get_map_buses():
1246
    """Returns a dictonary of foreign regions which are aggregated to another
1247
1248
    Returns
1249
    -------
1250
    Combination of aggregated regions
1251
1252
1253
    """
1254
    return {
1255
        "DK00": "DKW1",
1256
        "DKKF": "DKE1",
1257
        "FR15": "FR00",
1258
        "NON1": "NOM1",
1259
        "NOS0": "NOM1",
1260
        "NOS1": "NOM1",
1261
        "PLE0": "PL00",
1262
        "PLI0": "PL00",
1263
        "SE00": "SE02",
1264
        "SE01": "SE02",
1265
        "SE03": "SE02",
1266
        "SE04": "SE02",
1267
        "RU": "RU00",
1268
    }
1269
1270
1271
def tyndp_generation():
1272
    """Insert data from TYNDP 2020 accordning to NEP 2021
1273
    Scenario 'Distributed Energy', linear interpolate between 2030 and 2040
1274
1275
    Returns
1276
    -------
1277
    None.
1278
    """
1279
1280
    capacities = calc_capacities()
1281
1282
    insert_generators_tyndp(capacities)
1283
1284
    insert_storage_tyndp(capacities)
1285
1286
1287
def tyndp_demand():
1288
    """Copy load timeseries data from TYNDP 2020.
1289
    According to NEP 2021, the data for 2030 and 2040 is interpolated linearly.
1290
1291
    Returns
1292
    -------
1293
    None.
1294
1295
    """
1296
    map_buses = get_map_buses()
1297
1298
    sources = config.datasets()["electrical_neighbours"]["sources"]
1299
    targets = config.datasets()["electrical_neighbours"]["targets"]
1300
1301
    # Delete existing data
1302
    db.execute_sql(
1303
        f"""
1304
        DELETE FROM {targets['loads']['schema']}.
1305
        {targets['loads']['table']}
1306
        WHERE
1307
        scn_name = 'eGon2035'
1308
        AND carrier = 'AC'
1309
        AND bus NOT IN (
1310
            SELECT bus_i
1311
            FROM  {sources['osmtgmod_bus']['schema']}.
1312
            {sources['osmtgmod_bus']['table']})
1313
        """
1314
    )
1315
1316
    # Connect to database
1317
    engine = db.engine()
1318
    session = sessionmaker(bind=engine)()
1319
1320
    nodes = [
1321
        "AT00",
1322
        "BE00",
1323
        "CH00",
1324
        "CZ00",
1325
        "DKE1",
1326
        "DKW1",
1327
        "FR00",
1328
        "NL00",
1329
        "LUB1",
1330
        "LUF1",
1331
        "LUG1",
1332
        "NOM1",
1333
        "NON1",
1334
        "NOS0",
1335
        "SE01",
1336
        "SE02",
1337
        "SE03",
1338
        "SE04",
1339
        "PL00",
1340
        "UK00",
1341
        "UKNI",
1342
    ]
1343
    # Assign etrago bus_id to TYNDP nodes
1344
    buses = pd.DataFrame({"nodes": nodes})
1345
    buses.loc[buses[buses.nodes.isin(map_buses.keys())].index, "nodes"] = (
1346
        buses[buses.nodes.isin(map_buses.keys())].nodes.map(map_buses)
1347
    )
1348
    buses.loc[:, "bus"] = (
1349
        get_foreign_bus_id(scenario="eGon2035")
1350
        .loc[buses.loc[:, "nodes"]]
1351
        .values
1352
    )
1353
    buses.set_index("nodes", inplace=True)
1354
    buses = buses[~buses.index.duplicated(keep="first")]
1355
1356
    # Read in data from TYNDP for 2030 and 2040
1357
    dataset_2030 = pd.read_excel(
1358
        f"tyndp/{sources['tyndp_demand_2030']}", sheet_name=nodes, skiprows=10
1359
    )
1360
1361
    dataset_2040 = pd.read_excel(
1362
        f"tyndp/{sources['tyndp_demand_2040']}", sheet_name=None, skiprows=10
1363
    )
1364
1365
    # Transform map_buses to pandas.Series and select only used values
1366
    map_series = pd.Series(map_buses)
1367
    map_series = map_series[map_series.index.isin(nodes)]
1368
1369
    # Calculate and insert demand timeseries per etrago bus_id
1370
    for bus in buses.index:
1371
        nodes = [bus]
1372
1373
        if bus in map_series.values:
1374
            nodes.extend(list(map_series[map_series == bus].index.values))
1375
1376
        load_id = db.next_etrago_id("load")
1377
1378
        # Some etrago bus_ids represent multiple TYNDP nodes,
1379
        # in this cases the loads are summed
1380
        data_2030 = pd.Series(index=range(8760), data=0.0)
1381
        for node in nodes:
1382
            data_2030 = dataset_2030[node][2011] + data_2030
1383
1384
        try:
1385
            data_2040 = pd.Series(index=range(8760), data=0.0)
1386
1387
            for node in nodes:
1388
                data_2040 = dataset_2040[node][2011] + data_2040
1389
        except:
1390
            data_2040 = data_2030
1391
1392
        # According to the NEP, data for 2030 and 2040 is linear interpolated
1393
        data_2035 = ((data_2030 + data_2040) / 2)[:8760]
1394
1395
        entry = etrago.EgonPfHvLoad(
1396
            scn_name="eGon2035",
1397
            load_id=int(load_id),
1398
            carrier="AC",
1399
            bus=int(buses.bus[bus]),
1400
        )
1401
1402
        entry_ts = etrago.EgonPfHvLoadTimeseries(
1403
            scn_name="eGon2035",
1404
            load_id=int(load_id),
1405
            temp_id=1,
1406
            p_set=list(data_2035.values),
1407
        )
1408
1409
        session.add(entry)
1410
        session.add(entry_ts)
1411
        session.commit()
1412
1413
1414
def get_entsoe_token():
1415
    """Check for token in home dir. If not exists, check in working dir"""
1416
    token_path = path.join(path.expanduser("~"), ".entsoe-token")
1417
    if not os.path.isfile(token_path):
1418
        logger.info(
1419
            f"Token file not found at {token_path}. Will check in working directory."
1420
        )
1421
        token_path = Path(".entsoe-token")
1422
        if os.path.isfile(token_path):
1423
            logger.info(f"Token found at {token_path}")
1424
    entsoe_token = open(token_path, "r").read(36)
1425
    if entsoe_token is None:
1426
        raise FileNotFoundError("No entsoe-token found.")
1427
    return entsoe_token
1428
1429
1430
def entsoe_historic_generation_capacities(
1431
    year_start="20190101", year_end="20200101"
1432
):
1433
    entsoe_token = get_entsoe_token()
1434
    client = entsoe.EntsoePandasClient(api_key=entsoe_token)
1435
1436
    start = pd.Timestamp(year_start, tz="Europe/Brussels")
1437
    end = pd.Timestamp(year_end, tz="Europe/Brussels")
1438
    start_gb = pd.Timestamp(year_start, tz="Europe/London")
1439
    end_gb = pd.Timestamp(year_end, tz="Europe/London")
1440
    countries = [
1441
        "LU",
1442
        "AT",
1443
        "FR",
1444
        "NL",
1445
        "CZ",
1446
        "DK_1",
1447
        "DK_2",
1448
        "PL",
1449
        "CH",
1450
        "NO",
1451
        "BE",
1452
        "SE",
1453
        "GB",
1454
    ]
1455
    # No GB data after Brexit
1456
    if int(year_start[:4]) > 2021:
1457
        logger.warning(
1458
            "No GB data after Brexit. GB is dropped from entsoe query!"
1459
        )
1460
        countries = [c for c in countries if c != "GB"]
1461
    # todo: define wanted countries
1462
1463
    not_retrieved = []
1464
    dfs = []
1465
    for country in countries:
1466
        if country == "GB":
1467
            kwargs = dict(start=start_gb, end=end_gb)
1468
        else:
1469
            kwargs = dict(start=start, end=end)
1470
        try:
1471
            country_data = client.query_installed_generation_capacity(
1472
                country, **kwargs
1473
            )
1474
            dfs.append(country_data)
1475
        except (entsoe.exceptions.NoMatchingDataError, requests.HTTPError):
1476
            logger.warning(
1477
                f"Data for country: {country} could not be retrieved."
1478
            )
1479
            not_retrieved.append(country)
1480
            pass
1481
1482
    if dfs:
1483
        df = pd.concat(dfs)
1484
        df["country"] = [c for c in countries if c not in not_retrieved]
1485
        df.set_index("country", inplace=True)
1486
        if int(year_start[:4]) == 2023:
1487
            # https://www.bmreports.com/bmrs/?q=foregeneration/capacityaggregated
1488
            # could probably somehow be automised
1489
            # https://www.elexonportal.co.uk/category/view/178
1490
            # in MW
1491
            installed_capacity_gb = pd.Series(
1492
                {
1493
                    "Biomass": 4438,
1494
                    "Fossil Gas": 37047,
1495
                    "Fossil Hard coal": 1491,
1496
                    "Hydro Pumped Storage": 5603,
1497
                    "Hydro Run-of-river and poundage": 2063,
1498
                    "Nuclear": 4950,
1499
                    "Other": 3313,
1500
                    "Other renewable": 1462,
1501
                    "Solar": 14518,
1502
                    "Wind Offshore": 13038,
1503
                    "Wind Onshore": 13907,
1504
                },
1505
                name="GB",
1506
            )
1507
            df = pd.concat([df.T, installed_capacity_gb], axis=1).T
1508
            logger.info("Manually added generation capacities for GB 2023.")
1509
            not_retrieved = [c for c in not_retrieved if c != "GB"]
1510
        df.fillna(0, inplace=True)
1511
    else:
1512
        df = pd.DataFrame()
1513
    return df, not_retrieved
1514
1515
1516
def entsoe_historic_demand(year_start="20190101", year_end="20200101"):
1517
    entsoe_token = get_entsoe_token()
1518
    client = entsoe.EntsoePandasClient(api_key=entsoe_token)
1519
1520
    start = pd.Timestamp(year_start, tz="Europe/Brussels")
1521
    end = pd.Timestamp(year_end, tz="Europe/Brussels")
1522
    start_gb = start.tz_convert("Europe/London")
1523
    end_gb = end.tz_convert("Europe/London")
1524
1525
    countries = [
1526
        "LU",
1527
        "AT",
1528
        "FR",
1529
        "NL",
1530
        "CZ",
1531
        "DK_1",
1532
        "DK_2",
1533
        "PL",
1534
        "CH",
1535
        "NO",
1536
        "BE",
1537
        "SE",
1538
        "GB",
1539
    ]
1540
1541
    # todo: define wanted countries
1542
1543
    not_retrieved = []
1544
    dfs = []
1545
1546
    for country in countries:
1547
        if country == "GB":
1548
            kwargs = dict(start=start_gb, end=end_gb)
1549
        else:
1550
            kwargs = dict(start=start, end=end)
1551
        try:
1552
            country_data = (
1553
                client.query_load(country, **kwargs)
1554
                .resample("H")["Actual Load"]
1555
                .mean()
1556
            )
1557
            if country == "GB":
1558
                country_data.index = country_data.index.tz_convert(
1559
                    "Europe/Brussels"
1560
                )
1561
            dfs.append(country_data)
1562
        except (entsoe.exceptions.NoMatchingDataError, requests.HTTPError):
1563
            not_retrieved.append(country)
1564
            logger.warning(
1565
                f"Data for country: {country} could not be retrieved."
1566
            )
1567
            pass
1568
1569
    if dfs:
1570
        df = pd.concat(dfs, axis=1)
1571
        df.columns = [c for c in countries if c not in not_retrieved]
1572
        df.index = pd.date_range(year_start, periods=8760, freq="H")
1573
    else:
1574
        df = pd.DataFrame()
1575
    return df, not_retrieved
1576
1577
1578
def map_carriers_entsoe():
1579
    """Map carriers from entsoe-data to carriers used in eGon
1580
    Returns
1581
    -------
1582
    dict
1583
        Carrier from entsoe to eGon
1584
    """
1585
    return {
1586
        "Biomass": "biomass",
1587
        "Fossil Brown coal/Lignite": "lignite",
1588
        "Fossil Coal-derived gas": "coal",
1589
        "Fossil Gas": "OCGT",
1590
        "Fossil Hard coal": "coal",
1591
        "Fossil Oil": "oil",
1592
        "Fossil Oil shale": "oil",
1593
        "Fossil Peat": "others",
1594
        "Geothermal": "geo_thermal",
1595
        "Hydro Pumped Storage": "Hydro Pumped Storage",
1596
        "Hydro Run-of-river and poundage": "run_of_river",
1597
        "Hydro Water Reservoir": "reservoir",
1598
        "Marine": "others",
1599
        "Nuclear": "nuclear",
1600
        "Other": "others",
1601
        "Other renewable": "others",
1602
        "Solar": "solar",
1603
        "Waste": "others",
1604
        "Wind Offshore": "wind_offshore",
1605
        "Wind Onshore": "wind_onshore",
1606
    }
1607
1608
1609
def entsoe_to_bus_etrago(scenario="status2019"):
1610
    map_entsoe = pd.Series(
1611
        {
1612
            "LU": "LU00",
1613
            "AT": "AT00",
1614
            "FR": "FR00",
1615
            "NL": "NL00",
1616
            "DK_1": "DK00",
1617
            "DK_2": "DKE1",
1618
            "PL": "PL00",
1619
            "CH": "CH00",
1620
            "NO": "NO00",
1621
            "BE": "BE00",
1622
            "SE": "SE00",
1623
            "GB": "UK00",
1624
            "CZ": "CZ00",
1625
        }
1626
    )
1627
1628
    for_bus = get_foreign_bus_id(scenario=scenario)
1629
1630
    return map_entsoe.map(for_bus)
1631
1632
1633
def save_entsoe_data(df: pd.DataFrame, file_path: Path):
1634
    os.makedirs(file_path.parent, exist_ok=True)
1635
    if not df.empty:
1636
        df.to_csv(file_path, index_label="Index")
1637
        logger.info(
1638
            f"Saved entsoe data for {file_path.stem} "
1639
            f"to {file_path.parent} for countries: {df.index}"
1640
        )
1641
1642
1643
def fill_by_backup_data_from_former_runs(df_sq, file_path, not_retrieved):
1644
    """
1645
    Fills missing data from former runs
1646
    Parameters
1647
    ----------
1648
    df_sq: pd.DataFrame
1649
    file_path: str, Path
1650
    not_retrieved: list
1651
1652
    Returns
1653
    -------
1654
    df_sq, not_retrieved
1655
1656
    """
1657
    sq_backup = pd.read_csv(file_path, index_col="Index")
1658
    # check for missing columns in backup (former runs)
1659
    c_backup = [c for c in sq_backup.columns if c in not_retrieved]
1660
    # remove columns, if found in backup
1661
    not_retrieved = [c for c in not_retrieved if c not in c_backup]
1662
    if c_backup:
1663
        df_sq = pd.concat([df_sq, sq_backup.loc[:, c_backup]], axis=1)
1664
        logger.info(f"Appended data from former runs for {c_backup}")
1665
    return df_sq, not_retrieved
1666
1667
1668
def insert_storage_units_sq(scn_name="status2019"):
1669
    """
1670
    Insert storage_units for foreign countries based on ENTSO-E data
1671
1672
    Parameters
1673
    ----------
1674
    scn_name : str
1675
        Scenario to which the foreign storage units will be assigned.
1676
        The default is "status2019".
1677
1678
    Returns
1679
    -------
1680
    None.
1681
1682
    """
1683
    if "status" in scn_name:
1684
        year = int(scn_name.split("status")[-1])
1685
        year_start_end = {
1686
            "year_start": f"{year}0101",
1687
            "year_end": f"{year+1}0101",
1688
        }
1689
    else:
1690
        raise ValueError("No valid scenario name!")
1691
1692
    df_gen_sq, not_retrieved = entsoe_historic_generation_capacities(
1693
        **year_start_end
1694
    )
1695
1696 View Code Duplication
    if not_retrieved:
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
1697
        logger.warning("Generation data from entsoe could not be retrieved.")
1698
        # check for generation backup from former runs
1699
        file_path = Path(
1700
            "./",
1701
            "data_bundle_egon_data",
1702
            "entsoe",
1703
            f"gen_entsoe_{scn_name}.csv",
1704
        ).resolve()
1705
        if os.path.isfile(file_path):
1706
            df_gen_sq, not_retrieved = fill_by_backup_data_from_former_runs(
1707
                df_gen_sq, file_path, not_retrieved
1708
            )
1709
        save_entsoe_data(df_gen_sq, file_path=file_path)
1710
1711
    sto_sq = df_gen_sq.loc[:, df_gen_sq.columns == "Hydro Pumped Storage"]
1712
    sto_sq.rename(columns={"Hydro Pumped Storage": "p_nom"}, inplace=True)
1713
1714
    targets = config.datasets()["electrical_neighbours"]["targets"]
1715
1716
    # Delete existing data
1717
    db.execute_sql(
1718
        f"""
1719
        DELETE FROM {targets['storage']['schema']}.{targets['storage']['table']}
1720
        WHERE bus IN (
1721
            SELECT bus_id FROM
1722
            {targets['buses']['schema']}.{targets['buses']['table']}
1723
            WHERE country != 'DE'
1724
            AND scn_name = '{scn_name}')
1725
        AND scn_name = '{scn_name}'
1726
        """
1727
    )
1728
1729
    # Add missing information suitable for eTraGo selected from scenario_parameter table
1730
    parameters_pumped_hydro = get_sector_parameters(
1731
        sector="electricity", scenario=scn_name
1732
    )["efficiency"]["pumped_hydro"]
1733
1734
    # Set bus_id
1735
    entsoe_to_bus = entsoe_to_bus_etrago(scenario=scn_name)
1736
    sto_sq["bus"] = sto_sq.index.map(entsoe_to_bus)
1737
1738
    # Insert carrier specific parameters
1739
    sto_sq["carrier"] = "pumped_hydro"
1740
    sto_sq["scn_name"] = scn_name
1741
    sto_sq["dispatch"] = parameters_pumped_hydro["dispatch"]
1742
    sto_sq["store"] = parameters_pumped_hydro["store"]
1743
    sto_sq["standing_loss"] = parameters_pumped_hydro["standing_loss"]
1744
    sto_sq["max_hours"] = parameters_pumped_hydro["max_hours"]
1745
    sto_sq["cyclic_state_of_charge"] = parameters_pumped_hydro[
1746
        "cyclic_state_of_charge"
1747
    ]
1748
1749
    next_id = int(db.next_etrago_id("storage"))
1750
    sto_sq["storage_id"] = range(next_id, next_id + len(sto_sq))
1751
1752
    # Delete entrances without any installed capacity
1753
    sto_sq = sto_sq[sto_sq["p_nom"] > 0]
1754
1755
    # insert data pumped_hydro storage
1756
1757
    with session_scope() as session:
1758
        for i, row in sto_sq.iterrows():
1759
            entry = etrago.EgonPfHvStorage(
1760
                scn_name=scn_name,
1761
                storage_id=row.storage_id,
1762
                bus=row.bus,
1763
                max_hours=row.max_hours,
1764
                efficiency_store=row.store,
1765
                efficiency_dispatch=row.dispatch,
1766
                standing_loss=row.standing_loss,
1767
                carrier=row.carrier,
1768
                p_nom=row.p_nom,
1769
                cyclic_state_of_charge=row.cyclic_state_of_charge,
1770
            )
1771
            session.add(entry)
1772
            session.commit()
1773
1774
    # big scale batteries
1775
    # info based on EASE data. https://ease-storage.eu/publication/emmes-7-0-march-2023/
1776
    # batteries smaller than 100MW are neglected
1777
1778
    # TODO: include capacities between 2020 and 2023
1779
    bat_per_country = {
1780
        "LU": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1781
        "AT": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1782
        "FR": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1783
        "NL": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1784
        "DK_1": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1785
        "DK_2": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1786
        "PL": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1787
        "CH": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1788
        "NO": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1789
        "BE": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1790
        "SE": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1791
        "GB": [723.8, 952.3, 1380.9, 2333.3, 3928.5],
1792
        "CZ": [0, pd.NA, pd.NA, pd.NA, pd.NA],
1793
    }
1794
    bat_sq = pd.DataFrame(bat_per_country).T.set_axis(
1795
        ["2019", "2020", "2021", "2022", "2023"], axis=1
1796
    )
1797
1798
    # Select year of interest
1799
    bat_sq = bat_sq[[str(year)]]
1800
    bat_sq.rename(columns={str(year): "p_nom"}, inplace=True)
1801
1802
    # Add missing information suitable for eTraGo selected from scenario_parameter table
1803
    parameters_batteries = get_sector_parameters(
1804
        sector="electricity", scenario=scn_name
1805
    )["efficiency"]["battery"]
1806
1807
    # Set bus_id
1808
    entsoe_to_bus = entsoe_to_bus_etrago()
1809
    bat_sq["bus"] = bat_sq.index.map(entsoe_to_bus)
1810
1811
    # Insert carrier specific parameters
1812
    bat_sq["carrier"] = "battery"
1813
    bat_sq["scn_name"] = scn_name
1814
    bat_sq["dispatch"] = parameters_batteries["dispatch"]
1815
    bat_sq["store"] = parameters_batteries["store"]
1816
    bat_sq["standing_loss"] = parameters_batteries["standing_loss"]
1817
    bat_sq["max_hours"] = parameters_batteries["max_hours"]
1818
    bat_sq["cyclic_state_of_charge"] = parameters_batteries[
1819
        "cyclic_state_of_charge"
1820
    ]
1821
1822
    next_id = int(db.next_etrago_id("storage"))
1823
    bat_sq["storage_id"] = range(next_id, next_id + len(bat_sq))
1824
1825
    # Delete entrances without any installed capacity
1826
    bat_sq = bat_sq[bat_sq["p_nom"] > 0]
1827
1828
    # insert data pumped_hydro storage
1829
    with db.session_scope() as session:
1830
        for i, row in bat_sq.iterrows():
1831
            entry = etrago.EgonPfHvStorage(
1832
                scn_name=scn_name,
1833
                storage_id=row.storage_id,
1834
                bus=row.bus,
1835
                max_hours=row.max_hours,
1836
                efficiency_store=row.store,
1837
                efficiency_dispatch=row.dispatch,
1838
                standing_loss=row.standing_loss,
1839
                carrier=row.carrier,
1840
                p_nom=row.p_nom,
1841
                cyclic_state_of_charge=row.cyclic_state_of_charge,
1842
            )
1843
            session.add(entry)
1844
            session.commit()
1845
1846
1847
def insert_generators_sq(scn_name="status2019"):
1848
    """
1849
    Insert generators for foreign countries based on ENTSO-E data
1850
1851
    Parameters
1852
    ----------
1853
    gen_sq : pandas dataframe
1854
        df with all the foreign generators produced by the function
1855
        entsoe_historic_generation_capacities
1856
    scn_name : str
1857
        The default is "status2019".
1858
1859
    Returns
1860
    -------
1861
    None.
1862
1863
    """
1864
    if "status" in scn_name:
1865
        year = int(scn_name.split("status")[-1])
1866
        year_start_end = {
1867
            "year_start": f"{year}0101",
1868
            "year_end": f"{year+1}0101",
1869
        }
1870
    else:
1871
        raise ValueError("No valid scenario name!")
1872
1873
    df_gen_sq, not_retrieved = entsoe_historic_generation_capacities(
1874
        **year_start_end
1875
    )
1876
1877 View Code Duplication
    if not_retrieved:
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
1878
        logger.warning("Generation data from entsoe could not be retrieved.")
1879
        # check for generation backup from former runs
1880
        file_path = Path(
1881
            "./",
1882
            "data_bundle_egon_data",
1883
            "entsoe",
1884
            f"gen_entsoe_{scn_name}.csv",
1885
        ).resolve()
1886
        if os.path.isfile(file_path):
1887
            df_gen_sq, not_retrieved = fill_by_backup_data_from_former_runs(
1888
                df_gen_sq, file_path, not_retrieved
1889
            )
1890
        save_entsoe_data(df_gen_sq, file_path=file_path)
1891
1892
    targets = config.datasets()["electrical_neighbours"]["targets"]
1893
    # Delete existing data
1894
    db.execute_sql(
1895
        f"""
1896
        DELETE FROM
1897
        {targets['generators']['schema']}.{targets['generators']['table']}
1898
        WHERE bus IN (
1899
            SELECT bus_id FROM
1900
            {targets['buses']['schema']}.{targets['buses']['table']}
1901
            WHERE country != 'DE'
1902
            AND scn_name = '{scn_name}')
1903
        AND scn_name = '{scn_name}'
1904
        AND carrier != 'CH4'
1905
        """
1906
    )
1907
1908
    db.execute_sql(
1909
        f"""
1910
        DELETE FROM
1911
        {targets['generators_timeseries']['schema']}.
1912
        {targets['generators_timeseries']['table']}
1913
        WHERE generator_id NOT IN (
1914
            SELECT generator_id FROM
1915
            {targets['generators']['schema']}.{targets['generators']['table']}
1916
        )
1917
        AND scn_name = '{scn_name}'
1918
        """
1919
    )
1920
    entsoe_to_bus = entsoe_to_bus_etrago(scn_name)
1921
    carrier_entsoe = map_carriers_entsoe()
1922
    df_gen_sq = df_gen_sq.groupby(axis=1, by=carrier_entsoe).sum()
1923
1924
    # Filter generators modeled as storage and geothermal
1925
    df_gen_sq = df_gen_sq.loc[
1926
        :, ~df_gen_sq.columns.isin(["Hydro Pumped Storage", "geo_thermal"])
1927
    ]
1928
1929
    list_gen_sq = pd.DataFrame(
1930
        dtype=int, columns=["carrier", "country", "capacity"]
1931
    )
1932
    for carrier in df_gen_sq.columns:
1933
        gen_carry = df_gen_sq[carrier]
1934
        for country, cap in gen_carry.items():
1935
            gen = pd.DataFrame(
1936
                {"carrier": carrier, "country": country, "capacity": cap},
1937
                index=[1],
1938
            )
1939
            # print(gen)
1940
            list_gen_sq = pd.concat([list_gen_sq, gen], ignore_index=True)
1941
1942
    list_gen_sq = list_gen_sq[list_gen_sq.capacity > 0]
1943
    list_gen_sq["scenario"] = scn_name
1944
1945
    # Add marginal costs
1946
    list_gen_sq = add_marginal_costs(list_gen_sq)
1947
1948
    # Find foreign bus to assign the generator
1949
    list_gen_sq["bus"] = list_gen_sq.country.map(entsoe_to_bus)
1950
1951
    # insert generators data
1952
    session = sessionmaker(bind=db.engine())()
1953
    for i, row in list_gen_sq.iterrows():
1954
        entry = etrago.EgonPfHvGenerator(
1955
            scn_name=row.scenario,
1956
            generator_id=int(db.next_etrago_id("generator")),
1957
            bus=row.bus,
1958
            carrier=row.carrier,
1959
            p_nom=row.capacity,
1960
            marginal_cost=row.marginal_cost,
1961
        )
1962
1963
        session.add(entry)
1964
        session.commit()
1965
1966
    renewable_timeseries_pypsaeur(scn_name)
1967
1968
1969
def renewable_timeseries_pypsaeur(scn_name):
1970
    # select generators from database to get index values
1971
    foreign_re_generators = db.select_dataframe(
1972
        f"""
1973
        SELECT generator_id, a.carrier, country, x, y
1974
        FROM grid.egon_etrago_generator a
1975
        JOIN  grid.egon_etrago_bus b
1976
        ON a.bus = b.bus_id
1977
        WHERE a.scn_name = '{scn_name}'
1978
        AND  b.scn_name = '{scn_name}'
1979
        AND b.carrier = 'AC'
1980
        AND b.country != 'DE'
1981
        AND a.carrier IN ('wind_onshore', 'wind_offshore', 'solar')
1982
        """
1983
    )
1984
1985
    # Import prepared network from pypsa-eur
1986
    network = prepared_network()
1987
1988
    # Select fluctuating renewable generators
1989
    generators_pypsa_eur = network.generators.loc[
1990
        network.generators[
1991
            network.generators.carrier.isin(["onwind", "offwind-ac", "solar"])
1992
        ].index,
1993
        ["bus", "carrier"],
1994
    ]
1995
1996
    # Align carrier names for wind turbines
1997
    generators_pypsa_eur.loc[
1998
        generators_pypsa_eur[generators_pypsa_eur.carrier == "onwind"].index,
1999
        "carrier",
2000
    ] = "wind_onshore"
2001
    generators_pypsa_eur.loc[
2002
        generators_pypsa_eur[
2003
            generators_pypsa_eur.carrier == "offwind-ac"
2004
        ].index,
2005
        "carrier",
2006
    ] = "wind_offshore"
2007
2008
    # Set coordinates from bus table
2009
    generators_pypsa_eur["x"] = network.buses.loc[
2010
        generators_pypsa_eur.bus.values, "x"
2011
    ].values
2012
    generators_pypsa_eur["y"] = network.buses.loc[
2013
        generators_pypsa_eur.bus.values, "y"
2014
    ].values
2015
2016
    # Get p_max_pu time series from pypsa-eur
2017
    generators_pypsa_eur["p_max_pu"] = network.generators_t.p_max_pu[
2018
        generators_pypsa_eur.index
2019
    ].T.values.tolist()
2020
2021
    session = sessionmaker(bind=db.engine())()
2022
2023
    # Insert p_max_pu timeseries based on geometry and carrier
2024
    for gen in foreign_re_generators.index:
2025
        entry = etrago.EgonPfHvGeneratorTimeseries(
2026
            scn_name=scn_name,
2027
            generator_id=foreign_re_generators.loc[gen, "generator_id"],
2028
            temp_id=1,
2029
            p_max_pu=generators_pypsa_eur[
2030
                (
2031
                    (
2032
                        generators_pypsa_eur.x
2033
                        - foreign_re_generators.loc[gen, "x"]
2034
                    ).abs()
2035
                    < 0.01
2036
                )
2037
                & (
2038
                    (
2039
                        generators_pypsa_eur.y
2040
                        - foreign_re_generators.loc[gen, "y"]
2041
                    ).abs()
2042
                    < 0.01
2043
                )
2044
                & (
2045
                    generators_pypsa_eur.carrier
2046
                    == foreign_re_generators.loc[gen, "carrier"]
2047
                )
2048
            ].p_max_pu.iloc[0],
2049
        )
2050
2051
        session.add(entry)
2052
        session.commit()
2053
2054
2055
def insert_loads_sq(scn_name="status2019"):
2056
    """
2057
    Copy load timeseries data from entso-e.
2058
2059
    Returns
2060
    -------
2061
    None.
2062
2063
    """
2064
    sources = config.datasets()["electrical_neighbours"]["sources"]
2065
    targets = config.datasets()["electrical_neighbours"]["targets"]
2066
2067
    if scn_name == "status2019":
2068
        year_start_end = {"year_start": "20190101", "year_end": "20200101"}
2069
    elif scn_name == "status2023":
2070
        year_start_end = {"year_start": "20230101", "year_end": "20240101"}
2071
    else:
2072
        raise ValueError("No valid scenario name!")
2073
2074
    df_load_sq, not_retrieved = entsoe_historic_demand(**year_start_end)
2075
2076
    if not_retrieved:
2077
        logger.warning("Demand data from entsoe could not be retrieved.")
2078
        # check for generation backup from former runs
2079
        file_path = Path(
2080
            "./",
2081
            "data_bundle_egon_data",
2082
            "entsoe",
2083
            f"load_entsoe_{scn_name}.csv",
2084
        ).resolve()
2085
        if os.path.isfile(file_path):
2086
            df_load_sq, not_retrieved = fill_by_backup_data_from_former_runs(
2087
                df_load_sq, file_path, not_retrieved
2088
            )
2089
        save_entsoe_data(df_load_sq, file_path=file_path)
2090
2091
    # Delete existing data
2092
    db.execute_sql(
2093
        f"""
2094
        DELETE FROM {targets['load_timeseries']['schema']}.
2095
        {targets['load_timeseries']['table']}
2096
        WHERE
2097
        scn_name = '{scn_name}'
2098
        AND load_id IN (
2099
        SELECT load_id FROM {targets['loads']['schema']}.
2100
        {targets['loads']['table']}
2101
        WHERE
2102
        scn_name = '{scn_name}'
2103
        AND carrier = 'AC'
2104
        AND bus NOT IN (
2105
            SELECT bus_i
2106
            FROM  {sources['osmtgmod_bus']['schema']}.
2107
            {sources['osmtgmod_bus']['table']}))
2108
        """
2109
    )
2110
2111
    db.execute_sql(
2112
        f"""
2113
        DELETE FROM {targets['loads']['schema']}.
2114
        {targets['loads']['table']}
2115
        WHERE
2116
        scn_name = '{scn_name}'
2117
        AND carrier = 'AC'
2118
        AND bus NOT IN (
2119
            SELECT bus_i
2120
            FROM  {sources['osmtgmod_bus']['schema']}.
2121
            {sources['osmtgmod_bus']['table']})
2122
        """
2123
    )
2124
2125
    # get the corresponding bus per foreign country
2126
    entsoe_to_bus = entsoe_to_bus_etrago(scn_name)
2127
2128
    # Calculate and insert demand timeseries per etrago bus_id
2129
    with session_scope() as session:
2130
        for country in df_load_sq.columns:
2131
            load_id = db.next_etrago_id("load")
2132
2133
            entry = etrago.EgonPfHvLoad(
2134
                scn_name=scn_name,
2135
                load_id=int(load_id),
2136
                carrier="AC",
2137
                bus=int(entsoe_to_bus[country]),
2138
            )
2139
2140
            entry_ts = etrago.EgonPfHvLoadTimeseries(
2141
                scn_name=scn_name,
2142
                load_id=int(load_id),
2143
                temp_id=1,
2144
                p_set=list(df_load_sq[country]),
2145
            )
2146
2147
            session.add(entry)
2148
            session.add(entry_ts)
2149
            session.commit()
2150
2151
2152
tasks = (grid,)
2153
2154
insert_per_scenario = set()
2155
2156
for scn_name in config.settings()["egon-data"]["--scenarios"]:
2157
2158
    if scn_name == "eGon2035":
2159
        insert_per_scenario.update([tyndp_generation, tyndp_demand])
2160
2161
    if "status" in scn_name:
2162
        postfix = f"_{scn_name.split('status')[-1]}"
2163
        insert_per_scenario.update(
2164
            [
2165
                wrapped_partial(
2166
                    insert_generators_sq, scn_name=scn_name, postfix=postfix
2167
                ),
2168
                wrapped_partial(
2169
                    insert_loads_sq, scn_name=scn_name, postfix=postfix
2170
                ),
2171
                wrapped_partial(
2172
                    insert_storage_units_sq, scn_name=scn_name, postfix=postfix
2173
                ),
2174
            ]
2175
        )
2176
2177
tasks = tasks + (insert_per_scenario,)
2178
2179
2180
class ElectricalNeighbours(Dataset):
2181
    """
2182
    Add lines, loads, generation and storage for electrical neighbours
2183
2184
    This dataset creates data for modelling the considered foreign countries and writes
2185
    that data into the database tables that can be read by the eTraGo tool.
2186
    Neighbouring countries are modelled in a lower spatial resolution, in general one node per
2187
    country is considered.
2188
    Defined load timeseries as well as generatrion and storage capacities are connected to these nodes.
2189
    The nodes are connected by AC and DC transmission lines with the German grid and other neighbouring countries
2190
    considering the grid topology from ENTSO-E.
2191
2192
2193
    *Dependencies*
2194
      * :py:class:`Tyndp <egon.data.datasets.tyndp.Tyndp>`
2195
      * :py:class:`PypsaEurSec <egon.data.datasets.pypsaeursec.PypsaEurSec>`
2196
2197
2198
    *Resulting tables*
2199
      * :py:class:`grid.egon_etrago_bus <egon.data.datasets.etrago_setup.EgonPfHvBus>` is extended
2200
      * :py:class:`grid.egon_etrago_link <egon.data.datasets.etrago_setup.EgonPfHvLink>` is extended
2201
      * :py:class:`grid.egon_etrago_line <egon.data.datasets.etrago_setup.EgonPfHvLine>` is extended
2202
      * :py:class:`grid.egon_etrago_load <egon.data.datasets.etrago_setup.EgonPfHvLoad>` is extended
2203
      * :py:class:`grid.egon_etrago_load_timeseries <egon.data.datasets.etrago_setup.EgonPfHvLoadTimeseries>` is extended
2204
      * :py:class:`grid.egon_etrago_storage <egon.data.datasets.etrago_setup.EgonPfHvStorageUnit>` is extended
2205
      * :py:class:`grid.egon_etrago_generator <egon.data.datasets.etrago_setup.EgonPfHvGenerator>` is extended
2206
      * :py:class:`grid.egon_etrago_generator_timeseries <egon.data.datasets.etrago_setup.EgonPfHvGeneratorTimeseries>` is extended
2207
      * :py:class:`grid.egon_etrago_transformer <egon.data.datasets.etrago_setup.EgonPfHvTransformer>` is extended
2208
2209
    """
2210
2211
    #:
2212
    name: str = "ElectricalNeighbours"
2213
    #:
2214
    version: str = "0.0.11"
2215
2216
    def __init__(self, dependencies):
2217
        super().__init__(
2218
            name=self.name,
2219
            version=self.version,
2220
            dependencies=dependencies,
2221
            tasks=tasks,
2222
        )
2223