Passed
Pull Request — dev (#1068)
by
unknown
02:05
created

data.datasets.gas_grid.insert_gas_pipeline_list()   A

Complexity

Conditions 1

Size

Total Lines 45
Code Lines 20

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
eloc 20
dl 0
loc 45
rs 9.4
c 0
b 0
f 0
cc 1
nop 2
1
# -*- coding: utf-8 -*-
2
"""
3
The central module containing all code dealing with importing data from SciGRID_gas IGGIELGN data
4
"""
5
from pathlib import Path
6
from urllib.request import urlretrieve
7
from zipfile import ZipFile
8
import ast
9
import json
10
import os
11
12
from geoalchemy2.types import Geometry
13
from shapely import geometry
14
import geopandas
15
import numpy as np
16
import pandas as pd
17
18
from egon.data import config, db
19
from egon.data.config import settings
20
from egon.data.datasets import Dataset
21
from egon.data.datasets.electrical_neighbours import central_buses_egon100
22
from egon.data.datasets.etrago_helpers import copy_and_modify_buses
23
from egon.data.datasets.scenario_parameters import get_sector_parameters
24
25
26
class GasNodesandPipes(Dataset):
27
    def __init__(self, dependencies):
28
        super().__init__(
29
            name="GasNodesandPipes",
30
            version="0.0.8",
31
            dependencies=dependencies,
32
            tasks=(insert_gas_data, insert_gas_data_eGon100RE),
33
        )
34
35
36
def download_SciGRID_gas_data():
37
    """
38
    Download SciGRID_gas IGGIELGN data from Zenodo
39
40
    """
41
    path = Path(".") / "datasets" / "gas_data"
42
    os.makedirs(path, exist_ok=True)
43
44
    basename = "IGGIELGN"
45
    zip_file = Path(".") / "datasets" / "gas_data" / "IGGIELGN.zip"
46
    zenodo_zip_file_url = (
47
        "https://zenodo.org/record/4767098/files/" + basename + ".zip"
48
    )
49
    if not os.path.isfile(zip_file):
50
        urlretrieve(zenodo_zip_file_url, zip_file)
51
52
    components = [
53
        "Nodes",
54
        "PipeSegments",
55
        "Productions",
56
        "Storages",
57
        "LNGs",
58
    ]  #'Compressors'
59
    files = []
60
    for i in components:
61
        files.append("data/" + basename + "_" + i + ".csv")
62
63
    with ZipFile(zip_file, "r") as zipObj:
64
        listOfFileNames = zipObj.namelist()
65
        for fileName in listOfFileNames:
66
            if fileName in files:
67
                zipObj.extract(fileName, path)
68
69
70
def define_gas_nodes_list():
71
    """Define list of gas nodes from SciGRID_gas IGGIELGN data
72
73
    Returns
74
    -------
75
    gas_nodes_list : dataframe
76
        Dataframe containing the gas nodes (Europe)
77
78
    """
79
    # Select next id value
80
    new_id = db.next_etrago_id("bus")
81
82
    target_file = (
83
        Path(".") / "datasets" / "gas_data" / "data" / "IGGIELGN_Nodes.csv"
84
    )
85
86
    gas_nodes_list = pd.read_csv(
87
        target_file,
88
        delimiter=";",
89
        decimal=".",
90
        usecols=["lat", "long", "id", "country_code", "param"],
91
    )
92
93
    # Correct non valid neighbouring country nodes
94
    gas_nodes_list.loc[
95
        gas_nodes_list["id"] == "INET_N_1182", "country_code"
96
    ] = "AT"
97
    gas_nodes_list.loc[
98
        gas_nodes_list["id"] == "SEQ_10608_p", "country_code"
99
    ] = "NL"
100
    gas_nodes_list.loc[
101
        gas_nodes_list["id"] == "N_88_NS_LMGN", "country_code"
102
    ] = "XX"
103
104
    gas_nodes_list = gas_nodes_list.rename(columns={"lat": "y", "long": "x"})
105
106
    gas_nodes_list["bus_id"] = range(new_id, new_id + len(gas_nodes_list))
107
    gas_nodes_list = gas_nodes_list.set_index("id")
108
109
    return gas_nodes_list
110
111
112
def ch4_nodes_number_G(gas_nodes_list):
113
    """Insert list of CH4 nodes from SciGRID_gas IGGIELGN data
114
        Parameters
115
    ----------
116
    gas_nodes_list : dataframe
117
        Dataframe containing the gas nodes (Europe)
118
    Returns
119
    -------
120
        N_ch4_nodes_G : int
121
            Number of CH4 buses in Germany (independantly from the mode used)
122
    """
123
124
    ch4_nodes_list = gas_nodes_list[
125
        gas_nodes_list["country_code"].str.match("DE")
126
    ]
127
    N_ch4_nodes_G = len(ch4_nodes_list)
128
129
    return N_ch4_nodes_G
130
131
132
def insert_CH4_nodes_list(gas_nodes_list):
133
    """Insert list of CH4 nodes from SciGRID_gas IGGIELGN data
134
135
    Insert detailled description
136
137
    Parameters
138
    ----------
139
    gas_nodes_list : dataframe
140
        Dataframe containing the gas nodes (Europe)
141
142
    Returns
143
    -------
144
    None
145
146
    """
147
    # Connect to local database
148
    engine = db.engine()
149
150
    gas_nodes_list = gas_nodes_list[
151
        gas_nodes_list["country_code"].str.match("DE")
152
    ]  # To eventually replace with a test if the nodes are in the german boundaries.
153
154
    # Cut data to federal state if in testmode
155
    NUTS1 = []
156
    for index, row in gas_nodes_list.iterrows():
157
        param = ast.literal_eval(row["param"])
158
        NUTS1.append(param["nuts_id_1"])
159
    gas_nodes_list = gas_nodes_list.assign(NUTS1=NUTS1)
160
161
    boundary = settings()["egon-data"]["--dataset-boundary"]
162
    if boundary != "Everything":
163
        map_states = {
164
            "Baden-Württemberg": "DE1",
165
            "Nordrhein-Westfalen": "DEA",
166
            "Hessen": "DE7",
167
            "Brandenburg": "DE4",
168
            "Bremen": "DE5",
169
            "Rheinland-Pfalz": "DEB",
170
            "Sachsen-Anhalt": "DEE",
171
            "Schleswig-Holstein": "DEF",
172
            "Mecklenburg-Vorpommern": "DE8",
173
            "Thüringen": "DEG",
174
            "Niedersachsen": "DE9",
175
            "Sachsen": "DED",
176
            "Hamburg": "DE6",
177
            "Saarland": "DEC",
178
            "Berlin": "DE3",
179
            "Bayern": "DE2",
180
        }
181
182
        gas_nodes_list = gas_nodes_list[
183
            gas_nodes_list["NUTS1"].isin([map_states[boundary], np.nan])
184
        ]
185
186
        # A completer avec nodes related to pipelines which have an end in the selected area et evt deplacer ds define_gas_nodes_list
187
188
    # Add missing columns
189
    c = {"scn_name": "eGon2035", "carrier": "CH4"}
190
    gas_nodes_list = gas_nodes_list.assign(**c)
191
192
    gas_nodes_list = geopandas.GeoDataFrame(
193
        gas_nodes_list,
194
        geometry=geopandas.points_from_xy(
195
            gas_nodes_list["x"], gas_nodes_list["y"]
196
        ),
197
    )
198
    gas_nodes_list = gas_nodes_list.rename(
199
        columns={"geometry": "geom"}
200
    ).set_geometry("geom", crs=4326)
201
202
    gas_nodes_list = gas_nodes_list.reset_index(drop=True)
203
    gas_nodes_list = gas_nodes_list.drop(
204
        columns=["NUTS1", "param", "country_code"]
205
    )
206
207
    # Insert data to db
208
    db.execute_sql(
209
        f"""
210
    DELETE FROM grid.egon_etrago_bus WHERE "carrier" = 'CH4' AND
211
    scn_name = '{c['scn_name']}' AND country = 'DE';
212
    """
213
    )
214
215
    # Insert CH4 data to db
216
    print(gas_nodes_list)
217
    gas_nodes_list.to_postgis(
218
        "egon_etrago_bus",
219
        engine,
220
        schema="grid",
221
        index=False,
222
        if_exists="append",
223
        dtype={"geom": Geometry()},
224
    )
225
226
227
def insert_gas_buses_abroad(scn_name="eGon2035"):
228
    """Insert central CH4 buses in foreign countries for eGon2035
229
230
    Detailled description to be completed:
231
    Insert central gas buses in foreign countries to db, same buses
232
    than the foreign AC buses
233
234
    Parameters
235
    ----------
236
    scn_name : str
237
        Name of the scenario
238
239
    Returns
240
    -------
241
    gdf_abroad_buses : dataframe
242
        Dataframe containing the CH4 buses in the neighbouring countries
243
        and one in the center of Germany in test mode
244
    """
245
    # Select sources and targets from dataset configuration
246
    sources = config.datasets()["electrical_neighbours"]["sources"]
247
248
    main_gas_carrier = get_sector_parameters("gas", scenario=scn_name)[
249
        "main_gas_carrier"
250
    ]
251
252
    # Connect to local database
253
    engine = db.engine()
254
    db.execute_sql(
255
        f"""
256
    DELETE FROM grid.egon_etrago_bus WHERE "carrier" = '{main_gas_carrier}' AND
257
    scn_name = '{scn_name}' AND country != 'DE';
258
    """
259
    )
260
261
    # Select the foreign buses
262
    gdf_abroad_buses = central_buses_egon100(sources)
263
    gdf_abroad_buses = gdf_abroad_buses.drop_duplicates(subset=["country"])
264
265
    # Select next id value
266
    new_id = db.next_etrago_id("bus")
267
268
    gdf_abroad_buses = gdf_abroad_buses.drop(
269
        columns=[
270
            "v_nom",
271
            "v_mag_pu_set",
272
            "v_mag_pu_min",
273
            "v_mag_pu_max",
274
            "geom",
275
        ]
276
    )
277
    gdf_abroad_buses["scn_name"] = "eGon2035"
278
    gdf_abroad_buses["carrier"] = main_gas_carrier
279
    gdf_abroad_buses["bus_id"] = range(new_id, new_id + len(gdf_abroad_buses))
280
281
    # Add central bus in Russia
282
    gdf_abroad_buses = gdf_abroad_buses.append(
283
        {
284
            "scn_name": scn_name,
285
            "bus_id": (new_id + len(gdf_abroad_buses) + 1),
286
            "x": 41,
287
            "y": 55,
288
            "country": "RU",
289
            "carrier": main_gas_carrier,
290
        },
291
        ignore_index=True,
292
    )
293
    # if in test mode, add bus in center of Germany
294
    boundary = settings()["egon-data"]["--dataset-boundary"]
295
296
    if boundary != "Everything":
297
        gdf_abroad_buses = gdf_abroad_buses.append(
298
            {
299
                "scn_name": scn_name,
300
                "bus_id": (new_id + len(gdf_abroad_buses) + 1),
301
                "x": 10.4234469,
302
                "y": 51.0834196,
303
                "country": "DE",
304
                "carrier": main_gas_carrier,
305
            },
306
            ignore_index=True,
307
        )
308
309
    gdf_abroad_buses = geopandas.GeoDataFrame(
310
        gdf_abroad_buses,
311
        geometry=geopandas.points_from_xy(
312
            gdf_abroad_buses["x"], gdf_abroad_buses["y"]
313
        ),
314
    )
315
    gdf_abroad_buses = gdf_abroad_buses.rename(
316
        columns={"geometry": "geom"}
317
    ).set_geometry("geom", crs=4326)
318
319
    # Insert to db
320
    print(gdf_abroad_buses)
321
    gdf_abroad_buses.to_postgis(
322
        "egon_etrago_bus",
323
        engine,
324
        schema="grid",
325
        index=False,
326
        if_exists="append",
327
        dtype={"geom": Geometry()},
328
    )
329
    return gdf_abroad_buses
330
331
332
def define_gas_pipeline_list(
333
    gas_nodes_list, abroad_gas_nodes_list, scn_name="eGon2035"
334
):
335
    """Define gas pipelines in Germany from SciGRID_gas IGGIELGN data
336
337
    Insert detailled description
338
339
    Parameters
340
    ----------
341
    gas_nodes_list : dataframe
342
        description missing
343
    abroad_gas_nodes_list: dataframe
344
        description missing
345
    scn_name : str
346
        Name of the scenario
347
348
    Returns
349
    -------
350
    gas_pipelines_list : pandas.DataFrame
351
        Dataframe containing the gas pipelines in Germany
352
353
    """
354
    abroad_gas_nodes_list = abroad_gas_nodes_list.set_index("country")
355
356
    main_gas_carrier = get_sector_parameters("gas", scenario=scn_name)[
357
        "main_gas_carrier"
358
    ]
359
360
    # Select next id value
361
    new_id = db.next_etrago_id("link")
362
363
    classifiaction_file = (
364
        Path(".")
365
        / "data_bundle_egon_data"
366
        / "pipeline_classification_gas"
367
        / "pipeline_classification.csv"
368
    )
369
370
    classification = pd.read_csv(
371
        classifiaction_file,
372
        delimiter=",",
373
        usecols=["classification", "max_transport_capacity_Gwh/d"],
374
    )
375
376
    target_file = (
377
        Path(".")
378
        / "datasets"
379
        / "gas_data"
380
        / "data"
381
        / "IGGIELGN_PipeSegments.csv"
382
    )
383
384
    gas_pipelines_list = pd.read_csv(
385
        target_file,
386
        delimiter=";",
387
        decimal=".",
388
        usecols=["id", "node_id", "lat", "long", "country_code", "param"],
389
    )
390
391
    # Select the links having at least one bus in Germany
392
    gas_pipelines_list = gas_pipelines_list[
393
        gas_pipelines_list["country_code"].str.contains("DE")
394
    ]
395
396
    # Remove links disconnected of the rest of the grid
397
    # Remove manually for disconnected link EntsoG_Map__ST_195
398
    gas_pipelines_list = gas_pipelines_list[
399
        ~gas_pipelines_list["id"].str.match("EntsoG_Map__ST_195")
400
    ]
401
402
    gas_pipelines_list["link_id"] = range(
403
        new_id, new_id + len(gas_pipelines_list)
404
    )
405
    gas_pipelines_list["link_id"] = gas_pipelines_list["link_id"].astype(int)
406
407
    # Cut data to federal state if in testmode
408
    NUTS1 = []
409
    for index, row in gas_pipelines_list.iterrows():
410
        param = ast.literal_eval(row["param"])
411
        NUTS1.append(param["nuts_id_1"])
412
    gas_pipelines_list["NUTS1"] = NUTS1
413
414
    map_states = {
415
        "Baden-Württemberg": "DE1",
416
        "Nordrhein-Westfalen": "DEA",
417
        "Hessen": "DE7",
418
        "Brandenburg": "DE4",
419
        "Bremen": "DE5",
420
        "Rheinland-Pfalz": "DEB",
421
        "Sachsen-Anhalt": "DEE",
422
        "Schleswig-Holstein": "DEF",
423
        "Mecklenburg-Vorpommern": "DE8",
424
        "Thüringen": "DEG",
425
        "Niedersachsen": "DE9",
426
        "Sachsen": "DED",
427
        "Hamburg": "DE6",
428
        "Saarland": "DEC",
429
        "Berlin": "DE3",
430
        "Bayern": "DE2",
431
        "Everything": "Nan",
432
    }
433
    gas_pipelines_list["NUTS1_0"] = [x[0] for x in gas_pipelines_list["NUTS1"]]
434
    gas_pipelines_list["NUTS1_1"] = [x[1] for x in gas_pipelines_list["NUTS1"]]
435
436
    boundary = settings()["egon-data"]["--dataset-boundary"]
437
438
    if boundary != "Everything":
439
440
        gas_pipelines_list = gas_pipelines_list[
441
            gas_pipelines_list["NUTS1_0"].str.contains(map_states[boundary])
442
            | gas_pipelines_list["NUTS1_1"].str.contains(map_states[boundary])
443
        ]
444
445
    # Add missing columns
446
    gas_pipelines_list["scn_name"] = scn_name
447
    gas_pipelines_list["carrier"] = main_gas_carrier
448
    gas_pipelines_list["p_nom_extendable"] = False
449
450
    diameter = []
451
    geom = []
452
    topo = []
453
    length_km = []
454
455
    for index, row in gas_pipelines_list.iterrows():
456
457
        param = ast.literal_eval(row["param"])
458
        diameter.append(param["diameter_mm"])
459
        length_km.append(param["length_km"])
460
461
        long_e = json.loads(row["long"])
462
        lat_e = json.loads(row["lat"])
463
        crd_e = list(zip(long_e, lat_e))
464
        topo.append(geometry.LineString(crd_e))
465
466
        long_path = param["path_long"]
467
        lat_path = param["path_lat"]
468
        crd = list(zip(long_path, lat_path))
469
        crd.insert(0, crd_e[0])
470
        crd.append(crd_e[1])
471
        lines = []
472
        for i in range(len(crd) - 1):
473
            lines.append(geometry.LineString([crd[i], crd[i + 1]]))
474
        geom.append(geometry.MultiLineString(lines))
475
476
    gas_pipelines_list["diameter"] = diameter
477
    gas_pipelines_list["geom"] = geom
478
    gas_pipelines_list["topo"] = topo
479
    gas_pipelines_list["length_km"] = length_km
480
    gas_pipelines_list = gas_pipelines_list.set_geometry("geom", crs=4326)
481
482
    country_0 = []
483
    country_1 = []
484
    for index, row in gas_pipelines_list.iterrows():
485
        c = ast.literal_eval(row["country_code"])
486
        country_0.append(c[0])
487
        country_1.append(c[1])
488
489
    gas_pipelines_list["country_0"] = country_0
490
    gas_pipelines_list["country_1"] = country_1
491
492
    # Correct non valid neighbouring country nodes
493
    gas_pipelines_list.loc[
494
        gas_pipelines_list["country_0"] == "XX", "country_0"
495
    ] = "NO"
496
    gas_pipelines_list.loc[
497
        gas_pipelines_list["country_1"] == "FI", "country_1"
498
    ] = "RU"
499
    gas_pipelines_list.loc[
500
        gas_pipelines_list["id"] == "ST_2612_Seg_0_Seg_0", "country_0"
501
    ] = "AT"  # bus "INET_N_1182" DE -> AT
502
    gas_pipelines_list.loc[
503
        gas_pipelines_list["id"] == "INET_PL_385_EE_3_Seg_0_Seg_1", "country_1"
504
    ] = "AT"  # "INET_N_1182" DE -> AT
505
    gas_pipelines_list.loc[
506
        gas_pipelines_list["id"] == "LKD_PS_0_Seg_0_Seg_3", "country_0"
507
    ] = "NL"  # bus "SEQ_10608_p" DE -> NL
508
509
    # Remove uncorrect pipelines
510
    gas_pipelines_list = gas_pipelines_list[
511
        (gas_pipelines_list["id"] != "PLNG_2637_Seg_0_Seg_0_Seg_0")
512
        & (gas_pipelines_list["id"] != "NSG_6650_Seg_2_Seg_0")
513
        & (gas_pipelines_list["id"] != "NSG_6734_Seg_2_Seg_0")
514
    ]
515
516
    # Remove link test if length = 0
517
    gas_pipelines_list = gas_pipelines_list[
518
        gas_pipelines_list["length_km"] != 0
519
    ]
520
521
    # Adjust columns
522
    bus0 = []
523
    bus1 = []
524
    geom_adjusted = []
525
    topo_adjusted = []
526
    length_adjusted = []
527
    pipe_class = []
528
529
    for index, row in gas_pipelines_list.iterrows():
530
        buses = row["node_id"].strip("][").split(", ")
531
532
        if (
533
            (boundary != "Everything")
534
            & (row["NUTS1_0"] != map_states[boundary])
535
            & (row["country_0"] == "DE")
536
        ):
537
            bus0.append(abroad_gas_nodes_list.loc["DE", "bus_id"])
538
            bus1.append(gas_nodes_list.loc[buses[1][1:-1], "bus_id"])
539
            long_e = [
540
                abroad_gas_nodes_list.loc["DE", "x"],
541
                json.loads(row["long"])[1],
542
            ]
543
            lat_e = [
544
                abroad_gas_nodes_list.loc["DE", "y"],
545
                json.loads(row["lat"])[1],
546
            ]
547
            geom_pipe = geometry.MultiLineString(
548
                [geometry.LineString(list(zip(long_e, lat_e)))]
549
            )
550
            topo_adjusted.append(geometry.LineString(list(zip(long_e, lat_e))))
551
552
        elif row["country_0"] != "DE":
553
            country = str(row["country_0"])
554
            bus0.append(abroad_gas_nodes_list.loc[country, "bus_id"])
555
            bus1.append(gas_nodes_list.loc[buses[1][1:-1], "bus_id"])
556
            long_e = [
557
                abroad_gas_nodes_list.loc[country, "x"],
558
                json.loads(row["long"])[1],
559
            ]
560
            lat_e = [
561
                abroad_gas_nodes_list.loc[country, "y"],
562
                json.loads(row["lat"])[1],
563
            ]
564
            geom_pipe = geometry.MultiLineString(
565
                [geometry.LineString(list(zip(long_e, lat_e)))]
566
            )
567
            topo_adjusted.append(geometry.LineString(list(zip(long_e, lat_e))))
568
569
        elif (
570
            (boundary != "Everything")
571
            & (row["NUTS1_1"] != map_states[boundary])
572
            & (row["country_1"] == "DE")
573
        ):
574
            bus0.append(gas_nodes_list.loc[buses[0][1:-1], "bus_id"])
575
            bus1.append(abroad_gas_nodes_list.loc["DE", "bus_id"])
576
            long_e = [
577
                json.loads(row["long"])[0],
578
                abroad_gas_nodes_list.loc["DE", "x"],
579
            ]
580
            lat_e = [
581
                json.loads(row["lat"])[0],
582
                abroad_gas_nodes_list.loc["DE", "y"],
583
            ]
584
            geom_pipe = geometry.MultiLineString(
585
                [geometry.LineString(list(zip(long_e, lat_e)))]
586
            )
587
            topo_adjusted.append(geometry.LineString(list(zip(long_e, lat_e))))
588
589
        elif row["country_1"] != "DE":
590
            country = str(row["country_1"])
591
            bus0.append(gas_nodes_list.loc[buses[0][1:-1], "bus_id"])
592
            bus1.append(abroad_gas_nodes_list.loc[country, "bus_id"])
593
            long_e = [
594
                json.loads(row["long"])[0],
595
                abroad_gas_nodes_list.loc[country, "x"],
596
            ]
597
            lat_e = [
598
                json.loads(row["lat"])[0],
599
                abroad_gas_nodes_list.loc[country, "y"],
600
            ]
601
            geom_pipe = geometry.MultiLineString(
602
                [geometry.LineString(list(zip(long_e, lat_e)))]
603
            )
604
            topo_adjusted.append(geometry.LineString(list(zip(long_e, lat_e))))
605
606
        else:
607
            bus0.append(gas_nodes_list.loc[buses[0][1:-1], "bus_id"])
608
            bus1.append(gas_nodes_list.loc[buses[1][1:-1], "bus_id"])
609
            geom_pipe = row["geom"]
610
            topo_adjusted.append(row["topo"])
611
612
        geom_adjusted.append(geom_pipe)
613
        length_adjusted.append(geom_pipe.length)
614
615
        if row["diameter"] >= 1000:
616
            pipe_class = "A"
617
        elif 700 <= row["diameter"] <= 1000:
618
            pipe_class = "B"
619
        elif 500 <= row["diameter"] <= 700:
620
            pipe_class = "C"
621
        elif 350 <= row["diameter"] <= 500:
622
            pipe_class = "D"
623
        elif 200 <= row["diameter"] <= 350:
624
            pipe_class = "E"
625
        elif 100 <= row["diameter"] <= 200:
626
            pipe_class = "F"
627
        elif row["diameter"] <= 100:
628
            pipe_class = "G"
629
630
    gas_pipelines_list["bus0"] = bus0
631
    gas_pipelines_list["bus1"] = bus1
632
    gas_pipelines_list["geom"] = geom_adjusted
633
    gas_pipelines_list["topo"] = topo_adjusted
634
    gas_pipelines_list["length"] = length_adjusted
635
    gas_pipelines_list["pipe_class"] = pipe_class
636
637
    # Remove pipes having the same node for start and end
638
    gas_pipelines_list = gas_pipelines_list[
639
        gas_pipelines_list["bus0"] != gas_pipelines_list["bus1"]
640
    ]
641
642
    gas_pipelines_list = gas_pipelines_list.merge(
643
        classification,
644
        how="left",
645
        left_on="pipe_class",
646
        right_on="classification",
647
    )
648
    gas_pipelines_list["p_nom"] = gas_pipelines_list[
649
        "max_transport_capacity_Gwh/d"
650
    ] * (1000 / 24)
651
652
    # Remove useless columns
653
    gas_pipelines_list = gas_pipelines_list.drop(
654
        columns=[
655
            "id",
656
            "node_id",
657
            "param",
658
            "NUTS1",
659
            "NUTS1_0",
660
            "NUTS1_1",
661
            "country_code",
662
            "diameter",
663
            "pipe_class",
664
            "classification",
665
            "max_transport_capacity_Gwh/d",
666
            "lat",
667
            "long",
668
            "length_km",
669
        ]
670
    )
671
672
    return gas_pipelines_list
673
674
675
def insert_gas_pipeline_list(gas_pipelines_list, scn_name="eGon2035"):
676
    """Insert list of gas pipelines in the database
677
678
    Insert detailled description
679
680
    Parameters
681
    ----------
682
    gas_pipelines_list : pandas.DataFrame
683
        Dataframe containing the gas pipelines in Germany
684
    scn_name : str
685
        Name of the scenario
686
687
    """
688
    main_gas_carrier = get_sector_parameters("gas", scenario=scn_name)[
689
        "main_gas_carrier"
690
    ]
691
    engine = db.engine()
692
    gas_pipelines_list = gas_pipelines_list.drop(
693
        columns=[
694
            "country_0",
695
            "country_1",
696
        ]
697
    )
698
699
    # Clean db
700
    db.execute_sql(
701
        f"""DELETE FROM grid.egon_etrago_link
702
        WHERE "carrier" = '{main_gas_carrier}'
703
        AND scn_name = '{scn_name}';
704
        """
705
    )
706
707
    print(gas_pipelines_list)
708
    # Insert data to db
709
    gas_pipelines_list.to_postgis(
710
        "egon_etrago_gas_link",
711
        engine,
712
        schema="grid",
713
        index=False,
714
        if_exists="replace",
715
        dtype={"geom": Geometry(), "topo": Geometry()},
716
    )
717
718
    db.execute_sql(
719
        """
720
    select UpdateGeometrySRID('grid', 'egon_etrago_gas_link', 'topo', 4326) ;
721
722
    INSERT INTO grid.egon_etrago_link (scn_name,
723
                                              link_id, carrier,
724
                                              bus0, bus1,
725
                                              p_nom, p_nom_extendable, length,
726
                                              geom, topo)
727
    SELECT scn_name,
728
                link_id, carrier,
729
                bus0, bus1,
730
                p_nom, p_nom_extendable, length,
731
                geom, topo
732
733
    FROM grid.egon_etrago_gas_link;
734
735
    DROP TABLE grid.egon_etrago_gas_link;
736
        """
737
    )
738
739
740
def remove_isolated_gas_buses():
741
    """Delete gas buses which are not connected to the gas grid.
742
    Returns
743
    -------
744
    None.
745
    """
746
    targets = config.datasets()["gas_grid"]["targets"]
747
748
    db.execute_sql(
749
        f"""
750
        DELETE FROM {targets['buses']['schema']}.{targets['buses']['table']}
751
        WHERE "carrier" = 'CH4'
752
        AND scn_name = 'eGon2035'
753
        AND country = 'DE'
754
        AND "bus_id" NOT IN
755
            (SELECT bus0 FROM {targets['links']['schema']}.{targets['links']['table']}
756
            WHERE scn_name = 'eGon2035'
757
            AND carrier = 'CH4')
758
        AND "bus_id" NOT IN
759
            (SELECT bus1 FROM {targets['links']['schema']}.{targets['links']['table']}
760
            WHERE scn_name = 'eGon2035'
761
            AND carrier = 'CH4');
762
    """
763
    )
764
765
766
def insert_gas_data():
767
    """Overall function for importing gas data from SciGRID_gas
768
    Returns
769
    -------
770
    None.
771
    """
772
    download_SciGRID_gas_data()
773
774
    gas_nodes_list = define_gas_nodes_list()
775
776
    insert_CH4_nodes_list(gas_nodes_list)
777
    abroad_gas_nodes_list = insert_gas_buses_abroad()
778
779
    gas_pipeline_list = define_gas_pipeline_list(
780
        gas_nodes_list, abroad_gas_nodes_list
781
    )
782
    insert_gas_pipeline_list(gas_pipeline_list)
783
    remove_isolated_gas_buses()
784
785
786
def insert_gas_data_eGon100RE():
787
    """Overall function for importing gas data from SciGRID_gas
788
    Returns
789
    -------
790
    None.
791
    """
792
    # copy buses
793
    copy_and_modify_buses("eGon2035", "eGon100RE", {"carrier": ["CH4"]})
794
795
    # get CH4 pipelines and modify their nominal capacity with the
796
    # retrofitting factor
797
    gdf = db.select_geodataframe(
798
        f"""
799
        SELECT * FROM grid.egon_etrago_link
800
        WHERE carrier = 'CH4' AND scn_name = 'eGon2035' AND
801
        bus0 IN (
802
            SELECT bus_id FROM grid.egon_etrago_bus
803
            WHERE scn_name = 'eGon2035' AND country = 'DE'
804
        ) AND bus1 IN (
805
            SELECT bus_id FROM grid.egon_etrago_bus
806
            WHERE scn_name = 'eGon2035' AND country = 'DE'
807
        );
808
        """,
809
        epsg=4326,
810
        geom_col="topo",
811
    )
812
813
    # Update scenario specific information
814
    scn_name = "eGon100RE"
815
    gdf["scn_name"] = scn_name
816
    scn_params = get_sector_parameters("gas", scn_name)
817
818
    for param in ["capital_cost", "marginal_cost", "efficiency"]:
819
        try:
820
            gdf.loc[:, param] = scn_params[param]["CH4"]
821
        except KeyError:
822
            pass
823
824
    # remaining CH4 share is 1 - retroffited pipeline share
825
    gdf["p_nom"] *= (
826
        1 - scn_params["retrofitted_CH4pipeline-to-H2pipeline_share"]
827
    )
828
829
    # delete old entries
830
    db.execute_sql(
831
        f"""
832
        DELETE FROM grid.egon_etrago_link
833
        WHERE carrier = 'CH4' AND scn_name = '{scn_name}' AND
834
        bus0 NOT IN (
835
            SELECT bus_id FROM grid.egon_etrago_bus
836
            WHERE scn_name = '{scn_name}' AND country != 'DE'
837
        ) AND bus1 NOT IN (
838
            SELECT bus_id FROM grid.egon_etrago_bus
839
            WHERE scn_name = '{scn_name}' AND country != 'DE'
840
        );
841
        """
842
    )
843
844
    gdf.to_postgis(
845
        "egon_etrago_link",
846
        schema="grid",
847
        if_exists="append",
848
        con=db.engine(),
849
        index=False,
850
        dtype={"geom": Geometry(), "topo": Geometry()},
851
    )
852