Passed
Pull Request — dev (#1006)
by
unknown
02:01
created

data.datasets.gas_grid   A

Complexity

Total Complexity 38

Size/Duplication

Total Lines 850
Duplicated Lines 0 %

Importance

Changes 0
Metric Value
wmc 38
eloc 479
dl 0
loc 850
rs 9.36
c 0
b 0
f 0

10 Functions

Rating   Name   Duplication   Size   Complexity  
A ch4_nodes_number_G() 0 18 1
B download_SciGRID_gas_data() 0 32 6
A define_gas_nodes_list() 0 40 1
B insert_CH4_nodes_list() 0 92 3
A insert_gas_pipeline_list() 0 45 1
A insert_gas_data() 0 18 1
A insert_gas_data_eGon100RE() 0 65 3
B insert_gas_buses_abroad() 0 102 2
A remove_isolated_gas_buses() 0 10 1
F define_gas_pipeline_list() 0 341 18

1 Method

Rating   Name   Duplication   Size   Complexity  
A GasNodesandPipes.__init__() 0 6 1
1
# -*- coding: utf-8 -*-
2
"""
3
The central module containing all code dealing with importing data from SciGRID_gas IGGIELGN data
4
"""
5
from pathlib import Path
6
from urllib.request import urlretrieve
7
from zipfile import ZipFile
8
import ast
9
import json
10
import os
11
12
from geoalchemy2.types import Geometry
13
from shapely import geometry
14
import geopandas
15
import numpy as np
16
import pandas as pd
17
18
from egon.data import config, db
19
from egon.data.config import settings
20
from egon.data.datasets import Dataset
21
from egon.data.datasets.electrical_neighbours import central_buses_egon100
22
from egon.data.datasets.etrago_helpers import copy_and_modify_buses
23
from egon.data.datasets.scenario_parameters import get_sector_parameters
24
25
26
class GasNodesandPipes(Dataset):
27
    def __init__(self, dependencies):
28
        super().__init__(
29
            name="GasNodesandPipes",
30
            version="0.0.8",
31
            dependencies=dependencies,
32
            tasks=(insert_gas_data, insert_gas_data_eGon100RE),
33
        )
34
35
36
def download_SciGRID_gas_data():
37
    """
38
    Download SciGRID_gas IGGIELGN data from Zenodo
39
40
    """
41
    path = Path(".") / "datasets" / "gas_data"
42
    os.makedirs(path, exist_ok=True)
43
44
    basename = "IGGIELGN"
45
    zip_file = Path(".") / "datasets" / "gas_data" / "IGGIELGN.zip"
46
    zenodo_zip_file_url = (
47
        "https://zenodo.org/record/4767098/files/" + basename + ".zip"
48
    )
49
    if not os.path.isfile(zip_file):
50
        urlretrieve(zenodo_zip_file_url, zip_file)
51
52
    components = [
53
        "Nodes",
54
        "PipeSegments",
55
        "Productions",
56
        "Storages",
57
        "LNGs",
58
    ]  #'Compressors'
59
    files = []
60
    for i in components:
61
        files.append("data/" + basename + "_" + i + ".csv")
62
63
    with ZipFile(zip_file, "r") as zipObj:
64
        listOfFileNames = zipObj.namelist()
65
        for fileName in listOfFileNames:
66
            if fileName in files:
67
                zipObj.extract(fileName, path)
68
69
70
def define_gas_nodes_list():
71
    """Define list of gas nodes from SciGRID_gas IGGIELGN data
72
73
    Returns
74
    -------
75
    gas_nodes_list : dataframe
76
        Dataframe containing the gas nodes (Europe)
77
78
    """
79
    # Select next id value
80
    new_id = db.next_etrago_id("bus")
81
82
    target_file = (
83
        Path(".") / "datasets" / "gas_data" / "data" / "IGGIELGN_Nodes.csv"
84
    )
85
86
    gas_nodes_list = pd.read_csv(
87
        target_file,
88
        delimiter=";",
89
        decimal=".",
90
        usecols=["lat", "long", "id", "country_code", "param"],
91
    )
92
93
    # Correct non valid neighbouring country nodes
94
    gas_nodes_list.loc[
95
        gas_nodes_list["id"] == "INET_N_1182", "country_code"
96
    ] = "AT"
97
    gas_nodes_list.loc[
98
        gas_nodes_list["id"] == "SEQ_10608_p", "country_code"
99
    ] = "NL"
100
    gas_nodes_list.loc[
101
        gas_nodes_list["id"] == "N_88_NS_LMGN", "country_code"
102
    ] = "XX"
103
104
    gas_nodes_list = gas_nodes_list.rename(columns={"lat": "y", "long": "x"})
105
106
    gas_nodes_list["bus_id"] = range(new_id, new_id + len(gas_nodes_list))
107
    gas_nodes_list = gas_nodes_list.set_index("id")
108
109
    return gas_nodes_list
110
111
112
def ch4_nodes_number_G(gas_nodes_list):
113
    """Insert list of CH4 nodes from SciGRID_gas IGGIELGN data
114
        Parameters
115
    ----------
116
    gas_nodes_list : dataframe
117
        Dataframe containing the gas nodes (Europe)
118
    Returns
119
    -------
120
        N_ch4_nodes_G : int
121
            Number of CH4 buses in Germany (independantly from the mode used)
122
    """
123
124
    ch4_nodes_list = gas_nodes_list[
125
        gas_nodes_list["country_code"].str.match("DE")
126
    ]
127
    N_ch4_nodes_G = len(ch4_nodes_list)
128
129
    return N_ch4_nodes_G
130
131
132
def insert_CH4_nodes_list(gas_nodes_list):
133
    """Insert list of CH4 nodes from SciGRID_gas IGGIELGN data
134
135
    Insert detailled description
136
137
    Parameters
138
    ----------
139
    gas_nodes_list : dataframe
140
        Dataframe containing the gas nodes (Europe)
141
142
    Returns
143
    -------
144
    None
145
146
    """
147
    # Connect to local database
148
    engine = db.engine()
149
150
    gas_nodes_list = gas_nodes_list[
151
        gas_nodes_list["country_code"].str.match("DE")
152
    ]  # To eventually replace with a test if the nodes are in the german boundaries.
153
154
    # Cut data to federal state if in testmode
155
    NUTS1 = []
156
    for index, row in gas_nodes_list.iterrows():
157
        param = ast.literal_eval(row["param"])
158
        NUTS1.append(param["nuts_id_1"])
159
    gas_nodes_list = gas_nodes_list.assign(NUTS1=NUTS1)
160
161
    boundary = settings()["egon-data"]["--dataset-boundary"]
162
    if boundary != "Everything":
163
        map_states = {
164
            "Baden-Württemberg": "DE1",
165
            "Nordrhein-Westfalen": "DEA",
166
            "Hessen": "DE7",
167
            "Brandenburg": "DE4",
168
            "Bremen": "DE5",
169
            "Rheinland-Pfalz": "DEB",
170
            "Sachsen-Anhalt": "DEE",
171
            "Schleswig-Holstein": "DEF",
172
            "Mecklenburg-Vorpommern": "DE8",
173
            "Thüringen": "DEG",
174
            "Niedersachsen": "DE9",
175
            "Sachsen": "DED",
176
            "Hamburg": "DE6",
177
            "Saarland": "DEC",
178
            "Berlin": "DE3",
179
            "Bayern": "DE2",
180
        }
181
182
        gas_nodes_list = gas_nodes_list[
183
            gas_nodes_list["NUTS1"].isin([map_states[boundary], np.nan])
184
        ]
185
186
        # A completer avec nodes related to pipelines which have an end in the selected area et evt deplacer ds define_gas_nodes_list
187
188
    # Add missing columns
189
    c = {"scn_name": "eGon2035", "carrier": "CH4"}
190
    gas_nodes_list = gas_nodes_list.assign(**c)
191
192
    gas_nodes_list = geopandas.GeoDataFrame(
193
        gas_nodes_list,
194
        geometry=geopandas.points_from_xy(
195
            gas_nodes_list["x"], gas_nodes_list["y"]
196
        ),
197
    )
198
    gas_nodes_list = gas_nodes_list.rename(
199
        columns={"geometry": "geom"}
200
    ).set_geometry("geom", crs=4326)
201
202
    gas_nodes_list = gas_nodes_list.reset_index(drop=True)
203
    gas_nodes_list = gas_nodes_list.drop(
204
        columns=["NUTS1", "param", "country_code"]
205
    )
206
207
    # Insert data to db
208
    db.execute_sql(
209
        f"""
210
    DELETE FROM grid.egon_etrago_bus WHERE "carrier" = 'CH4' AND
211
    scn_name = '{c['scn_name']}' AND country = 'DE';
212
    """
213
    )
214
215
    # Insert CH4 data to db
216
    print(gas_nodes_list)
217
    gas_nodes_list.to_postgis(
218
        "egon_etrago_bus",
219
        engine,
220
        schema="grid",
221
        index=False,
222
        if_exists="append",
223
        dtype={"geom": Geometry()},
224
    )
225
226
227
def insert_gas_buses_abroad(scn_name="eGon2035"):
228
    """Insert central CH4 buses in foreign countries for eGon2035
229
230
    Detailled description to be completed:
231
    Insert central gas buses in foreign countries to db, same buses
232
    than the foreign AC buses
233
234
    Parameters
235
    ----------
236
    scn_name : str
237
        Name of the scenario
238
239
    Returns
240
    -------
241
    gdf_abroad_buses : dataframe
242
        Dataframe containing the CH4 buses in the neighbouring countries
243
        and one in the center of Germany in test mode
244
    """
245
    # Select sources and targets from dataset configuration
246
    sources = config.datasets()["electrical_neighbours"]["sources"]
247
248
    main_gas_carrier = get_sector_parameters("gas", scenario=scn_name)[
249
        "main_gas_carrier"
250
    ]
251
252
    # Connect to local database
253
    engine = db.engine()
254
    db.execute_sql(
255
        f"""
256
    DELETE FROM grid.egon_etrago_bus WHERE "carrier" = '{main_gas_carrier}' AND
257
    scn_name = '{scn_name}' AND country != 'DE';
258
    """
259
    )
260
261
    # Select the foreign buses
262
    gdf_abroad_buses = central_buses_egon100(sources)
263
    gdf_abroad_buses = gdf_abroad_buses.drop_duplicates(subset=["country"])
264
265
    # Select next id value
266
    new_id = db.next_etrago_id("bus")
267
268
    gdf_abroad_buses = gdf_abroad_buses.drop(
269
        columns=[
270
            "v_nom",
271
            "v_mag_pu_set",
272
            "v_mag_pu_min",
273
            "v_mag_pu_max",
274
            "geom",
275
        ]
276
    )
277
    gdf_abroad_buses["scn_name"] = "eGon2035"
278
    gdf_abroad_buses["carrier"] = main_gas_carrier
279
    gdf_abroad_buses["bus_id"] = range(new_id, new_id + len(gdf_abroad_buses))
280
281
    # Add central bus in Russia
282
    gdf_abroad_buses = gdf_abroad_buses.append(
283
        {
284
            "scn_name": scn_name,
285
            "bus_id": (new_id + len(gdf_abroad_buses) + 1),
286
            "x": 41,
287
            "y": 55,
288
            "country": "RU",
289
            "carrier": main_gas_carrier,
290
        },
291
        ignore_index=True,
292
    )
293
    # if in test mode, add bus in center of Germany
294
    boundary = settings()["egon-data"]["--dataset-boundary"]
295
296
    if boundary != "Everything":
297
        gdf_abroad_buses = gdf_abroad_buses.append(
298
            {
299
                "scn_name": scn_name,
300
                "bus_id": (new_id + len(gdf_abroad_buses) + 1),
301
                "x": 10.4234469,
302
                "y": 51.0834196,
303
                "country": "DE",
304
                "carrier": main_gas_carrier,
305
            },
306
            ignore_index=True,
307
        )
308
309
    gdf_abroad_buses = geopandas.GeoDataFrame(
310
        gdf_abroad_buses,
311
        geometry=geopandas.points_from_xy(
312
            gdf_abroad_buses["x"], gdf_abroad_buses["y"]
313
        ),
314
    )
315
    gdf_abroad_buses = gdf_abroad_buses.rename(
316
        columns={"geometry": "geom"}
317
    ).set_geometry("geom", crs=4326)
318
319
    # Insert to db
320
    gdf_abroad_buses.to_postgis(
321
        "egon_etrago_bus",
322
        engine,
323
        schema="grid",
324
        index=False,
325
        if_exists="append",
326
        dtype={"geom": Geometry()},
327
    )
328
    return gdf_abroad_buses
329
330
331
def define_gas_pipeline_list(
332
    gas_nodes_list, abroad_gas_nodes_list, scn_name="eGon2035"
333
):
334
    """Define gas pipelines in Germany from SciGRID_gas IGGIELGN data
335
336
    Insert detailled description
337
338
    Parameters
339
    ----------
340
    gas_nodes_list : dataframe
341
        description missing
342
    abroad_gas_nodes_list: dataframe
343
        description missing
344
    scn_name : str
345
        Name of the scenario
346
347
    Returns
348
    -------
349
    gas_pipelines_list : pandas.DataFrame
350
        Dataframe containing the gas pipelines in Germany
351
352
    """
353
    abroad_gas_nodes_list = abroad_gas_nodes_list.set_index("country")
354
355
    main_gas_carrier = get_sector_parameters("gas", scenario=scn_name)[
356
        "main_gas_carrier"
357
    ]
358
359
    # Select next id value
360
    new_id = db.next_etrago_id("link")
361
362
    classifiaction_file = (
363
        Path(".")
364
        / "data_bundle_egon_data"
365
        / "pipeline_classification_gas"
366
        / "pipeline_classification.csv"
367
    )
368
369
    classification = pd.read_csv(
370
        classifiaction_file,
371
        delimiter=",",
372
        usecols=["classification", "max_transport_capacity_Gwh/d"],
373
    )
374
375
    target_file = (
376
        Path(".")
377
        / "datasets"
378
        / "gas_data"
379
        / "data"
380
        / "IGGIELGN_PipeSegments.csv"
381
    )
382
383
    gas_pipelines_list = pd.read_csv(
384
        target_file,
385
        delimiter=";",
386
        decimal=".",
387
        usecols=["id", "node_id", "lat", "long", "country_code", "param"],
388
    )
389
390
    # Select the links having at least one bus in Germany
391
    gas_pipelines_list = gas_pipelines_list[
392
        gas_pipelines_list["country_code"].str.contains("DE")
393
    ]
394
395
    # Remove links disconnected of the rest of the grid
396
    # Remove manually for disconnected link EntsoG_Map__ST_195
397
    gas_pipelines_list = gas_pipelines_list[
398
        ~gas_pipelines_list["id"].str.match("EntsoG_Map__ST_195")
399
    ]
400
401
    gas_pipelines_list["link_id"] = range(
402
        new_id, new_id + len(gas_pipelines_list)
403
    )
404
    gas_pipelines_list["link_id"] = gas_pipelines_list["link_id"].astype(int)
405
406
    # Cut data to federal state if in testmode
407
    NUTS1 = []
408
    for index, row in gas_pipelines_list.iterrows():
409
        param = ast.literal_eval(row["param"])
410
        NUTS1.append(param["nuts_id_1"])
411
    gas_pipelines_list["NUTS1"] = NUTS1
412
413
    map_states = {
414
        "Baden-Württemberg": "DE1",
415
        "Nordrhein-Westfalen": "DEA",
416
        "Hessen": "DE7",
417
        "Brandenburg": "DE4",
418
        "Bremen": "DE5",
419
        "Rheinland-Pfalz": "DEB",
420
        "Sachsen-Anhalt": "DEE",
421
        "Schleswig-Holstein": "DEF",
422
        "Mecklenburg-Vorpommern": "DE8",
423
        "Thüringen": "DEG",
424
        "Niedersachsen": "DE9",
425
        "Sachsen": "DED",
426
        "Hamburg": "DE6",
427
        "Saarland": "DEC",
428
        "Berlin": "DE3",
429
        "Bayern": "DE2",
430
        "Everything": "Nan",
431
    }
432
    gas_pipelines_list["NUTS1_0"] = [x[0] for x in gas_pipelines_list["NUTS1"]]
433
    gas_pipelines_list["NUTS1_1"] = [x[1] for x in gas_pipelines_list["NUTS1"]]
434
435
    boundary = settings()["egon-data"]["--dataset-boundary"]
436
437
    if boundary != "Everything":
438
439
        gas_pipelines_list = gas_pipelines_list[
440
            gas_pipelines_list["NUTS1_0"].str.contains(map_states[boundary])
441
            | gas_pipelines_list["NUTS1_1"].str.contains(map_states[boundary])
442
        ]
443
444
    # Add missing columns
445
    gas_pipelines_list["scn_name"] = scn_name
446
    gas_pipelines_list["carrier"] = main_gas_carrier
447
    gas_pipelines_list["p_nom_extendable"] = False
448
449
    diameter = []
450
    geom = []
451
    topo = []
452
    length_km = []
453
454
    for index, row in gas_pipelines_list.iterrows():
455
456
        param = ast.literal_eval(row["param"])
457
        diameter.append(param["diameter_mm"])
458
        length_km.append(param["length_km"])
459
460
        long_e = json.loads(row["long"])
461
        lat_e = json.loads(row["lat"])
462
        crd_e = list(zip(long_e, lat_e))
463
        topo.append(geometry.LineString(crd_e))
464
465
        long_path = param["path_long"]
466
        lat_path = param["path_lat"]
467
        crd = list(zip(long_path, lat_path))
468
        crd.insert(0, crd_e[0])
469
        crd.append(crd_e[1])
470
        lines = []
471
        for i in range(len(crd) - 1):
472
            lines.append(geometry.LineString([crd[i], crd[i + 1]]))
473
        geom.append(geometry.MultiLineString(lines))
474
475
    gas_pipelines_list["diameter"] = diameter
476
    gas_pipelines_list["geom"] = geom
477
    gas_pipelines_list["topo"] = topo
478
    gas_pipelines_list["length_km"] = length_km
479
    gas_pipelines_list = gas_pipelines_list.set_geometry("geom", crs=4326)
480
481
    country_0 = []
482
    country_1 = []
483
    for index, row in gas_pipelines_list.iterrows():
484
        c = ast.literal_eval(row["country_code"])
485
        country_0.append(c[0])
486
        country_1.append(c[1])
487
488
    gas_pipelines_list["country_0"] = country_0
489
    gas_pipelines_list["country_1"] = country_1
490
491
    # Correct non valid neighbouring country nodes
492
    gas_pipelines_list.loc[
493
        gas_pipelines_list["country_0"] == "XX", "country_0"
494
    ] = "NO"
495
    gas_pipelines_list.loc[
496
        gas_pipelines_list["country_1"] == "FI", "country_1"
497
    ] = "RU"
498
    gas_pipelines_list.loc[
499
        gas_pipelines_list["id"] == "ST_2612_Seg_0_Seg_0", "country_0"
500
    ] = "AT"  # bus "INET_N_1182" DE -> AT
501
    gas_pipelines_list.loc[
502
        gas_pipelines_list["id"] == "INET_PL_385_EE_3_Seg_0_Seg_1", "country_1"
503
    ] = "AT"  # "INET_N_1182" DE -> AT
504
    gas_pipelines_list.loc[
505
        gas_pipelines_list["id"] == "LKD_PS_0_Seg_0_Seg_3", "country_0"
506
    ] = "NL"  # bus "SEQ_10608_p" DE -> NL
507
508
    # Remove uncorrect pipelines
509
    gas_pipelines_list = gas_pipelines_list[
510
        (gas_pipelines_list["id"] != "PLNG_2637_Seg_0_Seg_0_Seg_0")
511
        & (gas_pipelines_list["id"] != "NSG_6650_Seg_2_Seg_0")
512
        & (gas_pipelines_list["id"] != "NSG_6734_Seg_2_Seg_0")
513
    ]
514
515
    # Remove link test if length = 0
516
    gas_pipelines_list = gas_pipelines_list[
517
        gas_pipelines_list["length_km"] != 0
518
    ]
519
520
    # Adjust columns
521
    bus0 = []
522
    bus1 = []
523
    geom_adjusted = []
524
    topo_adjusted = []
525
    length_adjusted = []
526
    pipe_class = []
527
528
    for index, row in gas_pipelines_list.iterrows():
529
        buses = row["node_id"].strip("][").split(", ")
530
531
        if (
532
            (boundary != "Everything")
533
            & (row["NUTS1_0"] != map_states[boundary])
534
            & (row["country_0"] == "DE")
535
        ):
536
            bus0.append(abroad_gas_nodes_list.loc["DE", "bus_id"])
537
            bus1.append(gas_nodes_list.loc[buses[1][1:-1], "bus_id"])
538
            long_e = [
539
                abroad_gas_nodes_list.loc["DE", "x"],
540
                json.loads(row["long"])[1],
541
            ]
542
            lat_e = [
543
                abroad_gas_nodes_list.loc["DE", "y"],
544
                json.loads(row["lat"])[1],
545
            ]
546
            geom_pipe = geometry.MultiLineString(
547
                [geometry.LineString(list(zip(long_e, lat_e)))]
548
            )
549
            topo_adjusted.append(geometry.LineString(list(zip(long_e, lat_e))))
550
551
        elif row["country_0"] != "DE":
552
            country = str(row["country_0"])
553
            bus0.append(abroad_gas_nodes_list.loc[country, "bus_id"])
554
            bus1.append(gas_nodes_list.loc[buses[1][1:-1], "bus_id"])
555
            long_e = [
556
                abroad_gas_nodes_list.loc[country, "x"],
557
                json.loads(row["long"])[1],
558
            ]
559
            lat_e = [
560
                abroad_gas_nodes_list.loc[country, "y"],
561
                json.loads(row["lat"])[1],
562
            ]
563
            geom_pipe = geometry.MultiLineString(
564
                [geometry.LineString(list(zip(long_e, lat_e)))]
565
            )
566
            topo_adjusted.append(geometry.LineString(list(zip(long_e, lat_e))))
567
568
        elif (
569
            (boundary != "Everything")
570
            & (row["NUTS1_1"] != map_states[boundary])
571
            & (row["country_1"] == "DE")
572
        ):
573
            bus0.append(gas_nodes_list.loc[buses[0][1:-1], "bus_id"])
574
            bus1.append(abroad_gas_nodes_list.loc["DE", "bus_id"])
575
            long_e = [
576
                json.loads(row["long"])[0],
577
                abroad_gas_nodes_list.loc["DE", "x"],
578
            ]
579
            lat_e = [
580
                json.loads(row["lat"])[0],
581
                abroad_gas_nodes_list.loc["DE", "y"],
582
            ]
583
            geom_pipe = geometry.MultiLineString(
584
                [geometry.LineString(list(zip(long_e, lat_e)))]
585
            )
586
            topo_adjusted.append(geometry.LineString(list(zip(long_e, lat_e))))
587
588
        elif row["country_1"] != "DE":
589
            country = str(row["country_1"])
590
            bus0.append(gas_nodes_list.loc[buses[0][1:-1], "bus_id"])
591
            bus1.append(abroad_gas_nodes_list.loc[country, "bus_id"])
592
            long_e = [
593
                json.loads(row["long"])[0],
594
                abroad_gas_nodes_list.loc[country, "x"],
595
            ]
596
            lat_e = [
597
                json.loads(row["lat"])[0],
598
                abroad_gas_nodes_list.loc[country, "y"],
599
            ]
600
            geom_pipe = geometry.MultiLineString(
601
                [geometry.LineString(list(zip(long_e, lat_e)))]
602
            )
603
            topo_adjusted.append(geometry.LineString(list(zip(long_e, lat_e))))
604
605
        else:
606
            bus0.append(gas_nodes_list.loc[buses[0][1:-1], "bus_id"])
607
            bus1.append(gas_nodes_list.loc[buses[1][1:-1], "bus_id"])
608
            geom_pipe = row["geom"]
609
            topo_adjusted.append(row["topo"])
610
611
        geom_adjusted.append(geom_pipe)
612
        length_adjusted.append(geom_pipe.length)
613
614
        if row["diameter"] >= 1000:
615
            pipe_class = "A"
616
        elif 700 <= row["diameter"] <= 1000:
617
            pipe_class = "B"
618
        elif 500 <= row["diameter"] <= 700:
619
            pipe_class = "C"
620
        elif 350 <= row["diameter"] <= 500:
621
            pipe_class = "D"
622
        elif 200 <= row["diameter"] <= 350:
623
            pipe_class = "E"
624
        elif 100 <= row["diameter"] <= 200:
625
            pipe_class = "F"
626
        elif row["diameter"] <= 100:
627
            pipe_class = "G"
628
629
    gas_pipelines_list["bus0"] = bus0
630
    gas_pipelines_list["bus1"] = bus1
631
    gas_pipelines_list["geom"] = geom_adjusted
632
    gas_pipelines_list["topo"] = topo_adjusted
633
    gas_pipelines_list["length"] = length_adjusted
634
    gas_pipelines_list["pipe_class"] = pipe_class
635
636
    # Remove pipes having the same node for start and end
637
    gas_pipelines_list = gas_pipelines_list[
638
        gas_pipelines_list["bus0"] != gas_pipelines_list["bus1"]
639
    ]
640
641
    gas_pipelines_list = gas_pipelines_list.merge(
642
        classification,
643
        how="left",
644
        left_on="pipe_class",
645
        right_on="classification",
646
    )
647
    gas_pipelines_list["p_nom"] = gas_pipelines_list[
648
        "max_transport_capacity_Gwh/d"
649
    ] * (1000 / 24)
650
651
    # Remove useless columns
652
    gas_pipelines_list = gas_pipelines_list.drop(
653
        columns=[
654
            "id",
655
            "node_id",
656
            "param",
657
            "NUTS1",
658
            "NUTS1_0",
659
            "NUTS1_1",
660
            "country_code",
661
            "diameter",
662
            "pipe_class",
663
            "classification",
664
            "max_transport_capacity_Gwh/d",
665
            "lat",
666
            "long",
667
            "length_km",
668
        ]
669
    )
670
671
    return gas_pipelines_list
672
673
674
def insert_gas_pipeline_list(gas_pipelines_list, scn_name="eGon2035"):
675
    """Insert list of gas pipelines in the database
676
677
    Insert detailled description
678
679
    Parameters
680
    ----------
681
    gas_pipelines_list : pandas.DataFrame
682
        Dataframe containing the gas pipelines in Germany
683
    scn_name : str
684
        Name of the scenario
685
686
    """
687
    main_gas_carrier = get_sector_parameters("gas", scenario=scn_name)[
688
        "main_gas_carrier"
689
    ]
690
    engine = db.engine()
691
    gas_pipelines_list = gas_pipelines_list.drop(
692
        columns=[
693
            "country_0",
694
            "country_1",
695
        ]
696
    )
697
698
    # Clean db
699
    db.execute_sql(
700
        f"""DELETE FROM grid.egon_etrago_link
701
        WHERE "carrier" = '{main_gas_carrier}'
702
        AND scn_name = '{scn_name}';
703
        """
704
    )
705
706
    print(gas_pipelines_list)
707
    # Insert data to db
708
    gas_pipelines_list.to_postgis(
709
        "egon_etrago_gas_link",
710
        engine,
711
        schema="grid",
712
        index=False,
713
        if_exists="replace",
714
        dtype={"geom": Geometry(), "topo": Geometry()},
715
    )
716
717
    db.execute_sql(
718
        """
719
    select UpdateGeometrySRID('grid', 'egon_etrago_gas_link', 'topo', 4326) ;
720
721
    INSERT INTO grid.egon_etrago_link (scn_name,
722
                                              link_id, carrier,
723
                                              bus0, bus1,
724
                                              p_nom, p_nom_extendable, length,
725
                                              geom, topo)
726
    SELECT scn_name,
727
                link_id, carrier,
728
                bus0, bus1,
729
                p_nom, p_nom_extendable, length,
730
                geom, topo
731
732
    FROM grid.egon_etrago_gas_link;
733
734
    DROP TABLE grid.egon_etrago_gas_link;
735
        """
736
    )
737
738
739
def remove_isolated_gas_buses():
740
    """Delete gas buses which are not connected to the gas grid.
741
    Returns
742
    -------
743
    None.
744
    """
745
    targets = config.datasets()["gas_grid"]["targets"]
746
747
    db.execute_sql(
748
        f"""
749
        DELETE FROM {targets['buses']['schema']}.{targets['buses']['table']}
750
        WHERE "carrier" = 'CH4'
751
        AND scn_name = 'eGon2035'
752
        AND country = 'DE'
753
        AND "bus_id" NOT IN
754
            (SELECT bus0 FROM {targets['links']['schema']}.{targets['links']['table']}
755
            WHERE scn_name = 'eGon2035'
756
            AND carrier = 'CH4')
757
        AND "bus_id" NOT IN
758
            (SELECT bus1 FROM {targets['links']['schema']}.{targets['links']['table']}
759
            WHERE scn_name = 'eGon2035'
760
            AND carrier = 'CH4');
761
    """
762
    )
763
764
765
def insert_gas_data():
766
    """Overall function for importing gas data from SciGRID_gas
767
    Returns
768
    -------
769
    None.
770
    """
771
    download_SciGRID_gas_data()
772
773
    gas_nodes_list = define_gas_nodes_list()
774
775
    insert_CH4_nodes_list(gas_nodes_list)
776
    abroad_gas_nodes_list = insert_gas_buses_abroad()
777
778
    gas_pipeline_list = define_gas_pipeline_list(
779
        gas_nodes_list, abroad_gas_nodes_list
780
    )
781
    insert_gas_pipeline_list(gas_pipeline_list)
782
    remove_isolated_gas_buses()
783
784
785
def insert_gas_data_eGon100RE():
786
    """Overall function for importing gas data from SciGRID_gas
787
    Returns
788
    -------
789
    None.
790
    """
791
    # copy buses
792
    copy_and_modify_buses("eGon2035", "eGon100RE", {"carrier": ["CH4"]})
793
794
    # get CH4 pipelines and modify their nominal capacity with the
795
    # retrofitting factor
796
    gdf = db.select_geodataframe(
797
        f"""
798
        SELECT * FROM grid.egon_etrago_link
799
        WHERE carrier = 'CH4' AND scn_name = 'eGon2035' AND
800
        bus0 IN (
801
            SELECT bus_id FROM grid.egon_etrago_bus
802
            WHERE scn_name = 'eGon2035' AND country = 'DE'
803
        ) AND bus1 IN (
804
            SELECT bus_id FROM grid.egon_etrago_bus
805
            WHERE scn_name = 'eGon2035' AND country = 'DE'
806
        );
807
        """,
808
        epsg=4326,
809
        geom_col="topo",
810
    )
811
812
    # Update scenario specific information
813
    scn_name = "eGon100RE"
814
    gdf["scn_name"] = scn_name
815
    scn_params = get_sector_parameters("gas", scn_name)
816
817
    for param in ["capital_cost", "marginal_cost", "efficiency"]:
818
        try:
819
            gdf.loc[:, param] = scn_params[param]["CH4"]
820
        except KeyError:
821
            pass
822
823
    # remaining CH4 share is 1 - retroffited pipeline share
824
    gdf["p_nom"] *= (
825
        1 - scn_params["retrofitted_CH4pipeline-to-H2pipeline_share"]
826
    )
827
828
    # delete old entries
829
    db.execute_sql(
830
        f"""
831
        DELETE FROM grid.egon_etrago_link
832
        WHERE carrier = 'CH4' AND scn_name = '{scn_name}' AND
833
        bus0 NOT IN (
834
            SELECT bus_id FROM grid.egon_etrago_bus
835
            WHERE scn_name = '{scn_name}' AND country != 'DE'
836
        ) AND bus1 NOT IN (
837
            SELECT bus_id FROM grid.egon_etrago_bus
838
            WHERE scn_name = '{scn_name}' AND country != 'DE'
839
        );
840
        """
841
    )
842
843
    gdf.to_postgis(
844
        "egon_etrago_link",
845
        schema="grid",
846
        if_exists="append",
847
        con=db.engine(),
848
        index=False,
849
        dtype={"geom": Geometry(), "topo": Geometry()},
850
    )
851