data.datasets.gas_neighbours.gas_abroad.insert_gas_grid_capacities() - Code Metrics - Inspection of "Biogas generators for eGon100RE" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#931)

unknown

created 2023-02-22 11:31 UTC

insert_gas_grid_capacities() A

↳ Parent: data.datasets.gas_neighbours.gas_abroad

Complexity

Conditions

Size

Total Lines	107
Code Lines	26

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	26
dl	0
loc	107
rs	9.256
c	0
b	0
f	0
cc	4
nop	2

How to fix Long Method

"""Module containing functions to insert gas abroad

In this module, functions useful to insert the gas components (H2 and
CH4) abroad for eGon2035 and eGon100RE are defined.

"""
import zipfile

from geoalchemy2.types import Geometry
import geopandas as gpd
import pandas as pd

from egon.data import config, db
from egon.data.datasets.electrical_neighbours import get_map_buses
from egon.data.datasets.scenario_parameters import get_sector_parameters


def insert_generators(gen, scn_name):
    """Insert gas generators for foreign countries in database

    This function inserts the gas generators for foreign countries
    with the following steps:
      * Clean the database
      * Receive the gas production capacities per foreign node
      * For eGon2035, put them in the righ format
      * Add missing columns (generator_id and carrier)
      * Insert the table into the database

    Parameters
    ----------
    gen : pandas.DataFrame
        Gas production capacities per foreign node
    scn_name : str
        Name of the scenario

    Returns
    -------
    None

    """
    carrier = "CH4"
    sources = config.datasets()["gas_neighbours"]["sources"]
    targets = config.datasets()["gas_neighbours"]["targets"]

    # Delete existing data
    db.execute_sql(
        f"""
        DELETE FROM
        {targets['generators']['schema']}.{targets['generators']['table']}
        WHERE bus IN (
            SELECT bus_id FROM
            {sources['buses']['schema']}.{sources['buses']['table']}
            WHERE country != 'DE'
            AND scn_name = '{scn_name}')
        AND scn_name = '{scn_name}'
        AND carrier = '{carrier}';
        """
    )

    if scn_name == "eGon2035":
        map_buses = get_map_buses()
        scn_params = get_sector_parameters("gas", scn_name)

        # Set bus_id
        gen.loc[
            gen[gen["index"].isin(map_buses.keys())].index, "index"
        ] = gen.loc[
            gen[gen["index"].isin(map_buses.keys())].index, "index"
        ].map(
            map_buses
        )
        gen.loc[:, "bus"] = (
            get_foreign_gas_bus_id().loc[gen.loc[:, "index"]].values
        )

        gen["p_nom"] = gen["cap_2035"]
        gen["marginal_cost"] = (
        gen["share_LNG_2035"] * scn_params["marginal_cost"]["CH4"] * 1.3
        + gen["share_conv_pipe_2035"] * scn_params["marginal_cost"]["CH4"]
        + gen["share_bio_2035"] * scn_params["marginal_cost"]["biogas"]
        )
        gen["scn_name"] = scn_name

        # Remove useless columns
        gen = gen.drop(
            columns=[
                "index",
                "share_LNG_2035",
                "share_conv_pipe_2035",
                "share_bio_2035",
                "cap_2035",
            ]
    )

    # Add missing columns
    new_id = db.next_etrago_id("generator")
    gen["generator_id"] = range(new_id, new_id + len(gen))

    gen["carrier"] = carrier

    # Insert data to db
    gen.to_sql(
        targets["generators"]["table"],
        db.engine(),
        schema=targets["generators"]["schema"],
        index=False,
        if_exists="append",
    )


def insert_gas_grid_capacities(Neighbouring_pipe_capacities_list, scn_name):
    """Insert crossbordering gas pipelines in the database

    This function insert a list of crossbordering gas pipelines after
    cleaning the database.
    For eGon2035, all the CH4 crossbordering pipelines are inserted
    there (no H2 grid in this scenario).
    For eGon100RE, only the the crossbordering pipelines with Germany
    are inserted there (the other ones are inerted in PypsaEurSec),
    but in this scenario there are H2 and CH4 pipelines.

    Parameters
    ----------
    Neighbouring_pipe_capacities_list : pandas.DataFrame
        List of the crossbordering gas pipelines
    scn_name : str
        Name of the scenario

    Returns
    -------
    None

    """
    sources = config.datasets()["gas_neighbours"]["sources"]
    targets = config.datasets()["gas_neighbours"]["targets"]

    # Delete existing data
    if scn_name == "eGon2035":
        carrier_link = "CH4"
        carrier_bus = "CH4"

        db.execute_sql(
            f"""
            DELETE FROM 
            {sources['links']['schema']}.{sources['links']['table']}
            WHERE "bus0" IN (
                SELECT bus_id FROM 
                {sources['buses']['schema']}.{sources['buses']['table']}
                    WHERE country != 'DE'
                    AND carrier = '{carrier_bus}'
                    AND scn_name = '{scn_name}')
            OR "bus1" IN (
                SELECT bus_id FROM 
                {sources['buses']['schema']}.{sources['buses']['table']}
                    WHERE country != 'DE'
                    AND carrier = '{carrier_bus}' 
                    AND scn_name = '{scn_name}')
            AND scn_name = '{scn_name}'
            AND carrier = '{carrier_link}'            
            ;
            """
        )

    carriers = {
        "CH4": {"bus_inDE": "CH4", "bus_abroad": "CH4"},
        "H2_retrofit": {"bus_inDE": "H2_grid", "bus_abroad": "H2"},
    }

    if scn_name == "eGon100RE":
        for c in carriers:
            db.execute_sql(
                f"""
                DELETE FROM
                {sources['links']['schema']}.{sources['links']['table']}
                WHERE ("bus0" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country != 'DE'
                        AND carrier = '{carriers[c]["bus_abroad"]}'
                        AND scn_name = '{scn_name}')
                    AND "bus1" IN (SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country = 'DE'
                        AND carrier = '{carriers[c]["bus_inDE"]}' 
                        AND scn_name = '{scn_name}'))
                OR ("bus0" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country = 'DE'
                        AND carrier = '{carriers[c]["bus_inDE"]}'
                        AND scn_name = '{scn_name}')
                AND "bus1" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country != 'DE'
                        AND carrier = '{carriers[c]["bus_abroad"]}' 
                        AND scn_name = '{scn_name}'))
                AND scn_name = '{scn_name}'
                AND carrier = '{c.index}'            
                ;
                """
            )

    # Insert data to db
    Neighbouring_pipe_capacities_list.set_geometry(
        "geom", crs=4326
    ).to_postgis(
        "egon_etrago_gas_link",
        db.engine(),
        schema="grid",
        index=False,
        if_exists="replace",
        dtype={"geom": Geometry(), "topo": Geometry()},
    )

    db.execute_sql(
        f"""
    select UpdateGeometrySRID('grid', 'egon_etrago_gas_link', 'topo', 4326) ;

    INSERT INTO {targets['links']['schema']}.{targets['links']['table']} (
        scn_name, link_id, carrier,
        bus0, bus1, p_nom, p_min_pu, length, geom, topo)
    
    SELECT scn_name, link_id, carrier, bus0, bus1, p_nom, p_min_pu, length, geom, topo

    FROM grid.egon_etrago_gas_link;

    DROP TABLE grid.egon_etrago_gas_link;
        """
    )


def get_foreign_gas_bus_id(scn_name="eGon2035", carrier="CH4"):

    """Calculate the etrago bus id based on the geometry for eGon2035

    Map node_ids from TYNDP and etragos bus_id

    Parameters
    ----------
    scn_name : str
        Name of the scenario
    carrier : str
        Name of the carrier

    Returns
    -------
    pandas.Series
        List of mapped node_ids from TYNDP and etragos bus_id

    """
    sources = config.datasets()["gas_neighbours"]["sources"]

    bus_id = db.select_geodataframe(
        f"""
        SELECT bus_id, ST_Buffer(geom, 1) as geom, country
        FROM grid.egon_etrago_bus
        WHERE scn_name = '{scn_name}'
        AND carrier = '{carrier}'
        AND country != 'DE'
        """,
        epsg=3035,
    )

    # insert installed capacities
    file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}")

    # Select buses in neighbouring countries as geodataframe
    buses = pd.read_excel(
        file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(),
        sheet_name="Nodes - Dict",
    ).query("longitude==longitude")
    buses = gpd.GeoDataFrame(
        buses,
        crs=4326,
        geometry=gpd.points_from_xy(buses.longitude, buses.latitude),
    ).to_crs(3035)

    buses["bus_id"] = 0

    # Select bus_id from etrago with shortest distance to TYNDP node
    for i, row in buses.iterrows():
        distance = bus_id.set_index("bus_id").geom.distance(row.geometry)
        buses.loc[i, "bus_id"] = distance[
            distance == distance.min()
        ].index.values[0]

    return buses.set_index("node_id").bus_id


1		"""Module containing functions to insert gas abroad
2
3		In this module, functions useful to insert the gas components (H2 and
4		CH4) abroad for eGon2035 and eGon100RE are defined.
5
6		"""
7		import zipfile
8
9		from geoalchemy2.types import Geometry
10		import geopandas as gpd
11		import pandas as pd
12
13		from egon.data import config, db
14		from egon.data.datasets.electrical_neighbours import get_map_buses
15		from egon.data.datasets.scenario_parameters import get_sector_parameters
16
17
18		def insert_generators(gen, scn_name):
19		"""Insert gas generators for foreign countries in database
20
21		This function inserts the gas generators for foreign countries
22		with the following steps:
23		* Clean the database
24		* Receive the gas production capacities per foreign node
25		* For eGon2035, put them in the righ format
26		* Add missing columns (generator_id and carrier)
27		* Insert the table into the database
28
29		Parameters
30		----------
31		gen : pandas.DataFrame
32		Gas production capacities per foreign node
33		scn_name : str
34		Name of the scenario
35
36		Returns
37		-------
38		None
39
40		"""
41		carrier = "CH4"
42		sources = config.datasets()["gas_neighbours"]["sources"]
43		targets = config.datasets()["gas_neighbours"]["targets"]
44
45		# Delete existing data
46		db.execute_sql(
47		f"""
48		DELETE FROM
49		{targets['generators']['schema']}.{targets['generators']['table']}
50		WHERE bus IN (
51		SELECT bus_id FROM
52		{sources['buses']['schema']}.{sources['buses']['table']}
53		WHERE country != 'DE'
54		AND scn_name = '{scn_name}')
55		AND scn_name = '{scn_name}'
56		AND carrier = '{carrier}';
57		"""
58		)
59
60		if scn_name == "eGon2035":
61		map_buses = get_map_buses()
62		scn_params = get_sector_parameters("gas", scn_name)
63
64		# Set bus_id
65		gen.loc[
66		gen[gen["index"].isin(map_buses.keys())].index, "index"
67		] = gen.loc[
68		gen[gen["index"].isin(map_buses.keys())].index, "index"
69		].map(
70		map_buses
71		)
72		gen.loc[:, "bus"] = (
73		get_foreign_gas_bus_id().loc[gen.loc[:, "index"]].values
74		)
75
76		gen["p_nom"] = gen["cap_2035"]
77		gen["marginal_cost"] = (
78		gen["share_LNG_2035"] * scn_params["marginal_cost"]["CH4"] * 1.3
79		+ gen["share_conv_pipe_2035"] * scn_params["marginal_cost"]["CH4"]
80		+ gen["share_bio_2035"] * scn_params["marginal_cost"]["biogas"]
81		)
82		gen["scn_name"] = scn_name
83
84		# Remove useless columns
85		gen = gen.drop(
86		columns=[
87		"index",
88		"share_LNG_2035",
89		"share_conv_pipe_2035",
90		"share_bio_2035",
91		"cap_2035",
92		]
93		)
94
95		# Add missing columns
96		new_id = db.next_etrago_id("generator")
97		gen["generator_id"] = range(new_id, new_id + len(gen))
98
99		gen["carrier"] = carrier
100
101		# Insert data to db
102		gen.to_sql(
103		targets["generators"]["table"],
104		db.engine(),
105		schema=targets["generators"]["schema"],
106		index=False,
107		if_exists="append",
108		)
109
110
111		def insert_gas_grid_capacities(Neighbouring_pipe_capacities_list, scn_name):
112		"""Insert crossbordering gas pipelines in the database
113
114		This function insert a list of crossbordering gas pipelines after
115		cleaning the database.
116		For eGon2035, all the CH4 crossbordering pipelines are inserted
117		there (no H2 grid in this scenario).
118		For eGon100RE, only the the crossbordering pipelines with Germany
119		are inserted there (the other ones are inerted in PypsaEurSec),
120		but in this scenario there are H2 and CH4 pipelines.
121
122		Parameters
123		----------
124		Neighbouring_pipe_capacities_list : pandas.DataFrame
125		List of the crossbordering gas pipelines
126		scn_name : str
127		Name of the scenario
128
129		Returns
130		-------
131		None
132
133		"""
134		sources = config.datasets()["gas_neighbours"]["sources"]
135		targets = config.datasets()["gas_neighbours"]["targets"]
136
137		# Delete existing data
138		if scn_name == "eGon2035":
139		carrier_link = "CH4"
140		carrier_bus = "CH4"
141
142		db.execute_sql(
143		f"""
144		DELETE FROM
145		{sources['links']['schema']}.{sources['links']['table']}
146		WHERE "bus0" IN (
147		SELECT bus_id FROM
148		{sources['buses']['schema']}.{sources['buses']['table']}
149		WHERE country != 'DE'
150		AND carrier = '{carrier_bus}'
151		AND scn_name = '{scn_name}')
152		OR "bus1" IN (
153		SELECT bus_id FROM
154		{sources['buses']['schema']}.{sources['buses']['table']}
155		WHERE country != 'DE'
156		AND carrier = '{carrier_bus}'
157		AND scn_name = '{scn_name}')
158		AND scn_name = '{scn_name}'
159		AND carrier = '{carrier_link}'
160		;
161		"""
162		)
163
164		carriers = {
165		"CH4": {"bus_inDE": "CH4", "bus_abroad": "CH4"},
166		"H2_retrofit": {"bus_inDE": "H2_grid", "bus_abroad": "H2"},
167		}
168
169		if scn_name == "eGon100RE":
170		for c in carriers:
171		db.execute_sql(
172		f"""
173		DELETE FROM
174		{sources['links']['schema']}.{sources['links']['table']}
175		WHERE ("bus0" IN (
176		SELECT bus_id FROM
177		{sources['buses']['schema']}.{sources['buses']['table']}
178		WHERE country != 'DE'
179		AND carrier = '{carriers[c]["bus_abroad"]}'
180		AND scn_name = '{scn_name}')
181		AND "bus1" IN (SELECT bus_id FROM
182		{sources['buses']['schema']}.{sources['buses']['table']}
183		WHERE country = 'DE'
184		AND carrier = '{carriers[c]["bus_inDE"]}'
185		AND scn_name = '{scn_name}'))
186		OR ("bus0" IN (
187		SELECT bus_id FROM
188		{sources['buses']['schema']}.{sources['buses']['table']}
189		WHERE country = 'DE'
190		AND carrier = '{carriers[c]["bus_inDE"]}'
191		AND scn_name = '{scn_name}')
192		AND "bus1" IN (
193		SELECT bus_id FROM
194		{sources['buses']['schema']}.{sources['buses']['table']}
195		WHERE country != 'DE'
196		AND carrier = '{carriers[c]["bus_abroad"]}'
197		AND scn_name = '{scn_name}'))
198		AND scn_name = '{scn_name}'
199		AND carrier = '{c.index}'
200		;
201		"""
202		)
203
204		# Insert data to db
205		Neighbouring_pipe_capacities_list.set_geometry(
206		"geom", crs=4326
207		).to_postgis(
208		"egon_etrago_gas_link",
209		db.engine(),
210		schema="grid",
211		index=False,
212		if_exists="replace",
213		dtype={"geom": Geometry(), "topo": Geometry()},
214		)
215
216		db.execute_sql(
217		f"""
218		select UpdateGeometrySRID('grid', 'egon_etrago_gas_link', 'topo', 4326) ;
219
220		INSERT INTO {targets['links']['schema']}.{targets['links']['table']} (
221		scn_name, link_id, carrier,
222		bus0, bus1, p_nom, p_min_pu, length, geom, topo)
223
224		SELECT scn_name, link_id, carrier, bus0, bus1, p_nom, p_min_pu, length, geom, topo
225
226		FROM grid.egon_etrago_gas_link;
227
228		DROP TABLE grid.egon_etrago_gas_link;
229		"""
230		)
231
232
233	View Code Duplication	def get_foreign_gas_bus_id(scn_name="eGon2035", carrier="CH4"):
		0 ignored issues – show Duplication introduced 2022-09-12 14:50 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
234		"""Calculate the etrago bus id based on the geometry for eGon2035
235
236		Map node_ids from TYNDP and etragos bus_id
237
238		Parameters
239		----------
240		scn_name : str
241		Name of the scenario
242		carrier : str
243		Name of the carrier
244
245		Returns
246		-------
247		pandas.Series
248		List of mapped node_ids from TYNDP and etragos bus_id
249
250		"""
251		sources = config.datasets()["gas_neighbours"]["sources"]
252
253		bus_id = db.select_geodataframe(
254		f"""
255		SELECT bus_id, ST_Buffer(geom, 1) as geom, country
256		FROM grid.egon_etrago_bus
257		WHERE scn_name = '{scn_name}'
258		AND carrier = '{carrier}'
259		AND country != 'DE'
260		""",
261		epsg=3035,
262		)
263
264		# insert installed capacities
265		file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}")
266
267		# Select buses in neighbouring countries as geodataframe
268		buses = pd.read_excel(
269		file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(),
270		sheet_name="Nodes - Dict",
271		).query("longitude==longitude")
272		buses = gpd.GeoDataFrame(
273		buses,
274		crs=4326,
275		geometry=gpd.points_from_xy(buses.longitude, buses.latitude),
276		).to_crs(3035)
277
278		buses["bus_id"] = 0
279
280		# Select bus_id from etrago with shortest distance to TYNDP node
281		for i, row in buses.iterrows():
282		distance = bus_id.set_index("bus_id").geom.distance(row.geometry)
283		buses.loc[i, "bus_id"] = distance[
284		distance == distance.min()
285		].index.values[0]
286
287		return buses.set_index("node_id").bus_id
288

openego / eGon-data

Pull Request — dev (#931)

insert_gas_grid_capacities() A

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like