data.datasets.gas_neighbours.gas_abroad.insert_gas_grid_capacities() - Code Metrics - Inspection of "Gas sanity checks" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#1006)

unknown

created 2022-12-14 18:40 UTC

insert_gas_grid_capacities() A

↳ Parent: data.datasets.gas_neighbours.gas_abroad

Complexity

Conditions

Size

Total Lines	107
Code Lines	26

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	26
dl	0
loc	107
rs	9.256
c	0
b	0
f	0
cc	4
nop	2

How to fix Long Method

"""Module containing functions to insert gas abroad

In this module, functions useful to insert the gas components (H2 and
CH4) abroad for eGon2035 and eGon100RE are defined.

"""
import zipfile

from geoalchemy2.types import Geometry
import geopandas as gpd
import pandas as pd

from egon.data import config, db
from egon.data.datasets.electrical_neighbours import get_map_buses
from egon.data.datasets.scenario_parameters import get_sector_parameters


def insert_generators(gen, scn_name):
    """Insert gas generators for foreign countries in database

    This function inserts the gas generators for foreign countries
    with the following steps:
      * Clean the database
      * Receive the gas production capacities per foreign node
      * For eGon2035, put them in the righ format
      * Add missing columns (generator_id and carrier)
      * Insert the table into the database

    Parameters
    ----------
    gen : pandas.DataFrame
        Gas production capacities per foreign node
    scn_name : str
        Name of the scenario

    Returns
    -------
    None

    """
    carrier = "CH4"
    sources = config.datasets()["gas_neighbours"]["sources"]
    targets = config.datasets()["gas_neighbours"]["targets"]

    # Delete existing data
    db.execute_sql(
        f"""
        DELETE FROM
        {targets['generators']['schema']}.{targets['generators']['table']}
        WHERE bus IN (
            SELECT bus_id FROM
            {sources['buses']['schema']}.{sources['buses']['table']}
            WHERE country != 'DE'
            AND scn_name = '{scn_name}')
        AND scn_name = '{scn_name}'
        AND carrier = '{carrier}';
        """
    )

    if scn_name == "eGon2035":
        map_buses = get_map_buses()
        scn_params = get_sector_parameters("gas", scn_name)

        # Set bus_id
        gen.loc[
            gen[gen["index"].isin(map_buses.keys())].index, "index"
        ] = gen.loc[
            gen[gen["index"].isin(map_buses.keys())].index, "index"
        ].map(
            map_buses
        )
        gen.loc[:, "bus"] = (
            get_foreign_gas_bus_id().loc[gen.loc[:, "index"]].values
        )

        gen["p_nom"] = gen["cap_2035"]
        gen["marginal_cost"] = (
            gen["ratioConv_2035"] * scn_params["marginal_cost"]["CH4"]
            + (1 - gen["ratioConv_2035"])
            * scn_params["marginal_cost"]["biogas"]
        )

        gen["scn_name"] = scn_name

        # Remove useless columns
        gen = gen.drop(columns=["index", "ratioConv_2035", "cap_2035"])

    # Add missing columns
    new_id = db.next_etrago_id("generator")
    gen["generator_id"] = range(new_id, new_id + len(gen))

    gen["carrier"] = carrier

    # Insert data to db
    gen.to_sql(
        targets["generators"]["table"],
        db.engine(),
        schema=targets["generators"]["schema"],
        index=False,
        if_exists="append",
    )


def insert_gas_grid_capacities(Neighbouring_pipe_capacities_list, scn_name):
    """Insert crossbordering gas pipelines in the database

    This function insert a list of crossbordering gas pipelines after
    cleaning the database.
    For eGon2035, all the CH4 crossbordering pipelines are inserted
    there (no H2 grid in this scenario).
    For eGon100RE, only the the crossbordering pipelines with Germany
    are inserted there (the other ones are inerted in PypsaEurSec),
    but in this scenario there are H2 and CH4 pipelines.

    Parameters
    ----------
    Neighbouring_pipe_capacities_list : pandas.DataFrame
        List of the crossbordering gas pipelines
    scn_name : str
        Name of the scenario

    Returns
    -------
    None

    """
    sources = config.datasets()["gas_neighbours"]["sources"]
    targets = config.datasets()["gas_neighbours"]["targets"]

    # Delete existing data
    if scn_name == "eGon2035":
        carrier_link = "CH4"
        carrier_bus = "CH4"

        db.execute_sql(
            f"""
            DELETE FROM 
            {sources['links']['schema']}.{sources['links']['table']}
            WHERE "bus0" IN (
                SELECT bus_id FROM 
                {sources['buses']['schema']}.{sources['buses']['table']}
                    WHERE country != 'DE'
                    AND carrier = '{carrier_bus}'
                    AND scn_name = '{scn_name}')
            OR "bus1" IN (
                SELECT bus_id FROM 
                {sources['buses']['schema']}.{sources['buses']['table']}
                    WHERE country != 'DE'
                    AND carrier = '{carrier_bus}' 
                    AND scn_name = '{scn_name}')
            AND scn_name = '{scn_name}'
            AND carrier = '{carrier_link}'            
            ;
            """
        )

    carriers = {
        "CH4": {"bus_inDE": "CH4", "bus_abroad": "CH4"},
        "H2_retrofit": {"bus_inDE": "H2_grid", "bus_abroad": "H2"},
    }

    if scn_name == "eGon100RE":
        for c in carriers:
            db.execute_sql(
                f"""
                DELETE FROM
                {sources['links']['schema']}.{sources['links']['table']}
                WHERE ("bus0" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country != 'DE'
                        AND carrier = '{carriers[c]["bus_abroad"]}'
                        AND scn_name = '{scn_name}')
                    AND "bus1" IN (SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country = 'DE'
                        AND carrier = '{carriers[c]["bus_inDE"]}' 
                        AND scn_name = '{scn_name}'))
                OR ("bus0" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country = 'DE'
                        AND carrier = '{carriers[c]["bus_inDE"]}'
                        AND scn_name = '{scn_name}')
                AND "bus1" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country != 'DE'
                        AND carrier = '{carriers[c]["bus_abroad"]}' 
                        AND scn_name = '{scn_name}'))
                AND scn_name = '{scn_name}'
                AND carrier = '{c.index}'            
                ;
                """
            )

    # Insert data to db
    Neighbouring_pipe_capacities_list.set_geometry(
        "geom", crs=4326
    ).to_postgis(
        "egon_etrago_gas_link",
        db.engine(),
        schema="grid",
        index=False,
        if_exists="replace",
        dtype={"geom": Geometry(), "topo": Geometry()},
    )

    db.execute_sql(
        f"""
    select UpdateGeometrySRID('grid', 'egon_etrago_gas_link', 'topo', 4326) ;

    INSERT INTO {targets['links']['schema']}.{targets['links']['table']} (
        scn_name, link_id, carrier,
        bus0, bus1, p_nom, length, geom, topo)
    
    SELECT scn_name, link_id, carrier, bus0, bus1, p_nom, length, geom, topo

    FROM grid.egon_etrago_gas_link;

    DROP TABLE grid.egon_etrago_gas_link;
        """
    )


def get_foreign_gas_bus_id(scn_name="eGon2035", carrier="CH4"):

    """Calculate the etrago bus id based on the geometry for eGon2035

    Map node_ids from TYNDP and etragos bus_id

    Parameters
    ----------
    scn_name : str
        Name of the scenario
    carrier : str
        Name of the carrier

    Returns
    -------
    pandas.Series
        List of mapped node_ids from TYNDP and etragos bus_id

    """
    sources = config.datasets()["gas_neighbours"]["sources"]

    bus_id = db.select_geodataframe(
        f"""
        SELECT bus_id, ST_Buffer(geom, 1) as geom, country
        FROM grid.egon_etrago_bus
        WHERE scn_name = '{scn_name}'
        AND carrier = '{carrier}'
        AND country != 'DE'
        """,
        epsg=3035,
    )

    # insert installed capacities
    file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}")

    # Select buses in neighbouring countries as geodataframe
    buses = pd.read_excel(
        file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(),
        sheet_name="Nodes - Dict",
    ).query("longitude==longitude")
    buses = gpd.GeoDataFrame(
        buses,
        crs=4326,
        geometry=gpd.points_from_xy(buses.longitude, buses.latitude),
    ).to_crs(3035)

    buses["bus_id"] = 0

    # Select bus_id from etrago with shortest distance to TYNDP node
    for i, row in buses.iterrows():
        distance = bus_id.set_index("bus_id").geom.distance(row.geometry)
        buses.loc[i, "bus_id"] = distance[
            distance == distance.min()
        ].index.values[0]

    return buses.set_index("node_id").bus_id


1		"""Module containing functions to insert gas abroad
2
3		In this module, functions useful to insert the gas components (H2 and
4		CH4) abroad for eGon2035 and eGon100RE are defined.
5
6		"""
7		import zipfile
8
9		from geoalchemy2.types import Geometry
10		import geopandas as gpd
11		import pandas as pd
12
13		from egon.data import config, db
14		from egon.data.datasets.electrical_neighbours import get_map_buses
15		from egon.data.datasets.scenario_parameters import get_sector_parameters
16
17
18		def insert_generators(gen, scn_name):
19		"""Insert gas generators for foreign countries in database
20
21		This function inserts the gas generators for foreign countries
22		with the following steps:
23		* Clean the database
24		* Receive the gas production capacities per foreign node
25		* For eGon2035, put them in the righ format
26		* Add missing columns (generator_id and carrier)
27		* Insert the table into the database
28
29		Parameters
30		----------
31		gen : pandas.DataFrame
32		Gas production capacities per foreign node
33		scn_name : str
34		Name of the scenario
35
36		Returns
37		-------
38		None
39
40		"""
41		carrier = "CH4"
42		sources = config.datasets()["gas_neighbours"]["sources"]
43		targets = config.datasets()["gas_neighbours"]["targets"]
44
45		# Delete existing data
46		db.execute_sql(
47		f"""
48		DELETE FROM
49		{targets['generators']['schema']}.{targets['generators']['table']}
50		WHERE bus IN (
51		SELECT bus_id FROM
52		{sources['buses']['schema']}.{sources['buses']['table']}
53		WHERE country != 'DE'
54		AND scn_name = '{scn_name}')
55		AND scn_name = '{scn_name}'
56		AND carrier = '{carrier}';
57		"""
58		)
59
60		if scn_name == "eGon2035":
61		map_buses = get_map_buses()
62		scn_params = get_sector_parameters("gas", scn_name)
63
64		# Set bus_id
65		gen.loc[
66		gen[gen["index"].isin(map_buses.keys())].index, "index"
67		] = gen.loc[
68		gen[gen["index"].isin(map_buses.keys())].index, "index"
69		].map(
70		map_buses
71		)
72		gen.loc[:, "bus"] = (
73		get_foreign_gas_bus_id().loc[gen.loc[:, "index"]].values
74		)
75
76		gen["p_nom"] = gen["cap_2035"]
77		gen["marginal_cost"] = (
78		gen["ratioConv_2035"] * scn_params["marginal_cost"]["CH4"]
79		+ (1 - gen["ratioConv_2035"])
80		* scn_params["marginal_cost"]["biogas"]
81		)
82
83		gen["scn_name"] = scn_name
84
85		# Remove useless columns
86		gen = gen.drop(columns=["index", "ratioConv_2035", "cap_2035"])
87
88		# Add missing columns
89		new_id = db.next_etrago_id("generator")
90		gen["generator_id"] = range(new_id, new_id + len(gen))
91
92		gen["carrier"] = carrier
93
94		# Insert data to db
95		gen.to_sql(
96		targets["generators"]["table"],
97		db.engine(),
98		schema=targets["generators"]["schema"],
99		index=False,
100		if_exists="append",
101		)
102
103
104		def insert_gas_grid_capacities(Neighbouring_pipe_capacities_list, scn_name):
105		"""Insert crossbordering gas pipelines in the database
106
107		This function insert a list of crossbordering gas pipelines after
108		cleaning the database.
109		For eGon2035, all the CH4 crossbordering pipelines are inserted
110		there (no H2 grid in this scenario).
111		For eGon100RE, only the the crossbordering pipelines with Germany
112		are inserted there (the other ones are inerted in PypsaEurSec),
113		but in this scenario there are H2 and CH4 pipelines.
114
115		Parameters
116		----------
117		Neighbouring_pipe_capacities_list : pandas.DataFrame
118		List of the crossbordering gas pipelines
119		scn_name : str
120		Name of the scenario
121
122		Returns
123		-------
124		None
125
126		"""
127		sources = config.datasets()["gas_neighbours"]["sources"]
128		targets = config.datasets()["gas_neighbours"]["targets"]
129
130		# Delete existing data
131		if scn_name == "eGon2035":
132		carrier_link = "CH4"
133		carrier_bus = "CH4"
134
135		db.execute_sql(
136		f"""
137		DELETE FROM
138		{sources['links']['schema']}.{sources['links']['table']}
139		WHERE "bus0" IN (
140		SELECT bus_id FROM
141		{sources['buses']['schema']}.{sources['buses']['table']}
142		WHERE country != 'DE'
143		AND carrier = '{carrier_bus}'
144		AND scn_name = '{scn_name}')
145		OR "bus1" IN (
146		SELECT bus_id FROM
147		{sources['buses']['schema']}.{sources['buses']['table']}
148		WHERE country != 'DE'
149		AND carrier = '{carrier_bus}'
150		AND scn_name = '{scn_name}')
151		AND scn_name = '{scn_name}'
152		AND carrier = '{carrier_link}'
153		;
154		"""
155		)
156
157		carriers = {
158		"CH4": {"bus_inDE": "CH4", "bus_abroad": "CH4"},
159		"H2_retrofit": {"bus_inDE": "H2_grid", "bus_abroad": "H2"},
160		}
161
162		if scn_name == "eGon100RE":
163		for c in carriers:
164		db.execute_sql(
165		f"""
166		DELETE FROM
167		{sources['links']['schema']}.{sources['links']['table']}
168		WHERE ("bus0" IN (
169		SELECT bus_id FROM
170		{sources['buses']['schema']}.{sources['buses']['table']}
171		WHERE country != 'DE'
172		AND carrier = '{carriers[c]["bus_abroad"]}'
173		AND scn_name = '{scn_name}')
174		AND "bus1" IN (SELECT bus_id FROM
175		{sources['buses']['schema']}.{sources['buses']['table']}
176		WHERE country = 'DE'
177		AND carrier = '{carriers[c]["bus_inDE"]}'
178		AND scn_name = '{scn_name}'))
179		OR ("bus0" IN (
180		SELECT bus_id FROM
181		{sources['buses']['schema']}.{sources['buses']['table']}
182		WHERE country = 'DE'
183		AND carrier = '{carriers[c]["bus_inDE"]}'
184		AND scn_name = '{scn_name}')
185		AND "bus1" IN (
186		SELECT bus_id FROM
187		{sources['buses']['schema']}.{sources['buses']['table']}
188		WHERE country != 'DE'
189		AND carrier = '{carriers[c]["bus_abroad"]}'
190		AND scn_name = '{scn_name}'))
191		AND scn_name = '{scn_name}'
192		AND carrier = '{c.index}'
193		;
194		"""
195		)
196
197		# Insert data to db
198		Neighbouring_pipe_capacities_list.set_geometry(
199		"geom", crs=4326
200		).to_postgis(
201		"egon_etrago_gas_link",
202		db.engine(),
203		schema="grid",
204		index=False,
205		if_exists="replace",
206		dtype={"geom": Geometry(), "topo": Geometry()},
207		)
208
209		db.execute_sql(
210		f"""
211		select UpdateGeometrySRID('grid', 'egon_etrago_gas_link', 'topo', 4326) ;
212
213		INSERT INTO {targets['links']['schema']}.{targets['links']['table']} (
214		scn_name, link_id, carrier,
215		bus0, bus1, p_nom, length, geom, topo)
216
217		SELECT scn_name, link_id, carrier, bus0, bus1, p_nom, length, geom, topo
218
219		FROM grid.egon_etrago_gas_link;
220
221		DROP TABLE grid.egon_etrago_gas_link;
222		"""
223		)
224
225
226	View Code Duplication	def get_foreign_gas_bus_id(scn_name="eGon2035", carrier="CH4"):
		0 ignored issues – show Duplication introduced 2022-09-12 14:50 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
227		"""Calculate the etrago bus id based on the geometry for eGon2035
228
229		Map node_ids from TYNDP and etragos bus_id
230
231		Parameters
232		----------
233		scn_name : str
234		Name of the scenario
235		carrier : str
236		Name of the carrier
237
238		Returns
239		-------
240		pandas.Series
241		List of mapped node_ids from TYNDP and etragos bus_id
242
243		"""
244		sources = config.datasets()["gas_neighbours"]["sources"]
245
246		bus_id = db.select_geodataframe(
247		f"""
248		SELECT bus_id, ST_Buffer(geom, 1) as geom, country
249		FROM grid.egon_etrago_bus
250		WHERE scn_name = '{scn_name}'
251		AND carrier = '{carrier}'
252		AND country != 'DE'
253		""",
254		epsg=3035,
255		)
256
257		# insert installed capacities
258		file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}")
259
260		# Select buses in neighbouring countries as geodataframe
261		buses = pd.read_excel(
262		file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(),
263		sheet_name="Nodes - Dict",
264		).query("longitude==longitude")
265		buses = gpd.GeoDataFrame(
266		buses,
267		crs=4326,
268		geometry=gpd.points_from_xy(buses.longitude, buses.latitude),
269		).to_crs(3035)
270
271		buses["bus_id"] = 0
272
273		# Select bus_id from etrago with shortest distance to TYNDP node
274		for i, row in buses.iterrows():
275		distance = bus_id.set_index("bus_id").geom.distance(row.geometry)
276		buses.loc[i, "bus_id"] = distance[
277		distance == distance.min()
278		].index.values[0]
279
280		return buses.set_index("node_id").bus_id
281

openego / eGon-data

Pull Request — dev (#1006)

insert_gas_grid_capacities() A

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like