data.datasets.gas_neighbours.gas_abroad.insert_ch4_stores() - Code Metrics - Inspection of "Add missing gas stores in eGon100RE" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#936)

unknown

created 2023-03-31 08:57 UTC

insert_ch4_stores() A

↳ Parent: data.datasets.gas_neighbours.gas_abroad

Complexity

Conditions

Size

Total Lines	56
Code Lines	18

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	18
dl	0
loc	56
rs	9.5
c	0
b	0
f	0
cc	1
nop	2

How to fix Long Method

"""Module containing functions to insert gas abroad

In this module, functions useful to insert the gas components (H2 and
CH4) abroad for eGon2035 and eGon100RE are defined.

"""
import zipfile

from geoalchemy2.types import Geometry
import geopandas as gpd
import pandas as pd

from egon.data import config, db
from egon.data.datasets.electrical_neighbours import get_map_buses
from egon.data.datasets.scenario_parameters import get_sector_parameters


def insert_generators(gen, scn_name):
    """Insert gas generators for foreign countries in database

    This function inserts the gas generators for foreign countries
    with the following steps:
      * Clean the database
      * Receive the gas production capacities per foreign node
      * For eGon2035, put them in the righ format
      * Add missing columns (generator_id and carrier)
      * Insert the table into the database

    Parameters
    ----------
    gen : pandas.DataFrame
        Gas production capacities per foreign node
    scn_name : str
        Name of the scenario

    Returns
    -------
    None

    """
    carrier = "CH4"
    sources = config.datasets()["gas_neighbours"]["sources"]
    targets = config.datasets()["gas_neighbours"]["targets"]

    # Delete existing data
    db.execute_sql(
        f"""
        DELETE FROM
        {targets['generators']['schema']}.{targets['generators']['table']}
        WHERE bus IN (
            SELECT bus_id FROM
            {sources['buses']['schema']}.{sources['buses']['table']}
            WHERE country != 'DE'
            AND scn_name = '{scn_name}')
        AND scn_name = '{scn_name}'
        AND carrier = '{carrier}';
        """
    )

    if scn_name == "eGon2035":
        map_buses = get_map_buses()
        scn_params = get_sector_parameters("gas", scn_name)

        # Set bus_id
        gen.loc[
            gen[gen["index"].isin(map_buses.keys())].index, "index"
        ] = gen.loc[
            gen[gen["index"].isin(map_buses.keys())].index, "index"
        ].map(
            map_buses
        )
        gen.loc[:, "bus"] = (
            get_foreign_gas_bus_id().loc[gen.loc[:, "index"]].values
        )

        gen["p_nom"] = gen["cap_2035"]
        gen["marginal_cost"] = (
            gen["ratioConv_2035"] * scn_params["marginal_cost"]["CH4"]
            + (1 - gen["ratioConv_2035"])
            * scn_params["marginal_cost"]["biogas"]
        )

        gen["scn_name"] = scn_name

        # Remove useless columns
        gen = gen.drop(columns=["index", "ratioConv_2035", "cap_2035"])

    # Add missing columns
    new_id = db.next_etrago_id("generator")
    gen["generator_id"] = range(new_id, new_id + len(gen))

    gen["carrier"] = carrier

    # Insert data to db
    gen.to_sql(
        targets["generators"]["table"],
        db.engine(),
        schema=targets["generators"]["schema"],
        index=False,
        if_exists="append",
    )


def insert_ch4_stores(ch4_storage_capacities, scn_name):
    """Insert CH4 stores for foreign countries in database

    This function inserts the CH4 stores for foreign countries
    with the following steps:
      * Receive as argument the CH4 store capacities per foreign node
      * Clean the database
      * Add missing columns (scn_name, carrier and store_id)
      * Insert the table into the database

    Parameters
    ----------
    ch4_storage_capacities : pandas.DataFrame
        CH4 store capacities per foreign node
    scn_name : str
        Name of the scenario

    Returns
    -------
    None

    """
    carrier = "CH4"
    sources = config.datasets()["gas_neighbours"]["sources"]
    targets = config.datasets()["gas_neighbours"]["targets"]

    # Clean table
    db.execute_sql(
        f"""
        DELETE FROM {targets['stores']['schema']}.{targets['stores']['table']}
        WHERE "carrier" = '{carrier}'
        AND scn_name = '{scn_name}'
        AND bus IN (
            SELECT bus_id FROM {sources['buses']['schema']}.{sources['buses']['table']}
            WHERE scn_name = '{scn_name}'
            AND country != 'DE'
            );
        """
    )
    # Add missing columns
    c = {"scn_name": scn_name, "carrier": carrier, "e_cyclic": True}
    ch4_storage_capacities = ch4_storage_capacities.assign(**c)

    new_id = db.next_etrago_id("store")
    ch4_storage_capacities["store_id"] = range(
        new_id, new_id + len(ch4_storage_capacities)
    )

    ch4_storage_capacities = ch4_storage_capacities.reset_index(drop=True)
    # Insert data to db
    ch4_storage_capacities.to_sql(
        targets["stores"]["table"],
        db.engine(),
        schema=targets["stores"]["schema"],
        index=False,
        if_exists="append",
    )


def insert_gas_grid_capacities(Neighbouring_pipe_capacities_list, scn_name):
    """Insert crossbordering gas pipelines in the database

    This function insert a list of crossbordering gas pipelines after
    cleaning the database.
    For eGon2035, all the CH4 crossbordering pipelines are inserted
    there (no H2 grid in this scenario).
    For eGon100RE, only the the crossbordering pipelines with Germany
    are inserted there (the other ones are inerted in PypsaEurSec),
    but in this scenario there are H2 and CH4 pipelines.

    Parameters
    ----------
    Neighbouring_pipe_capacities_list : pandas.DataFrame
        List of the crossbordering gas pipelines
    scn_name : str
        Name of the scenario

    Returns
    -------
    None

    """
    sources = config.datasets()["gas_neighbours"]["sources"]
    targets = config.datasets()["gas_neighbours"]["targets"]

    # Delete existing data
    if scn_name == "eGon2035":
        carrier_link = "CH4"
        carrier_bus = "CH4"

        db.execute_sql(
            f"""
            DELETE FROM 
            {sources['links']['schema']}.{sources['links']['table']}
            WHERE "bus0" IN (
                SELECT bus_id FROM 
                {sources['buses']['schema']}.{sources['buses']['table']}
                    WHERE country != 'DE'
                    AND carrier = '{carrier_bus}'
                    AND scn_name = '{scn_name}')
            OR "bus1" IN (
                SELECT bus_id FROM 
                {sources['buses']['schema']}.{sources['buses']['table']}
                    WHERE country != 'DE'
                    AND carrier = '{carrier_bus}' 
                    AND scn_name = '{scn_name}')
            AND scn_name = '{scn_name}'
            AND carrier = '{carrier_link}'            
            ;
            """
        )

    carriers = {"CH4": "CH4", "H2_retrofit": "H2_grid"}

    if scn_name == "eGon100RE":
        for c in carriers:
            db.execute_sql(
                f"""
                DELETE FROM
                {sources['links']['schema']}.{sources['links']['table']}
                WHERE ("bus0" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country != 'DE'
                        AND carrier = '{carriers[c]}'
                        AND scn_name = '{scn_name}')
                    AND "bus1" IN (SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country = 'DE'
                        AND carrier = '{carriers[c]}'
                        AND scn_name = '{scn_name}'))
                OR ("bus0" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country = 'DE'
                        AND carrier = '{carriers[c]}'
                        AND scn_name = '{scn_name}')
                    AND "bus1" IN (
                        SELECT bus_id FROM 
                        {sources['buses']['schema']}.{sources['buses']['table']}
                        WHERE country != 'DE'
                        AND carrier = '{carriers[c]}'
                        AND scn_name = '{scn_name}'))
                AND scn_name = '{scn_name}'
                AND carrier = '{c}'
                ;
                """
            )

    # Insert data to db
    Neighbouring_pipe_capacities_list.set_geometry(
        "geom", crs=4326
    ).to_postgis(
        "egon_etrago_gas_link",
        db.engine(),
        schema="grid",
        index=False,
        if_exists="replace",
        dtype={"geom": Geometry(), "topo": Geometry()},
    )

    db.execute_sql(
        f"""
    select UpdateGeometrySRID('grid', 'egon_etrago_gas_link', 'topo', 4326) ;

    INSERT INTO {targets['links']['schema']}.{targets['links']['table']} (
        scn_name, link_id, carrier,
        bus0, bus1, p_nom, p_min_pu, length, geom, topo)
    
    SELECT scn_name, link_id, carrier, bus0, bus1, p_nom, p_min_pu, length, geom, topo

    FROM grid.egon_etrago_gas_link;

    DROP TABLE grid.egon_etrago_gas_link;
        """
    )


def get_foreign_gas_bus_id(scn_name="eGon2035", carrier="CH4"):

    """Calculate the etrago bus id based on the geometry for eGon2035

    Map node_ids from TYNDP and etragos bus_id

    Parameters
    ----------
    scn_name : str
        Name of the scenario
    carrier : str
        Name of the carrier

    Returns
    -------
    pandas.Series
        List of mapped node_ids from TYNDP and etragos bus_id

    """
    sources = config.datasets()["gas_neighbours"]["sources"]

    bus_id = db.select_geodataframe(
        f"""
        SELECT bus_id, ST_Buffer(geom, 1) as geom, country
        FROM grid.egon_etrago_bus
        WHERE scn_name = '{scn_name}'
        AND carrier = '{carrier}'
        AND country != 'DE'
        """,
        epsg=3035,
    )

    # insert installed capacities
    file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}")

    # Select buses in neighbouring countries as geodataframe
    buses = pd.read_excel(
        file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(),
        sheet_name="Nodes - Dict",
    ).query("longitude==longitude")
    buses = gpd.GeoDataFrame(
        buses,
        crs=4326,
        geometry=gpd.points_from_xy(buses.longitude, buses.latitude),
    ).to_crs(3035)

    buses["bus_id"] = 0

    # Select bus_id from etrago with shortest distance to TYNDP node
    for i, row in buses.iterrows():
        distance = bus_id.set_index("bus_id").geom.distance(row.geometry)
        buses.loc[i, "bus_id"] = distance[
            distance == distance.min()
        ].index.values[0]

    return buses.set_index("node_id").bus_id


1		"""Module containing functions to insert gas abroad
2
3		In this module, functions useful to insert the gas components (H2 and
4		CH4) abroad for eGon2035 and eGon100RE are defined.
5
6		"""
7		import zipfile
8
9		from geoalchemy2.types import Geometry
10		import geopandas as gpd
11		import pandas as pd
12
13		from egon.data import config, db
14		from egon.data.datasets.electrical_neighbours import get_map_buses
15		from egon.data.datasets.scenario_parameters import get_sector_parameters
16
17
18		def insert_generators(gen, scn_name):
19		"""Insert gas generators for foreign countries in database
20
21		This function inserts the gas generators for foreign countries
22		with the following steps:
23		* Clean the database
24		* Receive the gas production capacities per foreign node
25		* For eGon2035, put them in the righ format
26		* Add missing columns (generator_id and carrier)
27		* Insert the table into the database
28
29		Parameters
30		----------
31		gen : pandas.DataFrame
32		Gas production capacities per foreign node
33		scn_name : str
34		Name of the scenario
35
36		Returns
37		-------
38		None
39
40		"""
41		carrier = "CH4"
42		sources = config.datasets()["gas_neighbours"]["sources"]
43		targets = config.datasets()["gas_neighbours"]["targets"]
44
45		# Delete existing data
46		db.execute_sql(
47		f"""
48		DELETE FROM
49		{targets['generators']['schema']}.{targets['generators']['table']}
50		WHERE bus IN (
51		SELECT bus_id FROM
52		{sources['buses']['schema']}.{sources['buses']['table']}
53		WHERE country != 'DE'
54		AND scn_name = '{scn_name}')
55		AND scn_name = '{scn_name}'
56		AND carrier = '{carrier}';
57		"""
58		)
59
60		if scn_name == "eGon2035":
61		map_buses = get_map_buses()
62		scn_params = get_sector_parameters("gas", scn_name)
63
64		# Set bus_id
65		gen.loc[
66		gen[gen["index"].isin(map_buses.keys())].index, "index"
67		] = gen.loc[
68		gen[gen["index"].isin(map_buses.keys())].index, "index"
69		].map(
70		map_buses
71		)
72		gen.loc[:, "bus"] = (
73		get_foreign_gas_bus_id().loc[gen.loc[:, "index"]].values
74		)
75
76		gen["p_nom"] = gen["cap_2035"]
77		gen["marginal_cost"] = (
78		gen["ratioConv_2035"] * scn_params["marginal_cost"]["CH4"]
79		+ (1 - gen["ratioConv_2035"])
80		* scn_params["marginal_cost"]["biogas"]
81		)
82
83		gen["scn_name"] = scn_name
84
85		# Remove useless columns
86		gen = gen.drop(columns=["index", "ratioConv_2035", "cap_2035"])
87
88		# Add missing columns
89		new_id = db.next_etrago_id("generator")
90		gen["generator_id"] = range(new_id, new_id + len(gen))
91
92		gen["carrier"] = carrier
93
94		# Insert data to db
95		gen.to_sql(
96		targets["generators"]["table"],
97		db.engine(),
98		schema=targets["generators"]["schema"],
99		index=False,
100		if_exists="append",
101		)
102
103
104		def insert_ch4_stores(ch4_storage_capacities, scn_name):
105		"""Insert CH4 stores for foreign countries in database
106
107		This function inserts the CH4 stores for foreign countries
108		with the following steps:
109		* Receive as argument the CH4 store capacities per foreign node
110		* Clean the database
111		* Add missing columns (scn_name, carrier and store_id)
112		* Insert the table into the database
113
114		Parameters
115		----------
116		ch4_storage_capacities : pandas.DataFrame
117		CH4 store capacities per foreign node
118		scn_name : str
119		Name of the scenario
120
121		Returns
122		-------
123		None
124
125		"""
126		carrier = "CH4"
127		sources = config.datasets()["gas_neighbours"]["sources"]
128		targets = config.datasets()["gas_neighbours"]["targets"]
129
130		# Clean table
131		db.execute_sql(
132		f"""
133		DELETE FROM {targets['stores']['schema']}.{targets['stores']['table']}
134		WHERE "carrier" = '{carrier}'
135		AND scn_name = '{scn_name}'
136		AND bus IN (
137		SELECT bus_id FROM {sources['buses']['schema']}.{sources['buses']['table']}
138		WHERE scn_name = '{scn_name}'
139		AND country != 'DE'
140		);
141		"""
142		)
143		# Add missing columns
144		c = {"scn_name": scn_name, "carrier": carrier, "e_cyclic": True}
145		ch4_storage_capacities = ch4_storage_capacities.assign(**c)
146
147		new_id = db.next_etrago_id("store")
148		ch4_storage_capacities["store_id"] = range(
149		new_id, new_id + len(ch4_storage_capacities)
150		)
151
152		ch4_storage_capacities = ch4_storage_capacities.reset_index(drop=True)
153		# Insert data to db
154		ch4_storage_capacities.to_sql(
155		targets["stores"]["table"],
156		db.engine(),
157		schema=targets["stores"]["schema"],
158		index=False,
159		if_exists="append",
160		)
161
162
163		def insert_gas_grid_capacities(Neighbouring_pipe_capacities_list, scn_name):
164		"""Insert crossbordering gas pipelines in the database
165
166		This function insert a list of crossbordering gas pipelines after
167		cleaning the database.
168		For eGon2035, all the CH4 crossbordering pipelines are inserted
169		there (no H2 grid in this scenario).
170		For eGon100RE, only the the crossbordering pipelines with Germany
171		are inserted there (the other ones are inerted in PypsaEurSec),
172		but in this scenario there are H2 and CH4 pipelines.
173
174		Parameters
175		----------
176		Neighbouring_pipe_capacities_list : pandas.DataFrame
177		List of the crossbordering gas pipelines
178		scn_name : str
179		Name of the scenario
180
181		Returns
182		-------
183		None
184
185		"""
186		sources = config.datasets()["gas_neighbours"]["sources"]
187		targets = config.datasets()["gas_neighbours"]["targets"]
188
189		# Delete existing data
190		if scn_name == "eGon2035":
191		carrier_link = "CH4"
192		carrier_bus = "CH4"
193
194		db.execute_sql(
195		f"""
196		DELETE FROM
197		{sources['links']['schema']}.{sources['links']['table']}
198		WHERE "bus0" IN (
199		SELECT bus_id FROM
200		{sources['buses']['schema']}.{sources['buses']['table']}
201		WHERE country != 'DE'
202		AND carrier = '{carrier_bus}'
203		AND scn_name = '{scn_name}')
204		OR "bus1" IN (
205		SELECT bus_id FROM
206		{sources['buses']['schema']}.{sources['buses']['table']}
207		WHERE country != 'DE'
208		AND carrier = '{carrier_bus}'
209		AND scn_name = '{scn_name}')
210		AND scn_name = '{scn_name}'
211		AND carrier = '{carrier_link}'
212		;
213		"""
214		)
215
216		carriers = {"CH4": "CH4", "H2_retrofit": "H2_grid"}
217
218		if scn_name == "eGon100RE":
219		for c in carriers:
220		db.execute_sql(
221		f"""
222		DELETE FROM
223		{sources['links']['schema']}.{sources['links']['table']}
224		WHERE ("bus0" IN (
225		SELECT bus_id FROM
226		{sources['buses']['schema']}.{sources['buses']['table']}
227		WHERE country != 'DE'
228		AND carrier = '{carriers[c]}'
229		AND scn_name = '{scn_name}')
230		AND "bus1" IN (SELECT bus_id FROM
231		{sources['buses']['schema']}.{sources['buses']['table']}
232		WHERE country = 'DE'
233		AND carrier = '{carriers[c]}'
234		AND scn_name = '{scn_name}'))
235		OR ("bus0" IN (
236		SELECT bus_id FROM
237		{sources['buses']['schema']}.{sources['buses']['table']}
238		WHERE country = 'DE'
239		AND carrier = '{carriers[c]}'
240		AND scn_name = '{scn_name}')
241		AND "bus1" IN (
242		SELECT bus_id FROM
243		{sources['buses']['schema']}.{sources['buses']['table']}
244		WHERE country != 'DE'
245		AND carrier = '{carriers[c]}'
246		AND scn_name = '{scn_name}'))
247		AND scn_name = '{scn_name}'
248		AND carrier = '{c}'
249		;
250		"""
251		)
252
253		# Insert data to db
254		Neighbouring_pipe_capacities_list.set_geometry(
255		"geom", crs=4326
256		).to_postgis(
257		"egon_etrago_gas_link",
258		db.engine(),
259		schema="grid",
260		index=False,
261		if_exists="replace",
262		dtype={"geom": Geometry(), "topo": Geometry()},
263		)
264
265		db.execute_sql(
266		f"""
267		select UpdateGeometrySRID('grid', 'egon_etrago_gas_link', 'topo', 4326) ;
268
269		INSERT INTO {targets['links']['schema']}.{targets['links']['table']} (
270		scn_name, link_id, carrier,
271		bus0, bus1, p_nom, p_min_pu, length, geom, topo)
272
273		SELECT scn_name, link_id, carrier, bus0, bus1, p_nom, p_min_pu, length, geom, topo
274
275		FROM grid.egon_etrago_gas_link;
276
277		DROP TABLE grid.egon_etrago_gas_link;
278		"""
279		)
280
281
282	View Code Duplication	def get_foreign_gas_bus_id(scn_name="eGon2035", carrier="CH4"):
		0 ignored issues – show Duplication introduced 2022-09-12 14:50 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
283		"""Calculate the etrago bus id based on the geometry for eGon2035
284
285		Map node_ids from TYNDP and etragos bus_id
286
287		Parameters
288		----------
289		scn_name : str
290		Name of the scenario
291		carrier : str
292		Name of the carrier
293
294		Returns
295		-------
296		pandas.Series
297		List of mapped node_ids from TYNDP and etragos bus_id
298
299		"""
300		sources = config.datasets()["gas_neighbours"]["sources"]
301
302		bus_id = db.select_geodataframe(
303		f"""
304		SELECT bus_id, ST_Buffer(geom, 1) as geom, country
305		FROM grid.egon_etrago_bus
306		WHERE scn_name = '{scn_name}'
307		AND carrier = '{carrier}'
308		AND country != 'DE'
309		""",
310		epsg=3035,
311		)
312
313		# insert installed capacities
314		file = zipfile.ZipFile(f"tyndp/{sources['tyndp_capacities']}")
315
316		# Select buses in neighbouring countries as geodataframe
317		buses = pd.read_excel(
318		file.open("TYNDP-2020-Scenario-Datafile.xlsx").read(),
319		sheet_name="Nodes - Dict",
320		).query("longitude==longitude")
321		buses = gpd.GeoDataFrame(
322		buses,
323		crs=4326,
324		geometry=gpd.points_from_xy(buses.longitude, buses.latitude),
325		).to_crs(3035)
326
327		buses["bus_id"] = 0
328
329		# Select bus_id from etrago with shortest distance to TYNDP node
330		for i, row in buses.iterrows():
331		distance = bus_id.set_index("bus_id").geom.distance(row.geometry)
332		buses.loc[i, "bus_id"] = distance[
333		distance == distance.min()
334		].index.values[0]
335
336		return buses.set_index("node_id").bus_id
337

openego / eGon-data

Pull Request — dev (#936)

insert_ch4_stores() A

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like