data.datasets.ch4_prod.insert_ch4_generators() - Code Metrics - Inspection of "Add missing gas stores in eGon100RE" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#936)

unknown

created 2023-02-21 15:06 UTC

data.datasets.ch4_prod.insert_ch4_generators() A

↳ Parent: data.datasets.ch4_prod

Complexity

Conditions

Size

Total Lines	13
Code Lines	3

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	3
dl	0
loc	13
rs	10
c	0
b	0
f	0
cc	1
nop	0

# -*- coding: utf-8 -*-
"""
The central module containing all code dealing with importing CH4 production data
"""
from pathlib import Path
from urllib.request import urlretrieve
import ast
import json

import geopandas as gpd
import numpy as np
import pandas as pd

from egon.data import config, db
from egon.data.config import settings
from egon.data.datasets import Dataset
from egon.data.datasets.pypsaeursec import read_network
from egon.data.datasets.scenario_parameters import get_sector_parameters


class CH4Production(Dataset):
    def __init__(self, dependencies):
        super().__init__(
            name="CH4Production",
            version="0.0.8",
            dependencies=dependencies,
            tasks=(insert_ch4_generators),
        )


def load_NG_generators(scn_name="eGon2035"):
    """Define the natural CH4 production units in Germany

    Parameters
    ----------
    scn_name : str
        Name of the scenario.
    Returns
    -------
    CH4_generators_list :
        Dataframe containing the natural gas production units in Germany

    """
    # read carrier information from scnario parameter data
    scn_params = get_sector_parameters("gas", scn_name)

    target_file = (
        Path(".")
        / "datasets"
        / "gas_data"
        / "data"
        / "IGGIELGN_Productions.csv"
    )

    NG_generators_list = pd.read_csv(
        target_file,
        delimiter=";",
        decimal=".",
        usecols=["lat", "long", "country_code", "param"],
    )

    NG_generators_list = NG_generators_list[
        NG_generators_list["country_code"].str.match("DE")
    ]

    # Cut data to federal state if in testmode
    NUTS1 = []
    for index, row in NG_generators_list.iterrows():
        param = ast.literal_eval(row["param"])
        NUTS1.append(param["nuts_id_1"])
    NG_generators_list = NG_generators_list.assign(NUTS1=NUTS1)

    boundary = settings()["egon-data"]["--dataset-boundary"]
    if boundary != "Everything":
        map_states = {
            "Baden-Württemberg": "DE1",
            "Nordrhein-Westfalen": "DEA",
            "Hessen": "DE7",
            "Brandenburg": "DE4",
            "Bremen": "DE5",
            "Rheinland-Pfalz": "DEB",
            "Sachsen-Anhalt": "DEE",
            "Schleswig-Holstein": "DEF",
            "Mecklenburg-Vorpommern": "DE8",
            "Thüringen": "DEG",
            "Niedersachsen": "DE9",
            "Sachsen": "DED",
            "Hamburg": "DE6",
            "Saarland": "DEC",
            "Berlin": "DE3",
            "Bayern": "DE2",
        }

        NG_generators_list = NG_generators_list[
            NG_generators_list["NUTS1"].isin([map_states[boundary], np.nan])
        ]

    NG_generators_list = NG_generators_list.rename(
        columns={"lat": "y", "long": "x"}
    )
    NG_generators_list = gpd.GeoDataFrame(
        NG_generators_list,
        geometry=gpd.points_from_xy(
            NG_generators_list["x"], NG_generators_list["y"]
        ),
    )
    NG_generators_list = NG_generators_list.rename(
        columns={"geometry": "geom"}
    ).set_geometry("geom", crs=4326)

    # Insert p_nom
    p_nom = []
    for index, row in NG_generators_list.iterrows():
        param = ast.literal_eval(row["param"])
        p_nom.append(param["max_supply_M_m3_per_d"])

    conversion_factor = 437.5  # MCM/day to MWh/h
    NG_generators_list["p_nom"] = [i * conversion_factor for i in p_nom]

    # Add missing columns
    NG_generators_list["marginal_cost"] = scn_params["marginal_cost"]["CH4"]

    # Remove useless columns
    NG_generators_list = NG_generators_list.drop(
        columns=["x", "y", "param", "country_code", "NUTS1"]
    )

    return NG_generators_list


def download_biogas_data():
    """Download the biogas production units data in Germany

    Parameters
    ----------
    None

    Returns
    -------
    None

    """
    basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx"
    url = (
        "https://www.biogaspartner.de/fileadmin/Biogaspartner/Dokumente/Einspeiseatlas/"
        + basename
    )
    target_file = Path(".") / "datasets" / "gas_data" / basename

    urlretrieve(url, target_file)


def load_biogas_generators(scn_name):
    """Define the biogas production units in Germany

    Parameters
    ----------
    scn_name : str
        Name of the scenario.

    Returns
    -------
    CH4_generators_list :
        Dataframe containing the biogas production units in Germany

    """
    # read carrier information from scnario parameter data
    scn_params = get_sector_parameters("gas", scn_name)

    basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx"
    target_file = Path(".") / "datasets" / "gas_data" / basename

    # Read-in data from csv-file
    biogas_generators_list = pd.read_excel(
        target_file,
        usecols=["Koordinaten", "Einspeisung Biomethan [(N*m^3)/h)]"],
    )

    x = []
    y = []
    for index, row in biogas_generators_list.iterrows():
        coordinates = row["Koordinaten"].split(",")
        y.append(coordinates[0])
        x.append(coordinates[1])
    biogas_generators_list["x"] = x
    biogas_generators_list["y"] = y

    biogas_generators_list = gpd.GeoDataFrame(
        biogas_generators_list,
        geometry=gpd.points_from_xy(
            biogas_generators_list["x"], biogas_generators_list["y"]
        ),
    )
    biogas_generators_list = biogas_generators_list.rename(
        columns={"geometry": "geom"}
    ).set_geometry("geom", crs=4326)

    # Connect to local database
    engine = db.engine()

    # Cut data to federal state if in testmode
    boundary = settings()["egon-data"]["--dataset-boundary"]
    if boundary != "Everything":
        db.execute_sql(
            """
              DROP TABLE IF EXISTS grid.egon_biogas_generator CASCADE;
            """
        )
        biogas_generators_list.to_postgis(
            "egon_biogas_generator",
            engine,
            schema="grid",
            index=False,
            if_exists="replace",
        )

        sql = """SELECT *
            FROM grid.egon_biogas_generator, boundaries.vg250_sta_union as vg
            WHERE ST_Transform(vg.geometry,4326) && egon_biogas_generator.geom
            AND ST_Contains(ST_Transform(vg.geometry,4326), egon_biogas_generator.geom)"""

        biogas_generators_list = gpd.GeoDataFrame.from_postgis(
            sql, con=engine, geom_col="geom", crs=4326
        )
        biogas_generators_list = biogas_generators_list.drop(
            columns=["id", "bez", "area_ha", "geometry"]
        )
        db.execute_sql(
            """
              DROP TABLE IF EXISTS grid.egon_biogas_generator CASCADE;
            """
        )

    # Insert p_nom
    conversion_factor = 0.01083  # m^3/h to MWh/h
    biogas_generators_list["p_nom"] = [
        i * conversion_factor
        for i in biogas_generators_list["Einspeisung Biomethan [(N*m^3)/h)]"]
    ]

    # Add missing columns
    biogas_generators_list["marginal_cost"] = scn_params["marginal_cost"][
        "biogas"
    ]

    # Remove useless columns
    biogas_generators_list = biogas_generators_list.drop(
        columns=["x", "y", "Koordinaten", "Einspeisung Biomethan [(N*m^3)/h)]"]
    )
    return biogas_generators_list


def import_gas_generators(scn_name):
    """Insert list of gas production units in database

    Parameters
    ----------
    scn_name : str
        Name of the scenario.
    """
    carrier = "CH4"

    # Connect to local database
    engine = db.engine()

    # Select source and target from dataset configuration
    source = config.datasets()["gas_prod"]["source"]
    target = config.datasets()["gas_prod"]["target"]

    # Clean table
    db.execute_sql(
        f"""
        DELETE FROM {target['stores']['schema']}.{target['stores']['table']}
        WHERE "carrier" = '{carrier}' AND
        scn_name = '{scn_name}' AND bus not IN (
            SELECT bus_id FROM {source['buses']['schema']}.{source['buses']['table']}
            WHERE scn_name = '{scn_name}' AND country != 'DE'
        );
        """
    )

    if scn_name == "eGon2035":
        CH4_generators_list = pd.concat(
            [load_NG_generators(scn_name), load_biogas_generators(scn_name)]
        )

    elif scn_name == "eGon100RE":
        CH4_generators_list = load_biogas_generators(scn_name)
        overwrite_max_gas_generation_overtheyear(scn_name)

    # Add missing columns
    c = {"scn_name": scn_name, "carrier": carrier}
    CH4_generators_list = CH4_generators_list.assign(**c)


    # Match to associated CH4 bus
    CH4_generators_list = db.assign_gas_bus_id(
        CH4_generators_list, scn_name, carrier
    )

    # Remove useless columns
    CH4_generators_list = CH4_generators_list.drop(columns=["geom", "bus_id"])

    # Aggregate ch4 productions with same properties at the same bus
    CH4_generators_list = (
        CH4_generators_list.groupby(
            ["bus", "carrier", "scn_name", "marginal_cost"]
        )
        .agg({"p_nom": "sum"})
        .reset_index(drop=False)
    )

    new_id = db.next_etrago_id("generator")
    CH4_generators_list["generator_id"] = range(
        new_id, new_id + len(CH4_generators_list)
    )

    # Insert data to db
    CH4_generators_list.to_sql(
        target["stores"]["table"],
        engine,
        schema=target["stores"]["schema"],
        index=False,
        if_exists="append",
    )


def overwrite_max_gas_generation_overtheyear(scn_name):
    """Overright max_gas_generation_overtheyear in scenario parameter table

    Overright max_gas_generation_overtheyear in scenario parameter
    table if the value of this parameter has changed in the p-e-s run.

    Parameters
    ----------
    scn_name : str
        Name of the scenario

    """
    execute_pypsa_eur_sec = True  # False

    # Select source and target from dataset configuration
    target = config.datasets()["gas_prod"]["target"]

    if execute_pypsa_eur_sec:
        n = read_network()
        max_value = n.stores[n.stores["carrier"] == "biogas"].loc[
            "DE0 0 biogas", "e_initial"
        ]

        parameters = db.select_dataframe(
            f"""
            SELECT *
            FROM {target['scenario_parameters']['schema']}.{target['scenario_parameters']['table']}
            WHERE name = '{scn_name}'
            """
        )

        gas_param = parameters.loc[0, "gas_parameters"]
        gas_param["max_gas_generation_overtheyear"] = {"biogas": max_value}
        gas_param = json.dumps(gas_param)

        # Update data in db
        db.execute_sql(
            f"""
        UPDATE {target['scenario_parameters']['schema']}.{target['scenario_parameters']['table']}
        SET gas_parameters = '{gas_param}'
        WHERE name = '{scn_name}';
        """
        )


def insert_ch4_generators():
    """Insert gas production units in database for both scenarios

    Parameters
    ----------
    None

    Returns
    -------
    None
    """
    import_gas_generators("eGon2035")
    import_gas_generators("eGon100RE")


1			# -- coding: utf-8 --
2			"""
3			The central module containing all code dealing with importing CH4 production data
4			"""
5			from pathlib import Path
6			from urllib.request import urlretrieve
7			import ast
8			import json
9
10			import geopandas as gpd
11			import numpy as np
12			import pandas as pd
13
14			from egon.data import config, db
15			from egon.data.config import settings
16			from egon.data.datasets import Dataset
17			from egon.data.datasets.pypsaeursec import read_network
18			from egon.data.datasets.scenario_parameters import get_sector_parameters
19
20
21			class CH4Production(Dataset):
22			def __init__(self, dependencies):
23			super().__init__(
24			name="CH4Production",
25			version="0.0.8",
26			dependencies=dependencies,
27			tasks=(insert_ch4_generators),
28			)
29
30
31			def load_NG_generators(scn_name="eGon2035"):
32			"""Define the natural CH4 production units in Germany
33
34			Parameters
35			----------
36			scn_name : str
37			Name of the scenario.
38			Returns
39			-------
40			CH4_generators_list :
41			Dataframe containing the natural gas production units in Germany
42
43			"""
44			# read carrier information from scnario parameter data
45			scn_params = get_sector_parameters("gas", scn_name)
46
47			target_file = (
48			Path(".")
49			/ "datasets"
50			/ "gas_data"
51			/ "data"
52			/ "IGGIELGN_Productions.csv"
53			)
54
55			NG_generators_list = pd.read_csv(
56			target_file,
57			delimiter=";",
58			decimal=".",
59			usecols=["lat", "long", "country_code", "param"],
60			)
61
62			NG_generators_list = NG_generators_list[
63			NG_generators_list["country_code"].str.match("DE")
64			]
65
66			# Cut data to federal state if in testmode
67			NUTS1 = []
68			for index, row in NG_generators_list.iterrows():
69			param = ast.literal_eval(row["param"])
70			NUTS1.append(param["nuts_id_1"])
71			NG_generators_list = NG_generators_list.assign(NUTS1=NUTS1)
72
73			boundary = settings()["egon-data"]["--dataset-boundary"]
74			if boundary != "Everything":
75			map_states = {
76			"Baden-Württemberg": "DE1",
77			"Nordrhein-Westfalen": "DEA",
78			"Hessen": "DE7",
79			"Brandenburg": "DE4",
80			"Bremen": "DE5",
81			"Rheinland-Pfalz": "DEB",
82			"Sachsen-Anhalt": "DEE",
83			"Schleswig-Holstein": "DEF",
84			"Mecklenburg-Vorpommern": "DE8",
85			"Thüringen": "DEG",
86			"Niedersachsen": "DE9",
87			"Sachsen": "DED",
88			"Hamburg": "DE6",
89			"Saarland": "DEC",
90			"Berlin": "DE3",
91			"Bayern": "DE2",
92			}
93
94			NG_generators_list = NG_generators_list[
95			NG_generators_list["NUTS1"].isin([map_states[boundary], np.nan])
96			]
97
98			NG_generators_list = NG_generators_list.rename(
99			columns={"lat": "y", "long": "x"}
100			)
101			NG_generators_list = gpd.GeoDataFrame(
102			NG_generators_list,
103			geometry=gpd.points_from_xy(
104			NG_generators_list["x"], NG_generators_list["y"]
105			),
106			)
107			NG_generators_list = NG_generators_list.rename(
108			columns={"geometry": "geom"}
109			).set_geometry("geom", crs=4326)
110
111			# Insert p_nom
112			p_nom = []
113			for index, row in NG_generators_list.iterrows():
114			param = ast.literal_eval(row["param"])
115			p_nom.append(param["max_supply_M_m3_per_d"])
116
117			conversion_factor = 437.5 # MCM/day to MWh/h
118			NG_generators_list["p_nom"] = [i * conversion_factor for i in p_nom]
119
120			# Add missing columns
121			NG_generators_list["marginal_cost"] = scn_params["marginal_cost"]["CH4"]
122
123			# Remove useless columns
124			NG_generators_list = NG_generators_list.drop(
125			columns=["x", "y", "param", "country_code", "NUTS1"]
126			)
127
128			return NG_generators_list
129
130
131			def download_biogas_data():
132			"""Download the biogas production units data in Germany
133
134			Parameters
135			----------
136			None
137
138			Returns
139			-------
140			None
141
142			"""
143			basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx"
144			url = (
145			"https://www.biogaspartner.de/fileadmin/Biogaspartner/Dokumente/Einspeiseatlas/"
146			+ basename
147			)
148			target_file = Path(".") / "datasets" / "gas_data" / basename
149
150			urlretrieve(url, target_file)
151
152
153			def load_biogas_generators(scn_name):
154			"""Define the biogas production units in Germany
155
156			Parameters
157			----------
158			scn_name : str
159			Name of the scenario.
160
161			Returns
162			-------
163			CH4_generators_list :
164			Dataframe containing the biogas production units in Germany
165
166			"""
167			# read carrier information from scnario parameter data
168			scn_params = get_sector_parameters("gas", scn_name)
169
170			basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx"
171			target_file = Path(".") / "datasets" / "gas_data" / basename
172
173			# Read-in data from csv-file
174			biogas_generators_list = pd.read_excel(
175			target_file,
176			usecols=["Koordinaten", "Einspeisung Biomethan [(N*m^3)/h)]"],
177			)
178
179			x = []
180			y = []
181			for index, row in biogas_generators_list.iterrows():
182			coordinates = row["Koordinaten"].split(",")
183			y.append(coordinates[0])
184			x.append(coordinates[1])
185			biogas_generators_list["x"] = x
186			biogas_generators_list["y"] = y
187
188			biogas_generators_list = gpd.GeoDataFrame(
189			biogas_generators_list,
190			geometry=gpd.points_from_xy(
191			biogas_generators_list["x"], biogas_generators_list["y"]
192			),
193			)
194			biogas_generators_list = biogas_generators_list.rename(
195			columns={"geometry": "geom"}
196			).set_geometry("geom", crs=4326)
197
198			# Connect to local database
199			engine = db.engine()
200
201			# Cut data to federal state if in testmode
202			boundary = settings()["egon-data"]["--dataset-boundary"]
203			if boundary != "Everything":
204			db.execute_sql(
205			"""
206			DROP TABLE IF EXISTS grid.egon_biogas_generator CASCADE;
207			"""
208			)
209			biogas_generators_list.to_postgis(
210			"egon_biogas_generator",
211			engine,
212			schema="grid",
213			index=False,
214			if_exists="replace",
215			)
216
217			sql = """SELECT *
218			FROM grid.egon_biogas_generator, boundaries.vg250_sta_union as vg
219			WHERE ST_Transform(vg.geometry,4326) && egon_biogas_generator.geom
220			AND ST_Contains(ST_Transform(vg.geometry,4326), egon_biogas_generator.geom)"""
221
222			biogas_generators_list = gpd.GeoDataFrame.from_postgis(
223			sql, con=engine, geom_col="geom", crs=4326
224			)
225			biogas_generators_list = biogas_generators_list.drop(
226			columns=["id", "bez", "area_ha", "geometry"]
227			)
228			db.execute_sql(
229			"""
230			DROP TABLE IF EXISTS grid.egon_biogas_generator CASCADE;
231			"""
232			)
233
234			# Insert p_nom
235			conversion_factor = 0.01083 # m^3/h to MWh/h
236			biogas_generators_list["p_nom"] = [
237			i * conversion_factor
238			for i in biogas_generators_list["Einspeisung Biomethan [(N*m^3)/h)]"]
239			]
240
241			# Add missing columns
242			biogas_generators_list["marginal_cost"] = scn_params["marginal_cost"][
243			"biogas"
244			]
245
246			# Remove useless columns
247			biogas_generators_list = biogas_generators_list.drop(
248			columns=["x", "y", "Koordinaten", "Einspeisung Biomethan [(N*m^3)/h)]"]
249			)
250			return biogas_generators_list
251
252
253			def import_gas_generators(scn_name):
254			"""Insert list of gas production units in database
255
256			Parameters
257			----------
258			scn_name : str
259			Name of the scenario.
260			"""
261			carrier = "CH4"
262
263			# Connect to local database
264			engine = db.engine()
265
266			# Select source and target from dataset configuration
267			source = config.datasets()["gas_prod"]["source"]
268			target = config.datasets()["gas_prod"]["target"]
269
270			# Clean table
271			db.execute_sql(
272			f"""
273			DELETE FROM {target['stores']['schema']}.{target['stores']['table']}
274			WHERE "carrier" = '{carrier}' AND
275			scn_name = '{scn_name}' AND bus not IN (
276			SELECT bus_id FROM {source['buses']['schema']}.{source['buses']['table']}
277			WHERE scn_name = '{scn_name}' AND country != 'DE'
278			);
279			"""
280			)
281
282			if scn_name == "eGon2035":
283			CH4_generators_list = pd.concat(
284			[load_NG_generators(scn_name), load_biogas_generators(scn_name)]
285			)
286
287			elif scn_name == "eGon100RE":
288			CH4_generators_list = load_biogas_generators(scn_name)
289			overwrite_max_gas_generation_overtheyear(scn_name)
290
291			# Add missing columns
292			c = {"scn_name": scn_name, "carrier": carrier}
293			CH4_generators_list = CH4_generators_list.assign(**c)
			0 ignored issues – show introduced 2022-09-12 14:50 UTC by Report Bug Copy Issue Report The variable `CH4_generators_list` does not seem to be defined for all execution paths. Loading history...
294
295			# Match to associated CH4 bus
296			CH4_generators_list = db.assign_gas_bus_id(
297			CH4_generators_list, scn_name, carrier
298			)
299
300			# Remove useless columns
301			CH4_generators_list = CH4_generators_list.drop(columns=["geom", "bus_id"])
302
303			# Aggregate ch4 productions with same properties at the same bus
304			CH4_generators_list = (
305			CH4_generators_list.groupby(
306			["bus", "carrier", "scn_name", "marginal_cost"]
307			)
308			.agg({"p_nom": "sum"})
309			.reset_index(drop=False)
310			)
311
312			new_id = db.next_etrago_id("generator")
313			CH4_generators_list["generator_id"] = range(
314			new_id, new_id + len(CH4_generators_list)
315			)
316
317			# Insert data to db
318			CH4_generators_list.to_sql(
319			target["stores"]["table"],
320			engine,
321			schema=target["stores"]["schema"],
322			index=False,
323			if_exists="append",
324			)
325
326
327			def overwrite_max_gas_generation_overtheyear(scn_name):
328			"""Overright max_gas_generation_overtheyear in scenario parameter table
329
330			Overright max_gas_generation_overtheyear in scenario parameter
331			table if the value of this parameter has changed in the p-e-s run.
332
333			Parameters
334			----------
335			scn_name : str
336			Name of the scenario
337
338			"""
339			execute_pypsa_eur_sec = True # False
340
341			# Select source and target from dataset configuration
342			target = config.datasets()["gas_prod"]["target"]
343
344			if execute_pypsa_eur_sec:
345			n = read_network()
346			max_value = n.stores[n.stores["carrier"] == "biogas"].loc[
347			"DE0 0 biogas", "e_initial"
348			]
349
350			parameters = db.select_dataframe(
351			f"""
352			SELECT *
353			FROM {target['scenario_parameters']['schema']}.{target['scenario_parameters']['table']}
354			WHERE name = '{scn_name}'
355			"""
356			)
357
358			gas_param = parameters.loc[0, "gas_parameters"]
359			gas_param["max_gas_generation_overtheyear"] = {"biogas": max_value}
360			gas_param = json.dumps(gas_param)
361
362			# Update data in db
363			db.execute_sql(
364			f"""
365			UPDATE {target['scenario_parameters']['schema']}.{target['scenario_parameters']['table']}
366			SET gas_parameters = '{gas_param}'
367			WHERE name = '{scn_name}';
368			"""
369			)
370
371
372			def insert_ch4_generators():
373			"""Insert gas production units in database for both scenarios
374
375			Parameters
376			----------
377			None
378
379			Returns
380			-------
381			None
382			"""
383			import_gas_generators("eGon2035")
384			import_gas_generators("eGon100RE")
385

openego / eGon-data

Pull Request — dev (#936)

data.datasets.ch4_prod.insert_ch4_generators() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like