data.datasets.industrial_gas_demand.insert_new_entries() - Code Metrics - Inspection of "Merge pull request #899 from openego/fix/#871-gas-..." - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — dev ( 7c1955...864b08 )

unknown

created 2022-09-29 15:44 UTC

insert_new_entries() A

↳ Parent: data.datasets.industrial_gas_demand

Complexity

Conditions

Size

Total Lines	37
Code Lines	16

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	16
dl	0
loc	37
rs	9.6
c	0
b	0
f	0
cc	1
nop	2

# -*- coding: utf-8 -*-
"""
The central module containing all code dealing with importing gas industrial demand
"""
from pathlib import Path
import os

from geoalchemy2.types import Geometry
from shapely import wkt
import numpy as np
import pandas as pd
import requests

from egon.data import db
from egon.data.config import settings
from egon.data.datasets import Dataset
from egon.data.datasets.etrago_helpers import (
    finalize_bus_insertion,
    initialise_bus_insertion,
)
from egon.data.datasets.etrago_setup import link_geom_from_buses
from egon.data.datasets.pypsaeursec import read_network
from egon.data.datasets.scenario_parameters import get_sector_parameters


class IndustrialGasDemand(Dataset):
    def __init__(self, dependencies):
        super().__init__(
            name="IndustrialGasDemand",
            version="0.0.3",
            dependencies=dependencies,
            tasks=(download_industrial_gas_demand),
        )


class IndustrialGasDemandeGon2035(Dataset):
    def __init__(self, dependencies):
        super().__init__(
            name="IndustrialGasDemandeGon2035",
            version="0.0.2",
            dependencies=dependencies,
            tasks=(insert_industrial_gas_demand_egon2035),
        )


class IndustrialGasDemandeGon100RE(Dataset):
    def __init__(self, dependencies):
        super().__init__(
            name="IndustrialGasDemandeGon100RE",
            version="0.0.2",
            dependencies=dependencies,
            tasks=(insert_industrial_gas_demand_egon100RE),
        )


def read_industrial_demand(scn_name, carrier):
    """Read the gas demand data

    Parameters
    ----------
    scn_name : str
        Name of the scenario
    carrier : str
        Name of the gas carrier

    Returns
    -------
    df : pandas.core.frame.DataFrame
        Dataframe containing the industrial demand

    """
    target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
    df_corr = pd.read_json(target_file)
    df_corr = df_corr.loc[:, ["id_region", "name_short"]]
    df_corr.set_index("id_region", inplace=True)

    target_file = (
        Path(".")
        / "datasets/gas_data/demand"
        / (carrier + "_" + scn_name + ".json")
    )
    industrial_loads = pd.read_json(target_file)
    industrial_loads = industrial_loads.loc[:, ["id_region", "values"]]
    industrial_loads.set_index("id_region", inplace=True)

    # Match the id_region to obtain the NUT3 region names
    industrial_loads_list = pd.concat(
        [industrial_loads, df_corr], axis=1, join="inner"
    )
    industrial_loads_list["NUTS0"] = (industrial_loads_list["name_short"].str)[
        0:2
    ]
    industrial_loads_list["NUTS1"] = (industrial_loads_list["name_short"].str)[
        0:3
    ]
    industrial_loads_list = industrial_loads_list[
        industrial_loads_list["NUTS0"].str.match("DE")
    ]

    # Cut data to federal state if in testmode
    boundary = settings()["egon-data"]["--dataset-boundary"]
    if boundary != "Everything":
        map_states = {
            "Baden-Württemberg": "DE1",
            "Nordrhein-Westfalen": "DEA",
            "Hessen": "DE7",
            "Brandenburg": "DE4",
            "Bremen": "DE5",
            "Rheinland-Pfalz": "DEB",
            "Sachsen-Anhalt": "DEE",
            "Schleswig-Holstein": "DEF",
            "Mecklenburg-Vorpommern": "DE8",
            "Thüringen": "DEG",
            "Niedersachsen": "DE9",
            "Sachsen": "DED",
            "Hamburg": "DE6",
            "Saarland": "DEC",
            "Berlin": "DE3",
            "Bayern": "DE2",
        }

        industrial_loads_list = industrial_loads_list[
            industrial_loads_list["NUTS1"].isin([map_states[boundary], np.nan])
        ]

    industrial_loads_list = industrial_loads_list.rename(
        columns={"name_short": "nuts3", "values": "p_set"}
    )
    industrial_loads_list = industrial_loads_list.set_index("nuts3")

    # Add the centroid point to each NUTS3 area
    sql_vg250 = """SELECT nuts as nuts3, geometry as geom
                    FROM boundaries.vg250_krs
                    WHERE gf = 4 ;"""
    gdf_vg250 = db.select_geodataframe(sql_vg250, epsg=4326)

    point = []
    for index, row in gdf_vg250.iterrows():
        point.append(wkt.loads(str(row["geom"])).centroid)
    gdf_vg250["point"] = point
    gdf_vg250 = gdf_vg250.set_index("nuts3")
    gdf_vg250 = gdf_vg250.drop(columns=["geom"])

    # Match the load to the NUTS3 points
    industrial_loads_list = pd.concat(
        [industrial_loads_list, gdf_vg250], axis=1, join="inner"
    )
    return industrial_loads_list.rename(
        columns={"point": "geom"}
    ).set_geometry("geom", crs=4326)


def read_and_process_demand(scn_name="eGon2035", carrier=None, grid_carrier=None):
    """Assign the industrial demand in Germany to buses

    Parameters
    ----------
    scn_name : str
        Name of the scenario

    carrier : str
        Name of the carrier, the demand should hold

    grid_carrier : str
        Carrier name of the buses, the demand should be assigned to

    Returns
    -------
    industrial_demand :
        Dataframe containing the industrial demand in Germany

    """
    if grid_carrier is None:
        grid_carrier = carrier
    industrial_loads_list = read_industrial_demand(scn_name, carrier)
    number_loads = len(industrial_loads_list)

    # Match to associated gas bus
    industrial_loads_list = db.assign_gas_bus_id(
        industrial_loads_list, scn_name, grid_carrier
    )

    # Add carrier
    industrial_loads_list["carrier"] = carrier

    # Remove useless columns
    industrial_loads_list = industrial_loads_list.drop(
        columns=["geom", "NUTS0", "NUTS1", "bus_id"], errors="ignore"
    )

    msg = (
        "The number of load changed when assigning to the respective buses."
        f"It should be {number_loads} loads, but only"
        f"{len(industrial_loads_list)} got assigned to buses."
        f"scn_name: {scn_name}, load carrier: {carrier}, carrier of buses to"
        f"connect loads to: {grid_carrier}"
        )
    assert len(industrial_loads_list) == number_loads, msg

    return industrial_loads_list


def delete_old_entries(scn_name):
    """
    Delete loads and load timeseries.

    Parameters
    ----------
    scn_name : str
        Name of the scenario.
    """
    # Clean tables
    db.execute_sql(
        f"""
        DELETE FROM grid.egon_etrago_load_timeseries
        WHERE "load_id" IN (
            SELECT load_id FROM grid.egon_etrago_load
            WHERE "carrier" IN ('CH4', 'H2') AND
            scn_name = '{scn_name}' AND bus not IN (
                SELECT bus_id FROM grid.egon_etrago_bus
                WHERE scn_name = '{scn_name}' AND country != 'DE'
            )
        );
        """
    )

    db.execute_sql(
        f"""
        DELETE FROM grid.egon_etrago_load
        WHERE "load_id" IN (
            SELECT load_id FROM grid.egon_etrago_load
            WHERE "carrier" IN ('CH4', 'H2') AND
            scn_name = '{scn_name}' AND bus not IN (
                SELECT bus_id FROM grid.egon_etrago_bus
                WHERE scn_name = '{scn_name}' AND country != 'DE'
            )
        );
        """
    )


def insert_new_entries(industrial_gas_demand, scn_name):
    """
    Insert loads.

    Parameters
    ----------
    industrial_gas_demand : pandas.DataFrame
        Load data to insert.
    scn_name : str
        Name of the scenario.
    """

    new_id = db.next_etrago_id("load")
    industrial_gas_demand["load_id"] = range(
        new_id, new_id + len(industrial_gas_demand)
    )

    # Add missing columns
    c = {"scn_name": scn_name, "sign": -1}
    industrial_gas_demand = industrial_gas_demand.assign(**c)

    industrial_gas_demand = industrial_gas_demand.reset_index(drop=True)

    # Remove useless columns
    egon_etrago_load_gas = industrial_gas_demand.drop(columns=["p_set"])

    engine = db.engine()
    # Insert data to db
    egon_etrago_load_gas.to_sql(
        "egon_etrago_load",
        engine,
        schema="grid",
        index=False,
        if_exists="append",
    )

    return industrial_gas_demand


def insert_industrial_gas_demand_egon2035():
    """Insert list of industrial gas demand (one per NUTS3) in database

    Parameters
    ----------
    scn_name : str
        Name of the scenario

    Returns
    -------
        industrial_gas_demand : Dataframe containing the industrial gas demand
        in Germany
    """
    scn_name = "eGon2035"
    delete_old_entries(scn_name)

    industrial_gas_demand = pd.concat(
        [
            read_and_process_demand(scn_name=scn_name, carrier="CH4"),
            read_and_process_demand(
                scn_name=scn_name, carrier="H2", grid_carrier="H2_grid"
            ),
        ]
    )

    industrial_gas_demand = (
        industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
        .apply(lambda x: [sum(y) for y in zip(*x)])
        .reset_index(drop=False)
    )

    industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
    insert_industrial_gas_demand_time_series(industrial_gas_demand)


def insert_industrial_gas_demand_egon100RE():
    """Insert list of industrial gas demand (one per NUTS3) in database

    Parameters
    ----------
    scn_name : str
        Name of the scenario

    Returns
    -------
        industrial_gas_demand : Dataframe containing the industrial gas demand
        in Germany
    """
    scn_name = "eGon100RE"
    delete_old_entries(scn_name)

    # read demands
    industrial_gas_demand_CH4 = read_and_process_demand(
        scn_name=scn_name, carrier="CH4"
    )
    industrial_gas_demand_H2 = read_and_process_demand(
        scn_name=scn_name, carrier="H2", grid_carrier="H2_grid"
    )

    # adjust H2 and CH4 total demands (values from PES)
    # CH4 demand = 0 in 100RE, therefore scale H2 ts
    # fallback values see https://github.com/openego/eGon-data/issues/626
    n = read_network()

    try:
        H2_total_PES = (
            n.loads[n.loads["carrier"] == "H2 for industry"].loc[
                "DE0 0 H2 for industry", "p_set"
            ]
            * 8760
        )
    except KeyError:
        H2_total_PES = 42090000
        print("Could not find data from PES-run, assigning fallback number.")

    try:
        CH4_total_PES = (
            n.loads[n.loads["carrier"] == "gas for industry"].loc[
                "DE0 0 gas for industry", "p_set"
            ]
            * 8760
        )
    except KeyError:
        CH4_total_PES = 105490000
        print("Could not find data from PES-run, assigning fallback number.")

    boundary = settings()["egon-data"]["--dataset-boundary"]
    if boundary != "Everything":
        # modify values for test mode
        # the values are obtained by evaluating the share of H2 demand in
        # test region (NUTS1: DEF, Schleswig-Holstein) with respect to the H2
        # demand in full Germany model (NUTS0: DE). The task has been outsourced
        # to save processing cost
        H2_total_PES *= 0.01855683050330346
        CH4_total_PES *= 0.01855683050330346

    H2_total = industrial_gas_demand_H2["p_set"].apply(sum).astype(float).sum()

    industrial_gas_demand_CH4["p_set"] = industrial_gas_demand_H2[
        "p_set"
    ].apply(lambda x: [val / H2_total * CH4_total_PES for val in x])
    industrial_gas_demand_H2["p_set"] = industrial_gas_demand_H2[
        "p_set"
    ].apply(lambda x: [val / H2_total * H2_total_PES for val in x])

    # consistency check
    total_CH4_distributed = sum(
        [sum(x) for x in industrial_gas_demand_CH4["p_set"].to_list()]
    )
    total_H2_distributed = sum(
        [sum(x) for x in industrial_gas_demand_H2["p_set"].to_list()]
    )

    print(
        f"Total amount of industrial H2 demand distributed is "
        f"{total_H2_distributed} MWh. Total amount of industrial CH4 demand "
        f"distributed is {total_CH4_distributed} MWh."
    )
    msg = (
        f"Total amount of industrial H2 demand from P-E-S is equal to "
        f"{H2_total_PES}, which should be identical to the distributed amount "
        f"of {total_H2_distributed}, but it is not."
    )
    assert round(H2_total_PES) == round(total_H2_distributed), msg

    msg = (
        f"Total amount of industrial CH4 demand from P-E-S is equal to "
        f"{CH4_total_PES}, which should be identical to the distributed amount "
        f"of {total_CH4_distributed}, but it is not."
    )
    assert round(CH4_total_PES) == round(total_CH4_distributed), msg

    industrial_gas_demand = pd.concat(
        [
            industrial_gas_demand_CH4,
            industrial_gas_demand_H2,
        ]
    )
    industrial_gas_demand = (
        industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
        .apply(lambda x: [sum(y) for y in zip(*x)])
        .reset_index(drop=False)
    )

    industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
    insert_industrial_gas_demand_time_series(industrial_gas_demand)


def insert_industrial_gas_demand_time_series(egon_etrago_load_gas):
    """
    Insert list of industrial gas demand time series (one per NUTS3)
    """
    egon_etrago_load_gas_timeseries = egon_etrago_load_gas

    # Connect to local database
    engine = db.engine()

    # Adjust columns
    egon_etrago_load_gas_timeseries = egon_etrago_load_gas_timeseries.drop(
        columns=["carrier", "bus", "sign"]
    )
    egon_etrago_load_gas_timeseries["temp_id"] = 1

    # Insert data to db
    egon_etrago_load_gas_timeseries.to_sql(
        "egon_etrago_load_timeseries",
        engine,
        schema="grid",
        index=False,
        if_exists="append",
    )


def download_industrial_gas_demand():
    """Download the industrial gas demand data from opendata.ffe database."""
    correspondance_url = (
        "http://opendata.ffe.de:3000/region?id_region_type=eq.38"
    )

    # Read and save data
    result_corr = requests.get(correspondance_url)
    target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
    os.makedirs(os.path.dirname(target_file), exist_ok=True)
    pd.read_json(result_corr.content).to_json(target_file)

    carriers = {"H2": "2,162", "CH4": "2,11"}
    url = "http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq."

    for scn_name in ["eGon2035", "eGon100RE"]:
        year = str(
            get_sector_parameters("global", scn_name)["population_year"]
        )

        for carrier, internal_id in carriers.items():
            # Download the data
            datafilter = "&&internal_id=eq.{" + internal_id + "}"
            request = url + year + datafilter

            # Read and save data
            result = requests.get(request)
            target_file = (
                Path(".")
                / "datasets/gas_data/demand"
                / (carrier + "_" + scn_name + ".json")
            )
            pd.read_json(result.content).to_json(target_file)


1			# -- coding: utf-8 --
2			"""
3			The central module containing all code dealing with importing gas industrial demand
4			"""
5			from pathlib import Path
6			import os
7
8			from geoalchemy2.types import Geometry
9			from shapely import wkt
10			import numpy as np
11			import pandas as pd
12			import requests
13
14			from egon.data import db
15			from egon.data.config import settings
16			from egon.data.datasets import Dataset
17			from egon.data.datasets.etrago_helpers import (
18			finalize_bus_insertion,
19			initialise_bus_insertion,
20			)
21			from egon.data.datasets.etrago_setup import link_geom_from_buses
22			from egon.data.datasets.pypsaeursec import read_network
23			from egon.data.datasets.scenario_parameters import get_sector_parameters
24
25
26			class IndustrialGasDemand(Dataset):
27			def __init__(self, dependencies):
28			super().__init__(
29			name="IndustrialGasDemand",
30			version="0.0.3",
31			dependencies=dependencies,
32			tasks=(download_industrial_gas_demand),
33			)
34
35
36			class IndustrialGasDemandeGon2035(Dataset):
37			def __init__(self, dependencies):
38			super().__init__(
39			name="IndustrialGasDemandeGon2035",
40			version="0.0.2",
41			dependencies=dependencies,
42			tasks=(insert_industrial_gas_demand_egon2035),
43			)
44
45
46			class IndustrialGasDemandeGon100RE(Dataset):
47			def __init__(self, dependencies):
48			super().__init__(
49			name="IndustrialGasDemandeGon100RE",
50			version="0.0.2",
51			dependencies=dependencies,
52			tasks=(insert_industrial_gas_demand_egon100RE),
53			)
54
55
56			def read_industrial_demand(scn_name, carrier):
57			"""Read the gas demand data
58
59			Parameters
60			----------
61			scn_name : str
62			Name of the scenario
63			carrier : str
64			Name of the gas carrier
65
66			Returns
67			-------
68			df : pandas.core.frame.DataFrame
69			Dataframe containing the industrial demand
70
71			"""
72			target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
73			df_corr = pd.read_json(target_file)
74			df_corr = df_corr.loc[:, ["id_region", "name_short"]]
75			df_corr.set_index("id_region", inplace=True)
76
77			target_file = (
78			Path(".")
79			/ "datasets/gas_data/demand"
80			/ (carrier + "_" + scn_name + ".json")
81			)
82			industrial_loads = pd.read_json(target_file)
83			industrial_loads = industrial_loads.loc[:, ["id_region", "values"]]
84			industrial_loads.set_index("id_region", inplace=True)
85
86			# Match the id_region to obtain the NUT3 region names
87			industrial_loads_list = pd.concat(
88			[industrial_loads, df_corr], axis=1, join="inner"
89			)
90			industrial_loads_list["NUTS0"] = (industrial_loads_list["name_short"].str)[
91			0:2
92			]
93			industrial_loads_list["NUTS1"] = (industrial_loads_list["name_short"].str)[
94			0:3
95			]
96			industrial_loads_list = industrial_loads_list[
97			industrial_loads_list["NUTS0"].str.match("DE")
98			]
99
100			# Cut data to federal state if in testmode
101			boundary = settings()["egon-data"]["--dataset-boundary"]
102			if boundary != "Everything":
103			map_states = {
104			"Baden-Württemberg": "DE1",
105			"Nordrhein-Westfalen": "DEA",
106			"Hessen": "DE7",
107			"Brandenburg": "DE4",
108			"Bremen": "DE5",
109			"Rheinland-Pfalz": "DEB",
110			"Sachsen-Anhalt": "DEE",
111			"Schleswig-Holstein": "DEF",
112			"Mecklenburg-Vorpommern": "DE8",
113			"Thüringen": "DEG",
114			"Niedersachsen": "DE9",
115			"Sachsen": "DED",
116			"Hamburg": "DE6",
117			"Saarland": "DEC",
118			"Berlin": "DE3",
119			"Bayern": "DE2",
120			}
121
122			industrial_loads_list = industrial_loads_list[
123			industrial_loads_list["NUTS1"].isin([map_states[boundary], np.nan])
124			]
125
126			industrial_loads_list = industrial_loads_list.rename(
127			columns={"name_short": "nuts3", "values": "p_set"}
128			)
129			industrial_loads_list = industrial_loads_list.set_index("nuts3")
130
131			# Add the centroid point to each NUTS3 area
132			sql_vg250 = """SELECT nuts as nuts3, geometry as geom
133			FROM boundaries.vg250_krs
134			WHERE gf = 4 ;"""
135			gdf_vg250 = db.select_geodataframe(sql_vg250, epsg=4326)
136
137			point = []
138			for index, row in gdf_vg250.iterrows():
139			point.append(wkt.loads(str(row["geom"])).centroid)
140			gdf_vg250["point"] = point
141			gdf_vg250 = gdf_vg250.set_index("nuts3")
142			gdf_vg250 = gdf_vg250.drop(columns=["geom"])
143
144			# Match the load to the NUTS3 points
145			industrial_loads_list = pd.concat(
146			[industrial_loads_list, gdf_vg250], axis=1, join="inner"
147			)
148			return industrial_loads_list.rename(
149			columns={"point": "geom"}
150			).set_geometry("geom", crs=4326)
151
152
153			def read_and_process_demand(scn_name="eGon2035", carrier=None, grid_carrier=None):
154			"""Assign the industrial demand in Germany to buses
155
156			Parameters
157			----------
158			scn_name : str
159			Name of the scenario
160
161			carrier : str
162			Name of the carrier, the demand should hold
163
164			grid_carrier : str
165			Carrier name of the buses, the demand should be assigned to
166
167			Returns
168			-------
169			industrial_demand :
170			Dataframe containing the industrial demand in Germany
171
172			"""
173			if grid_carrier is None:
174			grid_carrier = carrier
175			industrial_loads_list = read_industrial_demand(scn_name, carrier)
176			number_loads = len(industrial_loads_list)
177
178			# Match to associated gas bus
179			industrial_loads_list = db.assign_gas_bus_id(
180			industrial_loads_list, scn_name, grid_carrier
181			)
182
183			# Add carrier
184			industrial_loads_list["carrier"] = carrier
185
186			# Remove useless columns
187			industrial_loads_list = industrial_loads_list.drop(
188			columns=["geom", "NUTS0", "NUTS1", "bus_id"], errors="ignore"
189			)
190
191			msg = (
192			"The number of load changed when assigning to the respective buses."
193			f"It should be {number_loads} loads, but only"
194			f"{len(industrial_loads_list)} got assigned to buses."
195			f"scn_name: {scn_name}, load carrier: {carrier}, carrier of buses to"
196			f"connect loads to: {grid_carrier}"
197			)
198			assert len(industrial_loads_list) == number_loads, msg
199
200			return industrial_loads_list
201
202
203			def delete_old_entries(scn_name):
204			"""
205			Delete loads and load timeseries.
206
207			Parameters
208			----------
209			scn_name : str
210			Name of the scenario.
211			"""
212			# Clean tables
213			db.execute_sql(
214			f"""
215			DELETE FROM grid.egon_etrago_load_timeseries
216			WHERE "load_id" IN (
217			SELECT load_id FROM grid.egon_etrago_load
218			WHERE "carrier" IN ('CH4', 'H2') AND
219			scn_name = '{scn_name}' AND bus not IN (
220			SELECT bus_id FROM grid.egon_etrago_bus
221			WHERE scn_name = '{scn_name}' AND country != 'DE'
222			)
223			);
224			"""
225			)
226
227			db.execute_sql(
228			f"""
229			DELETE FROM grid.egon_etrago_load
230			WHERE "load_id" IN (
231			SELECT load_id FROM grid.egon_etrago_load
232			WHERE "carrier" IN ('CH4', 'H2') AND
233			scn_name = '{scn_name}' AND bus not IN (
234			SELECT bus_id FROM grid.egon_etrago_bus
235			WHERE scn_name = '{scn_name}' AND country != 'DE'
236			)
237			);
238			"""
239			)
240
241
242			def insert_new_entries(industrial_gas_demand, scn_name):
243			"""
244			Insert loads.
245
246			Parameters
247			----------
248			industrial_gas_demand : pandas.DataFrame
249			Load data to insert.
250			scn_name : str
251			Name of the scenario.
252			"""
253
254			new_id = db.next_etrago_id("load")
255			industrial_gas_demand["load_id"] = range(
256			new_id, new_id + len(industrial_gas_demand)
257			)
258
259			# Add missing columns
260			c = {"scn_name": scn_name, "sign": -1}
261			industrial_gas_demand = industrial_gas_demand.assign(**c)
262
263			industrial_gas_demand = industrial_gas_demand.reset_index(drop=True)
264
265			# Remove useless columns
266			egon_etrago_load_gas = industrial_gas_demand.drop(columns=["p_set"])
267
268			engine = db.engine()
269			# Insert data to db
270			egon_etrago_load_gas.to_sql(
271			"egon_etrago_load",
272			engine,
273			schema="grid",
274			index=False,
275			if_exists="append",
276			)
277
278			return industrial_gas_demand
279
280
281			def insert_industrial_gas_demand_egon2035():
282			"""Insert list of industrial gas demand (one per NUTS3) in database
283
284			Parameters
285			----------
286			scn_name : str
287			Name of the scenario
288
289			Returns
290			-------
291			industrial_gas_demand : Dataframe containing the industrial gas demand
292			in Germany
293			"""
294			scn_name = "eGon2035"
295			delete_old_entries(scn_name)
296
297			industrial_gas_demand = pd.concat(
298			[
299			read_and_process_demand(scn_name=scn_name, carrier="CH4"),
300			read_and_process_demand(
301			scn_name=scn_name, carrier="H2", grid_carrier="H2_grid"
302			),
303			]
304			)
305
306			industrial_gas_demand = (
307			industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
308			.apply(lambda x: [sum(y) for y in zip(*x)])
309			.reset_index(drop=False)
310			)
311
312			industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
313			insert_industrial_gas_demand_time_series(industrial_gas_demand)
314
315
316			def insert_industrial_gas_demand_egon100RE():
317			"""Insert list of industrial gas demand (one per NUTS3) in database
318
319			Parameters
320			----------
321			scn_name : str
322			Name of the scenario
323
324			Returns
325			-------
326			industrial_gas_demand : Dataframe containing the industrial gas demand
327			in Germany
328			"""
329			scn_name = "eGon100RE"
330			delete_old_entries(scn_name)
331
332			# read demands
333			industrial_gas_demand_CH4 = read_and_process_demand(
334			scn_name=scn_name, carrier="CH4"
335			)
336			industrial_gas_demand_H2 = read_and_process_demand(
337			scn_name=scn_name, carrier="H2", grid_carrier="H2_grid"
338			)
339
340			# adjust H2 and CH4 total demands (values from PES)
341			# CH4 demand = 0 in 100RE, therefore scale H2 ts
342			# fallback values see https://github.com/openego/eGon-data/issues/626
343			n = read_network()
344
345			try:
346			H2_total_PES = (
347			n.loads[n.loads["carrier"] == "H2 for industry"].loc[
348			"DE0 0 H2 for industry", "p_set"
349			]
350			* 8760
351			)
352			except KeyError:
353			H2_total_PES = 42090000
354			print("Could not find data from PES-run, assigning fallback number.")
355
356			try:
357			CH4_total_PES = (
358			n.loads[n.loads["carrier"] == "gas for industry"].loc[
359			"DE0 0 gas for industry", "p_set"
360			]
361			* 8760
362			)
363			except KeyError:
364			CH4_total_PES = 105490000
365			print("Could not find data from PES-run, assigning fallback number.")
366
367			boundary = settings()["egon-data"]["--dataset-boundary"]
368			if boundary != "Everything":
369			# modify values for test mode
370			# the values are obtained by evaluating the share of H2 demand in
371			# test region (NUTS1: DEF, Schleswig-Holstein) with respect to the H2
372			# demand in full Germany model (NUTS0: DE). The task has been outsourced
373			# to save processing cost
374			H2_total_PES *= 0.01855683050330346
375			CH4_total_PES *= 0.01855683050330346
376
377			H2_total = industrial_gas_demand_H2["p_set"].apply(sum).astype(float).sum()
378
379			industrial_gas_demand_CH4["p_set"] = industrial_gas_demand_H2[
380			"p_set"
381			].apply(lambda x: [val / H2_total * CH4_total_PES for val in x])
382			industrial_gas_demand_H2["p_set"] = industrial_gas_demand_H2[
383			"p_set"
384			].apply(lambda x: [val / H2_total * H2_total_PES for val in x])
385
386			# consistency check
387			total_CH4_distributed = sum(
388			[sum(x) for x in industrial_gas_demand_CH4["p_set"].to_list()]
389			)
390			total_H2_distributed = sum(
391			[sum(x) for x in industrial_gas_demand_H2["p_set"].to_list()]
392			)
393
394			print(
395			f"Total amount of industrial H2 demand distributed is "
396			f"{total_H2_distributed} MWh. Total amount of industrial CH4 demand "
397			f"distributed is {total_CH4_distributed} MWh."
398			)
399			msg = (
400			f"Total amount of industrial H2 demand from P-E-S is equal to "
401			f"{H2_total_PES}, which should be identical to the distributed amount "
402			f"of {total_H2_distributed}, but it is not."
403			)
404			assert round(H2_total_PES) == round(total_H2_distributed), msg
405
406			msg = (
407			f"Total amount of industrial CH4 demand from P-E-S is equal to "
408			f"{CH4_total_PES}, which should be identical to the distributed amount "
409			f"of {total_CH4_distributed}, but it is not."
410			)
411			assert round(CH4_total_PES) == round(total_CH4_distributed), msg
412
413			industrial_gas_demand = pd.concat(
414			[
415			industrial_gas_demand_CH4,
416			industrial_gas_demand_H2,
417			]
418			)
419			industrial_gas_demand = (
420			industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
421			.apply(lambda x: [sum(y) for y in zip(*x)])
422			.reset_index(drop=False)
423			)
424
425			industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
426			insert_industrial_gas_demand_time_series(industrial_gas_demand)
427
428
429			def insert_industrial_gas_demand_time_series(egon_etrago_load_gas):
430			"""
431			Insert list of industrial gas demand time series (one per NUTS3)
432			"""
433			egon_etrago_load_gas_timeseries = egon_etrago_load_gas
434
435			# Connect to local database
436			engine = db.engine()
437
438			# Adjust columns
439			egon_etrago_load_gas_timeseries = egon_etrago_load_gas_timeseries.drop(
440			columns=["carrier", "bus", "sign"]
441			)
442			egon_etrago_load_gas_timeseries["temp_id"] = 1
443
444			# Insert data to db
445			egon_etrago_load_gas_timeseries.to_sql(
446			"egon_etrago_load_timeseries",
447			engine,
448			schema="grid",
449			index=False,
450			if_exists="append",
451			)
452
453
454			def download_industrial_gas_demand():
455			"""Download the industrial gas demand data from opendata.ffe database."""
456			correspondance_url = (
457			"http://opendata.ffe.de:3000/region?id_region_type=eq.38"
458			)
459
460			# Read and save data
461			result_corr = requests.get(correspondance_url)
462			target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
463			os.makedirs(os.path.dirname(target_file), exist_ok=True)
464			pd.read_json(result_corr.content).to_json(target_file)
465
466			carriers = {"H2": "2,162", "CH4": "2,11"}
467			url = "http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq."
468
469			for scn_name in ["eGon2035", "eGon100RE"]:
470			year = str(
471			get_sector_parameters("global", scn_name)["population_year"]
472			)
473
474			for carrier, internal_id in carriers.items():
475			# Download the data
476			datafilter = "&&internal_id=eq.{" + internal_id + "}"
477			request = url + year + datafilter
478
479			# Read and save data
480			result = requests.get(request)
481			target_file = (
482			Path(".")
483			/ "datasets/gas_data/demand"
484			/ (carrier + "_" + scn_name + ".json")
485			)
486			pd.read_json(result.content).to_json(target_file)
487

openego / eGon-data

Push — dev ( 7c1955...864b08 )

insert_new_entries() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like