data.datasets.industrial_gas_demand.read_industrial_CH4_demand() - Code Metrics - Inspection of "Fix/#871 gas load allocation" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#899)

unknown

created 2022-09-02 07:00 UTC

d() A

↳ Parent: data.datasets.industrial_gas_demand.read_and_process_demand()

Complexity

Conditions

Size

Total Lines	31
Code Lines	9

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	9
dl	0
loc	31
rs	9.95
c	0
b	0
f	0
cc	1
nop	1

# -*- coding: utf-8 -*-
"""
The central module containing all code dealing with importing gas industrial demand
"""
from pathlib import Path
import os

from geoalchemy2.types import Geometry
from shapely import wkt
import numpy as np
import pandas as pd
import requests

from egon.data import db
from egon.data.config import settings
from egon.data.datasets import Dataset
from egon.data.datasets.etrago_helpers import (
    finalize_bus_insertion,
    initialise_bus_insertion,
)
from egon.data.datasets.etrago_setup import link_geom_from_buses
from egon.data.datasets.pypsaeursec import read_network
from egon.data.datasets.scenario_parameters import get_sector_parameters


class IndustrialGasDemand(Dataset):
    def __init__(self, dependencies):
        super().__init__(
            name="IndustrialGasDemand",
            version="0.0.3",
            dependencies=dependencies,
            tasks=(download_industrial_gas_demand),
        )


class IndustrialGasDemandeGon2035(Dataset):
    def __init__(self, dependencies):
        super().__init__(
            name="IndustrialGasDemandeGon2035",
            version="0.0.2",
            dependencies=dependencies,
            tasks=(insert_industrial_gas_demand_egon2035),
        )


class IndustrialGasDemandeGon100RE(Dataset):
    def __init__(self, dependencies):
        super().__init__(
            name="IndustrialGasDemandeGon100RE",
            version="0.0.2",
            dependencies=dependencies,
            tasks=(insert_industrial_gas_demand_egon100RE),
        )


def read_industrial_demand(scn_name, carrier):
    """Read the gas demand data

    Parameters
    ----------
    scn_name : str
        Name of the scenario
    carrier : str
        Name of the gas carrier

    Returns
    -------
    df : pandas.core.frame.DataFrame
        Dataframe containing the industrial demand

    """
    target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
    df_corr = pd.read_json(target_file)
    df_corr = df_corr.loc[:, ["id_region", "name_short"]]
    df_corr.set_index("id_region", inplace=True)

    target_file = (
        Path(".")
        / "datasets/gas_data/demand"
        / (carrier + "_" + scn_name + ".json")
    )
    industrial_loads = pd.read_json(target_file)
    industrial_loads = industrial_loads.loc[:, ["id_region", "values"]]
    industrial_loads.set_index("id_region", inplace=True)

    # Match the id_region to obtain the NUT3 region names
    industrial_loads_list = pd.concat(
        [industrial_loads, df_corr], axis=1, join="inner"
    )
    industrial_loads_list["NUTS0"] = (industrial_loads_list["name_short"].str)[
        0:2
    ]
    industrial_loads_list["NUTS1"] = (industrial_loads_list["name_short"].str)[
        0:3
    ]
    industrial_loads_list = industrial_loads_list[
        industrial_loads_list["NUTS0"].str.match("DE")
    ]

    # Cut data to federal state if in testmode
    boundary = settings()["egon-data"]["--dataset-boundary"]
    if boundary != "Everything":
        map_states = {
            "Baden-Württemberg": "DE1",
            "Nordrhein-Westfalen": "DEA",
            "Hessen": "DE7",
            "Brandenburg": "DE4",
            "Bremen": "DE5",
            "Rheinland-Pfalz": "DEB",
            "Sachsen-Anhalt": "DEE",
            "Schleswig-Holstein": "DEF",
            "Mecklenburg-Vorpommern": "DE8",
            "Thüringen": "DEG",
            "Niedersachsen": "DE9",
            "Sachsen": "DED",
            "Hamburg": "DE6",
            "Saarland": "DEC",
            "Berlin": "DE3",
            "Bayern": "DE2",
        }

        industrial_loads_list = industrial_loads_list[
            industrial_loads_list["NUTS1"].isin([map_states[boundary], np.nan])
        ]

    industrial_loads_list = industrial_loads_list.rename(
        columns={"name_short": "nuts3", "values": "p_set"}
    )
    industrial_loads_list = industrial_loads_list.set_index("nuts3")

    # Add the centroid point to each NUTS3 area
    sql_vg250 = """SELECT nuts as nuts3, geometry as geom
                    FROM boundaries.vg250_krs
                    WHERE gf = 4 ;"""
    gdf_vg250 = db.select_geodataframe(sql_vg250, epsg=4326)

    point = []
    for index, row in gdf_vg250.iterrows():
        point.append(wkt.loads(str(row["geom"])).centroid)
    gdf_vg250["point"] = point
    gdf_vg250 = gdf_vg250.set_index("nuts3")
    gdf_vg250 = gdf_vg250.drop(columns=["geom"])

    # Match the load to the NUTS3 points
    industrial_loads_list = pd.concat(
        [industrial_loads_list, gdf_vg250], axis=1, join="inner"
    )
    return industrial_loads_list.rename(
        columns={"point": "geom"}
    ).set_geometry("geom", crs=4326)


def read_and_process_demand(scn_name="eGon2035", carrier=None, grid_carrier=None):
    """Assign the industrial demand in Germany to buses

    Parameters
    ----------
    scn_name : str
        Name of the scenario

    carrier : str
        Name of the carrier, the demand should hold

    grid_carrier : str
        Carrier name of the buses, the demand should be assigned to

    Returns
    -------
    industrial_demand :
        Dataframe containing the industrial demand in Germany

    """
    if grid_carrier is None:
        grid_carrier = carrier
    industrial_loads_list = read_industrial_demand(scn_name, carrier)

    # Match to associated gas bus
    industrial_loads_list = db.assign_gas_bus_id(
        industrial_loads_list, scn_name, carrier
    )

    # Add carrier
    industrial_loads_list["carrier"] = carrier

    # Remove useless columns
    industrial_loads_list = industrial_loads_list.drop(
        columns=["geom", "NUTS0", "NUTS1", "bus_id"], errors="ignore"
    )

    return industrial_loads_list


def delete_old_entries(scn_name):
    """
    Delete loads and load timeseries.

    Parameters
    ----------
    scn_name : str
        Name of the scenario.
    """
    # Clean tables
    db.execute_sql(
        f"""
        DELETE FROM grid.egon_etrago_load_timeseries
        WHERE "load_id" IN (
            SELECT load_id FROM grid.egon_etrago_load
            WHERE "carrier" IN ('CH4', 'H2') AND
            scn_name = '{scn_name}' AND bus not IN (
                SELECT bus_id FROM grid.egon_etrago_bus
                WHERE scn_name = '{scn_name}' AND country != 'DE'
            )
        );
        """
    )

    db.execute_sql(
        f"""
        DELETE FROM grid.egon_etrago_load
        WHERE "load_id" IN (
            SELECT load_id FROM grid.egon_etrago_load
            WHERE "carrier" IN ('CH4', 'H2') AND
            scn_name = '{scn_name}' AND bus not IN (
                SELECT bus_id FROM grid.egon_etrago_bus
                WHERE scn_name = '{scn_name}' AND country != 'DE'
            )
        );
        """
    )


def insert_new_entries(industrial_gas_demand, scn_name):
    """
    Insert loads.

    Parameters
    ----------
    industrial_gas_demand : pandas.DataFrame
        Load data to insert.
    scn_name : str
        Name of the scenario.
    """

    new_id = db.next_etrago_id("load")
    industrial_gas_demand["load_id"] = range(
        new_id, new_id + len(industrial_gas_demand)
    )

    # Add missing columns
    c = {"scn_name": scn_name, "sign": -1}
    industrial_gas_demand = industrial_gas_demand.assign(**c)

    industrial_gas_demand = industrial_gas_demand.reset_index(drop=True)

    # Remove useless columns
    egon_etrago_load_gas = industrial_gas_demand.drop(columns=["p_set"])

    engine = db.engine()
    # Insert data to db
    egon_etrago_load_gas.to_sql(
        "egon_etrago_load",
        engine,
        schema="grid",
        index=False,
        if_exists="append",
    )

    return industrial_gas_demand


def insert_industrial_gas_demand_egon2035():
    """Insert list of industrial gas demand (one per NUTS3) in database

    Parameters
    ----------
    scn_name : str
        Name of the scenario

    Returns
    -------
        industrial_gas_demand : Dataframe containing the industrial gas demand
        in Germany
    """
    scn_name = "eGon2035"
    delete_old_entries(scn_name)

    industrial_gas_demand = pd.concat(
        [
            read_and_process_demand(scn_name=scn_name, carrier="CH4"),
            read_and_process_demand(
                scn_name=scn_name, carrier="H2", grid_carrier="H2_grid"
            ),
        ]
    )

    industrial_gas_demand = (
        industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
        .apply(lambda x: [sum(y) for y in zip(*x)])
        .reset_index(drop=False)
    )

    industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
    insert_industrial_gas_demand_time_series(industrial_gas_demand)


def insert_industrial_gas_demand_egon100RE():
    """Insert list of industrial gas demand (one per NUTS3) in database

    Parameters
    ----------
    scn_name : str
        Name of the scenario

    Returns
    -------
        industrial_gas_demand : Dataframe containing the industrial gas demand
        in Germany
    """
    scn_name = "eGon100RE"
    delete_old_entries(scn_name)

    # read demands
    industrial_gas_demand_CH4 = read_and_process_demand(
        scn_name=scn_name, carrier="CH4"
    )
    industrial_gas_demand_H2 = read_and_process_demand(
        scn_name=scn_name, carrier="H2", grid_carrier="H2_grid"
    )

    # adjust H2 and CH4 total demands (values from PES)
    # CH4 demand = 0 in 100RE, therefore scale H2 ts
    # fallback values see https://github.com/openego/eGon-data/issues/626
    n = read_network()

    try:
        H2_total_PES = (
            n.loads[n.loads["carrier"] == "H2 for industry"].loc[
                "DE0 0 H2 for industry", "p_set"
            ]
            * 8760
        )
    except KeyError:
        H2_total_PES = 42090000

    try:
        CH4_total_PES = (
            n.loads[n.loads["carrier"] == "gas for industry"].loc[
                "DE0 0 gas for industry", "p_set"
            ]
            * 8760
        )
    except KeyError:
        CH4_total_PES = 105490000

    boundary = settings()["egon-data"]["--dataset-boundary"]
    if boundary != "Everything":
        # modify values for test mode
        # the values are obtained by evaluating the share of H2 demand in
        # test region (NUTS1: DEF, Schleswig-Holstein) with respect to the H2
        # demand in full Germany model (NUTS0: DE). The task has been outsourced
        # to save processing cost
        H2_total_PES *= 0.01855683050330346
        CH4_total_PES *= 0.01855683050330346

    H2_total = industrial_gas_demand_H2["p_set"].apply(sum).astype(float).sum()

    industrial_gas_demand_CH4["p_set"] = industrial_gas_demand_H2[
        "p_set"
    ].apply(lambda x: [val / H2_total * CH4_total_PES for val in x])
    industrial_gas_demand_H2["p_set"] = industrial_gas_demand_H2[
        "p_set"
    ].apply(lambda x: [val / H2_total * H2_total_PES for val in x])

    # consistency check
    total_CH4_distributed = sum(
        [sum(x) for x in industrial_gas_demand_CH4["p_set"].to_list()]
    )
    total_H2_distributed = sum(
        [sum(x) for x in industrial_gas_demand_H2["p_set"].to_list()]
    )

    print(
        f"Total amount of industrial H2 demand distributed is "
        f"{total_H2_distributed} MWh. Total amount of industrial CH4 demand "
        f"distributed is {total_CH4_distributed} MWh."
    )
    msg = (
        f"Total amount of industrial H2 demand from P-E-S is equal to "
        f"{H2_total_PES}, which should be identical to the distributed amount "
        f"of {total_H2_distributed}, but it is not."
    )
    assert round(H2_total_PES) == round(total_H2_distributed), msg

    msg = (
        f"Total amount of industrial CH4 demand from P-E-S is equal to "
        f"{CH4_total_PES}, which should be identical to the distributed amount "
        f"of {total_CH4_distributed}, but it is not."
    )
    assert round(CH4_total_PES) == round(total_CH4_distributed), msg

    industrial_gas_demand = pd.concat(
        [
            industrial_gas_demand_CH4,
            industrial_gas_demand_H2,
        ]
    )
    industrial_gas_demand = (
        industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
        .apply(lambda x: [sum(y) for y in zip(*x)])
        .reset_index(drop=False)
    )

    industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
    insert_industrial_gas_demand_time_series(industrial_gas_demand)


def insert_industrial_gas_demand_time_series(egon_etrago_load_gas):
    """
    Insert list of industrial gas demand time series (one per NUTS3)
    """
    egon_etrago_load_gas_timeseries = egon_etrago_load_gas

    # Connect to local database
    engine = db.engine()

    # Adjust columns
    egon_etrago_load_gas_timeseries = egon_etrago_load_gas_timeseries.drop(
        columns=["carrier", "bus", "sign"]
    )
    egon_etrago_load_gas_timeseries["temp_id"] = 1

    # Insert data to db
    egon_etrago_load_gas_timeseries.to_sql(
        "egon_etrago_load_timeseries",
        engine,
        schema="grid",
        index=False,
        if_exists="append",
    )


def download_industrial_gas_demand():
    """Download the industrial gas demand data from opendata.ffe database."""
    correspondance_url = (
        "http://opendata.ffe.de:3000/region?id_region_type=eq.38"
    )

    # Read and save data
    result_corr = requests.get(correspondance_url)
    target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
    os.makedirs(os.path.dirname(target_file), exist_ok=True)
    pd.read_json(result_corr.content).to_json(target_file)

    carriers = {"H2": "2,162", "CH4": "2,11"}
    url = "http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq."

    for scn_name in ["eGon2035", "eGon100RE"]:
        year = str(
            get_sector_parameters("global", scn_name)["population_year"]
        )

        for carrier, internal_id in carriers.items():
            # Download the data
            datafilter = "&&internal_id=eq.{" + internal_id + "}"
            request = url + year + datafilter

            # Read and save data
            result = requests.get(request)
            target_file = (
                Path(".")
                / "datasets/gas_data/demand"
                / (carrier + "_" + scn_name + ".json")
            )
            pd.read_json(result.content).to_json(target_file)


1			# -- coding: utf-8 --
2			"""
3			The central module containing all code dealing with importing gas industrial demand
4			"""
5			from pathlib import Path
6			import os
7
8			from geoalchemy2.types import Geometry
9			from shapely import wkt
10			import numpy as np
11			import pandas as pd
12			import requests
13
14			from egon.data import db
15			from egon.data.config import settings
16			from egon.data.datasets import Dataset
17			from egon.data.datasets.etrago_helpers import (
18			finalize_bus_insertion,
19			initialise_bus_insertion,
20			)
21			from egon.data.datasets.etrago_setup import link_geom_from_buses
22			from egon.data.datasets.pypsaeursec import read_network
23			from egon.data.datasets.scenario_parameters import get_sector_parameters
24
25
26			class IndustrialGasDemand(Dataset):
27			def __init__(self, dependencies):
28			super().__init__(
29			name="IndustrialGasDemand",
30			version="0.0.3",
31			dependencies=dependencies,
32			tasks=(download_industrial_gas_demand),
33			)
34
35
36			class IndustrialGasDemandeGon2035(Dataset):
37			def __init__(self, dependencies):
38			super().__init__(
39			name="IndustrialGasDemandeGon2035",
40			version="0.0.2",
41			dependencies=dependencies,
42			tasks=(insert_industrial_gas_demand_egon2035),
43			)
44
45
46			class IndustrialGasDemandeGon100RE(Dataset):
47			def __init__(self, dependencies):
48			super().__init__(
49			name="IndustrialGasDemandeGon100RE",
50			version="0.0.2",
51			dependencies=dependencies,
52			tasks=(insert_industrial_gas_demand_egon100RE),
53			)
54
55
56			def read_industrial_demand(scn_name, carrier):
57			"""Read the gas demand data
58
59			Parameters
60			----------
61			scn_name : str
62			Name of the scenario
63			carrier : str
64			Name of the gas carrier
65
66			Returns
67			-------
68			df : pandas.core.frame.DataFrame
69			Dataframe containing the industrial demand
70
71			"""
72			target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
73			df_corr = pd.read_json(target_file)
74			df_corr = df_corr.loc[:, ["id_region", "name_short"]]
75			df_corr.set_index("id_region", inplace=True)
76
77			target_file = (
78			Path(".")
79			/ "datasets/gas_data/demand"
80			/ (carrier + "_" + scn_name + ".json")
81			)
82			industrial_loads = pd.read_json(target_file)
83			industrial_loads = industrial_loads.loc[:, ["id_region", "values"]]
84			industrial_loads.set_index("id_region", inplace=True)
85
86			# Match the id_region to obtain the NUT3 region names
87			industrial_loads_list = pd.concat(
88			[industrial_loads, df_corr], axis=1, join="inner"
89			)
90			industrial_loads_list["NUTS0"] = (industrial_loads_list["name_short"].str)[
91			0:2
92			]
93			industrial_loads_list["NUTS1"] = (industrial_loads_list["name_short"].str)[
94			0:3
95			]
96			industrial_loads_list = industrial_loads_list[
97			industrial_loads_list["NUTS0"].str.match("DE")
98			]
99
100			# Cut data to federal state if in testmode
101			boundary = settings()["egon-data"]["--dataset-boundary"]
102			if boundary != "Everything":
103			map_states = {
104			"Baden-Württemberg": "DE1",
105			"Nordrhein-Westfalen": "DEA",
106			"Hessen": "DE7",
107			"Brandenburg": "DE4",
108			"Bremen": "DE5",
109			"Rheinland-Pfalz": "DEB",
110			"Sachsen-Anhalt": "DEE",
111			"Schleswig-Holstein": "DEF",
112			"Mecklenburg-Vorpommern": "DE8",
113			"Thüringen": "DEG",
114			"Niedersachsen": "DE9",
115			"Sachsen": "DED",
116			"Hamburg": "DE6",
117			"Saarland": "DEC",
118			"Berlin": "DE3",
119			"Bayern": "DE2",
120			}
121
122			industrial_loads_list = industrial_loads_list[
123			industrial_loads_list["NUTS1"].isin([map_states[boundary], np.nan])
124			]
125
126			industrial_loads_list = industrial_loads_list.rename(
127			columns={"name_short": "nuts3", "values": "p_set"}
128			)
129			industrial_loads_list = industrial_loads_list.set_index("nuts3")
130
131			# Add the centroid point to each NUTS3 area
132			sql_vg250 = """SELECT nuts as nuts3, geometry as geom
133			FROM boundaries.vg250_krs
134			WHERE gf = 4 ;"""
135			gdf_vg250 = db.select_geodataframe(sql_vg250, epsg=4326)
136
137			point = []
138			for index, row in gdf_vg250.iterrows():
139			point.append(wkt.loads(str(row["geom"])).centroid)
140			gdf_vg250["point"] = point
141			gdf_vg250 = gdf_vg250.set_index("nuts3")
142			gdf_vg250 = gdf_vg250.drop(columns=["geom"])
143
144			# Match the load to the NUTS3 points
145			industrial_loads_list = pd.concat(
146			[industrial_loads_list, gdf_vg250], axis=1, join="inner"
147			)
148			return industrial_loads_list.rename(
149			columns={"point": "geom"}
150			).set_geometry("geom", crs=4326)
151
152
153			def read_and_process_demand(scn_name="eGon2035", carrier=None, grid_carrier=None):
154			"""Assign the industrial demand in Germany to buses
155
156			Parameters
157			----------
158			scn_name : str
159			Name of the scenario
160
161			carrier : str
162			Name of the carrier, the demand should hold
163
164			grid_carrier : str
165			Carrier name of the buses, the demand should be assigned to
166
167			Returns
168			-------
169			industrial_demand :
170			Dataframe containing the industrial demand in Germany
171
172			"""
173			if grid_carrier is None:
174			grid_carrier = carrier
175			industrial_loads_list = read_industrial_demand(scn_name, carrier)
176
177			# Match to associated gas bus
178			industrial_loads_list = db.assign_gas_bus_id(
179			industrial_loads_list, scn_name, carrier
180			)
181
182			# Add carrier
183			industrial_loads_list["carrier"] = carrier
184
185			# Remove useless columns
186			industrial_loads_list = industrial_loads_list.drop(
187			columns=["geom", "NUTS0", "NUTS1", "bus_id"], errors="ignore"
188			)
189
190			return industrial_loads_list
191
192
193			def delete_old_entries(scn_name):
194			"""
195			Delete loads and load timeseries.
196
197			Parameters
198			----------
199			scn_name : str
200			Name of the scenario.
201			"""
202			# Clean tables
203			db.execute_sql(
204			f"""
205			DELETE FROM grid.egon_etrago_load_timeseries
206			WHERE "load_id" IN (
207			SELECT load_id FROM grid.egon_etrago_load
208			WHERE "carrier" IN ('CH4', 'H2') AND
209			scn_name = '{scn_name}' AND bus not IN (
210			SELECT bus_id FROM grid.egon_etrago_bus
211			WHERE scn_name = '{scn_name}' AND country != 'DE'
212			)
213			);
214			"""
215			)
216
217			db.execute_sql(
218			f"""
219			DELETE FROM grid.egon_etrago_load
220			WHERE "load_id" IN (
221			SELECT load_id FROM grid.egon_etrago_load
222			WHERE "carrier" IN ('CH4', 'H2') AND
223			scn_name = '{scn_name}' AND bus not IN (
224			SELECT bus_id FROM grid.egon_etrago_bus
225			WHERE scn_name = '{scn_name}' AND country != 'DE'
226			)
227			);
228			"""
229			)
230
231
232			def insert_new_entries(industrial_gas_demand, scn_name):
233			"""
234			Insert loads.
235
236			Parameters
237			----------
238			industrial_gas_demand : pandas.DataFrame
239			Load data to insert.
240			scn_name : str
241			Name of the scenario.
242			"""
243
244			new_id = db.next_etrago_id("load")
245			industrial_gas_demand["load_id"] = range(
246			new_id, new_id + len(industrial_gas_demand)
247			)
248
249			# Add missing columns
250			c = {"scn_name": scn_name, "sign": -1}
251			industrial_gas_demand = industrial_gas_demand.assign(**c)
252
253			industrial_gas_demand = industrial_gas_demand.reset_index(drop=True)
254
255			# Remove useless columns
256			egon_etrago_load_gas = industrial_gas_demand.drop(columns=["p_set"])
257
258			engine = db.engine()
259			# Insert data to db
260			egon_etrago_load_gas.to_sql(
261			"egon_etrago_load",
262			engine,
263			schema="grid",
264			index=False,
265			if_exists="append",
266			)
267
268			return industrial_gas_demand
269
270
271			def insert_industrial_gas_demand_egon2035():
272			"""Insert list of industrial gas demand (one per NUTS3) in database
273
274			Parameters
275			----------
276			scn_name : str
277			Name of the scenario
278
279			Returns
280			-------
281			industrial_gas_demand : Dataframe containing the industrial gas demand
282			in Germany
283			"""
284			scn_name = "eGon2035"
285			delete_old_entries(scn_name)
286
287			industrial_gas_demand = pd.concat(
288			[
289			read_and_process_demand(scn_name=scn_name, carrier="CH4"),
290			read_and_process_demand(
291			scn_name=scn_name, carrier="H2", grid_carrier="H2_grid"
292			),
293			]
294			)
295
296			industrial_gas_demand = (
297			industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
298			.apply(lambda x: [sum(y) for y in zip(*x)])
299			.reset_index(drop=False)
300			)
301
302			industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
303			insert_industrial_gas_demand_time_series(industrial_gas_demand)
304
305
306			def insert_industrial_gas_demand_egon100RE():
307			"""Insert list of industrial gas demand (one per NUTS3) in database
308
309			Parameters
310			----------
311			scn_name : str
312			Name of the scenario
313
314			Returns
315			-------
316			industrial_gas_demand : Dataframe containing the industrial gas demand
317			in Germany
318			"""
319			scn_name = "eGon100RE"
320			delete_old_entries(scn_name)
321
322			# read demands
323			industrial_gas_demand_CH4 = read_and_process_demand(
324			scn_name=scn_name, carrier="CH4"
325			)
326			industrial_gas_demand_H2 = read_and_process_demand(
327			scn_name=scn_name, carrier="H2", grid_carrier="H2_grid"
328			)
329
330			# adjust H2 and CH4 total demands (values from PES)
331			# CH4 demand = 0 in 100RE, therefore scale H2 ts
332			# fallback values see https://github.com/openego/eGon-data/issues/626
333			n = read_network()
334
335			try:
336			H2_total_PES = (
337			n.loads[n.loads["carrier"] == "H2 for industry"].loc[
338			"DE0 0 H2 for industry", "p_set"
339			]
340			* 8760
341			)
342			except KeyError:
343			H2_total_PES = 42090000
344
345			try:
346			CH4_total_PES = (
347			n.loads[n.loads["carrier"] == "gas for industry"].loc[
348			"DE0 0 gas for industry", "p_set"
349			]
350			* 8760
351			)
352			except KeyError:
353			CH4_total_PES = 105490000
354
355			boundary = settings()["egon-data"]["--dataset-boundary"]
356			if boundary != "Everything":
357			# modify values for test mode
358			# the values are obtained by evaluating the share of H2 demand in
359			# test region (NUTS1: DEF, Schleswig-Holstein) with respect to the H2
360			# demand in full Germany model (NUTS0: DE). The task has been outsourced
361			# to save processing cost
362			H2_total_PES *= 0.01855683050330346
363			CH4_total_PES *= 0.01855683050330346
364
365			H2_total = industrial_gas_demand_H2["p_set"].apply(sum).astype(float).sum()
366
367			industrial_gas_demand_CH4["p_set"] = industrial_gas_demand_H2[
368			"p_set"
369			].apply(lambda x: [val / H2_total * CH4_total_PES for val in x])
370			industrial_gas_demand_H2["p_set"] = industrial_gas_demand_H2[
371			"p_set"
372			].apply(lambda x: [val / H2_total * H2_total_PES for val in x])
373
374			# consistency check
375			total_CH4_distributed = sum(
376			[sum(x) for x in industrial_gas_demand_CH4["p_set"].to_list()]
377			)
378			total_H2_distributed = sum(
379			[sum(x) for x in industrial_gas_demand_H2["p_set"].to_list()]
380			)
381
382			print(
383			f"Total amount of industrial H2 demand distributed is "
384			f"{total_H2_distributed} MWh. Total amount of industrial CH4 demand "
385			f"distributed is {total_CH4_distributed} MWh."
386			)
387			msg = (
388			f"Total amount of industrial H2 demand from P-E-S is equal to "
389			f"{H2_total_PES}, which should be identical to the distributed amount "
390			f"of {total_H2_distributed}, but it is not."
391			)
392			assert round(H2_total_PES) == round(total_H2_distributed), msg
393
394			msg = (
395			f"Total amount of industrial CH4 demand from P-E-S is equal to "
396			f"{CH4_total_PES}, which should be identical to the distributed amount "
397			f"of {total_CH4_distributed}, but it is not."
398			)
399			assert round(CH4_total_PES) == round(total_CH4_distributed), msg
400
401			industrial_gas_demand = pd.concat(
402			[
403			industrial_gas_demand_CH4,
404			industrial_gas_demand_H2,
405			]
406			)
407			industrial_gas_demand = (
408			industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
409			.apply(lambda x: [sum(y) for y in zip(*x)])
410			.reset_index(drop=False)
411			)
412
413			industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
414			insert_industrial_gas_demand_time_series(industrial_gas_demand)
415
416
417			def insert_industrial_gas_demand_time_series(egon_etrago_load_gas):
418			"""
419			Insert list of industrial gas demand time series (one per NUTS3)
420			"""
421			egon_etrago_load_gas_timeseries = egon_etrago_load_gas
422
423			# Connect to local database
424			engine = db.engine()
425
426			# Adjust columns
427			egon_etrago_load_gas_timeseries = egon_etrago_load_gas_timeseries.drop(
428			columns=["carrier", "bus", "sign"]
429			)
430			egon_etrago_load_gas_timeseries["temp_id"] = 1
431
432			# Insert data to db
433			egon_etrago_load_gas_timeseries.to_sql(
434			"egon_etrago_load_timeseries",
435			engine,
436			schema="grid",
437			index=False,
438			if_exists="append",
439			)
440
441
442			def download_industrial_gas_demand():
443			"""Download the industrial gas demand data from opendata.ffe database."""
444			correspondance_url = (
445			"http://opendata.ffe.de:3000/region?id_region_type=eq.38"
446			)
447
448			# Read and save data
449			result_corr = requests.get(correspondance_url)
450			target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
451			os.makedirs(os.path.dirname(target_file), exist_ok=True)
452			pd.read_json(result_corr.content).to_json(target_file)
453
454			carriers = {"H2": "2,162", "CH4": "2,11"}
455			url = "http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq."
456
457			for scn_name in ["eGon2035", "eGon100RE"]:
458			year = str(
459			get_sector_parameters("global", scn_name)["population_year"]
460			)
461
462			for carrier, internal_id in carriers.items():
463			# Download the data
464			datafilter = "&&internal_id=eq.{" + internal_id + "}"
465			request = url + year + datafilter
466
467			# Read and save data
468			result = requests.get(request)
469			target_file = (
470			Path(".")
471			/ "datasets/gas_data/demand"
472			/ (carrier + "_" + scn_name + ".json")
473			)
474			pd.read_json(result.content).to_json(target_file)
475

openego / eGon-data

Pull Request — dev (#899)

d() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like