data.datasets.renewable_feedin.federal_states_per_weather_cell() - Code Metrics - Inspection of "feature/validation_integration" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#1375)

unknown

created 2026-01-16 09:14 UTC

federal_states_per_weather_cell() A

↳ Parent: data.datasets.renewable_feedin

Complexity

Conditions

Size

Total Lines	57
Code Lines	25

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	25
dl	0
loc	57
rs	9.28
c	0
b	0
f	0
cc	2
nop	0

How to fix Long Method

"""
Central module containing all code dealing with processing era5 weather data.
"""

import datetime
import json
import time

from sqlalchemy import Column, ForeignKey, Integer
from sqlalchemy.ext.declarative import declarative_base
import geopandas as gpd
import numpy as np
import pandas as pd

from egon.data import db
from egon.data.datasets import Dataset
from egon.data.datasets.era5 import (
    EgonEra5Cells,
    EgonRenewableFeedIn,
    import_cutout,
)
from egon.data.datasets.scenario_parameters import get_sector_parameters
from egon.data.datasets.zensus_vg250 import DestatisZensusPopulationPerHa
from egon.data.metadata import context, license_ccby, meta_metadata, sources
import egon.data.config

from egon_validation import(
    RowCountValidation,
    DataTypeValidation,
    NotNullAndNotNaNValidation,
    WholeTableNotNullAndNotNaNValidation,
    ValueSetValidation
)

class RenewableFeedin(Dataset):
    """
    Calculate possible feedin time series for renewable energy generators

    This dataset calculates possible feedin timeseries for fluctuation renewable generators
    and coefficient of performance time series for heat pumps. Relevant input is the
    downloaded weather data. Parameters for the time series calcultaion are also defined by
    representative types of pv plants and wind turbines that are selected within this dataset.
    The resulting profiles are stored in the database.


    *Dependencies*
      * :py:class:`WeatherData <egon.data.datasets.era5.WeatherData>`
      * :py:class:`Vg250 <egon.data.datasets.vg250.Vg250>`
      * :py:class:`ZensusVg250 <egon.data.datasets.zensus_vg250.ZensusVg250>`

    *Resulting tables*
      * :py:class:`supply.egon_era5_renewable_feedin <egon.data.datasets.era5.EgonRenewableFeedIn>` is filled

    """

    #:
    name: str = "RenewableFeedin"
    #:
    version: str = "0.0.8"

    def __init__(self, dependencies):
        super().__init__(
            name=self.name,
            version=self.version,
            dependencies=dependencies,
            tasks={
                wind,
                pv,
                solar_thermal,
                heat_pump_cop,
                wind_offshore,
                mapping_zensus_weather,
            },
            validation = {
                "data-quality": [
                    RowCountValidation(
                        table="supply.egon_era5_renewable_feedin",
                        rule_id="TEST_ROW_COUNT.egon_renewable_feedin",
                        expected_count=6102
                    ),
                    DataTypeValidation(
                        table="supply.egon_era5_renewable_feedin",
                        rule_id="TEST_DATA_MULTIPLE_TYPES.egon_era5_renewable_feedin",
                        column_types={
                            "w_id": "integer",
                            "weather_year": "integer",
                            "carrier": "character varying",
                            "feedin": "double precision[]"
                        }
                    ),
                    NotNullAndNotNaNValidation(
                        table="supply.egon_era5_renewable_feedin",
                        rule_id="TEST_NOT_NAN.egon_era5_renewable_feedin",
                        columns=["w_id", "weather_year", "carrier", "feedin"]
                    ),
                    WholeTableNotNullAndNotNaNValidation(
                        table="supply.egon_era5_renewable_feedin",
                        rule_id="TEST_WHOLE_TABLE_NOT_NAN.egon_era5_renewable_feedin"
                    ),
                    ValueSetValidation(
                        table="supply.egon_district_heating",
                        rule_id="VALUE_SET_VALIDATION_CARRIER.egon_district_heating",
                        column="carrier",
                        expected_values=["wind_onshore", "solar_thermal", "heat_pump_cop", "wind_offshore", "pv"]
                    ),

                ]
            },
            on_validation_failure = "continue"
        )


Base = declarative_base()
engine = db.engine()


class MapZensusWeatherCell(Base):
    __tablename__ = "egon_map_zensus_weather_cell"
    __table_args__ = {"schema": "boundaries"}

    zensus_population_id = Column(
        Integer,
        ForeignKey(DestatisZensusPopulationPerHa.id),
        primary_key=True,
        index=True,
    )
    w_id = Column(Integer, ForeignKey(EgonEra5Cells.w_id), index=True)


def weather_cells_in_germany(geom_column="geom"):
    """Get weather cells which intersect with Germany

    Returns
    -------
    GeoPandas.GeoDataFrame
        Index and points of weather cells inside Germany

    """

    cfg = egon.data.config.datasets()["renewable_feedin"]["sources"]

    return db.select_geodataframe(
        f"""SELECT w_id, geom_point, geom
        FROM {cfg['weather_cells']['schema']}.
        {cfg['weather_cells']['table']}
        WHERE ST_Intersects('SRID=4326;
        POLYGON((5 56, 15.5 56, 15.5 47, 5 47, 5 56))', geom)""",
        geom_col=geom_column,
        index_col="w_id",
    )


def offshore_weather_cells(geom_column="geom"):
    """Get weather cells which intersect with Germany

    Returns
    -------
    GeoPandas.GeoDataFrame
        Index and points of weather cells inside Germany

    """

    cfg = egon.data.config.datasets()["renewable_feedin"]["sources"]

    return db.select_geodataframe(
        f"""SELECT w_id, geom_point, geom
        FROM {cfg['weather_cells']['schema']}.
        {cfg['weather_cells']['table']}
        WHERE ST_Intersects('SRID=4326;
        POLYGON((5.5 55.5, 14.5 55.5, 14.5 53.5, 5.5 53.5, 5.5 55.5))',
         geom)""",
        geom_col=geom_column,
        index_col="w_id",
    )


def federal_states_per_weather_cell():
    """Assings a federal state to each weather cell in Germany.

    Sets the federal state to the weather celss using the centroid.
    Weather cells at the borders whoes centroid is not inside Germany
    are assinged to the closest federal state.

    Returns
    -------
    GeoPandas.GeoDataFrame
        Index, points and federal state of weather cells inside Germany

    """

    cfg = egon.data.config.datasets()["renewable_feedin"]["sources"]

    # Select weather cells and ferear states from database
    weather_cells = weather_cells_in_germany(geom_column="geom_point")

    federal_states = db.select_geodataframe(
        f"""SELECT gen, geometry
        FROM {cfg['vg250_lan_union']['schema']}.
        {cfg['vg250_lan_union']['table']}""",
        geom_col="geometry",
        index_col="gen",
    )

    # Map federal state and onshore wind turbine to weather cells
    weather_cells["federal_state"] = gpd.sjoin(
        weather_cells, federal_states
    ).gen

    # Assign a federal state to each cell inside Germany
    buffer = 1000

    while (buffer < 30000) & (
        len(weather_cells[weather_cells["federal_state"].isnull()]) > 0
    ):
        cells = weather_cells[weather_cells["federal_state"].isnull()]

        cells.loc[:, "geom_point"] = cells.geom_point.buffer(buffer)

        weather_cells.loc[cells.index, "federal_state"] = gpd.sjoin(
            cells, federal_states
        ).gen

        buffer += 200

        weather_cells = (
            weather_cells.reset_index()
            .drop_duplicates(subset="w_id", keep="first")
            .set_index("w_id")
        )

    weather_cells = weather_cells.dropna(axis=0, subset=["federal_state"])

    return weather_cells.to_crs(4326)


def turbine_per_weather_cell():
    """Assign wind onshore turbine types to weather cells

    Returns
    -------
    weather_cells : GeoPandas.GeoDataFrame
        Weather cells in Germany including turbine type

    """

    # Select representative onshore wind turbines per federal state
    map_federal_states_turbines = {
        "Schleswig-Holstein": "E-126",
        "Bremen": "E-126",
        "Hamburg": "E-126",
        "Mecklenburg-Vorpommern": "E-126",
        "Niedersachsen": "E-126",
        "Berlin": "E-141",
        "Brandenburg": "E-141",
        "Hessen": "E-141",
        "Nordrhein-Westfalen": "E-141",
        "Sachsen": "E-141",
        "Sachsen-Anhalt": "E-141",
        "Thüringen": "E-141",
        "Baden-Württemberg": "E-141",
        "Bayern": "E-141",
        "Rheinland-Pfalz": "E-141",
        "Saarland": "E-141",
    }

    # Select weather cells and federal states
    weather_cells = federal_states_per_weather_cell()

    # Assign turbine type per federal state
    weather_cells["wind_turbine"] = weather_cells["federal_state"].map(
        map_federal_states_turbines
    )

    return weather_cells


def feedin_per_turbine():
    """Calculate feedin timeseries per turbine type and weather cell

    Returns
    -------
    gdf : GeoPandas.GeoDataFrame
        Feed-in timeseries per turbine type and weather cell

    """

    # Select weather data for Germany
    cutout = import_cutout(boundary="Germany")

    gdf = gpd.GeoDataFrame(geometry=cutout.grid.geometry, crs=4326)

    # Calculate feedin-timeseries for E-141
    # source:
    # https://openenergy-platform.org/dataedit/view/supply/wind_turbine_library
    turbine_e141 = {
        "name": "E141 4200 kW",
        "hub_height": 129,
        "P": 4.200,
        "V": np.arange(1, 26, dtype=float),
        "POW": np.array(
            [
                0.0,
                0.022,
                0.104,
                0.26,
                0.523,
                0.92,
                1.471,
                2.151,
                2.867,
                3.481,
                3.903,
                4.119,
                4.196,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
            ]
        ),
    }
    ts_e141 = cutout.wind(
        turbine_e141, per_unit=True, shapes=cutout.grid.geometry
    )

    gdf["E-141"] = ts_e141.to_pandas().transpose().values.tolist()

    # Calculate feedin-timeseries for E-126
    # source:
    # https://openenergy-platform.org/dataedit/view/supply/wind_turbine_library
    turbine_e126 = {
        "name": "E126 4200 kW",
        "hub_height": 159,
        "P": 4.200,
        "V": np.arange(1, 26, dtype=float),
        "POW": np.array(
            [
                0.0,
                0.0,
                0.058,
                0.185,
                0.4,
                0.745,
                1.2,
                1.79,
                2.45,
                3.12,
                3.66,
                4.0,
                4.15,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
                4.2,
            ]
        ),
    }
    ts_e126 = cutout.wind(
        turbine_e126, per_unit=True, shapes=cutout.grid.geometry
    )

    gdf["E-126"] = ts_e126.to_pandas().transpose().values.tolist()

    return gdf


def wind():
    """Insert feed-in timeseries for wind onshore turbines to database

    Returns
    -------
    None.

    """

    cfg = egon.data.config.datasets()["renewable_feedin"]["targets"]

    # Get weather cells with turbine type
    weather_cells = turbine_per_weather_cell()
    weather_cells = weather_cells[weather_cells.wind_turbine.notnull()]

    # Calculate feedin timeseries per turbine and weather cell
    timeseries_per_turbine = feedin_per_turbine()

    # Join weather cells and feedin-timeseries
    timeseries = gpd.sjoin(weather_cells, timeseries_per_turbine)[
        ["E-141", "E-126"]
    ]

    weather_year = get_sector_parameters("global", "eGon2035")["weather_year"]

    df = pd.DataFrame(
        index=weather_cells.index,
        columns=["weather_year", "carrier", "feedin"],
        data={"weather_year": weather_year, "carrier": "wind_onshore"},
    )

    # Insert feedin for selected turbine per weather cell
    for turbine in ["E-126", "E-141"]:
        idx = weather_cells.index[
            (weather_cells.wind_turbine == turbine)
            & (weather_cells.index.isin(timeseries.index))
        ]
        df.loc[idx, "feedin"] = timeseries.loc[idx, turbine].values

    db.execute_sql(
        f"""
                   DELETE FROM {cfg['feedin_table']['schema']}.
                   {cfg['feedin_table']['table']}
                   WHERE carrier = 'wind_onshore'"""
    )

    # Insert values into database
    df.to_sql(
        cfg["feedin_table"]["table"],
        schema=cfg["feedin_table"]["schema"],
        con=db.engine(),
        if_exists="append",
    )


def wind_offshore():
    """Insert feed-in timeseries for wind offshore turbines to database

    Returns
    -------
    None.

    """

    # Get offshore weather cells arround Germany
    weather_cells = offshore_weather_cells()

    # Select weather data for German coast
    cutout = import_cutout(boundary="Germany-offshore")

    # Select weather year from cutout
    weather_year = cutout.name.split("-")[2]

    # Calculate feedin timeseries
    ts_wind_offshore = cutout.wind(
        "Vestas_V164_7MW_offshore",
        per_unit=True,
        shapes=weather_cells.to_crs(4326).geom,
    )

    # Create dataframe and insert to database
    insert_feedin(ts_wind_offshore, "wind_offshore", weather_year)


def pv():
    """Insert feed-in timeseries for pv plants to database

    Returns
    -------
    None.

    """

    # Get weather cells in Germany
    weather_cells = weather_cells_in_germany()

    # Select weather data for Germany
    cutout = import_cutout(boundary="Germany")

    # Select weather year from cutout
    weather_year = cutout.name.split("-")[1]

    # Calculate feedin timeseries
    ts_pv = cutout.pv(
        "CSi",
        orientation={"slope": 35.0, "azimuth": 180.0},
        per_unit=True,
        shapes=weather_cells.to_crs(4326).geom,
    )

    # Create dataframe and insert to database
    insert_feedin(ts_pv, "pv", weather_year)


def solar_thermal():
    """Insert feed-in timeseries for pv plants to database

    Returns
    -------
    None.

    """

    # Get weather cells in Germany
    weather_cells = weather_cells_in_germany()

    # Select weather data for Germany
    cutout = import_cutout(boundary="Germany")

    # Select weather year from cutout
    weather_year = cutout.name.split("-")[1]

    # Calculate feedin timeseries
    ts_solar_thermal = cutout.solar_thermal(
        clearsky_model="simple",
        orientation={"slope": 45.0, "azimuth": 180.0},
        per_unit=True,
        shapes=weather_cells.to_crs(4326).geom,
        capacity_factor=False,
    )

    # Create dataframe and insert to database
    insert_feedin(ts_solar_thermal, "solar_thermal", weather_year)


def heat_pump_cop():
    """
    Calculate coefficient of performance for heat pumps according to
    T. Brown et al: "Synergies of sector coupling and transmission
    reinforcement in a cost-optimised, highlyrenewable European energy system",
    2018, p. 8

    Returns
    -------
    None.

    """
    # Assume temperature of heating system to 55°C according to Brown et. al
    t_sink = 55

    carrier = "heat_pump_cop"

    # Load configuration
    cfg = egon.data.config.datasets()["renewable_feedin"]

    # Get weather cells in Germany
    weather_cells = weather_cells_in_germany()

    # Select weather data for Germany
    cutout = import_cutout(boundary="Germany")

    # Select weather year from cutout
    weather_year = cutout.name.split("-")[1]

    # Calculate feedin timeseries
    temperature = cutout.temperature(
        shapes=weather_cells.to_crs(4326).geom
    ).transpose()

    t_source = temperature.to_pandas()

    delta_t = t_sink - t_source

    # Calculate coefficient of performance for air sourced heat pumps
    # according to Brown et. al
    cop = 6.81 - 0.121 * delta_t + 0.00063 * delta_t**2

    df = pd.DataFrame(
        index=temperature.to_pandas().index,
        columns=["weather_year", "carrier", "feedin"],
        data={"weather_year": weather_year, "carrier": carrier},
    )

    df.feedin = cop.values.tolist()

    # Delete existing rows for carrier
    db.execute_sql(
        f"""
                   DELETE FROM {cfg['targets']['feedin_table']['schema']}.
                   {cfg['targets']['feedin_table']['table']}
                   WHERE carrier = '{carrier}'"""
    )

    # Insert values into database
    df.to_sql(
        cfg["targets"]["feedin_table"]["table"],
        schema=cfg["targets"]["feedin_table"]["schema"],
        con=db.engine(),
        if_exists="append",
    )


def insert_feedin(data, carrier, weather_year):
    """Insert feedin data into database

    Parameters
    ----------
    data : xarray.core.dataarray.DataArray
        Feedin timeseries data
    carrier : str
        Name of energy carrier
    weather_year : int
        Selected weather year

    Returns
    -------
    None.

    """
    # Transpose DataFrame
    data = data.transpose().to_pandas()

    # Load configuration
    cfg = egon.data.config.datasets()["renewable_feedin"]

    # Initialize DataFrame
    df = pd.DataFrame(
        index=data.index,
        columns=["weather_year", "carrier", "feedin"],
        data={"weather_year": weather_year, "carrier": carrier},
    )

    # Convert solar thermal data from W/m^2 to MW/(1000m^2) = kW/m^2
    if carrier == "solar_thermal":
        data *= 1e-3

    # Insert feedin into DataFrame
    df.feedin = data.values.tolist()

    # Delete existing rows for carrier
    db.execute_sql(
        f"""
                   DELETE FROM {cfg['targets']['feedin_table']['schema']}.
                   {cfg['targets']['feedin_table']['table']}
                   WHERE carrier = '{carrier}'"""
    )

    # Insert values into database
    df.to_sql(
        cfg["targets"]["feedin_table"]["table"],
        schema=cfg["targets"]["feedin_table"]["schema"],
        con=db.engine(),
        if_exists="append",
    )


def mapping_zensus_weather():
    """Perform mapping between era5 weather cell and zensus grid"""

    with db.session_scope() as session:
        cells_query = session.query(
            DestatisZensusPopulationPerHa.id.label("zensus_population_id"),
            DestatisZensusPopulationPerHa.geom_point,
        )

    gdf_zensus_population = gpd.read_postgis(
        cells_query.statement,
        cells_query.session.bind,
        index_col=None,
        geom_col="geom_point",
    )

    with db.session_scope() as session:
        cells_query = session.query(EgonEra5Cells.w_id, EgonEra5Cells.geom)

    gdf_weather_cell = gpd.read_postgis(
        cells_query.statement,
        cells_query.session.bind,
        index_col=None,
        geom_col="geom",
    )
    # CRS is 4326
    gdf_weather_cell = gdf_weather_cell.to_crs(epsg=3035)

    gdf_zensus_weather = gdf_zensus_population.sjoin(
        gdf_weather_cell, how="left", predicate="within"
    )

    MapZensusWeatherCell.__table__.drop(bind=engine, checkfirst=True)
    MapZensusWeatherCell.__table__.create(bind=engine, checkfirst=True)

    # Write mapping into db
    with db.session_scope() as session:
        session.bulk_insert_mappings(
            MapZensusWeatherCell,
            gdf_zensus_weather[["zensus_population_id", "w_id"]].to_dict(
                orient="records"
            ),
        )


def add_metadata():

    """Add metdata to supply.egon_era5_renewable_feedin

    Returns
    -------
    None.

    """

    # Import column names and datatypes
    fields = [
        {
            "description": "Weather cell index",
            "name": "w_id",
            "type": "integer",
            "unit": "none",
        },
        {
            "description": "Weather year",
            "name": "weather_year",
            "type": "integer",
            "unit": "none",
        },
        {
            "description": "Energy carrier",
            "name": "carrier",
            "type": "string",
            "unit": "none",
        },
        {
            "description": "Weather-dependent feedin timeseries",
            "name": "feedin",
            "type": "array",
            "unit": "p.u.",
        },
    ]

    meta = {
        "name": "supply.egon_era5_renewable_feedin",
        "title": "eGon feedin timeseries for RES",
        "id": "WILL_BE_SET_AT_PUBLICATION",
        "description": "Weather-dependent feedin timeseries for RES",
        "language": ["EN"],
        "publicationDate": datetime.date.today().isoformat(),
        "context": context(),
        "spatial": {
            "location": None,
            "extent": "Germany",
            "resolution": None,
        },
        "sources": [
            sources()["era5"],
            sources()["vg250"],
            sources()["egon-data"],
        ],
        "licenses": [
            license_ccby(
                "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert); "
                "© Copernicus Climate Change Service (C3S) Climate Data Store "
                "© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte",
            )
        ],
        "contributors": [
            {
                "title": "Clara Büttner",
                "email": "http://github.com/ClaraBuettner",
                "date": time.strftime("%Y-%m-%d"),
                "object": None,
                "comment": "Imported data",
            },
        ],
        "resources": [
            {
                "profile": "tabular-data-resource",
                "name": "supply.egon_scenario_capacities",
                "path": None,
                "format": "PostgreSQL",
                "encoding": "UTF-8",
                "schema": {
                    "fields": fields,
                    "primaryKey": ["index"],
                    "foreignKeys": [],
                },
                "dialect": {"delimiter": None, "decimalSeparator": "."},
            }
        ],
        "metaMetadata": meta_metadata(),
    }

    # Create json dump
    meta_json = "'" + json.dumps(meta) + "'"

    # Add metadata as a comment to the table
    db.submit_comment(
        meta_json,
        EgonRenewableFeedIn.__table__.schema,
        EgonRenewableFeedIn.__table__.name,
    )


1		"""
2		Central module containing all code dealing with processing era5 weather data.
3		"""
4
5		import datetime
6		import json
7		import time
8
9		from sqlalchemy import Column, ForeignKey, Integer
10		from sqlalchemy.ext.declarative import declarative_base
11		import geopandas as gpd
12		import numpy as np
13		import pandas as pd
14
15		from egon.data import db
16		from egon.data.datasets import Dataset
17		from egon.data.datasets.era5 import (
18		EgonEra5Cells,
19		EgonRenewableFeedIn,
20		import_cutout,
21		)
22		from egon.data.datasets.scenario_parameters import get_sector_parameters
23		from egon.data.datasets.zensus_vg250 import DestatisZensusPopulationPerHa
24		from egon.data.metadata import context, license_ccby, meta_metadata, sources
25		import egon.data.config
26
27		from egon_validation import(
28		RowCountValidation,
29		DataTypeValidation,
30		NotNullAndNotNaNValidation,
31		WholeTableNotNullAndNotNaNValidation,
32		ValueSetValidation
33		)
34
35		class RenewableFeedin(Dataset):
36		"""
37		Calculate possible feedin time series for renewable energy generators
38
39		This dataset calculates possible feedin timeseries for fluctuation renewable generators
40		and coefficient of performance time series for heat pumps. Relevant input is the
41		downloaded weather data. Parameters for the time series calcultaion are also defined by
42		representative types of pv plants and wind turbines that are selected within this dataset.
43		The resulting profiles are stored in the database.
44
45
46		Dependencies
47		* :py:class:`WeatherData <egon.data.datasets.era5.WeatherData>`
48		* :py:class:`Vg250 <egon.data.datasets.vg250.Vg250>`
49		* :py:class:`ZensusVg250 <egon.data.datasets.zensus_vg250.ZensusVg250>`
50
51		Resulting tables
52		* :py:class:`supply.egon_era5_renewable_feedin <egon.data.datasets.era5.EgonRenewableFeedIn>` is filled
53
54		"""
55
56		#:
57		name: str = "RenewableFeedin"
58		#:
59		version: str = "0.0.8"
60
61		def __init__(self, dependencies):
62		super().__init__(
63		name=self.name,
64		version=self.version,
65		dependencies=dependencies,
66		tasks={
67		wind,
68		pv,
69		solar_thermal,
70		heat_pump_cop,
71		wind_offshore,
72		mapping_zensus_weather,
73		},
74		validation = {
75		"data-quality": [
76		RowCountValidation(
77		table="supply.egon_era5_renewable_feedin",
78		rule_id="TEST_ROW_COUNT.egon_renewable_feedin",
79		expected_count=6102
80		),
81		DataTypeValidation(
82		table="supply.egon_era5_renewable_feedin",
83		rule_id="TEST_DATA_MULTIPLE_TYPES.egon_era5_renewable_feedin",
84		column_types={
85		"w_id": "integer",
86		"weather_year": "integer",
87		"carrier": "character varying",
88		"feedin": "double precision[]"
89		}
90		),
91		NotNullAndNotNaNValidation(
92		table="supply.egon_era5_renewable_feedin",
93		rule_id="TEST_NOT_NAN.egon_era5_renewable_feedin",
94		columns=["w_id", "weather_year", "carrier", "feedin"]
95		),
96		WholeTableNotNullAndNotNaNValidation(
97		table="supply.egon_era5_renewable_feedin",
98		rule_id="TEST_WHOLE_TABLE_NOT_NAN.egon_era5_renewable_feedin"
99		),
100		ValueSetValidation(
101		table="supply.egon_district_heating",
102		rule_id="VALUE_SET_VALIDATION_CARRIER.egon_district_heating",
103		column="carrier",
104		expected_values=["wind_onshore", "solar_thermal", "heat_pump_cop", "wind_offshore", "pv"]
105		),
106
107		]
108		},
109		on_validation_failure = "continue"
110		)
111
112
113		Base = declarative_base()
114		engine = db.engine()
115
116
117		class MapZensusWeatherCell(Base):
118		__tablename__ = "egon_map_zensus_weather_cell"
119		__table_args__ = {"schema": "boundaries"}
120
121		zensus_population_id = Column(
122		Integer,
123		ForeignKey(DestatisZensusPopulationPerHa.id),
124		primary_key=True,
125		index=True,
126		)
127		w_id = Column(Integer, ForeignKey(EgonEra5Cells.w_id), index=True)
128
129
130		def weather_cells_in_germany(geom_column="geom"):
131		"""Get weather cells which intersect with Germany
132
133		Returns
134		-------
135		GeoPandas.GeoDataFrame
136		Index and points of weather cells inside Germany
137
138		"""
139
140		cfg = egon.data.config.datasets()["renewable_feedin"]["sources"]
141
142		return db.select_geodataframe(
143		f"""SELECT w_id, geom_point, geom
144		FROM {cfg['weather_cells']['schema']}.
145		{cfg['weather_cells']['table']}
146		WHERE ST_Intersects('SRID=4326;
147		POLYGON((5 56, 15.5 56, 15.5 47, 5 47, 5 56))', geom)""",
148		geom_col=geom_column,
149		index_col="w_id",
150		)
151
152
153		def offshore_weather_cells(geom_column="geom"):
154		"""Get weather cells which intersect with Germany
155
156		Returns
157		-------
158		GeoPandas.GeoDataFrame
159		Index and points of weather cells inside Germany
160
161		"""
162
163		cfg = egon.data.config.datasets()["renewable_feedin"]["sources"]
164
165		return db.select_geodataframe(
166		f"""SELECT w_id, geom_point, geom
167		FROM {cfg['weather_cells']['schema']}.
168		{cfg['weather_cells']['table']}
169		WHERE ST_Intersects('SRID=4326;
170		POLYGON((5.5 55.5, 14.5 55.5, 14.5 53.5, 5.5 53.5, 5.5 55.5))',
171		geom)""",
172		geom_col=geom_column,
173		index_col="w_id",
174		)
175
176
177		def federal_states_per_weather_cell():
178		"""Assings a federal state to each weather cell in Germany.
179
180		Sets the federal state to the weather celss using the centroid.
181		Weather cells at the borders whoes centroid is not inside Germany
182		are assinged to the closest federal state.
183
184		Returns
185		-------
186		GeoPandas.GeoDataFrame
187		Index, points and federal state of weather cells inside Germany
188
189		"""
190
191		cfg = egon.data.config.datasets()["renewable_feedin"]["sources"]
192
193		# Select weather cells and ferear states from database
194		weather_cells = weather_cells_in_germany(geom_column="geom_point")
195
196		federal_states = db.select_geodataframe(
197		f"""SELECT gen, geometry
198		FROM {cfg['vg250_lan_union']['schema']}.
199		{cfg['vg250_lan_union']['table']}""",
200		geom_col="geometry",
201		index_col="gen",
202		)
203
204		# Map federal state and onshore wind turbine to weather cells
205		weather_cells["federal_state"] = gpd.sjoin(
206		weather_cells, federal_states
207		).gen
208
209		# Assign a federal state to each cell inside Germany
210		buffer = 1000
211
212		while (buffer < 30000) & (
213		len(weather_cells[weather_cells["federal_state"].isnull()]) > 0
214		):
215		cells = weather_cells[weather_cells["federal_state"].isnull()]
216
217		cells.loc[:, "geom_point"] = cells.geom_point.buffer(buffer)
218
219		weather_cells.loc[cells.index, "federal_state"] = gpd.sjoin(
220		cells, federal_states
221		).gen
222
223		buffer += 200
224
225		weather_cells = (
226		weather_cells.reset_index()
227		.drop_duplicates(subset="w_id", keep="first")
228		.set_index("w_id")
229		)
230
231		weather_cells = weather_cells.dropna(axis=0, subset=["federal_state"])
232
233		return weather_cells.to_crs(4326)
234
235
236		def turbine_per_weather_cell():
237		"""Assign wind onshore turbine types to weather cells
238
239		Returns
240		-------
241		weather_cells : GeoPandas.GeoDataFrame
242		Weather cells in Germany including turbine type
243
244		"""
245
246		# Select representative onshore wind turbines per federal state
247		map_federal_states_turbines = {
248		"Schleswig-Holstein": "E-126",
249		"Bremen": "E-126",
250		"Hamburg": "E-126",
251		"Mecklenburg-Vorpommern": "E-126",
252		"Niedersachsen": "E-126",
253		"Berlin": "E-141",
254		"Brandenburg": "E-141",
255		"Hessen": "E-141",
256		"Nordrhein-Westfalen": "E-141",
257		"Sachsen": "E-141",
258		"Sachsen-Anhalt": "E-141",
259		"Thüringen": "E-141",
260		"Baden-Württemberg": "E-141",
261		"Bayern": "E-141",
262		"Rheinland-Pfalz": "E-141",
263		"Saarland": "E-141",
264		}
265
266		# Select weather cells and federal states
267		weather_cells = federal_states_per_weather_cell()
268
269		# Assign turbine type per federal state
270		weather_cells["wind_turbine"] = weather_cells["federal_state"].map(
271		map_federal_states_turbines
272		)
273
274		return weather_cells
275
276
277		def feedin_per_turbine():
278		"""Calculate feedin timeseries per turbine type and weather cell
279
280		Returns
281		-------
282		gdf : GeoPandas.GeoDataFrame
283		Feed-in timeseries per turbine type and weather cell
284
285		"""
286
287		# Select weather data for Germany
288		cutout = import_cutout(boundary="Germany")
289
290		gdf = gpd.GeoDataFrame(geometry=cutout.grid.geometry, crs=4326)
291
292		# Calculate feedin-timeseries for E-141
293		# source:
294		# https://openenergy-platform.org/dataedit/view/supply/wind_turbine_library
295		turbine_e141 = {
296		"name": "E141 4200 kW",
297		"hub_height": 129,
298		"P": 4.200,
299		"V": np.arange(1, 26, dtype=float),
300		"POW": np.array(
301		[
302		0.0,
303		0.022,
304		0.104,
305		0.26,
306		0.523,
307		0.92,
308		1.471,
309		2.151,
310		2.867,
311		3.481,
312		3.903,
313		4.119,
314		4.196,
315		4.2,
316		4.2,
317		4.2,
318		4.2,
319		4.2,
320		4.2,
321		4.2,
322		4.2,
323		4.2,
324		4.2,
325		4.2,
326		4.2,
327		]
328		),
329		}
330		ts_e141 = cutout.wind(
331		turbine_e141, per_unit=True, shapes=cutout.grid.geometry
332		)
333
334		gdf["E-141"] = ts_e141.to_pandas().transpose().values.tolist()
335
336		# Calculate feedin-timeseries for E-126
337		# source:
338		# https://openenergy-platform.org/dataedit/view/supply/wind_turbine_library
339		turbine_e126 = {
340		"name": "E126 4200 kW",
341		"hub_height": 159,
342		"P": 4.200,
343		"V": np.arange(1, 26, dtype=float),
344		"POW": np.array(
345		[
346		0.0,
347		0.0,
348		0.058,
349		0.185,
350		0.4,
351		0.745,
352		1.2,
353		1.79,
354		2.45,
355		3.12,
356		3.66,
357		4.0,
358		4.15,
359		4.2,
360		4.2,
361		4.2,
362		4.2,
363		4.2,
364		4.2,
365		4.2,
366		4.2,
367		4.2,
368		4.2,
369		4.2,
370		4.2,
371		]
372		),
373		}
374		ts_e126 = cutout.wind(
375		turbine_e126, per_unit=True, shapes=cutout.grid.geometry
376		)
377
378		gdf["E-126"] = ts_e126.to_pandas().transpose().values.tolist()
379
380		return gdf
381
382
383		def wind():
384		"""Insert feed-in timeseries for wind onshore turbines to database
385
386		Returns
387		-------
388		None.
389
390		"""
391
392		cfg = egon.data.config.datasets()["renewable_feedin"]["targets"]
393
394		# Get weather cells with turbine type
395		weather_cells = turbine_per_weather_cell()
396		weather_cells = weather_cells[weather_cells.wind_turbine.notnull()]
397
398		# Calculate feedin timeseries per turbine and weather cell
399		timeseries_per_turbine = feedin_per_turbine()
400
401		# Join weather cells and feedin-timeseries
402		timeseries = gpd.sjoin(weather_cells, timeseries_per_turbine)[
403		["E-141", "E-126"]
404		]
405
406		weather_year = get_sector_parameters("global", "eGon2035")["weather_year"]
407
408		df = pd.DataFrame(
409		index=weather_cells.index,
410		columns=["weather_year", "carrier", "feedin"],
411		data={"weather_year": weather_year, "carrier": "wind_onshore"},
412		)
413
414		# Insert feedin for selected turbine per weather cell
415		for turbine in ["E-126", "E-141"]:
416		idx = weather_cells.index[
417		(weather_cells.wind_turbine == turbine)
418		& (weather_cells.index.isin(timeseries.index))
419		]
420		df.loc[idx, "feedin"] = timeseries.loc[idx, turbine].values
421
422		db.execute_sql(
423		f"""
424		DELETE FROM {cfg['feedin_table']['schema']}.
425		{cfg['feedin_table']['table']}
426		WHERE carrier = 'wind_onshore'"""
427		)
428
429		# Insert values into database
430		df.to_sql(
431		cfg["feedin_table"]["table"],
432		schema=cfg["feedin_table"]["schema"],
433		con=db.engine(),
434		if_exists="append",
435		)
436
437
438		def wind_offshore():
439		"""Insert feed-in timeseries for wind offshore turbines to database
440
441		Returns
442		-------
443		None.
444
445		"""
446
447		# Get offshore weather cells arround Germany
448		weather_cells = offshore_weather_cells()
449
450		# Select weather data for German coast
451		cutout = import_cutout(boundary="Germany-offshore")
452
453		# Select weather year from cutout
454		weather_year = cutout.name.split("-")[2]
455
456		# Calculate feedin timeseries
457		ts_wind_offshore = cutout.wind(
458		"Vestas_V164_7MW_offshore",
459		per_unit=True,
460		shapes=weather_cells.to_crs(4326).geom,
461		)
462
463		# Create dataframe and insert to database
464		insert_feedin(ts_wind_offshore, "wind_offshore", weather_year)
465
466
467		def pv():
468		"""Insert feed-in timeseries for pv plants to database
469
470		Returns
471		-------
472		None.
473
474		"""
475
476		# Get weather cells in Germany
477		weather_cells = weather_cells_in_germany()
478
479		# Select weather data for Germany
480		cutout = import_cutout(boundary="Germany")
481
482		# Select weather year from cutout
483		weather_year = cutout.name.split("-")[1]
484
485		# Calculate feedin timeseries
486		ts_pv = cutout.pv(
487		"CSi",
488		orientation={"slope": 35.0, "azimuth": 180.0},
489		per_unit=True,
490		shapes=weather_cells.to_crs(4326).geom,
491		)
492
493		# Create dataframe and insert to database
494		insert_feedin(ts_pv, "pv", weather_year)
495
496
497		def solar_thermal():
498		"""Insert feed-in timeseries for pv plants to database
499
500		Returns
501		-------
502		None.
503
504		"""
505
506		# Get weather cells in Germany
507		weather_cells = weather_cells_in_germany()
508
509		# Select weather data for Germany
510		cutout = import_cutout(boundary="Germany")
511
512		# Select weather year from cutout
513		weather_year = cutout.name.split("-")[1]
514
515		# Calculate feedin timeseries
516		ts_solar_thermal = cutout.solar_thermal(
517		clearsky_model="simple",
518		orientation={"slope": 45.0, "azimuth": 180.0},
519		per_unit=True,
520		shapes=weather_cells.to_crs(4326).geom,
521		capacity_factor=False,
522		)
523
524		# Create dataframe and insert to database
525		insert_feedin(ts_solar_thermal, "solar_thermal", weather_year)
526
527
528		def heat_pump_cop():
529		"""
530		Calculate coefficient of performance for heat pumps according to
531		T. Brown et al: "Synergies of sector coupling and transmission
532		reinforcement in a cost-optimised, highlyrenewable European energy system",
533		2018, p. 8
534
535		Returns
536		-------
537		None.
538
539		"""
540		# Assume temperature of heating system to 55°C according to Brown et. al
541		t_sink = 55
542
543		carrier = "heat_pump_cop"
544
545		# Load configuration
546		cfg = egon.data.config.datasets()["renewable_feedin"]
547
548		# Get weather cells in Germany
549		weather_cells = weather_cells_in_germany()
550
551		# Select weather data for Germany
552		cutout = import_cutout(boundary="Germany")
553
554		# Select weather year from cutout
555		weather_year = cutout.name.split("-")[1]
556
557		# Calculate feedin timeseries
558		temperature = cutout.temperature(
559		shapes=weather_cells.to_crs(4326).geom
560		).transpose()
561
562		t_source = temperature.to_pandas()
563
564		delta_t = t_sink - t_source
565
566		# Calculate coefficient of performance for air sourced heat pumps
567		# according to Brown et. al
568		cop = 6.81 - 0.121 * delta_t + 0.00063 * delta_t**2
569
570		df = pd.DataFrame(
571		index=temperature.to_pandas().index,
572		columns=["weather_year", "carrier", "feedin"],
573		data={"weather_year": weather_year, "carrier": carrier},
574		)
575
576		df.feedin = cop.values.tolist()
577
578		# Delete existing rows for carrier
579		db.execute_sql(
580		f"""
581		DELETE FROM {cfg['targets']['feedin_table']['schema']}.
582		{cfg['targets']['feedin_table']['table']}
583		WHERE carrier = '{carrier}'"""
584		)
585
586		# Insert values into database
587		df.to_sql(
588		cfg["targets"]["feedin_table"]["table"],
589		schema=cfg["targets"]["feedin_table"]["schema"],
590		con=db.engine(),
591		if_exists="append",
592		)
593
594
595		def insert_feedin(data, carrier, weather_year):
596		"""Insert feedin data into database
597
598		Parameters
599		----------
600		data : xarray.core.dataarray.DataArray
601		Feedin timeseries data
602		carrier : str
603		Name of energy carrier
604		weather_year : int
605		Selected weather year
606
607		Returns
608		-------
609		None.
610
611		"""
612		# Transpose DataFrame
613		data = data.transpose().to_pandas()
614
615		# Load configuration
616		cfg = egon.data.config.datasets()["renewable_feedin"]
617
618		# Initialize DataFrame
619		df = pd.DataFrame(
620		index=data.index,
621		columns=["weather_year", "carrier", "feedin"],
622		data={"weather_year": weather_year, "carrier": carrier},
623		)
624
625		# Convert solar thermal data from W/m^2 to MW/(1000m^2) = kW/m^2
626		if carrier == "solar_thermal":
627		data *= 1e-3
628
629		# Insert feedin into DataFrame
630		df.feedin = data.values.tolist()
631
632		# Delete existing rows for carrier
633		db.execute_sql(
634		f"""
635		DELETE FROM {cfg['targets']['feedin_table']['schema']}.
636		{cfg['targets']['feedin_table']['table']}
637		WHERE carrier = '{carrier}'"""
638		)
639
640		# Insert values into database
641		df.to_sql(
642		cfg["targets"]["feedin_table"]["table"],
643		schema=cfg["targets"]["feedin_table"]["schema"],
644		con=db.engine(),
645		if_exists="append",
646		)
647
648
649		def mapping_zensus_weather():
650		"""Perform mapping between era5 weather cell and zensus grid"""
651
652		with db.session_scope() as session:
653		cells_query = session.query(
654		DestatisZensusPopulationPerHa.id.label("zensus_population_id"),
655		DestatisZensusPopulationPerHa.geom_point,
656		)
657
658		gdf_zensus_population = gpd.read_postgis(
659		cells_query.statement,
660		cells_query.session.bind,
661		index_col=None,
662		geom_col="geom_point",
663		)
664
665		with db.session_scope() as session:
666		cells_query = session.query(EgonEra5Cells.w_id, EgonEra5Cells.geom)
667
668		gdf_weather_cell = gpd.read_postgis(
669		cells_query.statement,
670		cells_query.session.bind,
671		index_col=None,
672		geom_col="geom",
673		)
674		# CRS is 4326
675		gdf_weather_cell = gdf_weather_cell.to_crs(epsg=3035)
676
677		gdf_zensus_weather = gdf_zensus_population.sjoin(
678		gdf_weather_cell, how="left", predicate="within"
679		)
680
681		MapZensusWeatherCell.__table__.drop(bind=engine, checkfirst=True)
682		MapZensusWeatherCell.__table__.create(bind=engine, checkfirst=True)
683
684		# Write mapping into db
685		with db.session_scope() as session:
686		session.bulk_insert_mappings(
687		MapZensusWeatherCell,
688		gdf_zensus_weather[["zensus_population_id", "w_id"]].to_dict(
689		orient="records"
690		),
691		)
692
693
694	View Code Duplication	def add_metadata():
		0 ignored issues – show Duplication introduced 2021-12-13 16:43 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
695		"""Add metdata to supply.egon_era5_renewable_feedin
696
697		Returns
698		-------
699		None.
700
701		"""
702
703		# Import column names and datatypes
704		fields = [
705		{
706		"description": "Weather cell index",
707		"name": "w_id",
708		"type": "integer",
709		"unit": "none",
710		},
711		{
712		"description": "Weather year",
713		"name": "weather_year",
714		"type": "integer",
715		"unit": "none",
716		},
717		{
718		"description": "Energy carrier",
719		"name": "carrier",
720		"type": "string",
721		"unit": "none",
722		},
723		{
724		"description": "Weather-dependent feedin timeseries",
725		"name": "feedin",
726		"type": "array",
727		"unit": "p.u.",
728		},
729		]
730
731		meta = {
732		"name": "supply.egon_era5_renewable_feedin",
733		"title": "eGon feedin timeseries for RES",
734		"id": "WILL_BE_SET_AT_PUBLICATION",
735		"description": "Weather-dependent feedin timeseries for RES",
736		"language": ["EN"],
737		"publicationDate": datetime.date.today().isoformat(),
738		"context": context(),
739		"spatial": {
740		"location": None,
741		"extent": "Germany",
742		"resolution": None,
743		},
744		"sources": [
745		sources()["era5"],
746		sources()["vg250"],
747		sources()["egon-data"],
748		],
749		"licenses": [
750		license_ccby(
751		"© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert); "
752		"© Copernicus Climate Change Service (C3S) Climate Data Store "
753		"© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte",
754		)
755		],
756		"contributors": [
757		{
758		"title": "Clara Büttner",
759		"email": "http://github.com/ClaraBuettner",
760		"date": time.strftime("%Y-%m-%d"),
761		"object": None,
762		"comment": "Imported data",
763		},
764		],
765		"resources": [
766		{
767		"profile": "tabular-data-resource",
768		"name": "supply.egon_scenario_capacities",
769		"path": None,
770		"format": "PostgreSQL",
771		"encoding": "UTF-8",
772		"schema": {
773		"fields": fields,
774		"primaryKey": ["index"],
775		"foreignKeys": [],
776		},
777		"dialect": {"delimiter": None, "decimalSeparator": "."},
778		}
779		],
780		"metaMetadata": meta_metadata(),
781		}
782
783		# Create json dump
784		meta_json = "'" + json.dumps(meta) + "'"
785
786		# Add metadata as a comment to the table
787		db.submit_comment(
788		meta_json,
789		EgonRenewableFeedIn.__table__.schema,
790		EgonRenewableFeedIn.__table__.name,
791		)
792

openego / eGon-data

Pull Request — dev (#1375)

federal_states_per_weather_cell() A

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like