data.datasets.power_plants.mastr_db_classes.add_metadata() - Code Metrics - Inspection of "Features/#450 add metadata" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#568)

unknown

created 2025-04-16 13:02 UTC

add_metadata() B

↳ Parent: data.datasets.power_plants.mastr_db_classes

Complexity

Conditions

Size

Total Lines	148
Code Lines	96

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	96
dl	0
loc	148
rs	7.1309
c	0
b	0
f	0
cc	2
nop	0

How to fix Long Method

import datetime
import json

from geoalchemy2 import Geometry
from omi.dialects import get_dialect
from sqlalchemy import (
    Boolean,
    Column,
    DateTime,
    Float,
    Integer,
    Sequence,
    String,
)
from sqlalchemy.ext.declarative import declarative_base

from egon.data import config, db
from egon.data.metadata import (
    context,
    contributors,
    generate_resource_fields_from_db_table,
    license_dedl,
    meta_metadata,
    meta_metadata,
    sources,
)

Base = declarative_base()


class EgonMastrGeocoded(Base):
    __tablename__ = "egon_mastr_geocoded"
    __table_args__ = {"schema": "supply"}

    index = Column(
        Integer, Sequence("mastr_geocoded_seq"), primary_key=True, index=True
    )
    zip_and_municipality = Column(String)
    latitude = Column(Float)
    longitude = Column(Float)
    altitude = Column(Float)
    geometry = Column(Geometry("POINT", 4326))


class EgonPowerPlantsPv(Base):
    __tablename__ = "egon_power_plants_pv"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_pv_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    municipality = Column(String, nullable=True)  # Gemeinde
    federal_state = Column(String(31), nullable=True)  # Bundesland
    site = Column(String, nullable=True)  # Standort
    zip_and_municipality = Column(String, nullable=True)

    site_type = Column(String(69), nullable=True)  # Lage
    usage_sector = Column(String(36), nullable=True)  # Nutzungsbereich
    orientation_primary = Column(String(11), nullable=True)  # Hauptausrichtung
    orientation_primary_angle = Column(
        String(18), nullable=True
    )  # HauptausrichtungNeigungswinkel
    orientation_secondary = Column(
        String(11), nullable=True
    )  # Nebenausrichtung
    orientation_secondary_angle = Column(
        String(18), nullable=True
    )  # NebenausrichtungNeigungswinkel
    orientation_uniform = Column(
        Boolean, nullable=True
    )  # EinheitlicheAusrichtungUndNeigungswinkel
    module_count = Column(Float, nullable=True)  # AnzahlModule

    capacity = Column(Float, nullable=True)  # Nettonennleistung
    capacity_inverter = Column(
        Float, nullable=True
    )  # ZugeordneteWirkleistungWechselrichter in MW
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)
    voltage_level_inferred = Column(Boolean, nullable=True)

    geometry_geocoded = Column(Boolean)
    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsWind(Base):

    __tablename__ = "egon_power_plants_wind"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_wind_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    municipality = Column(String, nullable=True)  # Gemeinde
    federal_state = Column(String(31), nullable=True)  # Bundesland
    zip_and_municipality = Column(String, nullable=True)

    site_type = Column(String(17), nullable=True)  # Lage
    manufacturer_name = Column(String(100), nullable=True)  # Hersteller
    type_name = Column(String(100), nullable=True)  # Typenbezeichnung
    hub_height = Column(Float, nullable=True)  # Nabenhoehe
    rotor_diameter = Column(Float, nullable=True)  # Rotordurchmesser

    capacity = Column(Float, nullable=True)  # Nettonennleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)
    voltage_level_inferred = Column(Boolean, nullable=True)

    geometry_geocoded = Column(Boolean)
    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsBiomass(Base):

    __tablename__ = "egon_power_plants_biomass"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_biomass_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    municipality = Column(String, nullable=True)  # Gemeinde
    federal_state = Column(String(31), nullable=True)  # Bundesland
    zip_and_municipality = Column(String, nullable=True)

    technology = Column(String(45), nullable=True)  # Technologie
    main_fuel = Column(String(52), nullable=True)  # Hauptbrennstoff
    fuel_type = Column(String(19), nullable=True)  # Biomasseart

    capacity = Column(Float, nullable=True)  # Nettonennleistung
    th_capacity = Column(Float, nullable=True)  # ThermischeNutzleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)
    voltage_level_inferred = Column(Boolean, nullable=True)

    geometry_geocoded = Column(Boolean)
    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsHydro(Base):

    __tablename__ = "egon_power_plants_hydro"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_hydro_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    municipality = Column(String, nullable=True)  # Gemeinde
    federal_state = Column(String(31), nullable=True)  # Bundesland
    zip_and_municipality = Column(String, nullable=True)

    plant_type = Column(String(39), nullable=True)  # ArtDerWasserkraftanlage
    water_origin = Column(String(20), nullable=True)  # ArtDesZuflusses

    capacity = Column(Float, nullable=True)  # Nettonennleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)
    voltage_level_inferred = Column(Boolean, nullable=True)

    geometry_geocoded = Column(Boolean)
    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsCombustion(Base):

    __tablename__ = "egon_power_plants_combustion"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_combustion_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    municipality = Column(String, nullable=True)  # Gemeinde
    federal_state = Column(String(31), nullable=True)  # Bundesland
    zip_and_municipality = Column(String, nullable=True)

    carrier = Column(String)  # Energietraeger
    main_fuel = Column(String)  # Hauptbrennstoff
    other_main_fuel = Column(String)  # WeitererHauptbrennstoff
    technology = Column(String)  # Technologie

    capacity = Column(Float, nullable=True)  # Nettonennleistung
    th_capacity = Column(Float, nullable=True)  # ThermischeNutzleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)
    voltage_level_inferred = Column(Boolean, nullable=True)

    geometry_geocoded = Column(Boolean)
    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsGsgk(Base):

    __tablename__ = "egon_power_plants_gsgk"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_gsgk_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    municipality = Column(String, nullable=True)  # Gemeinde
    federal_state = Column(String(31), nullable=True)  # Bundesland
    zip_and_municipality = Column(String, nullable=True)

    carrier = Column(String)  # Energietraeger
    technology = Column(String)  # Technologie

    capacity = Column(Float, nullable=True)  # Nettonennleistung
    th_capacity = Column(Float, nullable=True)  # ThermischeNutzleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)
    voltage_level_inferred = Column(Boolean, nullable=True)

    geometry_geocoded = Column(Boolean)
    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsNuclear(Base):

    __tablename__ = "egon_power_plants_nuclear"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_nuclear_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    municipality = Column(String, nullable=True)  # Gemeinde
    federal_state = Column(String(31), nullable=True)  # Bundesland
    zip_and_municipality = Column(String, nullable=True)

    carrier = Column(String)  # Energietraeger
    technology = Column(String)  # Technologie

    capacity = Column(Float, nullable=True)  # Nettonennleistung
    th_capacity = Column(Float, nullable=True)  # ThermischeNutzleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)
    voltage_level_inferred = Column(Boolean, nullable=True)

    geometry_geocoded = Column(Boolean)
    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsStorage(Base):

    __tablename__ = "egon_power_plants_storage"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_storage_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    municipality = Column(String, nullable=True)  # Gemeinde
    federal_state = Column(String(31), nullable=True)  # Bundesland
    zip_and_municipality = Column(String, nullable=True)

    carrier = Column(String)  # Energietraeger
    technology = Column(String)  # Technologie
    battery_type = Column(String)  # Batterietechnologie
    pump_storage_type = Column(String)  # Pumpspeichertechnologie

    capacity = Column(Float, nullable=True)  # Nettonennleistung
    th_capacity = Column(Float, nullable=True)  # ThermischeNutzleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)
    voltage_level_inferred = Column(Boolean, nullable=True)

    geometry_geocoded = Column(Boolean)
    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


def add_metadata():
    technologies = config.datasets()["mastr_new"]["technologies"]

    target_tables = {
        "pv": EgonPowerPlantsPv,
        "wind": EgonPowerPlantsWind,
        "biomass": EgonPowerPlantsBiomass,
        "hydro": EgonPowerPlantsHydro,
        "combustion": EgonPowerPlantsCombustion,
        "gsgk": EgonPowerPlantsGsgk,
        "nuclear": EgonPowerPlantsNuclear,
        "storage": EgonPowerPlantsStorage,
    }

    deposit_id_data_bundle = config.datasets()["data-bundle"]["sources"][
        "zenodo"
    ]["deposit_id"]
    deposit_id_mastr = config.datasets()["mastr_new"]["deposit_id"]

    contris = contributors(["kh", "kh"])

    contris[0]["date"] = "2023-03-15"

    contris[0]["object"] = "metadata"
    contris[1]["object"] = "dataset"

    contris[0]["comment"] = "Add metadata to dataset."
    contris[1]["comment"] = "Add workflow to generate dataset."

    for technology in technologies:
        target_table = target_tables[technology]

        meta = {
            "name": (
                f"{target_table.__table_args__['schema']}."
                f"{target_table.__tablename__}"
            ),
            "title": f"eGon {technology} power plants",
            "id": "WILL_BE_SET_AT_PUBLICATION",
            "description": (
                f"eGon {technology} power plants status quo derived from MaStR"
            ),
            "language": "en-US",
            "keywords": [technology, "mastr"],
            "publicationDate": datetime.date.today().isoformat(),
            "context": context(),
            "spatial": {
                "location": "none",
                "extent": "Germany",
                "resolution": "1 m",
            },
            "temporal": {
                "referenceDate": (
                    config.datasets()["mastr_new"]["egon2021_date_max"].split(
                        " "
                    )[0]
                ),
                "timeseries": {},
            },
            "sources": [
                {
                    "title": "Data bundle for egon-data",
                    "description": (
                        "Data bundle for egon-data: A transparent and "
                        "reproducible data processing pipeline for energy "
                        "system modeling"
                    ),
                    "path": (
                        "https://zenodo.org/record/"
                        f"{deposit_id_data_bundle}#.Y_dWM4CZMVM"
                    ),
                    "licenses": [license_dedl(attribution="© Cußmann, Ilka")],
                },
                {
                    "title": (
                        "open-MaStR power unit registry for eGo^n project"
                    ),
                    "description": (
                        "Data from Marktstammdatenregister (MaStR) data using "
                        "the data dump from 2022-11-17 for eGon-data."
                    ),
                    "path": (
                        f"https://zenodo.org/record/{deposit_id_mastr}"
                    ),
                    "licenses": [license_dedl(attribution="© Amme, Jonathan")],
                },
                sources()["egon-data"],
            ],
            "licenses": [license_dedl(attribution="© eGon development team")],
            "contributors": contris,
            "resources": [
                {
                    "profile": "tabular-data-resource",
                    "name": (
                        f"{target_table.__table_args__['schema']}."
                        f"{target_table.__tablename__}"
                    ),
                    "path": "None",
                    "format": "PostgreSQL",
                    "encoding": "UTF-8",
                    "schema": {
                        "fields": generate_resource_fields_from_db_table(
                            target_table.__table_args__["schema"],
                            target_table.__tablename__,
                            geom_columns=["geom"],
                        ),
                        "primaryKey": "id",
                    },
                    "dialect": {"delimiter": "", "decimalSeparator": ""},
                }
            ],
            "review": {"path": "", "badge": ""},
            "metaMetadata": meta_metadata(),
            "_comment": {
                "metadata": (
                    "Metadata documentation and explanation (https://github."
                    "com/OpenEnergyPlatform/oemetadata/blob/master/metadata/"
                    "v141/metadata_key_description.md)"
                ),
                "dates": (
                    "Dates and time must follow the ISO8601 including time "
                    "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)"
                ),
                "units": "Use a space between numbers and units (100 m)",
                "languages": (
                    "Languages must follow the IETF (BCP47) format (en-GB, "
                    "en-US, de-DE)"
                ),
                "licenses": (
                    "License name must follow the SPDX License List "
                    "(https://spdx.org/licenses/)"
                ),
                "review": (
                    "Following the OEP Data Review (https://github.com/"
                    "OpenEnergyPlatform/data-preprocessing/wiki)"
                ),
                "none": "If not applicable use (none)",
            },
        }

        dialect = get_dialect(meta_metadata()["metadataVersion"])()

        meta = dialect.compile_and_render(dialect.parse(json.dumps(meta)))

        db.submit_comment(
            f"'{json.dumps(meta)}'",
            target_table.__table_args__["schema"],
            target_table.__tablename__,
        )


1		import datetime
2		import json
3
4		from geoalchemy2 import Geometry
5		from omi.dialects import get_dialect
6		from sqlalchemy import (
7		Boolean,
8		Column,
9		DateTime,
10		Float,
11		Integer,
12		Sequence,
13		String,
14		)
15		from sqlalchemy.ext.declarative import declarative_base
16
17		from egon.data import config, db
18		from egon.data.metadata import (
19		context,
20		contributors,
21		generate_resource_fields_from_db_table,
22		license_dedl,
23		meta_metadata,
24		meta_metadata,
25		sources,
26		)
27
28		Base = declarative_base()
29
30
31		class EgonMastrGeocoded(Base):
32		__tablename__ = "egon_mastr_geocoded"
33		__table_args__ = {"schema": "supply"}
34
35		index = Column(
36		Integer, Sequence("mastr_geocoded_seq"), primary_key=True, index=True
37		)
38		zip_and_municipality = Column(String)
39		latitude = Column(Float)
40		longitude = Column(Float)
41		altitude = Column(Float)
42		geometry = Column(Geometry("POINT", 4326))
43
44
45		class EgonPowerPlantsPv(Base):
46		__tablename__ = "egon_power_plants_pv"
47		__table_args__ = {"schema": "supply"}
48
49		id = Column(Integer, Sequence("pp_pv_seq"), primary_key=True)
50		bus_id = Column(Integer, nullable=True) # Grid district id
51		gens_id = Column(String, nullable=True) # EinheitMastrNummer
52
53		status = Column(String, nullable=True) # EinheitBetriebsstatus
54		commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
55		postcode = Column(String(5), nullable=True) # Postleitzahl
56		city = Column(String(50), nullable=True) # Ort
57		municipality = Column(String, nullable=True) # Gemeinde
58		federal_state = Column(String(31), nullable=True) # Bundesland
59		site = Column(String, nullable=True) # Standort
60		zip_and_municipality = Column(String, nullable=True)
61
62		site_type = Column(String(69), nullable=True) # Lage
63		usage_sector = Column(String(36), nullable=True) # Nutzungsbereich
64		orientation_primary = Column(String(11), nullable=True) # Hauptausrichtung
65		orientation_primary_angle = Column(
66		String(18), nullable=True
67		) # HauptausrichtungNeigungswinkel
68		orientation_secondary = Column(
69		String(11), nullable=True
70		) # Nebenausrichtung
71		orientation_secondary_angle = Column(
72		String(18), nullable=True
73		) # NebenausrichtungNeigungswinkel
74		orientation_uniform = Column(
75		Boolean, nullable=True
76		) # EinheitlicheAusrichtungUndNeigungswinkel
77		module_count = Column(Float, nullable=True) # AnzahlModule
78
79		capacity = Column(Float, nullable=True) # Nettonennleistung
80		capacity_inverter = Column(
81		Float, nullable=True
82		) # ZugeordneteWirkleistungWechselrichter in MW
83		feedin_type = Column(String(47), nullable=True) # Einspeisungsart
84		voltage_level = Column(Integer, nullable=True)
85		voltage_level_inferred = Column(Boolean, nullable=True)
86
87		geometry_geocoded = Column(Boolean)
88		geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
89
90
91	View Code Duplication	class EgonPowerPlantsWind(Base):
		0 ignored issues – show Duplication introduced 2023-02-24 15:00 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
92		__tablename__ = "egon_power_plants_wind"
93		__table_args__ = {"schema": "supply"}
94
95		id = Column(Integer, Sequence("pp_wind_seq"), primary_key=True)
96		bus_id = Column(Integer, nullable=True) # Grid district id
97		gens_id = Column(String, nullable=True) # EinheitMastrNummer
98
99		status = Column(String, nullable=True) # EinheitBetriebsstatus
100		commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
101		postcode = Column(String(5), nullable=True) # Postleitzahl
102		city = Column(String(50), nullable=True) # Ort
103		municipality = Column(String, nullable=True) # Gemeinde
104		federal_state = Column(String(31), nullable=True) # Bundesland
105		zip_and_municipality = Column(String, nullable=True)
106
107		site_type = Column(String(17), nullable=True) # Lage
108		manufacturer_name = Column(String(100), nullable=True) # Hersteller
109		type_name = Column(String(100), nullable=True) # Typenbezeichnung
110		hub_height = Column(Float, nullable=True) # Nabenhoehe
111		rotor_diameter = Column(Float, nullable=True) # Rotordurchmesser
112
113		capacity = Column(Float, nullable=True) # Nettonennleistung
114		feedin_type = Column(String(47), nullable=True) # Einspeisungsart
115		voltage_level = Column(Integer, nullable=True)
116		voltage_level_inferred = Column(Boolean, nullable=True)
117
118		geometry_geocoded = Column(Boolean)
119		geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
120
121
122	View Code Duplication	class EgonPowerPlantsBiomass(Base):
		0 ignored issues – show Duplication introduced 2022-12-07 16:34 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
123		__tablename__ = "egon_power_plants_biomass"
124		__table_args__ = {"schema": "supply"}
125
126		id = Column(Integer, Sequence("pp_biomass_seq"), primary_key=True)
127		bus_id = Column(Integer, nullable=True) # Grid district id
128		gens_id = Column(String, nullable=True) # EinheitMastrNummer
129
130		status = Column(String, nullable=True) # EinheitBetriebsstatus
131		commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
132		postcode = Column(String(5), nullable=True) # Postleitzahl
133		city = Column(String(50), nullable=True) # Ort
134		municipality = Column(String, nullable=True) # Gemeinde
135		federal_state = Column(String(31), nullable=True) # Bundesland
136		zip_and_municipality = Column(String, nullable=True)
137
138		technology = Column(String(45), nullable=True) # Technologie
139		main_fuel = Column(String(52), nullable=True) # Hauptbrennstoff
140		fuel_type = Column(String(19), nullable=True) # Biomasseart
141
142		capacity = Column(Float, nullable=True) # Nettonennleistung
143		th_capacity = Column(Float, nullable=True) # ThermischeNutzleistung
144		feedin_type = Column(String(47), nullable=True) # Einspeisungsart
145		voltage_level = Column(Integer, nullable=True)
146		voltage_level_inferred = Column(Boolean, nullable=True)
147
148		geometry_geocoded = Column(Boolean)
149		geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
150
151
152	View Code Duplication	class EgonPowerPlantsHydro(Base):
		0 ignored issues – show Duplication introduced 2023-03-02 10:58 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
153		__tablename__ = "egon_power_plants_hydro"
154		__table_args__ = {"schema": "supply"}
155
156		id = Column(Integer, Sequence("pp_hydro_seq"), primary_key=True)
157		bus_id = Column(Integer, nullable=True) # Grid district id
158		gens_id = Column(String, nullable=True) # EinheitMastrNummer
159
160		status = Column(String, nullable=True) # EinheitBetriebsstatus
161		commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
162		postcode = Column(String(5), nullable=True) # Postleitzahl
163		city = Column(String(50), nullable=True) # Ort
164		municipality = Column(String, nullable=True) # Gemeinde
165		federal_state = Column(String(31), nullable=True) # Bundesland
166		zip_and_municipality = Column(String, nullable=True)
167
168		plant_type = Column(String(39), nullable=True) # ArtDerWasserkraftanlage
169		water_origin = Column(String(20), nullable=True) # ArtDesZuflusses
170
171		capacity = Column(Float, nullable=True) # Nettonennleistung
172		feedin_type = Column(String(47), nullable=True) # Einspeisungsart
173		voltage_level = Column(Integer, nullable=True)
174		voltage_level_inferred = Column(Boolean, nullable=True)
175
176		geometry_geocoded = Column(Boolean)
177		geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
178
179
180	View Code Duplication	class EgonPowerPlantsCombustion(Base):
		0 ignored issues – show Duplication introduced 2023-03-02 10:58 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
181		__tablename__ = "egon_power_plants_combustion"
182		__table_args__ = {"schema": "supply"}
183
184		id = Column(Integer, Sequence("pp_combustion_seq"), primary_key=True)
185		bus_id = Column(Integer, nullable=True) # Grid district id
186		gens_id = Column(String, nullable=True) # EinheitMastrNummer
187
188		status = Column(String, nullable=True) # EinheitBetriebsstatus
189		commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
190		postcode = Column(String(5), nullable=True) # Postleitzahl
191		city = Column(String(50), nullable=True) # Ort
192		municipality = Column(String, nullable=True) # Gemeinde
193		federal_state = Column(String(31), nullable=True) # Bundesland
194		zip_and_municipality = Column(String, nullable=True)
195
196		carrier = Column(String) # Energietraeger
197		main_fuel = Column(String) # Hauptbrennstoff
198		other_main_fuel = Column(String) # WeitererHauptbrennstoff
199		technology = Column(String) # Technologie
200
201		capacity = Column(Float, nullable=True) # Nettonennleistung
202		th_capacity = Column(Float, nullable=True) # ThermischeNutzleistung
203		feedin_type = Column(String(47), nullable=True) # Einspeisungsart
204		voltage_level = Column(Integer, nullable=True)
205		voltage_level_inferred = Column(Boolean, nullable=True)
206
207		geometry_geocoded = Column(Boolean)
208		geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
209
210
211	View Code Duplication	class EgonPowerPlantsGsgk(Base):
		0 ignored issues – show Duplication introduced 2023-03-02 10:58 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
212		__tablename__ = "egon_power_plants_gsgk"
213		__table_args__ = {"schema": "supply"}
214
215		id = Column(Integer, Sequence("pp_gsgk_seq"), primary_key=True)
216		bus_id = Column(Integer, nullable=True) # Grid district id
217		gens_id = Column(String, nullable=True) # EinheitMastrNummer
218
219		status = Column(String, nullable=True) # EinheitBetriebsstatus
220		commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
221		postcode = Column(String(5), nullable=True) # Postleitzahl
222		city = Column(String(50), nullable=True) # Ort
223		municipality = Column(String, nullable=True) # Gemeinde
224		federal_state = Column(String(31), nullable=True) # Bundesland
225		zip_and_municipality = Column(String, nullable=True)
226
227		carrier = Column(String) # Energietraeger
228		technology = Column(String) # Technologie
229
230		capacity = Column(Float, nullable=True) # Nettonennleistung
231		th_capacity = Column(Float, nullable=True) # ThermischeNutzleistung
232		feedin_type = Column(String(47), nullable=True) # Einspeisungsart
233		voltage_level = Column(Integer, nullable=True)
234		voltage_level_inferred = Column(Boolean, nullable=True)
235
236		geometry_geocoded = Column(Boolean)
237		geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
238
239
240	View Code Duplication	class EgonPowerPlantsNuclear(Base):
		0 ignored issues – show Duplication introduced 2023-03-02 12:26 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
241		__tablename__ = "egon_power_plants_nuclear"
242		__table_args__ = {"schema": "supply"}
243
244		id = Column(Integer, Sequence("pp_nuclear_seq"), primary_key=True)
245		bus_id = Column(Integer, nullable=True) # Grid district id
246		gens_id = Column(String, nullable=True) # EinheitMastrNummer
247
248		status = Column(String, nullable=True) # EinheitBetriebsstatus
249		commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
250		postcode = Column(String(5), nullable=True) # Postleitzahl
251		city = Column(String(50), nullable=True) # Ort
252		municipality = Column(String, nullable=True) # Gemeinde
253		federal_state = Column(String(31), nullable=True) # Bundesland
254		zip_and_municipality = Column(String, nullable=True)
255
256		carrier = Column(String) # Energietraeger
257		technology = Column(String) # Technologie
258
259		capacity = Column(Float, nullable=True) # Nettonennleistung
260		th_capacity = Column(Float, nullable=True) # ThermischeNutzleistung
261		feedin_type = Column(String(47), nullable=True) # Einspeisungsart
262		voltage_level = Column(Integer, nullable=True)
263		voltage_level_inferred = Column(Boolean, nullable=True)
264
265		geometry_geocoded = Column(Boolean)
266		geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
267
268
269	View Code Duplication	class EgonPowerPlantsStorage(Base):
		0 ignored issues – show Duplication introduced 2023-03-02 12:26 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
270		__tablename__ = "egon_power_plants_storage"
271		__table_args__ = {"schema": "supply"}
272
273		id = Column(Integer, Sequence("pp_storage_seq"), primary_key=True)
274		bus_id = Column(Integer, nullable=True) # Grid district id
275		gens_id = Column(String, nullable=True) # EinheitMastrNummer
276
277		status = Column(String, nullable=True) # EinheitBetriebsstatus
278		commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
279		postcode = Column(String(5), nullable=True) # Postleitzahl
280		city = Column(String(50), nullable=True) # Ort
281		municipality = Column(String, nullable=True) # Gemeinde
282		federal_state = Column(String(31), nullable=True) # Bundesland
283		zip_and_municipality = Column(String, nullable=True)
284
285		carrier = Column(String) # Energietraeger
286		technology = Column(String) # Technologie
287		battery_type = Column(String) # Batterietechnologie
288		pump_storage_type = Column(String) # Pumpspeichertechnologie
289
290		capacity = Column(Float, nullable=True) # Nettonennleistung
291		th_capacity = Column(Float, nullable=True) # ThermischeNutzleistung
292		feedin_type = Column(String(47), nullable=True) # Einspeisungsart
293		voltage_level = Column(Integer, nullable=True)
294		voltage_level_inferred = Column(Boolean, nullable=True)
295
296		geometry_geocoded = Column(Boolean)
297		geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
298
299
300		def add_metadata():
301		technologies = config.datasets()["mastr_new"]["technologies"]
302
303		target_tables = {
304		"pv": EgonPowerPlantsPv,
305		"wind": EgonPowerPlantsWind,
306		"biomass": EgonPowerPlantsBiomass,
307		"hydro": EgonPowerPlantsHydro,
308		"combustion": EgonPowerPlantsCombustion,
309		"gsgk": EgonPowerPlantsGsgk,
310		"nuclear": EgonPowerPlantsNuclear,
311		"storage": EgonPowerPlantsStorage,
312		}
313
314		deposit_id_data_bundle = config.datasets()["data-bundle"]["sources"][
315		"zenodo"
316		]["deposit_id"]
317		deposit_id_mastr = config.datasets()["mastr_new"]["deposit_id"]
318
319		contris = contributors(["kh", "kh"])
320
321		contris[0]["date"] = "2023-03-15"
322
323		contris[0]["object"] = "metadata"
324		contris[1]["object"] = "dataset"
325
326		contris[0]["comment"] = "Add metadata to dataset."
327		contris[1]["comment"] = "Add workflow to generate dataset."
328
329		for technology in technologies:
330		target_table = target_tables[technology]
331
332		meta = {
333		"name": (
334		f"{target_table.__table_args__['schema']}."
335		f"{target_table.__tablename__}"
336		),
337		"title": f"eGon {technology} power plants",
338		"id": "WILL_BE_SET_AT_PUBLICATION",
339		"description": (
340		f"eGon {technology} power plants status quo derived from MaStR"
341		),
342		"language": "en-US",
343		"keywords": [technology, "mastr"],
344		"publicationDate": datetime.date.today().isoformat(),
345		"context": context(),
346		"spatial": {
347		"location": "none",
348		"extent": "Germany",
349		"resolution": "1 m",
350		},
351		"temporal": {
352		"referenceDate": (
353		config.datasets()["mastr_new"]["egon2021_date_max"].split(
354		" "
355		)[0]
356		),
357		"timeseries": {},
358		},
359		"sources": [
360		{
361		"title": "Data bundle for egon-data",
362		"description": (
363		"Data bundle for egon-data: A transparent and "
364		"reproducible data processing pipeline for energy "
365		"system modeling"
366		),
367		"path": (
368		"https://zenodo.org/record/"
369		f"{deposit_id_data_bundle}#.Y_dWM4CZMVM"
370		),
371		"licenses": [license_dedl(attribution="© Cußmann, Ilka")],
372		},
373		{
374		"title": (
375		"open-MaStR power unit registry for eGo^n project"
376		),
377		"description": (
378		"Data from Marktstammdatenregister (MaStR) data using "
379		"the data dump from 2022-11-17 for eGon-data."
380		),
381		"path": (
382		f"https://zenodo.org/record/{deposit_id_mastr}"
383		),
384		"licenses": [license_dedl(attribution="© Amme, Jonathan")],
385		},
386		sources()["egon-data"],
387		],
388		"licenses": [license_dedl(attribution="© eGon development team")],
389		"contributors": contris,
390		"resources": [
391		{
392		"profile": "tabular-data-resource",
393		"name": (
394		f"{target_table.__table_args__['schema']}."
395		f"{target_table.__tablename__}"
396		),
397		"path": "None",
398		"format": "PostgreSQL",
399		"encoding": "UTF-8",
400		"schema": {
401		"fields": generate_resource_fields_from_db_table(
402		target_table.__table_args__["schema"],
403		target_table.__tablename__,
404		geom_columns=["geom"],
405		),
406		"primaryKey": "id",
407		},
408		"dialect": {"delimiter": "", "decimalSeparator": ""},
409		}
410		],
411		"review": {"path": "", "badge": ""},
412		"metaMetadata": meta_metadata(),
413		"_comment": {
414		"metadata": (
415		"Metadata documentation and explanation (https://github."
416		"com/OpenEnergyPlatform/oemetadata/blob/master/metadata/"
417		"v141/metadata_key_description.md)"
418		),
419		"dates": (
420		"Dates and time must follow the ISO8601 including time "
421		"zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)"
422		),
423		"units": "Use a space between numbers and units (100 m)",
424		"languages": (
425		"Languages must follow the IETF (BCP47) format (en-GB, "
426		"en-US, de-DE)"
427		),
428		"licenses": (
429		"License name must follow the SPDX License List "
430		"(https://spdx.org/licenses/)"
431		),
432		"review": (
433		"Following the OEP Data Review (https://github.com/"
434		"OpenEnergyPlatform/data-preprocessing/wiki)"
435		),
436		"none": "If not applicable use (none)",
437		},
438		}
439
440		dialect = get_dialect(meta_metadata()["metadataVersion"])()
441
442		meta = dialect.compile_and_render(dialect.parse(json.dumps(meta)))
443
444		db.submit_comment(
445		f"'{json.dumps(meta)}'",
446		target_table.__table_args__["schema"],
447		target_table.__tablename__,
448		)
449

openego / eGon-data

Pull Request — dev (#568)

add_metadata() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like