data.datasets.power_plants.mastr.import_mastr() - Code Metrics - Inspection of "Features/#1051 add new mastr data to db" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#1054)

unknown

created 2022-11-28 11:46 UTC

data.datasets.power_plants.mastr.import_mastr() B

↳ Parent: data.datasets.power_plants.mastr

Complexity

Conditions

Size

Total Lines	71
Code Lines	56

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	56
dl	0
loc	71
rs	8.44
c	0
b	0
f	0
cc	2
nop	0

How to fix Long Method

"""Import MaStR dataset and write to DB tables"""
from geoalchemy2 import Geometry
from sqlalchemy import (
    BigInteger,
    Boolean,
    Column,
    DateTime,
    Float,
    Integer,
    Sequence,
    String,
)
from sqlalchemy.ext.declarative import declarative_base
import pandas as pd

import egon.data.config

Base = declarative_base()


class EgonPowerPlantsPv(Base):
    __tablename__ = "egon_power_plants_pv"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_pv_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    federal_state = Column(String(21), nullable=True)  # Bundesland

    site_type = Column(String(69), nullable=True)  # Lage
    usage_sector = Column(String(36), nullable=True)  # Nutzungsbereich
    orientation_primary = Column(String(11), nullable=True)  # Hauptausrichtung
    orientation_primary_angle = Column(
        String(18), nullable=True
    )  # HauptausrichtungNeigungswinkel
    orientation_secondary = Column(
        String(11), nullable=True
    )  # Nebenausrichtung
    orientation_secondary_angle = Column(
        String(18), nullable=True
    )  # NebenausrichtungNeigungswinkel
    orientation_uniform = Column(
        Boolean, nullable=True
    )  # EinheitlicheAusrichtungUndNeigungswinkel
    module_count = Column(Float, nullable=True)  # AnzahlModule

    capacity = Column(Float)  # Nettonennleistung
    capacity_inverter = Column(
        Float, nullable=True
    )  # ZugeordneteWirkleistungWechselrichter in MW
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)

    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsWind(Base):
    __tablename__ = "egon_power_plants_wind"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_wind_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    federal_state = Column(String(21), nullable=True)  # Bundesland

    site_type = Column(String(17), nullable=True)  # Lage
    manufacturer_name = Column(String(100), nullable=True)  # Hersteller
    type_name = Column(String(100), nullable=True)  # Typenbezeichnung
    hub_height = Column(Float, nullable=True)  # Nabenhoehe
    rotor_diameter = Column(Float, nullable=True)  # Rotordurchmesser

    capacity = Column(Float)  # Nettonennleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)

    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsBiomass(Base):
    __tablename__ = "egon_power_plants_biomass"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_biomass_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    federal_state = Column(String(21), nullable=True)  # Bundesland

    technology = Column(String(45), nullable=True)  # Technologie
    fuel_name = Column(String(52), nullable=True)  # Hauptbrennstoff
    fuel_type = Column(String(19), nullable=True)  # Biomasseart

    capacity = Column(Float)  # Nettonennleistung
    th_capacity = Column(Float, nullable=True)  # ThermischeNutzleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)

    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


class EgonPowerPlantsHydro(Base):
    __tablename__ = "egon_power_plants_hydro"
    __table_args__ = {"schema": "supply"}

    id = Column(Integer, Sequence("pp_hydro_seq"), primary_key=True)
    bus_id = Column(Integer, nullable=True)  # Grid district id
    gens_id = Column(String, nullable=True)  # EinheitMastrNummer

    status = Column(String, nullable=True)  # EinheitBetriebsstatus
    commissioning_date = Column(DateTime, nullable=True)  # Inbetriebnahmedatum
    postcode = Column(String(5), nullable=True)  # Postleitzahl
    city = Column(String(50), nullable=True)  # Ort
    federal_state = Column(String(21), nullable=True)  # Bundesland

    type = Column(Integer, nullable=True)  # ArtDerWasserkraftanlage
    water_origin = Column(String(20), nullable=True)  # ArtDesZuflusses

    capacity = Column(Float, nullable=False)  # Nettonennleistung
    feedin_type = Column(String(47), nullable=True)  # Einspeisungsart
    voltage_level = Column(Integer, nullable=True)

    geom = Column(Geometry("POINT", 4326), index=True, nullable=True)


def import_mastr():
    cfg = egon.data.config.datasets()["power_plants"]

    cols_mapping = {
        "all": {
            "EinheitMastrNummer": "gens_id",
            "EinheitBetriebsstatus": "status",
            "Inbetriebnahmedatum": "commissioning_date",
            "Postleitzahl": "postcode",
            "Ort": "city",
            "Bundesland": "federal_state",
            "Nettonennleistung": "capacity",
            "Einspeisungsart": "feedin_type",
        },
        "pv": {
            "Lage": "site_type",
            "Nutzungsbereich": "usage_sector",
            "Hauptausrichtung": "orientation_primary",
            "HauptausrichtungNeigungswinkel": "orientation_primary_angle",
            "Nebenausrichtung": "orientation_secondary",
            "NebenausrichtungNeigungswinkel": "orientation_secondary_angle",
            "EinheitlicheAusrichtungUndNeigungswinkel": "orientation_uniform",
            "AnzahlModule": "module_count",
            "ZugeordneteWirkleistungWechselrichter": "capacity_inverter",
        },
        "wind": {
            "Lage": "site_type",
            "Hersteller": "manufacturer_name",
            "Typenbezeichnung": "type_name",
            "Nabenhoehe": "hub_height",
            "Rotordurchmesser": "rotor_diameter",
        },
        "biomass": {
            "Technologie": "technology",
            "Hauptbrennstoff": "fuel_name",
            "Biomasseart": "fuel_type",
            "ThermischeNutzleistung": "th_capacity",
        },
        "hydro": {
            "ArtDerWasserkraftanlage": "type",
            "ArtDesZuflusses": "water_origin",
        },
    }

    source_files = {
        "pv": cfg["sources"]["mastr_pv"],
        "wind": cfg["sources"]["mastr_wind"],
        "biomass": cfg["sources"]["mastr_biomass"],
        "hydro": cfg["sources"]["mastr_hydro"],
    }
    target_tables = {
        "pv": EgonPowerPlantsPv,
        "wind": EgonPowerPlantsWind,
        "biomass": EgonPowerPlantsBiomass,
        "hydro": EgonPowerPlantsHydro,
    }

    # import locations
    locations = pd.read_csv(cfg["sources"]["mastr_location"], index_col=None)

    # import units
    technologies = ["pv", "wind", "biomass", "hydro"]
    for tech in technologies:
        units = pd.read_csv(
            source_files[tech],
            usecols=(
                list(cols_mapping["all"].keys())
                + list(cols_mapping[tech].keys())
            ),
            index_col=None,
        ).rename(columns=cols_mapping)


1			"""Import MaStR dataset and write to DB tables"""
2			from geoalchemy2 import Geometry
3			from sqlalchemy import (
4			BigInteger,
5			Boolean,
6			Column,
7			DateTime,
8			Float,
9			Integer,
10			Sequence,
11			String,
12			)
13			from sqlalchemy.ext.declarative import declarative_base
14			import pandas as pd
15
16			import egon.data.config
17
18			Base = declarative_base()
19
20
21			class EgonPowerPlantsPv(Base):
22			__tablename__ = "egon_power_plants_pv"
23			__table_args__ = {"schema": "supply"}
24
25			id = Column(Integer, Sequence("pp_pv_seq"), primary_key=True)
26			bus_id = Column(Integer, nullable=True) # Grid district id
27			gens_id = Column(String, nullable=True) # EinheitMastrNummer
28
29			status = Column(String, nullable=True) # EinheitBetriebsstatus
30			commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
31			postcode = Column(String(5), nullable=True) # Postleitzahl
32			city = Column(String(50), nullable=True) # Ort
33			federal_state = Column(String(21), nullable=True) # Bundesland
34
35			site_type = Column(String(69), nullable=True) # Lage
36			usage_sector = Column(String(36), nullable=True) # Nutzungsbereich
37			orientation_primary = Column(String(11), nullable=True) # Hauptausrichtung
38			orientation_primary_angle = Column(
39			String(18), nullable=True
40			) # HauptausrichtungNeigungswinkel
41			orientation_secondary = Column(
42			String(11), nullable=True
43			) # Nebenausrichtung
44			orientation_secondary_angle = Column(
45			String(18), nullable=True
46			) # NebenausrichtungNeigungswinkel
47			orientation_uniform = Column(
48			Boolean, nullable=True
49			) # EinheitlicheAusrichtungUndNeigungswinkel
50			module_count = Column(Float, nullable=True) # AnzahlModule
51
52			capacity = Column(Float) # Nettonennleistung
53			capacity_inverter = Column(
54			Float, nullable=True
55			) # ZugeordneteWirkleistungWechselrichter in MW
56			feedin_type = Column(String(47), nullable=True) # Einspeisungsart
57			voltage_level = Column(Integer, nullable=True)
58
59			geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
60
61
62			class EgonPowerPlantsWind(Base):
63			__tablename__ = "egon_power_plants_wind"
64			__table_args__ = {"schema": "supply"}
65
66			id = Column(Integer, Sequence("pp_wind_seq"), primary_key=True)
67			bus_id = Column(Integer, nullable=True) # Grid district id
68			gens_id = Column(String, nullable=True) # EinheitMastrNummer
69
70			status = Column(String, nullable=True) # EinheitBetriebsstatus
71			commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
72			postcode = Column(String(5), nullable=True) # Postleitzahl
73			city = Column(String(50), nullable=True) # Ort
74			federal_state = Column(String(21), nullable=True) # Bundesland
75
76			site_type = Column(String(17), nullable=True) # Lage
77			manufacturer_name = Column(String(100), nullable=True) # Hersteller
78			type_name = Column(String(100), nullable=True) # Typenbezeichnung
79			hub_height = Column(Float, nullable=True) # Nabenhoehe
80			rotor_diameter = Column(Float, nullable=True) # Rotordurchmesser
81
82			capacity = Column(Float) # Nettonennleistung
83			feedin_type = Column(String(47), nullable=True) # Einspeisungsart
84			voltage_level = Column(Integer, nullable=True)
85
86			geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
87
88
89			class EgonPowerPlantsBiomass(Base):
90			__tablename__ = "egon_power_plants_biomass"
91			__table_args__ = {"schema": "supply"}
92
93			id = Column(Integer, Sequence("pp_biomass_seq"), primary_key=True)
94			bus_id = Column(Integer, nullable=True) # Grid district id
95			gens_id = Column(String, nullable=True) # EinheitMastrNummer
96
97			status = Column(String, nullable=True) # EinheitBetriebsstatus
98			commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
99			postcode = Column(String(5), nullable=True) # Postleitzahl
100			city = Column(String(50), nullable=True) # Ort
101			federal_state = Column(String(21), nullable=True) # Bundesland
102
103			technology = Column(String(45), nullable=True) # Technologie
104			fuel_name = Column(String(52), nullable=True) # Hauptbrennstoff
105			fuel_type = Column(String(19), nullable=True) # Biomasseart
106
107			capacity = Column(Float) # Nettonennleistung
108			th_capacity = Column(Float, nullable=True) # ThermischeNutzleistung
109			feedin_type = Column(String(47), nullable=True) # Einspeisungsart
110			voltage_level = Column(Integer, nullable=True)
111
112			geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
113
114
115			class EgonPowerPlantsHydro(Base):
116			__tablename__ = "egon_power_plants_hydro"
117			__table_args__ = {"schema": "supply"}
118
119			id = Column(Integer, Sequence("pp_hydro_seq"), primary_key=True)
120			bus_id = Column(Integer, nullable=True) # Grid district id
121			gens_id = Column(String, nullable=True) # EinheitMastrNummer
122
123			status = Column(String, nullable=True) # EinheitBetriebsstatus
124			commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum
125			postcode = Column(String(5), nullable=True) # Postleitzahl
126			city = Column(String(50), nullable=True) # Ort
127			federal_state = Column(String(21), nullable=True) # Bundesland
128
129			type = Column(Integer, nullable=True) # ArtDerWasserkraftanlage
130			water_origin = Column(String(20), nullable=True) # ArtDesZuflusses
131
132			capacity = Column(Float, nullable=False) # Nettonennleistung
133			feedin_type = Column(String(47), nullable=True) # Einspeisungsart
134			voltage_level = Column(Integer, nullable=True)
135
136			geom = Column(Geometry("POINT", 4326), index=True, nullable=True)
137
138
139			def import_mastr():
140			cfg = egon.data.config.datasets()["power_plants"]
141
142			cols_mapping = {
143			"all": {
144			"EinheitMastrNummer": "gens_id",
145			"EinheitBetriebsstatus": "status",
146			"Inbetriebnahmedatum": "commissioning_date",
147			"Postleitzahl": "postcode",
148			"Ort": "city",
149			"Bundesland": "federal_state",
150			"Nettonennleistung": "capacity",
151			"Einspeisungsart": "feedin_type",
152			},
153			"pv": {
154			"Lage": "site_type",
155			"Nutzungsbereich": "usage_sector",
156			"Hauptausrichtung": "orientation_primary",
157			"HauptausrichtungNeigungswinkel": "orientation_primary_angle",
158			"Nebenausrichtung": "orientation_secondary",
159			"NebenausrichtungNeigungswinkel": "orientation_secondary_angle",
160			"EinheitlicheAusrichtungUndNeigungswinkel": "orientation_uniform",
161			"AnzahlModule": "module_count",
162			"ZugeordneteWirkleistungWechselrichter": "capacity_inverter",
163			},
164			"wind": {
165			"Lage": "site_type",
166			"Hersteller": "manufacturer_name",
167			"Typenbezeichnung": "type_name",
168			"Nabenhoehe": "hub_height",
169			"Rotordurchmesser": "rotor_diameter",
170			},
171			"biomass": {
172			"Technologie": "technology",
173			"Hauptbrennstoff": "fuel_name",
174			"Biomasseart": "fuel_type",
175			"ThermischeNutzleistung": "th_capacity",
176			},
177			"hydro": {
178			"ArtDerWasserkraftanlage": "type",
179			"ArtDesZuflusses": "water_origin",
180			},
181			}
182
183			source_files = {
184			"pv": cfg["sources"]["mastr_pv"],
185			"wind": cfg["sources"]["mastr_wind"],
186			"biomass": cfg["sources"]["mastr_biomass"],
187			"hydro": cfg["sources"]["mastr_hydro"],
188			}
189			target_tables = {
190			"pv": EgonPowerPlantsPv,
191			"wind": EgonPowerPlantsWind,
192			"biomass": EgonPowerPlantsBiomass,
193			"hydro": EgonPowerPlantsHydro,
194			}
195
196			# import locations
197			locations = pd.read_csv(cfg["sources"]["mastr_location"], index_col=None)
198
199			# import units
200			technologies = ["pv", "wind", "biomass", "hydro"]
201			for tech in technologies:
202			units = pd.read_csv(
203			source_files[tech],
204			usecols=(
205			list(cols_mapping["all"].keys())
206			+ list(cols_mapping[tech].keys())
207			),
208			index_col=None,
209			).rename(columns=cols_mapping)
210

openego / eGon-data

Pull Request — dev (#1054)

data.datasets.power_plants.mastr.import_mastr() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like