data.datasets.heat_supply.metadata() - Code Metrics - Inspection of "Features/#450 add metadata" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#568)

by Stephan

created 2023-03-18 16:00 UTC

data.datasets.heat_supply.metadata() B

↳ Parent: data.datasets.heat_supply

Complexity

Conditions

Size

Total Lines	167
Code Lines	123

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	123
dl	0
loc	167
rs	7
c	0
b	0
f	0
cc	1
nop	0

How to fix Long Method

"""The central module containing all code dealing with heat supply data

"""

import datetime
import json
import time

from geoalchemy2.types import Geometry
from sqlalchemy import Column, Float, ForeignKey, Integer, String
from sqlalchemy.ext.declarative import declarative_base
import pandas as pd

from egon.data import config, db
from egon.data.datasets import Dataset
from egon.data.datasets.district_heating_areas import EgonDistrictHeatingAreas
from egon.data.datasets.heat_supply.district_heating import (
    backup_gas_boilers,
    backup_resistive_heaters,
    cascade_heat_supply,
)
from egon.data.datasets.heat_supply.geothermal import potential_germany
from egon.data.datasets.heat_supply.individual_heating import (
    cascade_heat_supply_indiv,
)
from egon.data.metadata import (
    context,
    generate_resource_fields_from_sqla_model,
    license_ccby,
    license_egon_data_odbl,
    meta_metadata,
    sources,
)

# Will later be imported from another file.
Base = declarative_base()


class EgonDistrictHeatingSupply(Base):
    __tablename__ = "egon_district_heating"
    __table_args__ = {"schema": "supply"}
    index = Column(Integer, primary_key=True, autoincrement=True)
    district_heating_id = Column(
        Integer, ForeignKey(EgonDistrictHeatingAreas.id)
    )
    carrier = Column(String(25))
    category = Column(String(25))
    capacity = Column(Float)
    geometry = Column(Geometry("POINT", 3035))
    scenario = Column(String(50))


class EgonIndividualHeatingSupply(Base):
    __tablename__ = "egon_individual_heating"
    __table_args__ = {"schema": "supply"}
    index = Column(Integer, primary_key=True, autoincrement=True)
    mv_grid_id = Column(Integer)
    carrier = Column(String(25))
    category = Column(String(25))
    capacity = Column(Float)
    geometry = Column(Geometry("POINT", 3035))
    scenario = Column(String(50))


def create_tables():
    """Create tables for district heating areas

    Returns
    -------
        None
    """

    engine = db.engine()
    EgonDistrictHeatingSupply.__table__.drop(bind=engine, checkfirst=True)
    EgonDistrictHeatingSupply.__table__.create(bind=engine, checkfirst=True)
    EgonIndividualHeatingSupply.__table__.drop(bind=engine, checkfirst=True)
    EgonIndividualHeatingSupply.__table__.create(bind=engine, checkfirst=True)


def district_heating():
    """Insert supply for district heating areas

    Returns
    -------
    None.

    """
    sources = config.datasets()["heat_supply"]["sources"]
    targets = config.datasets()["heat_supply"]["targets"]

    db.execute_sql(
        f"""
        DELETE FROM {targets['district_heating_supply']['schema']}.
        {targets['district_heating_supply']['table']}
        """
    )

    supply_2035 = cascade_heat_supply("eGon2035", plotting=False)

    supply_2035["scenario"] = "eGon2035"

    supply_2035.to_postgis(
        targets["district_heating_supply"]["table"],
        schema=targets["district_heating_supply"]["schema"],
        con=db.engine(),
        if_exists="append",
    )

    # Compare target value with sum of distributed heat supply
    df_check = db.select_dataframe(
        f"""
        SELECT a.carrier,
        (SUM(a.capacity) - b.capacity) / SUM(a.capacity) as deviation
        FROM {targets['district_heating_supply']['schema']}.
        {targets['district_heating_supply']['table']} a,
        {sources['scenario_capacities']['schema']}.
        {sources['scenario_capacities']['table']} b
        WHERE a.scenario = 'eGon2035'
        AND b.scenario_name = 'eGon2035'
        AND b.carrier = CONCAT('urban_central_', a.carrier)
        GROUP BY (a.carrier,  b.capacity);
        """
    )
    # If the deviation is > 1%, throw an error
    assert (
        df_check.deviation.abs().max() < 1
    ), f"""Unexpected deviation between target value and distributed
        heat supply: {df_check}
        """

    # Add gas boilers as conventional backup capacities
    backup = backup_gas_boilers("eGon2035")

    backup.to_postgis(
        targets["district_heating_supply"]["table"],
        schema=targets["district_heating_supply"]["schema"],
        con=db.engine(),
        if_exists="append",
    )

    backup_rh = backup_resistive_heaters("eGon2035")

    if not backup_rh.empty:
        backup_rh.to_postgis(
            targets["district_heating_supply"]["table"],
            schema=targets["district_heating_supply"]["schema"],
            con=db.engine(),
            if_exists="append",
        )


def individual_heating():
    """Insert supply for individual heating

    Returns
    -------
    None.

    """
    targets = config.datasets()["heat_supply"]["targets"]

    db.execute_sql(
        f"""
        DELETE FROM {targets['individual_heating_supply']['schema']}.
        {targets['individual_heating_supply']['table']}
        """
    )

    supply_2035 = cascade_heat_supply_indiv(
        "eGon2035", distribution_level="federal_states", plotting=False
    )

    supply_2035["scenario"] = "eGon2035"

    supply_2035.to_postgis(
        targets["individual_heating_supply"]["table"],
        schema=targets["individual_heating_supply"]["schema"],
        con=db.engine(),
        if_exists="append",
    )


def metadata():
    """Write metadata for heat supply tables

    Returns
    -------
    None.

    """

    fields = generate_resource_fields_from_sqla_model(
        EgonDistrictHeatingSupply
    )

    fields_df = pd.DataFrame(data=fields).set_index("name")
    fields_df.loc["index", "description"] = "Unique identifyer"
    fields_df.loc[
        "district_heating_id", "description"
    ] = "Index of the corresponding district heating grid"
    fields_df.loc["carrier", "description"] = "Name of energy carrier"
    fields_df.loc[
        "category", "description"
    ] = "Size-category of district heating grid"
    fields_df.loc["capacity", "description"] = "Installed heating capacity"
    fields_df.loc[
        "geometry", "description"
    ] = "Location of thermal power plant"
    fields_df.loc["scenario", "description"] = "Name of corresponing scenario"

    fields_df.loc["capacity", "unit"] = "MW_th"
    fields_df.unit.fillna("none", inplace=True)

    fields = fields_df.reset_index().to_dict(orient="records")

    meta_district = {
        "name": "supply.egon_district_heating",
        "title": "eGon heat supply for district heating grids",
        "id": "WILL_BE_SET_AT_PUBLICATION",
        "description": "Heat supply technologies for district heating grids",
        "language": ["EN"],
        "publicationDate": datetime.date.today().isoformat(),
        "context": context(),
        "spatial": {
            "location": None,
            "extent": "Germany",
            "resolution": None,
        },
        "sources": [
            sources()["era5"],
            sources()["vg250"],
            sources()["egon-data"],
            sources()["egon-data_bundle"],
            sources()["openstreetmap"],
            sources()["mastr"],
            sources()["peta"],
        ],
        "licenses": [license_egon_data_odbl()],
        "contributors": [
            {
                "title": "Clara Büttner",
                "email": "http://github.com/ClaraBuettner",
                "date": time.strftime("%Y-%m-%d"),
                "object": None,
                "comment": "Imported data",
            },
        ],
        "resources": [
            {
                "profile": "tabular-data-resource",
                "name": "supply.egon_district_heating",
                "path": None,
                "format": "PostgreSQL",
                "encoding": "UTF-8",
                "schema": {
                    "fields": fields,
                    "primaryKey": ["index"],
                    "foreignKeys": [],
                },
                "dialect": {"delimiter": None, "decimalSeparator": "."},
            }
        ],
        "metaMetadata": meta_metadata(),
    }

    # Add metadata as a comment to the table
    db.submit_comment(
        "'" + json.dumps(meta_district) + "'",
        EgonDistrictHeatingSupply.__table__.schema,
        EgonDistrictHeatingSupply.__table__.name,
    )

    fields = generate_resource_fields_from_sqla_model(
        EgonIndividualHeatingSupply
    )

    fields_df = pd.DataFrame(data=fields).set_index("name")
    fields_df.loc["index", "description"] = "Unique identifyer"
    fields_df.loc[
        "mv_grid_id", "description"
    ] = "Index of the corresponding mv grid district"
    fields_df.loc["carrier", "description"] = "Name of energy carrier"
    fields_df.loc["category", "description"] = "Size-category"
    fields_df.loc["capacity", "description"] = "Installed heating capacity"
    fields_df.loc[
        "geometry", "description"
    ] = "Location of thermal power plant"
    fields_df.loc["scenario", "description"] = "Name of corresponing scenario"

    fields_df.loc["capacity", "unit"] = "MW_th"
    fields_df.unit.fillna("none", inplace=True)

    fields = fields_df.reset_index().to_dict(orient="records")

    meta_district = {
        "name": "supply.egon_individual_heating",
        "title": "eGon heat supply for individual supplied buildings",
        "id": "WILL_BE_SET_AT_PUBLICATION",
        "description": "Heat supply technologies for individual supplied buildings",
        "language": ["EN"],
        "publicationDate": datetime.date.today().isoformat(),
        "context": context(),
        "spatial": {
            "location": None,
            "extent": "Germany",
            "resolution": None,
        },
        "sources": [
            sources()["era5"],
            sources()["vg250"],
            sources()["egon-data"],
            sources()["egon-data_bundle"],
            sources()["openstreetmap"],
            sources()["mastr"],
            sources()["peta"],
        ],
        "licenses": [license_egon_data_odbl()],
        "contributors": [
            {
                "title": "Clara Büttner",
                "email": "http://github.com/ClaraBuettner",
                "date": time.strftime("%Y-%m-%d"),
                "object": None,
                "comment": "Imported data",
            },
        ],
        "resources": [
            {
                "profile": "tabular-data-resource",
                "name": "supply.egon_individual_heating",
                "path": None,
                "format": "PostgreSQL",
                "encoding": "UTF-8",
                "schema": {
                    "fields": fields,
                    "primaryKey": ["index"],
                    "foreignKeys": [],
                },
                "dialect": {"delimiter": None, "decimalSeparator": "."},
            }
        ],
        "metaMetadata": meta_metadata(),
    }

    # Add metadata as a comment to the table
    db.submit_comment(
        "'" + json.dumps(meta_district) + "'",
        EgonIndividualHeatingSupply.__table__.schema,
        EgonIndividualHeatingSupply.__table__.name,
    )


class HeatSupply(Dataset):
    def __init__(self, dependencies):
        super().__init__(
            name="HeatSupply",
            version="0.0.9",
            dependencies=dependencies,
            tasks=(
                create_tables,
                {
                    district_heating,
                    individual_heating,
                    potential_germany,
                },
                metadata,
            ),
        )


1			"""The central module containing all code dealing with heat supply data
2
3			"""
4
5			import datetime
6			import json
7			import time
8
9			from geoalchemy2.types import Geometry
10			from sqlalchemy import Column, Float, ForeignKey, Integer, String
11			from sqlalchemy.ext.declarative import declarative_base
12			import pandas as pd
13
14			from egon.data import config, db
15			from egon.data.datasets import Dataset
16			from egon.data.datasets.district_heating_areas import EgonDistrictHeatingAreas
17			from egon.data.datasets.heat_supply.district_heating import (
18			backup_gas_boilers,
19			backup_resistive_heaters,
20			cascade_heat_supply,
21			)
22			from egon.data.datasets.heat_supply.geothermal import potential_germany
23			from egon.data.datasets.heat_supply.individual_heating import (
24			cascade_heat_supply_indiv,
25			)
26			from egon.data.metadata import (
27			context,
28			generate_resource_fields_from_sqla_model,
29			license_ccby,
30			license_egon_data_odbl,
31			meta_metadata,
32			sources,
33			)
34
35			# Will later be imported from another file.
36			Base = declarative_base()
37
38
39			class EgonDistrictHeatingSupply(Base):
40			__tablename__ = "egon_district_heating"
41			__table_args__ = {"schema": "supply"}
42			index = Column(Integer, primary_key=True, autoincrement=True)
43			district_heating_id = Column(
44			Integer, ForeignKey(EgonDistrictHeatingAreas.id)
45			)
46			carrier = Column(String(25))
47			category = Column(String(25))
48			capacity = Column(Float)
49			geometry = Column(Geometry("POINT", 3035))
50			scenario = Column(String(50))
51
52
53			class EgonIndividualHeatingSupply(Base):
54			__tablename__ = "egon_individual_heating"
55			__table_args__ = {"schema": "supply"}
56			index = Column(Integer, primary_key=True, autoincrement=True)
57			mv_grid_id = Column(Integer)
58			carrier = Column(String(25))
59			category = Column(String(25))
60			capacity = Column(Float)
61			geometry = Column(Geometry("POINT", 3035))
62			scenario = Column(String(50))
63
64
65			def create_tables():
66			"""Create tables for district heating areas
67
68			Returns
69			-------
70			None
71			"""
72
73			engine = db.engine()
74			EgonDistrictHeatingSupply.__table__.drop(bind=engine, checkfirst=True)
75			EgonDistrictHeatingSupply.__table__.create(bind=engine, checkfirst=True)
76			EgonIndividualHeatingSupply.__table__.drop(bind=engine, checkfirst=True)
77			EgonIndividualHeatingSupply.__table__.create(bind=engine, checkfirst=True)
78
79
80			def district_heating():
81			"""Insert supply for district heating areas
82
83			Returns
84			-------
85			None.
86
87			"""
88			sources = config.datasets()["heat_supply"]["sources"]
89			targets = config.datasets()["heat_supply"]["targets"]
90
91			db.execute_sql(
92			f"""
93			DELETE FROM {targets['district_heating_supply']['schema']}.
94			{targets['district_heating_supply']['table']}
95			"""
96			)
97
98			supply_2035 = cascade_heat_supply("eGon2035", plotting=False)
99
100			supply_2035["scenario"] = "eGon2035"
101
102			supply_2035.to_postgis(
103			targets["district_heating_supply"]["table"],
104			schema=targets["district_heating_supply"]["schema"],
105			con=db.engine(),
106			if_exists="append",
107			)
108
109			# Compare target value with sum of distributed heat supply
110			df_check = db.select_dataframe(
111			f"""
112			SELECT a.carrier,
113			(SUM(a.capacity) - b.capacity) / SUM(a.capacity) as deviation
114			FROM {targets['district_heating_supply']['schema']}.
115			{targets['district_heating_supply']['table']} a,
116			{sources['scenario_capacities']['schema']}.
117			{sources['scenario_capacities']['table']} b
118			WHERE a.scenario = 'eGon2035'
119			AND b.scenario_name = 'eGon2035'
120			AND b.carrier = CONCAT('urban_central_', a.carrier)
121			GROUP BY (a.carrier, b.capacity);
122			"""
123			)
124			# If the deviation is > 1%, throw an error
125			assert (
126			df_check.deviation.abs().max() < 1
127			), f"""Unexpected deviation between target value and distributed
128			heat supply: {df_check}
129			"""
130
131			# Add gas boilers as conventional backup capacities
132			backup = backup_gas_boilers("eGon2035")
133
134			backup.to_postgis(
135			targets["district_heating_supply"]["table"],
136			schema=targets["district_heating_supply"]["schema"],
137			con=db.engine(),
138			if_exists="append",
139			)
140
141			backup_rh = backup_resistive_heaters("eGon2035")
142
143			if not backup_rh.empty:
144			backup_rh.to_postgis(
145			targets["district_heating_supply"]["table"],
146			schema=targets["district_heating_supply"]["schema"],
147			con=db.engine(),
148			if_exists="append",
149			)
150
151
152			def individual_heating():
153			"""Insert supply for individual heating
154
155			Returns
156			-------
157			None.
158
159			"""
160			targets = config.datasets()["heat_supply"]["targets"]
161
162			db.execute_sql(
163			f"""
164			DELETE FROM {targets['individual_heating_supply']['schema']}.
165			{targets['individual_heating_supply']['table']}
166			"""
167			)
168
169			supply_2035 = cascade_heat_supply_indiv(
170			"eGon2035", distribution_level="federal_states", plotting=False
171			)
172
173			supply_2035["scenario"] = "eGon2035"
174
175			supply_2035.to_postgis(
176			targets["individual_heating_supply"]["table"],
177			schema=targets["individual_heating_supply"]["schema"],
178			con=db.engine(),
179			if_exists="append",
180			)
181
182
183			def metadata():
184			"""Write metadata for heat supply tables
185
186			Returns
187			-------
188			None.
189
190			"""
191
192			fields = generate_resource_fields_from_sqla_model(
193			EgonDistrictHeatingSupply
194			)
195
196			fields_df = pd.DataFrame(data=fields).set_index("name")
197			fields_df.loc["index", "description"] = "Unique identifyer"
198			fields_df.loc[
199			"district_heating_id", "description"
200			] = "Index of the corresponding district heating grid"
201			fields_df.loc["carrier", "description"] = "Name of energy carrier"
202			fields_df.loc[
203			"category", "description"
204			] = "Size-category of district heating grid"
205			fields_df.loc["capacity", "description"] = "Installed heating capacity"
206			fields_df.loc[
207			"geometry", "description"
208			] = "Location of thermal power plant"
209			fields_df.loc["scenario", "description"] = "Name of corresponing scenario"
210
211			fields_df.loc["capacity", "unit"] = "MW_th"
212			fields_df.unit.fillna("none", inplace=True)
213
214			fields = fields_df.reset_index().to_dict(orient="records")
215
216			meta_district = {
217			"name": "supply.egon_district_heating",
218			"title": "eGon heat supply for district heating grids",
219			"id": "WILL_BE_SET_AT_PUBLICATION",
220			"description": "Heat supply technologies for district heating grids",
221			"language": ["EN"],
222			"publicationDate": datetime.date.today().isoformat(),
223			"context": context(),
224			"spatial": {
225			"location": None,
226			"extent": "Germany",
227			"resolution": None,
228			},
229			"sources": [
230			sources()["era5"],
231			sources()["vg250"],
232			sources()["egon-data"],
233			sources()["egon-data_bundle"],
234			sources()["openstreetmap"],
235			sources()["mastr"],
236			sources()["peta"],
237			],
238			"licenses": [license_egon_data_odbl()],
239			"contributors": [
240			{
241			"title": "Clara Büttner",
242			"email": "http://github.com/ClaraBuettner",
243			"date": time.strftime("%Y-%m-%d"),
244			"object": None,
245			"comment": "Imported data",
246			},
247			],
248			"resources": [
249			{
250			"profile": "tabular-data-resource",
251			"name": "supply.egon_district_heating",
252			"path": None,
253			"format": "PostgreSQL",
254			"encoding": "UTF-8",
255			"schema": {
256			"fields": fields,
257			"primaryKey": ["index"],
258			"foreignKeys": [],
259			},
260			"dialect": {"delimiter": None, "decimalSeparator": "."},
261			}
262			],
263			"metaMetadata": meta_metadata(),
264			}
265
266			# Add metadata as a comment to the table
267			db.submit_comment(
268			"'" + json.dumps(meta_district) + "'",
269			EgonDistrictHeatingSupply.__table__.schema,
270			EgonDistrictHeatingSupply.__table__.name,
271			)
272
273			fields = generate_resource_fields_from_sqla_model(
274			EgonIndividualHeatingSupply
275			)
276
277			fields_df = pd.DataFrame(data=fields).set_index("name")
278			fields_df.loc["index", "description"] = "Unique identifyer"
279			fields_df.loc[
280			"mv_grid_id", "description"
281			] = "Index of the corresponding mv grid district"
282			fields_df.loc["carrier", "description"] = "Name of energy carrier"
283			fields_df.loc["category", "description"] = "Size-category"
284			fields_df.loc["capacity", "description"] = "Installed heating capacity"
285			fields_df.loc[
286			"geometry", "description"
287			] = "Location of thermal power plant"
288			fields_df.loc["scenario", "description"] = "Name of corresponing scenario"
289
290			fields_df.loc["capacity", "unit"] = "MW_th"
291			fields_df.unit.fillna("none", inplace=True)
292
293			fields = fields_df.reset_index().to_dict(orient="records")
294
295			meta_district = {
296			"name": "supply.egon_individual_heating",
297			"title": "eGon heat supply for individual supplied buildings",
298			"id": "WILL_BE_SET_AT_PUBLICATION",
299			"description": "Heat supply technologies for individual supplied buildings",
300			"language": ["EN"],
301			"publicationDate": datetime.date.today().isoformat(),
302			"context": context(),
303			"spatial": {
304			"location": None,
305			"extent": "Germany",
306			"resolution": None,
307			},
308			"sources": [
309			sources()["era5"],
310			sources()["vg250"],
311			sources()["egon-data"],
312			sources()["egon-data_bundle"],
313			sources()["openstreetmap"],
314			sources()["mastr"],
315			sources()["peta"],
316			],
317			"licenses": [license_egon_data_odbl()],
318			"contributors": [
319			{
320			"title": "Clara Büttner",
321			"email": "http://github.com/ClaraBuettner",
322			"date": time.strftime("%Y-%m-%d"),
323			"object": None,
324			"comment": "Imported data",
325			},
326			],
327			"resources": [
328			{
329			"profile": "tabular-data-resource",
330			"name": "supply.egon_individual_heating",
331			"path": None,
332			"format": "PostgreSQL",
333			"encoding": "UTF-8",
334			"schema": {
335			"fields": fields,
336			"primaryKey": ["index"],
337			"foreignKeys": [],
338			},
339			"dialect": {"delimiter": None, "decimalSeparator": "."},
340			}
341			],
342			"metaMetadata": meta_metadata(),
343			}
344
345			# Add metadata as a comment to the table
346			db.submit_comment(
347			"'" + json.dumps(meta_district) + "'",
348			EgonIndividualHeatingSupply.__table__.schema,
349			EgonIndividualHeatingSupply.__table__.name,
350			)
351
352
353			class HeatSupply(Dataset):
354			def __init__(self, dependencies):
355			super().__init__(
356			name="HeatSupply",
357			version="0.0.9",
358			dependencies=dependencies,
359			tasks=(
360			create_tables,
361			{
362			district_heating,
363			individual_heating,
364			potential_germany,
365			},
366			metadata,
367			),
368			)
369

openego / eGon-data

Pull Request — dev (#568)

data.datasets.heat_supply.metadata() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like