data.datasets.heat_demand_timeseries.service_sector.CTS_demand_scale() - Code Metrics - Inspection of "Features/#857 store intermediate results of heat d..." - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#870)

unknown

created 2022-09-01 07:02 UTC

CTS_demand_scale() B

↳ Parent: data.datasets.heat_demand_timeseries.service_sector

Complexity

Conditions

Size

Total Lines	172
Code Lines	86

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	86
dl	0
loc	172
rs	7.4581
c	0
b	0
f	0
cc	3
nop	1

How to fix Long Method

import os

from sqlalchemy.ext.declarative import declarative_base


import pandas as pd

from egon.data import db


try:
    from disaggregator import temporal
except ImportError as e:
    pass


Base = declarative_base()


def cts_demand_per_aggregation_level(aggregation_level, scenario):
    """

    Description: Create dataframe assigining the CTS demand curve to individual zensus cell
    based on their respective NUTS3 CTS curve

    Parameters
    ----------
    aggregation_level : str
        if further processing is to be done in zensus cell level 'other'
        else 'dsitrict'

    Returns
    -------
    CTS_per_district : pandas.DataFrame
        if aggregation ='district'
            NUTS3 CTS profiles assigned to individual
            zensu cells and aggregated per district heat area id
        else
            empty dataframe
    CTS_per_grid : pandas.DataFrame
        if aggregation ='district'
            NUTS3 CTS profiles assigned to individual
            zensu cells and aggregated per mv grid subst id
        else
            empty dataframe
    CTS_per_zensus : pandas.DataFrame
        if aggregation ='district'
            empty dataframe
        else
            NUTS3 CTS profiles assigned to individual
            zensu population id

    """

    demand_nuts = db.select_dataframe(
        f"""
        SELECT demand, a.zensus_population_id, b.vg250_nuts3
        FROM demand.egon_peta_heat a 
        JOIN boundaries.egon_map_zensus_vg250 b 
        ON a.zensus_population_id = b.zensus_population_id
        
        WHERE a.sector = 'service'
        AND a.scenario = '{scenario}'
        ORDER BY a.zensus_population_id
        """
    )

    if os.path.isfile("CTS_heat_demand_profile_nuts3.csv"):
        df_CTS_gas_2011 = pd.read_csv(
            "CTS_heat_demand_profile_nuts3.csv", index_col=0
        )
        df_CTS_gas_2011.columns.name = "ags_lk"
        df_CTS_gas_2011.index = pd.to_datetime(df_CTS_gas_2011.index)
        df_CTS_gas_2011 = df_CTS_gas_2011.asfreq("H")
    else:
        df_CTS_gas_2011 = temporal.disagg_temporal_gas_CTS(
            use_nuts3code=True, year=2011
        )
        df_CTS_gas_2011.to_csv("CTS_heat_demand_profile_nuts3.csv")

    ags_lk = pd.read_csv(
        os.path.join(
            os.getcwd(),
            "demandregio-disaggregator/disaggregator/disaggregator/data_in/regional",
            "t_nuts3_lk.csv",
        ),
        index_col=0,
    )
    ags_lk = ags_lk.drop(
        ags_lk.columns.difference(["natcode_nuts3", "ags_lk"]), axis=1
    )

    CTS_profile = df_CTS_gas_2011.transpose()
    CTS_profile.reset_index(inplace=True)
    CTS_profile.ags_lk = CTS_profile.ags_lk.astype(int)
    CTS_profile = pd.merge(CTS_profile, ags_lk, on="ags_lk", how="inner")
    CTS_profile.set_index("natcode_nuts3", inplace=True)
    CTS_profile.drop("ags_lk", axis=1, inplace=True)

    CTS_per_zensus = pd.merge(
        demand_nuts[["zensus_population_id", "vg250_nuts3"]],
        CTS_profile,
        left_on="vg250_nuts3",
        right_on=CTS_profile.index,
        how="left",
    )

    CTS_per_zensus = CTS_per_zensus.drop("vg250_nuts3", axis=1)

    if aggregation_level == "district":
        district_heating = db.select_dataframe(
            f"""
            SELECT area_id, zensus_population_id
            FROM demand.egon_map_zensus_district_heating_areas
            WHERE scenario = '{scenario}'
            """
        )

        CTS_per_district = pd.merge(
            CTS_per_zensus,
            district_heating,
            on="zensus_population_id",
            how="inner",
        )
        CTS_per_district.set_index("area_id", inplace=True)
        CTS_per_district.drop("zensus_population_id", axis=1, inplace=True)

        CTS_per_district = CTS_per_district.groupby(lambda x: x, axis=0).sum()
        CTS_per_district = CTS_per_district.transpose()
        CTS_per_district = CTS_per_district.apply(lambda x: x / x.sum())
        CTS_per_district.columns.name = "area_id"
        CTS_per_district.reset_index(drop=True, inplace=True)

        # mv_grid = mv_grid.set_index("zensus_population_id")
        district_heating = district_heating.set_index("zensus_population_id")

        mv_grid_ind = db.select_dataframe(
            f"""
            SELECT bus_id, zensus_population_id
            FROM boundaries.egon_map_zensus_grid_districts
            WHERE zensus_population_id NOT IN 
            (SELECT zensus_population_id
             FROM demand.egon_map_zensus_district_heating_areas
             WHERE scenario = '{scenario}')
            """
        )

        CTS_per_grid = pd.merge(
            CTS_per_zensus,
            mv_grid_ind,
            on="zensus_population_id",
            how="inner",
        )
        CTS_per_grid.set_index("bus_id", inplace=True)
        CTS_per_grid.drop("zensus_population_id", axis=1, inplace=True)

        CTS_per_grid = CTS_per_grid.groupby(lambda x: x, axis=0).sum()
        CTS_per_grid = CTS_per_grid.transpose()
        CTS_per_grid = CTS_per_grid.apply(lambda x: x / x.sum())
        CTS_per_grid.columns.name = "bus_id"
        CTS_per_grid.reset_index(drop=True, inplace=True)

        CTS_per_zensus = pd.DataFrame()

    else:
        CTS_per_district = pd.DataFrame()
        CTS_per_grid = pd.DataFrame()
        CTS_per_zensus.set_index("zensus_population_id", inplace=True)

        CTS_per_zensus = CTS_per_zensus.groupby(lambda x: x, axis=0).sum()
        CTS_per_zensus = CTS_per_zensus.transpose()
        CTS_per_zensus = CTS_per_zensus.apply(lambda x: x / x.sum())
        CTS_per_zensus.columns.name = "zensus_population_id"
        CTS_per_zensus.reset_index(drop=True, inplace=True)

    return CTS_per_district, CTS_per_grid, CTS_per_zensus


def CTS_demand_scale(aggregation_level):
    """

    Description: caling the demand curves to the annual demand of the respective aggregation level


    Parameters
    ----------
    aggregation_level : str
        aggregation_level : str
        if further processing is to be done in zensus cell level 'other'
        else 'dsitrict'

    Returns
    -------
    CTS_per_district : pandas.DataFrame
        if aggregation ='district'
            Profiles scaled up to annual demand
        else
            0
    CTS_per_grid : pandas.DataFrame
        if aggregation ='district'
            Profiles scaled up to annual demandd
        else
            0
    CTS_per_zensus : pandas.DataFrame
        if aggregation ='district'
            0
        else
           Profiles scaled up to annual demand

    """
    scenarios = ["eGon2035", "eGon100RE"]

    CTS_district = pd.DataFrame()
    CTS_grid = pd.DataFrame()
    CTS_zensus = pd.DataFrame()

    for scenario in scenarios:
        (
            CTS_per_district,
            CTS_per_grid,
            CTS_per_zensus,
        ) = cts_demand_per_aggregation_level(aggregation_level, scenario)
        CTS_per_district = CTS_per_district.transpose()
        CTS_per_grid = CTS_per_grid.transpose()
        CTS_per_zensus = CTS_per_zensus.transpose()

        demand = db.select_dataframe(
            f"""
                SELECT demand, zensus_population_id
                FROM demand.egon_peta_heat                
                WHERE sector = 'service'
                AND scenario = '{scenario}'
                ORDER BY zensus_population_id
                """
        )

        if aggregation_level == "district":

            district_heating = db.select_dataframe(
                f"""
                SELECT area_id, zensus_population_id
                FROM demand.egon_map_zensus_district_heating_areas
                WHERE scenario = '{scenario}'
                """
            )

            CTS_demands_district = pd.merge(
                demand,
                district_heating,
                on="zensus_population_id",
                how="inner",
            )
            CTS_demands_district.drop(
                "zensus_population_id", axis=1, inplace=True
            )
            CTS_demands_district = CTS_demands_district.groupby(
                "area_id"
            ).sum()

            CTS_per_district = pd.merge(
                CTS_per_district,
                CTS_demands_district[["demand"]],
                how="inner",
                right_on=CTS_per_district.index,
                left_on=CTS_demands_district.index,
            )

            CTS_per_district = CTS_per_district.rename(
                columns={"key_0": "area_id"}
            )
            CTS_per_district.set_index("area_id", inplace=True)

            CTS_per_district = CTS_per_district[
                CTS_per_district.columns[:-1]
            ].multiply(CTS_per_district.demand, axis=0)

            CTS_per_district.insert(0, "scenario", scenario)

            CTS_district = CTS_district.append(CTS_per_district)
            CTS_district = CTS_district.sort_index()

            mv_grid_ind = db.select_dataframe(
                f"""
                SELECT bus_id, zensus_population_id
                FROM boundaries.egon_map_zensus_grid_districts
                WHERE zensus_population_id NOT IN 
                (SELECT zensus_population_id
                 FROM demand.egon_map_zensus_district_heating_areas
                 WHERE scenario = '{scenario}')
                """
            )

            CTS_demands_grid = pd.merge(
                demand,
                mv_grid_ind[["bus_id", "zensus_population_id"]],
                on="zensus_population_id",
                how="inner",
            )

            CTS_demands_grid.drop("zensus_population_id", axis=1, inplace=True)
            CTS_demands_grid = CTS_demands_grid.groupby("bus_id").sum()

            CTS_per_grid = pd.merge(
                CTS_per_grid,
                CTS_demands_grid[["demand"]],
                how="inner",
                right_on=CTS_per_grid.index,
                left_on=CTS_demands_grid.index,
            )

            CTS_per_grid = CTS_per_grid.rename(columns={"key_0": "bus_id"})
            CTS_per_grid.set_index("bus_id", inplace=True)

            CTS_per_grid = CTS_per_grid[CTS_per_grid.columns[:-1]].multiply(
                CTS_per_grid.demand, axis=0
            )

            CTS_per_grid.insert(0, "scenario", scenario)

            CTS_grid = CTS_grid.append(CTS_per_grid)
            CTS_grid = CTS_grid.sort_index()

            CTS_per_zensus = 0

        else:
            CTS_per_district = 0
            CTS_per_grid = 0

            CTS_per_zensus = pd.merge(
                CTS_per_zensus,
                demand,
                how="inner",
                right_on=CTS_per_zensus.index,
                left_on=demand.zensus_population_id,
            )
            CTS_per_zensus = CTS_per_zensus.drop("key_0", axis=1)
            CTS_per_zensus.set_index("zensus_population_id", inplace=True)

            CTS_per_zensus = CTS_per_zensus[
                CTS_per_zensus.columns[:-1]
            ].multiply(CTS_per_zensus.demand, axis=0)
            CTS_per_zensus.insert(0, "scenario", scenario)

            CTS_per_zensus.reset_index(inplace=True)

            CTS_zensus = CTS_zensus.append(CTS_per_grid)
            CTS_zensus = CTS_zensus.set_index("bus_id")
            CTS_zensus = CTS_zensus.sort_index()

    return CTS_district, CTS_grid, CTS_zensus


1			import os
2
3			from sqlalchemy.ext.declarative import declarative_base
4
5
6			import pandas as pd
7
8			from egon.data import db
9
10
11			try:
12			from disaggregator import temporal
13			except ImportError as e:
14			pass
15
16
17			Base = declarative_base()
18
19
20			def cts_demand_per_aggregation_level(aggregation_level, scenario):
21			"""
22
23			Description: Create dataframe assigining the CTS demand curve to individual zensus cell
24			based on their respective NUTS3 CTS curve
25
26			Parameters
27			----------
28			aggregation_level : str
29			if further processing is to be done in zensus cell level 'other'
30			else 'dsitrict'
31
32			Returns
33			-------
34			CTS_per_district : pandas.DataFrame
35			if aggregation ='district'
36			NUTS3 CTS profiles assigned to individual
37			zensu cells and aggregated per district heat area id
38			else
39			empty dataframe
40			CTS_per_grid : pandas.DataFrame
41			if aggregation ='district'
42			NUTS3 CTS profiles assigned to individual
43			zensu cells and aggregated per mv grid subst id
44			else
45			empty dataframe
46			CTS_per_zensus : pandas.DataFrame
47			if aggregation ='district'
48			empty dataframe
49			else
50			NUTS3 CTS profiles assigned to individual
51			zensu population id
52
53			"""
54
55			demand_nuts = db.select_dataframe(
56			f"""
57			SELECT demand, a.zensus_population_id, b.vg250_nuts3
58			FROM demand.egon_peta_heat a
59			JOIN boundaries.egon_map_zensus_vg250 b
60			ON a.zensus_population_id = b.zensus_population_id
61
62			WHERE a.sector = 'service'
63			AND a.scenario = '{scenario}'
64			ORDER BY a.zensus_population_id
65			"""
66			)
67
68			if os.path.isfile("CTS_heat_demand_profile_nuts3.csv"):
69			df_CTS_gas_2011 = pd.read_csv(
70			"CTS_heat_demand_profile_nuts3.csv", index_col=0
71			)
72			df_CTS_gas_2011.columns.name = "ags_lk"
73			df_CTS_gas_2011.index = pd.to_datetime(df_CTS_gas_2011.index)
74			df_CTS_gas_2011 = df_CTS_gas_2011.asfreq("H")
75			else:
76			df_CTS_gas_2011 = temporal.disagg_temporal_gas_CTS(
77			use_nuts3code=True, year=2011
78			)
79			df_CTS_gas_2011.to_csv("CTS_heat_demand_profile_nuts3.csv")
80
81			ags_lk = pd.read_csv(
82			os.path.join(
83			os.getcwd(),
84			"demandregio-disaggregator/disaggregator/disaggregator/data_in/regional",
85			"t_nuts3_lk.csv",
86			),
87			index_col=0,
88			)
89			ags_lk = ags_lk.drop(
90			ags_lk.columns.difference(["natcode_nuts3", "ags_lk"]), axis=1
91			)
92
93			CTS_profile = df_CTS_gas_2011.transpose()
94			CTS_profile.reset_index(inplace=True)
95			CTS_profile.ags_lk = CTS_profile.ags_lk.astype(int)
96			CTS_profile = pd.merge(CTS_profile, ags_lk, on="ags_lk", how="inner")
97			CTS_profile.set_index("natcode_nuts3", inplace=True)
98			CTS_profile.drop("ags_lk", axis=1, inplace=True)
99
100			CTS_per_zensus = pd.merge(
101			demand_nuts[["zensus_population_id", "vg250_nuts3"]],
102			CTS_profile,
103			left_on="vg250_nuts3",
104			right_on=CTS_profile.index,
105			how="left",
106			)
107
108			CTS_per_zensus = CTS_per_zensus.drop("vg250_nuts3", axis=1)
109
110			if aggregation_level == "district":
111			district_heating = db.select_dataframe(
112			f"""
113			SELECT area_id, zensus_population_id
114			FROM demand.egon_map_zensus_district_heating_areas
115			WHERE scenario = '{scenario}'
116			"""
117			)
118
119			CTS_per_district = pd.merge(
120			CTS_per_zensus,
121			district_heating,
122			on="zensus_population_id",
123			how="inner",
124			)
125			CTS_per_district.set_index("area_id", inplace=True)
126			CTS_per_district.drop("zensus_population_id", axis=1, inplace=True)
127
128			CTS_per_district = CTS_per_district.groupby(lambda x: x, axis=0).sum()
129			CTS_per_district = CTS_per_district.transpose()
130			CTS_per_district = CTS_per_district.apply(lambda x: x / x.sum())
131			CTS_per_district.columns.name = "area_id"
132			CTS_per_district.reset_index(drop=True, inplace=True)
133
134			# mv_grid = mv_grid.set_index("zensus_population_id")
135			district_heating = district_heating.set_index("zensus_population_id")
136
137			mv_grid_ind = db.select_dataframe(
138			f"""
139			SELECT bus_id, zensus_population_id
140			FROM boundaries.egon_map_zensus_grid_districts
141			WHERE zensus_population_id NOT IN
142			(SELECT zensus_population_id
143			FROM demand.egon_map_zensus_district_heating_areas
144			WHERE scenario = '{scenario}')
145			"""
146			)
147
148			CTS_per_grid = pd.merge(
149			CTS_per_zensus,
150			mv_grid_ind,
151			on="zensus_population_id",
152			how="inner",
153			)
154			CTS_per_grid.set_index("bus_id", inplace=True)
155			CTS_per_grid.drop("zensus_population_id", axis=1, inplace=True)
156
157			CTS_per_grid = CTS_per_grid.groupby(lambda x: x, axis=0).sum()
158			CTS_per_grid = CTS_per_grid.transpose()
159			CTS_per_grid = CTS_per_grid.apply(lambda x: x / x.sum())
160			CTS_per_grid.columns.name = "bus_id"
161			CTS_per_grid.reset_index(drop=True, inplace=True)
162
163			CTS_per_zensus = pd.DataFrame()
164
165			else:
166			CTS_per_district = pd.DataFrame()
167			CTS_per_grid = pd.DataFrame()
168			CTS_per_zensus.set_index("zensus_population_id", inplace=True)
169
170			CTS_per_zensus = CTS_per_zensus.groupby(lambda x: x, axis=0).sum()
171			CTS_per_zensus = CTS_per_zensus.transpose()
172			CTS_per_zensus = CTS_per_zensus.apply(lambda x: x / x.sum())
173			CTS_per_zensus.columns.name = "zensus_population_id"
174			CTS_per_zensus.reset_index(drop=True, inplace=True)
175
176			return CTS_per_district, CTS_per_grid, CTS_per_zensus
177
178
179			def CTS_demand_scale(aggregation_level):
180			"""
181
182			Description: caling the demand curves to the annual demand of the respective aggregation level
183
184
185			Parameters
186			----------
187			aggregation_level : str
188			aggregation_level : str
189			if further processing is to be done in zensus cell level 'other'
190			else 'dsitrict'
191
192			Returns
193			-------
194			CTS_per_district : pandas.DataFrame
195			if aggregation ='district'
196			Profiles scaled up to annual demand
197			else
198			0
199			CTS_per_grid : pandas.DataFrame
200			if aggregation ='district'
201			Profiles scaled up to annual demandd
202			else
203			0
204			CTS_per_zensus : pandas.DataFrame
205			if aggregation ='district'
206			0
207			else
208			Profiles scaled up to annual demand
209
210			"""
211			scenarios = ["eGon2035", "eGon100RE"]
212
213			CTS_district = pd.DataFrame()
214			CTS_grid = pd.DataFrame()
215			CTS_zensus = pd.DataFrame()
216
217			for scenario in scenarios:
218			(
219			CTS_per_district,
220			CTS_per_grid,
221			CTS_per_zensus,
222			) = cts_demand_per_aggregation_level(aggregation_level, scenario)
223			CTS_per_district = CTS_per_district.transpose()
224			CTS_per_grid = CTS_per_grid.transpose()
225			CTS_per_zensus = CTS_per_zensus.transpose()
226
227			demand = db.select_dataframe(
228			f"""
229			SELECT demand, zensus_population_id
230			FROM demand.egon_peta_heat
231			WHERE sector = 'service'
232			AND scenario = '{scenario}'
233			ORDER BY zensus_population_id
234			"""
235			)
236
237			if aggregation_level == "district":
238
239			district_heating = db.select_dataframe(
240			f"""
241			SELECT area_id, zensus_population_id
242			FROM demand.egon_map_zensus_district_heating_areas
243			WHERE scenario = '{scenario}'
244			"""
245			)
246
247			CTS_demands_district = pd.merge(
248			demand,
249			district_heating,
250			on="zensus_population_id",
251			how="inner",
252			)
253			CTS_demands_district.drop(
254			"zensus_population_id", axis=1, inplace=True
255			)
256			CTS_demands_district = CTS_demands_district.groupby(
257			"area_id"
258			).sum()
259
260			CTS_per_district = pd.merge(
261			CTS_per_district,
262			CTS_demands_district[["demand"]],
263			how="inner",
264			right_on=CTS_per_district.index,
265			left_on=CTS_demands_district.index,
266			)
267
268			CTS_per_district = CTS_per_district.rename(
269			columns={"key_0": "area_id"}
270			)
271			CTS_per_district.set_index("area_id", inplace=True)
272
273			CTS_per_district = CTS_per_district[
274			CTS_per_district.columns[:-1]
275			].multiply(CTS_per_district.demand, axis=0)
276
277			CTS_per_district.insert(0, "scenario", scenario)
278
279			CTS_district = CTS_district.append(CTS_per_district)
280			CTS_district = CTS_district.sort_index()
281
282			mv_grid_ind = db.select_dataframe(
283			f"""
284			SELECT bus_id, zensus_population_id
285			FROM boundaries.egon_map_zensus_grid_districts
286			WHERE zensus_population_id NOT IN
287			(SELECT zensus_population_id
288			FROM demand.egon_map_zensus_district_heating_areas
289			WHERE scenario = '{scenario}')
290			"""
291			)
292
293			CTS_demands_grid = pd.merge(
294			demand,
295			mv_grid_ind[["bus_id", "zensus_population_id"]],
296			on="zensus_population_id",
297			how="inner",
298			)
299
300			CTS_demands_grid.drop("zensus_population_id", axis=1, inplace=True)
301			CTS_demands_grid = CTS_demands_grid.groupby("bus_id").sum()
302
303			CTS_per_grid = pd.merge(
304			CTS_per_grid,
305			CTS_demands_grid[["demand"]],
306			how="inner",
307			right_on=CTS_per_grid.index,
308			left_on=CTS_demands_grid.index,
309			)
310
311			CTS_per_grid = CTS_per_grid.rename(columns={"key_0": "bus_id"})
312			CTS_per_grid.set_index("bus_id", inplace=True)
313
314			CTS_per_grid = CTS_per_grid[CTS_per_grid.columns[:-1]].multiply(
315			CTS_per_grid.demand, axis=0
316			)
317
318			CTS_per_grid.insert(0, "scenario", scenario)
319
320			CTS_grid = CTS_grid.append(CTS_per_grid)
321			CTS_grid = CTS_grid.sort_index()
322
323			CTS_per_zensus = 0
324
325			else:
326			CTS_per_district = 0
327			CTS_per_grid = 0
328
329			CTS_per_zensus = pd.merge(
330			CTS_per_zensus,
331			demand,
332			how="inner",
333			right_on=CTS_per_zensus.index,
334			left_on=demand.zensus_population_id,
335			)
336			CTS_per_zensus = CTS_per_zensus.drop("key_0", axis=1)
337			CTS_per_zensus.set_index("zensus_population_id", inplace=True)
338
339			CTS_per_zensus = CTS_per_zensus[
340			CTS_per_zensus.columns[:-1]
341			].multiply(CTS_per_zensus.demand, axis=0)
342			CTS_per_zensus.insert(0, "scenario", scenario)
343
344			CTS_per_zensus.reset_index(inplace=True)
345
346			CTS_zensus = CTS_zensus.append(CTS_per_grid)
347			CTS_zensus = CTS_zensus.set_index("bus_id")
348			CTS_zensus = CTS_zensus.sort_index()
349
350			return CTS_district, CTS_grid, CTS_zensus
351

openego / eGon-data

Pull Request — dev (#870)

CTS_demand_scale() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like