create_individual_heating_peak_loads()   A
last analyzed

Complexity

Conditions 3

Size

Total Lines 108
Code Lines 33

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
eloc 33
dl 0
loc 108
rs 9.0879
c 0
b 0
f 0
cc 3
nop 1

How to fix   Long Method   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

1
from datetime import date, datetime
2
from pathlib import Path
3
import json
4
import os
5
import time
6
import warnings
7
8
from sqlalchemy import ARRAY, Column, Float, Integer, String, Text
9
from sqlalchemy.ext.declarative import declarative_base
10
import geopandas as gpd
11
import numpy as np
12
import pandas as pd
13
14
from egon.data import config, db
15
import egon.data.datasets.era5 as era
16
17
try:
18
    from disaggregator import temporal
19
except ImportError as e:
20
    pass
21
22
from math import ceil
23
24
from egon.data.datasets import Dataset
25
from egon.data.datasets.heat_demand_timeseries.daily import (
26
    daily_demand_shares_per_climate_zone,
27
    map_climate_zones_to_zensus,
28
)
29
from egon.data.datasets.heat_demand_timeseries.idp_pool import create, select
30
from egon.data.datasets.heat_demand_timeseries.service_sector import (
31
    CTS_demand_scale,
32
)
33
from egon.data.metadata import (
34
    context,
35
    license_egon_data_odbl,
36
    meta_metadata,
37
    sources,
38
)
39
40
Base = declarative_base()
41
42
43
class EgonTimeseriesDistrictHeating(Base):
44
    __tablename__ = "egon_timeseries_district_heating"
45
    __table_args__ = {"schema": "demand"}
46
    area_id = Column(Integer, primary_key=True)
47
    scenario = Column(Text, primary_key=True)
48
    dist_aggregated_mw = Column(ARRAY(Float(53)))
49
50
51
class EgonEtragoTimeseriesIndividualHeating(Base):
52
    __tablename__ = "egon_etrago_timeseries_individual_heating"
53
    __table_args__ = {"schema": "demand"}
54
    bus_id = Column(Integer, primary_key=True)
55
    scenario = Column(Text, primary_key=True)
56
    dist_aggregated_mw = Column(ARRAY(Float(53)))
57
58
59
class EgonIndividualHeatingPeakLoads(Base):
60
    __tablename__ = "egon_individual_heating_peak_loads"
61
    __table_args__ = {"schema": "demand"}
62
    building_id = Column(Integer, primary_key=True)
63
    scenario = Column(Text, primary_key=True)
64
    w_th = Column(Float(53))
65
66
67
class EgonEtragoHeatCts(Base):
68
    __tablename__ = "egon_etrago_heat_cts"
69
    __table_args__ = {"schema": "demand"}
70
71
    bus_id = Column(Integer, primary_key=True)
72
    scn_name = Column(String, primary_key=True)
73
    p_set = Column(ARRAY(Float))
74
75
76
def create_timeseries_for_building(building_id, scenario):
77
    """Generates final heat demand timeseries for a specific building
78
79
    Parameters
80
    ----------
81
    building_id : int
82
        Index of the selected building
83
    scenario : str
84
        Name of the selected scenario.
85
86
    Returns
87
    -------
88
    pandas.DataFrame
89
        Hourly heat demand timeseries in MW for the selected building
90
91
    """
92
93
    return db.select_dataframe(
94
        f"""
95
        SELECT building_demand * UNNEST(idp) as demand
96
        FROM
97
        (
98
        SELECT
99
            demand.demand
100
            / building.count
101
            * daily_demand.daily_demand_share as building_demand,
102
            daily_demand.day_of_year
103
        FROM
104
105
        (SELECT demand FROM
106
        demand.egon_peta_heat
107
        WHERE scenario = '{scenario}'
108
        AND sector = 'residential'
109
        AND zensus_population_id IN(
110
        SELECT zensus_population_id FROM
111
        demand.egon_heat_timeseries_selected_profiles
112
        WHERE building_id  = {building_id})) as demand,
113
114
        (SELECT COUNT(building_id)
115
        FROM demand.egon_heat_timeseries_selected_profiles
116
        WHERE zensus_population_id IN(
117
        SELECT zensus_population_id FROM
118
        demand.egon_heat_timeseries_selected_profiles
119
        WHERE building_id  = {building_id})) as building,
120
121
        (SELECT daily_demand_share, day_of_year FROM
122
        demand.egon_daily_heat_demand_per_climate_zone
123
        WHERE climate_zone = (
124
            SELECT climate_zone FROM boundaries.egon_map_zensus_climate_zones
125
            WHERE zensus_population_id =
126
            (
127
                SELECT zensus_population_id
128
                FROM demand.egon_heat_timeseries_selected_profiles
129
                WHERE building_id = {building_id}
130
            )
131
        )) as daily_demand) as daily_demand
132
133
        JOIN (SELECT b.idp, ordinality as day
134
        FROM demand.egon_heat_timeseries_selected_profiles a,
135
        UNNEST (a.selected_idp_profiles) WITH ORDINALITY as selected_idp
136
        JOIN demand.egon_heat_idp_pool b
137
        ON selected_idp = b.index
138
        WHERE a.building_id = {building_id}) as demand_profile
139
        ON demand_profile.day = daily_demand.day_of_year
140
        """
141
    )
142
143
144
def create_district_heating_profile(scenario, area_id):
145
    """Create a heat demand profile for a district heating grid.
146
147
    The created heat demand profile includes the demands of households
148
    and the service sector.
149
150
    Parameters
151
    ----------
152
    scenario : str
153
        The name of the selected scenario.
154
    area_id : int
155
        The index of the selected district heating grid.
156
157
    Returns
158
    -------
159
    pd.DataFrame
160
        An hourly heat demand timeseries in MW for the selected district
161
        heating grid.
162
163
    """
164
165
    start_time = datetime.now()
166
167
    df = db.select_dataframe(
168
        f"""
169
170
        SELECT SUM(building_demand_per_hour) as demand_profile, hour_of_year
171
        FROM
172
173
        (
174
        SELECT demand.demand  *
175
        c.daily_demand_share * hourly_demand as building_demand_per_hour,
176
        ordinality + 24* (c.day_of_year-1) as hour_of_year,
177
        demand_profile.building_id,
178
        c.day_of_year,
179
        ordinality
180
181
        FROM
182
183
        (SELECT zensus_population_id, demand FROM
184
        demand.egon_peta_heat
185
        WHERE scenario = '{scenario}'
186
        AND sector = 'residential'
187
        AND zensus_population_id IN(
188
        SELECT zensus_population_id FROM
189
        demand.egon_map_zensus_district_heating_areas
190
        WHERE scenario = '{scenario}'
191
        AND area_id = {area_id}
192
        )) as demand
193
194
        JOIN boundaries.egon_map_zensus_climate_zones b
195
        ON demand.zensus_population_id = b.zensus_population_id
196
197
        JOIN demand.egon_daily_heat_demand_per_climate_zone c
198
        ON c.climate_zone = b.climate_zone
199
200
        JOIN (
201
        SELECT e.idp, ordinality as day, zensus_population_id, building_id
202
        FROM demand.egon_heat_timeseries_selected_profiles d,
203
        UNNEST (d.selected_idp_profiles) WITH ORDINALITY as selected_idp
204
        JOIN demand.egon_heat_idp_pool e
205
        ON selected_idp = e.index
206
        WHERE zensus_population_id IN (
207
        SELECT zensus_population_id FROM
208
        demand.egon_map_zensus_district_heating_areas
209
        WHERE scenario = '{scenario}'
210
        AND area_id = {area_id}
211
        ))  demand_profile
212
        ON (demand_profile.day = c.day_of_year AND
213
            demand_profile.zensus_population_id = b.zensus_population_id)
214
215
        JOIN (SELECT COUNT(building_id), zensus_population_id
216
        FROM demand.egon_heat_timeseries_selected_profiles
217
        WHERE zensus_population_id IN(
218
        SELECT zensus_population_id FROM
219
        demand.egon_heat_timeseries_selected_profiles
220
       WHERE zensus_population_id IN (
221
       SELECT zensus_population_id FROM
222
       demand.egon_map_zensus_district_heating_areas
223
       WHERE scenario = '{scenario}'
224
       AND area_id = {area_id}
225
       ))
226
        GROUP BY zensus_population_id) building
227
        ON building.zensus_population_id = b.zensus_population_id,
228
229
        UNNEST(demand_profile.idp) WITH ORDINALITY as hourly_demand
230
        )   result
231
232
233
        GROUP BY hour_of_year
234
235
        """
236
    )
237
238
    print(
239
        f"Time to create time series for district heating grid {scenario}"
240
        f" {area_id}:\n{datetime.now() - start_time}"
241
    )
242
243
    return df
244
245
246
def create_district_heating_profile_python_like(scenario="eGon2035"):
247
    """Creates profiles for all district heating grids in one scenario.
248
    Similar to create_district_heating_profile but faster and needs more RAM.
249
    The results are directly written into the database.
250
251
    Parameters
252
    ----------
253
    scenario : str
254
        Name of the selected scenario.
255
256
    Returns
257
    -------
258
    None.
259
260
    """
261
262
    start_time = datetime.now()
263
264
    idp_df = db.select_dataframe(
265
        """
266
        SELECT index, idp FROM demand.egon_heat_idp_pool
267
        """,
268
        index_col="index",
269
    )
270
271
    district_heating_grids = db.select_dataframe(
272
        f"""
273
        SELECT area_id
274
        FROM demand.egon_district_heating_areas
275
        WHERE scenario = '{scenario}'
276
        """
277
    )
278
279
    annual_demand = db.select_dataframe(
280
        f"""
281
        SELECT
282
            a.zensus_population_id,
283
            demand / c.count as per_building,
284
            area_id,
285
            demand as demand_total
286
        FROM
287
        demand.egon_peta_heat a
288
        INNER JOIN (
289
            SELECT * FROM demand.egon_map_zensus_district_heating_areas
290
            WHERE scenario = '{scenario}'
291
        ) b ON a.zensus_population_id = b.zensus_population_id
292
293
        JOIN (SELECT COUNT(building_id), zensus_population_id
294
        FROM demand.egon_heat_timeseries_selected_profiles
295
        WHERE zensus_population_id IN(
296
        SELECT zensus_population_id FROM
297
        demand.egon_heat_timeseries_selected_profiles
298
        WHERE zensus_population_id IN (
299
        SELECT zensus_population_id FROM
300
        boundaries.egon_map_zensus_grid_districts
301
       ))
302
        GROUP BY zensus_population_id)c
303
        ON a.zensus_population_id = c.zensus_population_id
304
305
        WHERE a.scenario = '{scenario}'
306
        AND a.sector = 'residential'
307
308
        """,
309
        index_col="zensus_population_id",
310
    )
311
312
    annual_demand = annual_demand[
313
        ~annual_demand.index.duplicated(keep="first")
314
    ]
315
316
    daily_demand_shares = db.select_dataframe(
317
        """
318
        SELECT climate_zone, day_of_year as day, daily_demand_share FROM
319
        demand.egon_daily_heat_demand_per_climate_zone
320
        """
321
    )
322
323
    CTS_demand_dist, CTS_demand_grid, CTS_demand_zensus = CTS_demand_scale(
324
        aggregation_level="district"
325
    )
326
327
    print(datetime.now() - start_time)
328
329
    start_time = datetime.now()
330
    for area in district_heating_grids.area_id.unique():
331
        with db.session_scope() as session:
332
            selected_profiles = db.select_dataframe(
333
                f"""
334
                SELECT a.zensus_population_id, building_id, c.climate_zone,
335
                selected_idp, ordinality as day, b.area_id
336
                FROM demand.egon_heat_timeseries_selected_profiles a
337
                INNER JOIN boundaries.egon_map_zensus_climate_zones c
338
                ON a.zensus_population_id = c.zensus_population_id
339
                INNER JOIN (
340
                    SELECT * FROM demand.egon_map_zensus_district_heating_areas
341
                    WHERE scenario = '{scenario}'
342
                    AND area_id = '{area}'
343
                ) b ON a.zensus_population_id = b.zensus_population_id        ,
344
345
                UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp
346
347
                """
348
            )
349
350
            if not selected_profiles.empty:
351
                df = pd.merge(
352
                    selected_profiles,
353
                    daily_demand_shares,
354
                    on=["day", "climate_zone"],
355
                )
356
357
                slice_df = pd.merge(
358
                    df[df.area_id == area],
359
                    idp_df,
360
                    left_on="selected_idp",
361
                    right_on="index",
362
                )
363
364
                # Drop cells without a demand or outside of MVGD
365
                slice_df = slice_df[
366
                    slice_df.zensus_population_id.isin(annual_demand.index)
367
                ]
368
369
                for hour in range(24):
370
                    slice_df[hour] = (
371
                        slice_df.idp.str[hour]
372
                        .mul(slice_df.daily_demand_share)
373
                        .mul(
374
                            annual_demand.loc[
375
                                slice_df.zensus_population_id.values,
376
                                "per_building",
377
                            ].values
378
                        )
379
                    )
380
381
                diff = (
382
                    slice_df[range(24)].sum().sum()
383
                    - annual_demand[
384
                        annual_demand.area_id == area
385
                    ].demand_total.sum()
386
                ) / (
387
                    annual_demand[annual_demand.area_id == area].demand_total.sum()
388
                )
389
390
                assert (
391
                    abs(diff) < 0.04
392
                ), f"""Deviation of residential heat demand time
393
                series for district heating grid {str(area)} is {diff}"""
394
395
                if abs(diff) > 0.03:
396
                    warnings.warn(
397
                        f"""Deviation of residential heat demand time
398
                    series for district heating grid {str(area)} is {diff}"""
399
                    )
400
401
                hh = np.concatenate(
402
                    slice_df.drop(
403
                        [
404
                            "zensus_population_id",
405
                            "building_id",
406
                            "climate_zone",
407
                            "selected_idp",
408
                            "area_id",
409
                            "daily_demand_share",
410
                            "idp",
411
                        ],
412
                        axis="columns",
413
                    )
414
                    .groupby("day")
415
                    .sum()[range(24)]
416
                    .values
417
                ).ravel()
418
419
            cts = CTS_demand_dist[
420
                (CTS_demand_dist.scenario == scenario)
421
                & (CTS_demand_dist.index == area)
422
            ].drop("scenario", axis="columns")
423
424
            if (not selected_profiles.empty) and not cts.empty:
425
                entry = EgonTimeseriesDistrictHeating(
426
                    area_id=int(area),
427
                    scenario=scenario,
428
                    dist_aggregated_mw=(hh + cts.values[0]).tolist(),
0 ignored issues
show
introduced by
The variable hh does not seem to be defined for all execution paths.
Loading history...
429
                )
430
            elif (not selected_profiles.empty) and cts.empty:
431
                entry = EgonTimeseriesDistrictHeating(
432
                    area_id=int(area),
433
                    scenario=scenario,
434
                    dist_aggregated_mw=(hh).tolist(),
435
                )
436
            elif not cts.empty:
437
                entry = EgonTimeseriesDistrictHeating(
438
                    area_id=int(area),
439
                    scenario=scenario,
440
                    dist_aggregated_mw=(cts.values[0]).tolist(),
441
                )
442
            else:
443
                entry = EgonTimeseriesDistrictHeating(
444
                    area_id=int(area),
445
                    scenario=scenario,
446
                    dist_aggregated_mw=np.repeat(0, 8760).tolist(),
447
                )
448
449
            session.add(entry)
450
        session.commit()
451
452
    print(
453
        f"Time to create time series for district heating scenario {scenario}"
454
    )
455
    print(datetime.now() - start_time)
456
457
458
def create_individual_heat_per_mv_grid(scenario="eGon2035", mv_grid_id=1564):
459
    start_time = datetime.now()
460
    df = db.select_dataframe(
461
        f"""
462
463
        SELECT SUM(building_demand_per_hour) as demand_profile, hour_of_year
464
        FROM
465
466
        (
467
        SELECT demand.demand  *
468
        c.daily_demand_share * hourly_demand as building_demand_per_hour,
469
        ordinality + 24* (c.day_of_year-1) as hour_of_year,
470
        demand_profile.building_id,
471
        c.day_of_year,
472
        ordinality
473
474
        FROM
475
476
        (SELECT zensus_population_id, demand FROM
477
        demand.egon_peta_heat
478
        WHERE scenario = '{scenario}'
479
        AND sector = 'residential'
480
        AND zensus_population_id IN (
481
        SELECT zensus_population_id FROM
482
        boundaries.egon_map_zensus_grid_districts
483
        WHERE bus_id = {mv_grid_id}
484
        )) as demand
485
486
        JOIN boundaries.egon_map_zensus_climate_zones b
487
        ON demand.zensus_population_id = b.zensus_population_id
488
489
        JOIN demand.egon_daily_heat_demand_per_climate_zone c
490
        ON c.climate_zone = b.climate_zone
491
492
        JOIN (
493
        SELECT
494
            e.idp, ordinality as day, zensus_population_id, building_id
495
        FROM demand.egon_heat_timeseries_selected_profiles d,
496
        UNNEST (d.selected_idp_profiles) WITH ORDINALITY as selected_idp
497
        JOIN demand.egon_heat_idp_pool e
498
        ON selected_idp = e.index
499
        WHERE zensus_population_id IN (
500
        SELECT zensus_population_id FROM
501
        boundaries.egon_map_zensus_grid_districts
502
        WHERE bus_id = {mv_grid_id}
503
        ))  demand_profile
504
        ON (demand_profile.day = c.day_of_year AND
505
            demand_profile.zensus_population_id = b.zensus_population_id)
506
507
        JOIN (SELECT COUNT(building_id), zensus_population_id
508
        FROM demand.egon_heat_timeseries_selected_profiles
509
        WHERE zensus_population_id IN(
510
        SELECT zensus_population_id FROM
511
        demand.egon_heat_timeseries_selected_profiles
512
        WHERE zensus_population_id IN (
513
        SELECT zensus_population_id FROM
514
        boundaries.egon_map_zensus_grid_districts
515
        WHERE bus_id = {mv_grid_id}
516
       ))
517
        GROUP BY zensus_population_id) building
518
        ON building.zensus_population_id = b.zensus_population_id,
519
520
        UNNEST(demand_profile.idp) WITH ORDINALITY as hourly_demand
521
        )   result
522
523
524
        GROUP BY hour_of_year
525
526
        """
527
    )
528
529
    print(f"Time to create time series for mv grid {scenario} {mv_grid_id}:")
530
    print(datetime.now() - start_time)
531
532
    return df
533
534
535
def calulate_peak_load(df, scenario):
536
    # peat load in W_th
537
    data = (
538
        df.groupby("building_id")
539
        .max()[range(24)]
540
        .max(axis=1)
541
        .mul(1000000)
542
        .astype(int)
543
        .reset_index()
544
    )
545
546
    data["scenario"] = scenario
547
548
    data.rename({0: "w_th"}, axis="columns", inplace=True)
549
550
    data.to_sql(
551
        EgonIndividualHeatingPeakLoads.__table__.name,
552
        schema=EgonIndividualHeatingPeakLoads.__table__.schema,
553
        con=db.engine(),
554
        if_exists="append",
555
        index=False,
556
    )
557
558
559
def create_individual_heating_peak_loads(scenario="eGon2035"):
560
    engine = db.engine()
561
562
    EgonIndividualHeatingPeakLoads.__table__.drop(bind=engine, checkfirst=True)
563
564
    EgonIndividualHeatingPeakLoads.__table__.create(
565
        bind=engine, checkfirst=True
566
    )
567
568
    start_time = datetime.now()
569
570
    idp_df = db.select_dataframe(
571
        """
572
        SELECT index, idp FROM demand.egon_heat_idp_pool
573
        """,
574
        index_col="index",
575
    )
576
577
    annual_demand = db.select_dataframe(
578
        f"""
579
        SELECT a.zensus_population_id, demand/c.count as per_building, bus_id
580
        FROM demand.egon_peta_heat a
581
582
583
        JOIN (SELECT COUNT(building_id), zensus_population_id
584
        FROM demand.egon_heat_timeseries_selected_profiles
585
        WHERE zensus_population_id IN(
586
        SELECT zensus_population_id FROM
587
        demand.egon_heat_timeseries_selected_profiles
588
        WHERE zensus_population_id IN (
589
        SELECT zensus_population_id FROM
590
        boundaries.egon_map_zensus_grid_districts
591
       ))
592
        GROUP BY zensus_population_id)c
593
        ON a.zensus_population_id = c.zensus_population_id
594
595
        JOIN boundaries.egon_map_zensus_grid_districts d
596
        ON a.zensus_population_id = d.zensus_population_id
597
598
        WHERE a.scenario = '{scenario}'
599
        AND a.sector = 'residential'
600
        AND a.zensus_population_id NOT IN (
601
            SELECT zensus_population_id
602
            FROM demand.egon_map_zensus_district_heating_areas
603
            WHERE scenario = '{scenario}'
604
        )
605
606
        """,
607
        index_col="zensus_population_id",
608
    )
609
610
    daily_demand_shares = db.select_dataframe(
611
        """
612
        SELECT climate_zone, day_of_year as day, daily_demand_share FROM
613
        demand.egon_daily_heat_demand_per_climate_zone
614
        """
615
    )
616
617
    start_time = datetime.now()
618
    for grid in annual_demand.bus_id.unique():
619
        selected_profiles = db.select_dataframe(
620
            f"""
621
            SELECT a.zensus_population_id, building_id, c.climate_zone,
622
            selected_idp, ordinality as day
623
            FROM demand.egon_heat_timeseries_selected_profiles a
624
            INNER JOIN boundaries.egon_map_zensus_climate_zones c
625
            ON a.zensus_population_id = c.zensus_population_id
626
            ,
627
628
            UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp
629
630
            WHERE a.zensus_population_id NOT IN (
631
                SELECT zensus_population_id
632
                FROM demand.egon_map_zensus_district_heating_areas
633
                WHERE scenario = '{scenario}'
634
            )
635
            AND a.zensus_population_id IN (
636
                SELECT zensus_population_id
637
                FROM boundaries.egon_map_zensus_grid_districts
638
                WHERE bus_id = '{grid}'
639
            )
640
641
            """
642
        )
643
644
        df = pd.merge(
645
            selected_profiles, daily_demand_shares, on=["day", "climate_zone"]
646
        )
647
648
        slice_df = pd.merge(
649
            df, idp_df, left_on="selected_idp", right_on="index"
650
        )
651
652
        for hour in range(24):
653
            slice_df[hour] = (
654
                slice_df.idp.str[hour]
655
                .mul(slice_df.daily_demand_share)
656
                .mul(
657
                    annual_demand.loc[
658
                        slice_df.zensus_population_id.values, "per_building"
659
                    ].values
660
                )
661
            )
662
663
        calulate_peak_load(slice_df, scenario)
664
665
    print(f"Time to create peak loads per building for {scenario}")
666
    print(datetime.now() - start_time)
667
668
669
def create_individual_heating_profile_python_like(scenario="eGon2035"):
670
    start_time = datetime.now()
671
672
    idp_df = db.select_dataframe(
673
        f"""
674
        SELECT index, idp FROM demand.egon_heat_idp_pool
675
        """,
676
        index_col="index",
677
    )
678
679
    annual_demand = db.select_dataframe(
680
        f"""
681
        SELECT
682
            a.zensus_population_id,
683
            demand / c.count as per_building,
684
            demand as demand_total,
685
            bus_id
686
        FROM demand.egon_peta_heat a
687
688
689
        JOIN (SELECT COUNT(building_id), zensus_population_id
690
        FROM demand.egon_heat_timeseries_selected_profiles
691
        WHERE zensus_population_id IN(
692
        SELECT zensus_population_id FROM
693
        demand.egon_heat_timeseries_selected_profiles
694
        WHERE zensus_population_id IN (
695
        SELECT zensus_population_id FROM
696
        boundaries.egon_map_zensus_grid_districts
697
       ))
698
        GROUP BY zensus_population_id)c
699
        ON a.zensus_population_id = c.zensus_population_id
700
701
        JOIN boundaries.egon_map_zensus_grid_districts d
702
        ON a.zensus_population_id = d.zensus_population_id
703
704
        WHERE a.scenario = '{scenario}'
705
        AND a.sector = 'residential'
706
        AND a.zensus_population_id NOT IN (
707
            SELECT zensus_population_id
708
            FROM demand.egon_map_zensus_district_heating_areas
709
            WHERE scenario = '{scenario}'
710
        )
711
712
        """,
713
        index_col="zensus_population_id",
714
    )
715
716
    daily_demand_shares = db.select_dataframe(
717
        """
718
        SELECT climate_zone, day_of_year as day, daily_demand_share FROM
719
        demand.egon_daily_heat_demand_per_climate_zone
720
        """
721
    )
722
723
    CTS_demand_dist, CTS_demand_grid, CTS_demand_zensus = CTS_demand_scale(
724
        aggregation_level="district"
725
    )
726
727
    # TODO: use session_scope!
728
    from sqlalchemy.orm import sessionmaker
729
730
    session = sessionmaker(bind=db.engine())()
731
732
    print(
733
        f"Time to create overhead for time series for district heating scenario {scenario}"
734
    )
735
    print(datetime.now() - start_time)
736
737
    start_time = datetime.now()
738
    for grid in annual_demand.bus_id.unique():
739
        selected_profiles = db.select_dataframe(
740
            f"""
741
            SELECT a.zensus_population_id, building_id, c.climate_zone,
742
            selected_idp, ordinality as day
743
            FROM demand.egon_heat_timeseries_selected_profiles a
744
            INNER JOIN boundaries.egon_map_zensus_climate_zones c
745
            ON a.zensus_population_id = c.zensus_population_id
746
            ,
747
748
            UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp
749
750
            WHERE a.zensus_population_id NOT IN (
751
                SELECT zensus_population_id FROM demand.egon_map_zensus_district_heating_areas
752
                WHERE scenario = '{scenario}'
753
            )
754
            AND a.zensus_population_id IN (
755
                SELECT zensus_population_id
756
                FROM boundaries.egon_map_zensus_grid_districts
757
                WHERE bus_id = '{grid}'
758
            )
759
760
            """
761
        )
762
763
        df = pd.merge(
764
            selected_profiles, daily_demand_shares, on=["day", "climate_zone"]
765
        )
766
767
        slice_df = pd.merge(
768
            df, idp_df, left_on="selected_idp", right_on="index"
769
        )
770
771
        for hour in range(24):
772
            slice_df[hour] = (
773
                slice_df.idp.str[hour]
774
                .mul(slice_df.daily_demand_share)
775
                .mul(
776
                    annual_demand.loc[
777
                        slice_df.zensus_population_id.values, "per_building"
778
                    ].values
779
                )
780
            )
781
782
        cts = CTS_demand_grid[
783
            (CTS_demand_grid.scenario == scenario)
784
            & (CTS_demand_grid.index == grid)
785
        ].drop("scenario", axis="columns")
786
787
        hh = np.concatenate(
788
            slice_df.groupby("day").sum()[range(24)].values
789
        ).ravel()
790
791
        diff = (
792
            slice_df.groupby("day").sum()[range(24)].sum().sum()
793
            - annual_demand[annual_demand.bus_id == grid].demand_total.sum()
794
        ) / (annual_demand[annual_demand.bus_id == grid].demand_total.sum())
795
796
        assert abs(diff) < 0.03, (
797
            "Deviation of residential heat demand time series for mv"
798
            f" grid {grid} is {diff}"
799
        )
800
801
        if not (slice_df[hour].empty or cts.empty):
0 ignored issues
show
introduced by
The variable hour does not seem to be defined for all execution paths.
Loading history...
802
            entry = EgonEtragoTimeseriesIndividualHeating(
803
                bus_id=int(grid),
804
                scenario=scenario,
805
                dist_aggregated_mw=(hh + cts.values[0]).tolist(),
806
            )
807
        elif not slice_df[hour].empty:
808
            entry = EgonEtragoTimeseriesIndividualHeating(
809
                bus_id=int(grid),
810
                scenario=scenario,
811
                dist_aggregated_mw=(hh).tolist(),
812
            )
813
        elif not cts.empty:
814
            entry = EgonEtragoTimeseriesIndividualHeating(
815
                bus_id=int(grid),
816
                scenario=scenario,
817
                dist_aggregated_mw=(cts).tolist(),
818
            )
819
820
        session.add(entry)
0 ignored issues
show
introduced by
The variable entry does not seem to be defined for all execution paths.
Loading history...
821
822
    session.commit()
823
824
    print(
825
        f"Time to create time series for district heating scenario {scenario}"
826
    )
827
    print(datetime.now() - start_time)
828
829
830
def district_heating(method="python"):
831
    engine = db.engine()
832
    EgonTimeseriesDistrictHeating.__table__.drop(bind=engine, checkfirst=True)
833
    EgonTimeseriesDistrictHeating.__table__.create(
834
        bind=engine, checkfirst=True
835
    )
836
837
    if method == "python":
838
        for scenario in config.settings()["egon-data"]["--scenarios"]:
839
            create_district_heating_profile_python_like(scenario)
840
841
    else:
842
        CTS_demand_dist, CTS_demand_grid, CTS_demand_zensus = CTS_demand_scale(
843
            aggregation_level="district"
844
        )
845
846
        ids = db.select_dataframe(
847
            """
848
            SELECT area_id, scenario
849
            FROM demand.egon_district_heating_areas
850
            """
851
        )
852
853
        df = pd.DataFrame(
854
            columns=["area_id", "scenario", "dist_aggregated_mw"]
855
        )
856
857
        for index, row in ids.iterrows():
858
            series = create_district_heating_profile(
859
                scenario=row.scenario, area_id=row.area_id
860
            )
861
862
            cts = (
863
                CTS_demand_dist[
864
                    (CTS_demand_dist.scenario == row.scenario)
865
                    & (CTS_demand_dist.index == row.area_id)
866
                ]
867
                .drop("scenario", axis="columns")
868
                .transpose()
869
            )
870
871
            if not cts.empty:
872
                data = (
873
                    cts[row.area_id] + series.demand_profile
874
                ).values.tolist()
875
            else:
876
                data = series.demand_profile.values.tolist()
877
878
            df = df.append(
879
                pd.Series(
880
                    data={
881
                        "area_id": row.area_id,
882
                        "scenario": row.scenario,
883
                        "dist_aggregated_mw": data,
884
                    },
885
                ),
886
                ignore_index=True,
887
            )
888
889
        df.to_sql(
890
            "egon_timeseries_district_heating",
891
            schema="demand",
892
            con=db.engine(),
893
            if_exists="append",
894
            index=False,
895
        )
896
897
898
def individual_heating_per_mv_grid_tables(method="python"):
899
    engine = db.engine()
900
    EgonEtragoTimeseriesIndividualHeating.__table__.drop(
901
        bind=engine, checkfirst=True
902
    )
903
    EgonEtragoTimeseriesIndividualHeating.__table__.create(
904
        bind=engine, checkfirst=True
905
    )
906
907
908
def individual_heating_per_mv_grid_2035(method="python"):
909
    create_individual_heating_profile_python_like("eGon2035")
910
911
912
def individual_heating_per_mv_grid_100(method="python"):
913
    create_individual_heating_profile_python_like("eGon100RE")
914
915
916
def individual_heating_per_mv_grid(method="python"):
917
    if method == "python":
918
        engine = db.engine()
919
        EgonEtragoTimeseriesIndividualHeating.__table__.drop(
920
            bind=engine, checkfirst=True
921
        )
922
        EgonEtragoTimeseriesIndividualHeating.__table__.create(
923
            bind=engine, checkfirst=True
924
        )
925
926
        create_individual_heating_profile_python_like("eGon2035")
927
        create_individual_heating_profile_python_like("eGon100RE")
928
929
    else:
930
        engine = db.engine()
931
        EgonEtragoTimeseriesIndividualHeating.__table__.drop(
932
            bind=engine, checkfirst=True
933
        )
934
        EgonEtragoTimeseriesIndividualHeating.__table__.create(
935
            bind=engine, checkfirst=True
936
        )
937
938
        CTS_demand_dist, CTS_demand_grid, CTS_demand_zensus = CTS_demand_scale(
939
            aggregation_level="district"
940
        )
941
        df = pd.DataFrame(columns=["bus_id", "scenario", "dist_aggregated_mw"])
942
943
        ids = db.select_dataframe(
944
            """
945
            SELECT bus_id
946
            FROM grid.egon_mv_grid_district
947
            """
948
        )
949
950
        for index, row in ids.iterrows():
951
            for scenario in ["eGon2035", "eGon100RE"]:
952
                series = create_individual_heat_per_mv_grid(
953
                    scenario, row.bus_id
954
                )
955
                cts = (
956
                    CTS_demand_grid[
957
                        (CTS_demand_grid.scenario == scenario)
958
                        & (CTS_demand_grid.index == row.bus_id)
959
                    ]
960
                    .drop("scenario", axis="columns")
961
                    .transpose()
962
                )
963
                if not cts.empty:
964
                    data = (
965
                        cts[row.bus_id] + series.demand_profile
966
                    ).values.tolist()
967
                else:
968
                    data = series.demand_profile.values.tolist()
969
970
                df = df.append(
971
                    pd.Series(
972
                        data={
973
                            "bus_id": row.bus_id,
974
                            "scenario": scenario,
975
                            "dist_aggregated_mw": data,
976
                        },
977
                    ),
978
                    ignore_index=True,
979
                )
980
981
        df.to_sql(
982
            "egon_etrago_timeseries_individual_heating",
983
            schema="demand",
984
            con=db.engine(),
985
            if_exists="append",
986
            index=False,
987
        )
988
989
990
def store_national_profiles():
991
    scenario = "eGon100RE"
992
993
    df = db.select_dataframe(
994
        f"""
995
996
        SELECT SUM(building_demand_per_hour) as "residential rural"
997
        FROM
998
999
        (
1000
        SELECT demand.demand  / building.count *
1001
        c.daily_demand_share * hourly_demand as building_demand_per_hour,
1002
        ordinality + 24* (c.day_of_year-1) as hour_of_year,
1003
        demand_profile.building_id,
1004
        c.day_of_year,
1005
        ordinality
1006
1007
        FROM
1008
1009
        (SELECT zensus_population_id, demand FROM
1010
        demand.egon_peta_heat
1011
        WHERE scenario = '{scenario}'
1012
        AND sector = 'residential'
1013
       ) as demand
1014
1015
        JOIN boundaries.egon_map_zensus_climate_zones b
1016
        ON demand.zensus_population_id = b.zensus_population_id
1017
1018
        JOIN demand.egon_daily_heat_demand_per_climate_zone c
1019
        ON c.climate_zone = b.climate_zone
1020
1021
        JOIN (
1022
        SELECT e.idp, ordinality as day, zensus_population_id, building_id
1023
        FROM demand.egon_heat_timeseries_selected_profiles d,
1024
        UNNEST (d.selected_idp_profiles) WITH ORDINALITY as selected_idp
1025
        JOIN demand.egon_heat_idp_pool e
1026
        ON selected_idp = e.index
1027
        )  demand_profile
1028
        ON (demand_profile.day = c.day_of_year AND
1029
            demand_profile.zensus_population_id = b.zensus_population_id)
1030
1031
        JOIN (SELECT COUNT(building_id), zensus_population_id
1032
        FROM demand.egon_heat_timeseries_selected_profiles
1033
        WHERE zensus_population_id IN(
1034
        SELECT zensus_population_id FROM
1035
        demand.egon_heat_timeseries_selected_profiles
1036
        )
1037
        GROUP BY zensus_population_id) building
1038
        ON building.zensus_population_id = b.zensus_population_id,
1039
1040
        UNNEST(demand_profile.idp) WITH ORDINALITY as hourly_demand
1041
        )   result
1042
1043
1044
        GROUP BY hour_of_year
1045
1046
        """
1047
    )
1048
1049
    CTS_demand_dist, CTS_demand_grid, CTS_demand_zensus = CTS_demand_scale(
1050
        aggregation_level="district"
1051
    )
1052
1053
    df["service rural"] = (
1054
        CTS_demand_dist.loc[CTS_demand_dist.scenario == scenario]
1055
        .drop("scenario", axis=1)
1056
        .sum()
1057
    )
1058
1059
    df["urban central"] = db.select_dataframe(
1060
        f"""
1061
        SELECT sum(nullif(demand, 'NaN')) as "urban central"
1062
1063
        FROM demand.egon_timeseries_district_heating,
1064
        UNNEST (dist_aggregated_mw) WITH ORDINALITY as demand
1065
1066
        WHERE scenario = '{scenario}'
1067
1068
        GROUP BY ordinality
1069
1070
        """
1071
    )
1072
1073
    folder = Path(".") / "input-pypsa-eur-sec"
1074
    # Create the folder, if it does not exists already
1075
    if not os.path.exists(folder):
1076
        os.mkdir(folder)
1077
1078
    df.to_csv(folder / f"heat_demand_timeseries_DE_{scenario}.csv")
1079
1080
1081
def export_etrago_cts_heat_profiles():
1082
    """Export heat cts load profiles at mv substation level
1083
    to etrago-table in the database
1084
1085
    Returns
1086
    -------
1087
    None.
1088
1089
    """
1090
1091
    # Calculate cts heat profiles at substation
1092
    _, CTS_grid, _ = CTS_demand_scale("district")
1093
1094
    # Change format
1095
    data = CTS_grid.drop(columns="scenario")
1096
    df_etrago_cts_heat_profiles = pd.DataFrame(
1097
        index=data.index, columns=["scn_name", "p_set"]
1098
    )
1099
    df_etrago_cts_heat_profiles.p_set = data.values.tolist()
1100
    df_etrago_cts_heat_profiles.scn_name = CTS_grid["scenario"]
1101
    df_etrago_cts_heat_profiles.reset_index(inplace=True)
1102
1103
    # Drop and recreate Table if exists
1104
    EgonEtragoHeatCts.__table__.drop(bind=db.engine(), checkfirst=True)
1105
    EgonEtragoHeatCts.__table__.create(bind=db.engine(), checkfirst=True)
1106
1107
    # Write heat ts into db
1108
    with db.session_scope() as session:
1109
        session.bulk_insert_mappings(
1110
            EgonEtragoHeatCts,
1111
            df_etrago_cts_heat_profiles.to_dict(orient="records"),
1112
        )
1113
1114
1115
def metadata():
1116
    fields = [
1117
        {
1118
            "description": "Index of corresponding district heating area",
1119
            "name": "area_id",
1120
            "type": "integer",
1121
            "unit": "none",
1122
        },
1123
        {
1124
            "description": "Name of scenario",
1125
            "name": "scenario",
1126
            "type": "str",
1127
            "unit": "none",
1128
        },
1129
        {
1130
            "description": "Heat demand time series",
1131
            "name": "dist_aggregated_mw",
1132
            "type": "array of floats",
1133
            "unit": "MW",
1134
        },
1135
    ]
1136
1137
    meta_district = {
1138
        "name": "demand.egon_timeseries_district_heating",
1139
        "title": "eGon heat demand time series for district heating grids",
1140
        "id": "WILL_BE_SET_AT_PUBLICATION",
1141
        "description": "Heat demand time series for district heating grids",
1142
        "language": ["EN"],
1143
        "publicationDate": date.today().isoformat(),
1144
        "context": context(),
1145
        "spatial": {
1146
            "location": None,
1147
            "extent": "Germany",
1148
            "resolution": None,
1149
        },
1150
        "sources": [
1151
            sources()["era5"],
1152
            sources()["vg250"],
1153
            sources()["egon-data"],
1154
            sources()["egon-data_bundle"],
1155
            sources()["peta"],
1156
        ],
1157
        "licenses": [license_egon_data_odbl()],
1158
        "contributors": [
1159
            {
1160
                "title": "Clara Büttner",
1161
                "email": "http://github.com/ClaraBuettner",
1162
                "date": time.strftime("%Y-%m-%d"),
1163
                "object": None,
1164
                "comment": "Imported data",
1165
            },
1166
        ],
1167
        "resources": [
1168
            {
1169
                "profile": "tabular-data-resource",
1170
                "name": "demand.egon_timeseries_district_heating",
1171
                "path": None,
1172
                "format": "PostgreSQL",
1173
                "encoding": "UTF-8",
1174
                "schema": {
1175
                    "fields": fields,
1176
                    "primaryKey": ["index"],
1177
                    "foreignKeys": [],
1178
                },
1179
                "dialect": {"delimiter": None, "decimalSeparator": "."},
1180
            }
1181
        ],
1182
        "metaMetadata": meta_metadata(),
1183
    }
1184
1185
    # Add metadata as a comment to the table
1186
    db.submit_comment(
1187
        "'" + json.dumps(meta_district) + "'",
1188
        EgonTimeseriesDistrictHeating.__table__.schema,
1189
        EgonTimeseriesDistrictHeating.__table__.name,
1190
    )
1191
1192
1193
class HeatTimeSeries(Dataset):
1194
    """
1195
    Chooses heat demand profiles for each residential and CTS building
1196
1197
    This dataset creates heat demand profiles in an hourly resoultion.
1198
    Time series for CTS buildings are created using the SLP-gas method implemented
1199
    in the demandregio disagregator with the function :py:func:`export_etrago_cts_heat_profiles`
1200
    and stored in the database.
1201
    Time series for residential buildings are created based on a variety of synthetical created
1202
    individual demand profiles that are part of :py:class:`DataBundle <egon.data.datasets.data_bundle.DataBundle>`.
1203
    This method is desribed within the functions and in this publication:
1204
1205
    C. Büttner, J. Amme, J. Endres, A. Malla, B. Schachler, I. Cußmann,
1206
    Open modeling of electricity and heat demand curves for all
1207
    residential buildings in Germany, Energy Informatics 5 (1) (2022) 21.
1208
    doi:10.1186/s42162-022-00201-y.
1209
1210
1211
    *Dependencies*
1212
      * :py:class:`DataBundle <egon.data.datasets.data_bundle.DataBundle>`
1213
      * :py:class:`DemandRegio <egon.data.datasets.demandregio.DemandRegio>`
1214
      * :py:class:`HeatDemandImport <egon.data.datasets.heat_demand.HeatDemandImport>`
1215
      * :py:class:`DistrictHeatingAreas <egon.data.datasets.district_heating_areas.DistrictHeatingAreas>`
1216
      * :py:class:`Vg250 <egon.data.datasets.vg250.Vg250>`
1217
      * :py:class:`ZensusMvGridDistricts <egon.data.datasets.zensus_mv_grid_districts.ZensusMvGridDistricts>`
1218
      * :py:func:`hh_demand_buildings_setup <egon.data.datasets.electricity_demand_timeseries.hh_buildings.map_houseprofiles_to_buildings>`
1219
      * :py:class:`WeatherData <egon.data.datasets.era5.WeatherData>`
1220
1221
1222
    *Resulting tables*
1223
      * :py:class:`demand.egon_timeseries_district_heating <egon.data.datasets.heat_demand_timeseries.EgonTimeseriesDistrictHeating>` is created and filled
1224
      * :py:class:`demand.egon_etrago_heat_cts <egon.data.datasets.heat_demand_timeseries.EgonEtragoHeatCts>` is created and filled
1225
      * :py:class:`demand.egon_heat_timeseries_selected_profiles <egon.data.datasets.heat_demand_timeseries.idp_pool.EgonHeatTimeseries>` is created and filled
1226
      * :py:class:`demand.egon_daily_heat_demand_per_climate_zone <egon.data.datasets.heat_demand_timeseries.daily.EgonDailyHeatDemandPerClimateZone>`
1227
        is created and filled
1228
      * :py:class:`boundaries.egon_map_zensus_climate_zones <egon.data.datasets.heat_demand_timeseries.daily.EgonMapZensusClimateZones>` is created and filled
1229
1230
    """
1231
1232
    #:
1233
    name: str = "HeatTimeSeries"
1234
    #:
1235
    version: str = "0.0.12"
1236
1237
    def __init__(self, dependencies):
1238
        super().__init__(
1239
            name=self.name,
1240
            version=self.version,
1241
            dependencies=dependencies,
1242
            tasks=(
1243
                {
1244
                    export_etrago_cts_heat_profiles,
1245
                    map_climate_zones_to_zensus,
1246
                    daily_demand_shares_per_climate_zone,
1247
                    create,
1248
                },
1249
                select,
1250
                district_heating,
1251
                metadata,
1252
                store_national_profiles,
1253
            ),
1254
        )
1255