Passed
Pull Request — dev (#934)
by
unknown
01:36
created

sanitycheck_eGon2035_heat()   B

Complexity

Conditions 1

Size

Total Lines 217
Code Lines 86

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
eloc 86
dl 0
loc 217
rs 7.4581
c 0
b 0
f 0
cc 1
nop 0

How to fix   Long Method   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

1
"""
2
This module does sanity checks for both the eGon2035 and the eGon100RE scenario seperately where a percentage
3
error is given to showcase difference in output and input values. Please note that there are missing input technologies in the supply tables.
4
 Authors: @ALonso, @dana
5
"""
6
from math import isclose
7
from pathlib import Path
8
9
from loguru import logger
10
from sqlalchemy import Numeric
11
from sqlalchemy.sql import and_, cast, func, or_
12
import matplotlib.pyplot as plt
13
import numpy as np
14
import pandas as pd
15
import seaborn as sns
16
17
from egon.data import config, db
18
from egon.data.datasets import Dataset
19
from egon.data.datasets.emobility.motorized_individual_travel.db_classes import (
20
    EgonEvCountMunicipality,
21
    EgonEvCountMvGridDistrict,
22
    EgonEvCountRegistrationDistrict,
23
    EgonEvMvGridDistrict,
24
    EgonEvPool,
25
    EgonEvTrip,
26
)
27
from egon.data.datasets.emobility.motorized_individual_travel.helpers import (
28
    DATASET_CFG,
29
    read_simbev_metadata_file,
30
)
31
from egon.data.datasets.etrago_setup import (
32
    EgonPfHvLink,
33
    EgonPfHvLinkTimeseries,
34
    EgonPfHvLoad,
35
    EgonPfHvLoadTimeseries,
36
    EgonPfHvStore,
37
    EgonPfHvStoreTimeseries,
38
)
39
from egon.data.datasets.power_plants.pv_rooftop_buildings import (
40
    municipality_data,
41
    osm_buildings,
42
    scenario_data,
43
)
44
from egon.data.datasets.scenario_parameters import get_sector_parameters
45
import egon.data
46
47
TESTMODE_OFF = (
48
    config.settings()["egon-data"]["--dataset-boundary"] == "Everything"
49
)
50
51
52
class SanityChecks(Dataset):
53
    def __init__(self, dependencies):
54
        super().__init__(
55
            name="SanityChecks",
56
            version="0.0.4",
57
            dependencies=dependencies,
58
            tasks={
59
                sanitycheck_eGon2035_electricity,
60
                sanitycheck_eGon2035_heat,
61
                sanitycheck_emobility_mit,
62
            },
63
        )
64
65
66
def sanitycheck_eGon2035_electricity():
67
    """Execute basic sanity checks.
68
69
    Returns print statements as sanity checks for the electricity sector in
70
    the eGon2035 scenario.
71
72
    Parameters
73
    ----------
74
    None
75
76
    Returns
77
    -------
78
    None
79
    """
80
81
    scn = "eGon2035"
82
83
    # Section to check generator capacities
84
    print(f"Sanity checks for scenario {scn}")
85
    print(
86
        "For German electricity generators the following deviations between the inputs and outputs can be observed:"
87
    )
88
89
    carriers_electricity = [
90
        "other_non_renewable",
91
        "other_renewable",
92
        "reservoir",
93
        "run_of_river",
94
        "oil",
95
        "wind_onshore",
96
        "wind_offshore",
97
        "solar",
98
        "solar_rooftop",
99
        "biomass",
100
    ]
101
102
    for carrier in carriers_electricity:
103
104
        if carrier == "biomass":
105
            sum_output = db.select_dataframe(
106
                """SELECT scn_name, SUM(p_nom::numeric) as output_capacity_mw
107
                    FROM grid.egon_etrago_generator
108
                    WHERE bus IN (
109
                        SELECT bus_id FROM grid.egon_etrago_bus
110
                        WHERE scn_name = 'eGon2035'
111
                        AND country = 'DE')
112
                    AND carrier IN ('biomass', 'industrial_biomass_CHP', 'central_biomass_CHP')
113
                    GROUP BY (scn_name);
114
                """,
115
                warning=False,
116
            )
117
118
        else:
119
            sum_output = db.select_dataframe(
120
                f"""SELECT scn_name, SUM(p_nom::numeric) as output_capacity_mw
121
                         FROM grid.egon_etrago_generator
122
                         WHERE scn_name = '{scn}'
123
                         AND carrier IN ('{carrier}')
124
                         AND bus IN
125
                             (SELECT bus_id
126
                               FROM grid.egon_etrago_bus
127
                               WHERE scn_name = 'eGon2035'
128
                               AND country = 'DE')
129
                         GROUP BY (scn_name);
130
                    """,
131
                warning=False,
132
            )
133
134
        sum_input = db.select_dataframe(
135
            f"""SELECT carrier, SUM(capacity::numeric) as input_capacity_mw
136
                     FROM supply.egon_scenario_capacities
137
                     WHERE carrier= '{carrier}'
138
                     AND scenario_name ='{scn}'
139
                     GROUP BY (carrier);
140
                """,
141
            warning=False,
142
        )
143
144 View Code Duplication
        if (
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
145
            sum_output.output_capacity_mw.sum() == 0
146
            and sum_input.input_capacity_mw.sum() == 0
147
        ):
148
            print(
149
                f"No capacity for carrier '{carrier}' needed to be distributed. "
150
                f"Everything is fine"
151
            )
152
153
        elif (
154
            sum_input.input_capacity_mw.sum() > 0
155
            and sum_output.output_capacity_mw.sum() == 0
156
        ):
157
            print(
158
                f"Error: Capacity for carrier '{carrier}' was not distributed at all!"
159
            )
160
161
        elif (
162
            sum_output.output_capacity_mw.sum() > 0
163
            and sum_input.input_capacity_mw.sum() == 0
164
        ):
165
            print(
166
                f"Error: Eventhough no input capacity was provided for carrier '{carrier}' a capacity got distributed!"
167
            )
168
169
        else:
170
            sum_input["error"] = (
171
                (sum_output.output_capacity_mw - sum_input.input_capacity_mw)
172
                / sum_input.input_capacity_mw
173
            ) * 100
174
            g = sum_input["error"].values[0]
175
176
            print(f"{carrier}: " + str(round(g, 2)) + " %")
177
178
    # Section to check storage units
179
180
    print(f"Sanity checks for scenario {scn}")
181
    print(
182
        "For German electrical storage units the following deviations between the inputs and outputs can be observed:"
183
    )
184
185
    carriers_electricity = ["pumped_hydro"]
186
187
    for carrier in carriers_electricity:
188
189
        sum_output = db.select_dataframe(
190
            f"""SELECT scn_name, SUM(p_nom::numeric) as output_capacity_mw
191
                         FROM grid.egon_etrago_storage
192
                         WHERE scn_name = '{scn}'
193
                         AND carrier IN ('{carrier}')
194
                         AND bus IN
195
                             (SELECT bus_id
196
                               FROM grid.egon_etrago_bus
197
                               WHERE scn_name = 'eGon2035'
198
                               AND country = 'DE')
199
                         GROUP BY (scn_name);
200
                    """,
201
            warning=False,
202
        )
203
204
        sum_input = db.select_dataframe(
205
            f"""SELECT carrier, SUM(capacity::numeric) as input_capacity_mw
206
                     FROM supply.egon_scenario_capacities
207
                     WHERE carrier= '{carrier}'
208
                     AND scenario_name ='{scn}'
209
                     GROUP BY (carrier);
210
                """,
211
            warning=False,
212
        )
213
214 View Code Duplication
        if (
0 ignored issues
show
Duplication introduced by
This code seems to be duplicated in your project.
Loading history...
215
            sum_output.output_capacity_mw.sum() == 0
216
            and sum_input.input_capacity_mw.sum() == 0
217
        ):
218
            print(
219
                f"No capacity for carrier '{carrier}' needed to be distributed. Everything is fine"
220
            )
221
222
        elif (
223
            sum_input.input_capacity_mw.sum() > 0
224
            and sum_output.output_capacity_mw.sum() == 0
225
        ):
226
            print(
227
                f"Error: Capacity for carrier '{carrier}' was not distributed at all!"
228
            )
229
230
        elif (
231
            sum_output.output_capacity_mw.sum() > 0
232
            and sum_input.input_capacity_mw.sum() == 0
233
        ):
234
            print(
235
                f"Error: Eventhough no input capacity was provided for carrier '{carrier}' a capacity got distributed!"
236
            )
237
238
        else:
239
            sum_input["error"] = (
240
                (sum_output.output_capacity_mw - sum_input.input_capacity_mw)
241
                / sum_input.input_capacity_mw
242
            ) * 100
243
            g = sum_input["error"].values[0]
244
245
            print(f"{carrier}: " + str(round(g, 2)) + " %")
246
247
    # Section to check loads
248
249
    print(
250
        "For German electricity loads the following deviations between the input and output can be observed:"
251
    )
252
253
    output_demand = db.select_dataframe(
254
        """SELECT a.scn_name, a.carrier,  SUM((SELECT SUM(p) FROM UNNEST(b.p_set) p))/1000000::numeric as load_twh
255
            FROM grid.egon_etrago_load a
256
            JOIN grid.egon_etrago_load_timeseries b
257
            ON (a.load_id = b.load_id)
258
            JOIN grid.egon_etrago_bus c
259
            ON (a.bus=c.bus_id)
260
            AND b.scn_name = 'eGon2035'
261
            AND a.scn_name = 'eGon2035'
262
            AND a.carrier = 'AC'
263
            AND c.scn_name= 'eGon2035'
264
            AND c.country='DE'
265
            GROUP BY (a.scn_name, a.carrier);
266
267
    """,
268
        warning=False,
269
    )["load_twh"].values[0]
270
271
    input_cts_ind = db.select_dataframe(
272
        """SELECT scenario, SUM(demand::numeric/1000000) as demand_mw_regio_cts_ind
273
            FROM demand.egon_demandregio_cts_ind
274
            WHERE scenario= 'eGon2035'
275
            AND year IN ('2035')
276
            GROUP BY (scenario);
277
278
        """,
279
        warning=False,
280
    )["demand_mw_regio_cts_ind"].values[0]
281
282
    input_hh = db.select_dataframe(
283
        """SELECT scenario, SUM(demand::numeric/1000000) as demand_mw_regio_hh
284
            FROM demand.egon_demandregio_hh
285
            WHERE scenario= 'eGon2035'
286
            AND year IN ('2035')
287
            GROUP BY (scenario);
288
        """,
289
        warning=False,
290
    )["demand_mw_regio_hh"].values[0]
291
292
    input_demand = input_hh + input_cts_ind
293
294
    e = round((output_demand - input_demand) / input_demand, 2) * 100
295
296
    print(f"electricity demand: {e} %")
297
298
299
def sanitycheck_eGon2035_heat():
300
    """Execute basic sanity checks.
301
302
    Returns print statements as sanity checks for the heat sector in
303
    the eGon2035 scenario.
304
305
    Parameters
306
    ----------
307
    None
308
309
    Returns
310
    -------
311
    None
312
    """
313
314
    # Check input and output values for the carriers "other_non_renewable",
315
    # "other_renewable", "reservoir", "run_of_river" and "oil"
316
317
    scn = "eGon2035"
318
319
    # Section to check generator capacities
320
    print(f"Sanity checks for scenario {scn}")
321
    print(
322
        "For German heat demands the following deviations between the inputs and outputs can be observed:"
323
    )
324
325
    # Sanity checks for heat demand
326
327
    output_heat_demand = db.select_dataframe(
328
        """SELECT a.scn_name,  (SUM((SELECT SUM(p) FROM UNNEST(b.p_set) p))/1000000)::numeric as load_twh
329
            FROM grid.egon_etrago_load a
330
            JOIN grid.egon_etrago_load_timeseries b
331
            ON (a.load_id = b.load_id)
332
            JOIN grid.egon_etrago_bus c
333
            ON (a.bus=c.bus_id)
334
            AND b.scn_name = 'eGon2035'
335
            AND a.scn_name = 'eGon2035'
336
            AND c.scn_name= 'eGon2035'
337
            AND c.country='DE'
338
            AND a.carrier IN ('rural_heat', 'central_heat')
339
            GROUP BY (a.scn_name);
340
        """,
341
        warning=False,
342
    )["load_twh"].values[0]
343
344
    input_heat_demand = db.select_dataframe(
345
        """SELECT scenario, SUM(demand::numeric/1000000) as demand_mw_peta_heat
346
            FROM demand.egon_peta_heat
347
            WHERE scenario= 'eGon2035'
348
            GROUP BY (scenario);
349
        """,
350
        warning=False,
351
    )["demand_mw_peta_heat"].values[0]
352
353
    e_demand = (
354
        round((output_heat_demand - input_heat_demand) / input_heat_demand, 2)
355
        * 100
356
    )
357
358
    print(f"heat demand: {e_demand} %")
359
360
    # Sanity checks for heat supply
361
362
    print(
363
        "For German heat supplies the following deviations between the inputs and "
364
        "outputs can be observed:"
365
    )
366
367
    # Comparison for central heat pumps
368
    heat_pump_input = db.select_dataframe(
369
        """SELECT carrier, SUM(capacity::numeric) as Urban_central_heat_pump_mw
370
            FROM supply.egon_scenario_capacities
371
            WHERE carrier= 'urban_central_heat_pump'
372
            AND scenario_name IN ('eGon2035')
373
            GROUP BY (carrier);
374
        """,
375
        warning=False,
376
    )["urban_central_heat_pump_mw"].values[0]
377
378
    heat_pump_output = db.select_dataframe(
379
        """SELECT carrier, SUM(p_nom::numeric) as Central_heat_pump_mw
380
            FROM grid.egon_etrago_link
381
            WHERE carrier= 'central_heat_pump'
382
            AND scn_name IN ('eGon2035')
383
            GROUP BY (carrier);
384
    """,
385
        warning=False,
386
    )["central_heat_pump_mw"].values[0]
387
388
    e_heat_pump = (
389
        round((heat_pump_output - heat_pump_input) / heat_pump_output, 2) * 100
390
    )
391
392
    print(f"'central_heat_pump': {e_heat_pump} % ")
393
394
    # Comparison for residential heat pumps
395
396
    input_residential_heat_pump = db.select_dataframe(
397
        """SELECT carrier, SUM(capacity::numeric) as residential_heat_pump_mw
398
            FROM supply.egon_scenario_capacities
399
            WHERE carrier= 'residential_rural_heat_pump'
400
            AND scenario_name IN ('eGon2035')
401
            GROUP BY (carrier);
402
        """,
403
        warning=False,
404
    )["residential_heat_pump_mw"].values[0]
405
406
    output_residential_heat_pump = db.select_dataframe(
407
        """SELECT carrier, SUM(p_nom::numeric) as rural_heat_pump_mw
408
            FROM grid.egon_etrago_link
409
            WHERE carrier= 'rural_heat_pump'
410
            AND scn_name IN ('eGon2035')
411
            GROUP BY (carrier);
412
    """,
413
        warning=False,
414
    )["rural_heat_pump_mw"].values[0]
415
416
    e_residential_heat_pump = (
417
        round(
418
            (output_residential_heat_pump - input_residential_heat_pump)
419
            / input_residential_heat_pump,
420
            2,
421
        )
422
        * 100
423
    )
424
    print(f"'residential heat pumps': {e_residential_heat_pump} %")
425
426
    # Comparison for resistive heater
427
    resistive_heater_input = db.select_dataframe(
428
        """SELECT carrier, SUM(capacity::numeric) as Urban_central_resistive_heater_MW
429
            FROM supply.egon_scenario_capacities
430
            WHERE carrier= 'urban_central_resistive_heater'
431
            AND scenario_name IN ('eGon2035')
432
            GROUP BY (carrier);
433
        """,
434
        warning=False,
435
    )["urban_central_resistive_heater_mw"].values[0]
436
437
    resistive_heater_output = db.select_dataframe(
438
        """SELECT carrier, SUM(p_nom::numeric) as central_resistive_heater_MW
439
            FROM grid.egon_etrago_link
440
            WHERE carrier= 'central_resistive_heater'
441
            AND scn_name IN ('eGon2035')
442
            GROUP BY (carrier);
443
        """,
444
        warning=False,
445
    )["central_resistive_heater_mw"].values[0]
446
447
    e_resistive_heater = (
448
        round(
449
            (resistive_heater_output - resistive_heater_input)
450
            / resistive_heater_input,
451
            2,
452
        )
453
        * 100
454
    )
455
456
    print(f"'resistive heater': {e_resistive_heater} %")
457
458
    # Comparison for solar thermal collectors
459
460
    input_solar_thermal = db.select_dataframe(
461
        """SELECT carrier, SUM(capacity::numeric) as solar_thermal_collector_mw
462
            FROM supply.egon_scenario_capacities
463
            WHERE carrier= 'urban_central_solar_thermal_collector'
464
            AND scenario_name IN ('eGon2035')
465
            GROUP BY (carrier);
466
        """,
467
        warning=False,
468
    )["solar_thermal_collector_mw"].values[0]
469
470
    output_solar_thermal = db.select_dataframe(
471
        """SELECT carrier, SUM(p_nom::numeric) as solar_thermal_collector_mw
472
            FROM grid.egon_etrago_generator
473
            WHERE carrier= 'solar_thermal_collector'
474
            AND scn_name IN ('eGon2035')
475
            GROUP BY (carrier);
476
        """,
477
        warning=False,
478
    )["solar_thermal_collector_mw"].values[0]
479
480
    e_solar_thermal = (
481
        round(
482
            (output_solar_thermal - input_solar_thermal) / input_solar_thermal,
483
            2,
484
        )
485
        * 100
486
    )
487
    print(f"'solar thermal collector': {e_solar_thermal} %")
488
489
    # Comparison for geothermal
490
491
    input_geo_thermal = db.select_dataframe(
492
        """SELECT carrier, SUM(capacity::numeric) as Urban_central_geo_thermal_MW
493
            FROM supply.egon_scenario_capacities
494
            WHERE carrier= 'urban_central_geo_thermal'
495
            AND scenario_name IN ('eGon2035')
496
            GROUP BY (carrier);
497
        """,
498
        warning=False,
499
    )["urban_central_geo_thermal_mw"].values[0]
500
501
    output_geo_thermal = db.select_dataframe(
502
        """SELECT carrier, SUM(p_nom::numeric) as geo_thermal_MW
503
            FROM grid.egon_etrago_generator
504
            WHERE carrier= 'geo_thermal'
505
            AND scn_name IN ('eGon2035')
506
            GROUP BY (carrier);
507
    """,
508
        warning=False,
509
    )["geo_thermal_mw"].values[0]
510
511
    e_geo_thermal = (
512
        round((output_geo_thermal - input_geo_thermal) / input_geo_thermal, 2)
513
        * 100
514
    )
515
    print(f"'geothermal': {e_geo_thermal} %")
516
517
518
def sanitycheck_pv_rooftop_buildings():
519
    def egon_power_plants_pv_roof_building():
520
        sql = """
521
        SELECT *
522
        FROM supply.egon_power_plants_pv_roof_building
523
        """
524
525
        return db.select_dataframe(sql, index_col="index")
526
527
    pv_roof_df = egon_power_plants_pv_roof_building()
528
529
    municipalities_gdf = municipality_data()
530
531
    osm_buildings_gdf = osm_buildings(municipalities_gdf.crs)
532
533
    merge_df = pv_roof_df.merge(
534
        osm_buildings_gdf[["area"]],
535
        how="left",
536
        left_on="building_id",
537
        right_index=True,
538
    )
539
540
    assert len(merge_df.loc[merge_df.area.isna()]) == 0
541
542
    scenarios = ["status_quo", "eGon2035"]
543
544
    base_path = Path(egon.data.__path__[0]).resolve()
545
546
    res_dir = base_path / "sanity_checks"
547
548
    res_dir.mkdir(parents=True, exist_ok=True)
549
550
    for scenario in scenarios:
551
        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 8))
552
553
        scenario_df = merge_df.loc[merge_df.scenario == scenario]
554
555
        logger.info(
556
            scenario + " Capacity:\n" + str(scenario_df.capacity.describe())
557
        )
558
559
        small_gens_df = scenario_df.loc[scenario_df.capacity < 100]
560
561
        sns.histplot(data=small_gens_df, x="capacity", ax=ax1).set_title(
562
            scenario
563
        )
564
565
        sns.scatterplot(
566
            data=small_gens_df, x="capacity", y="area", ax=ax2
567
        ).set_title(scenario)
568
569
        plt.tight_layout()
570
571
        plt.savefig(
572
            res_dir / f"{scenario}_pv_rooftop_distribution.png",
573
            bbox_inches="tight",
574
        )
575
576
    scenarios = ["eGon2035"]  # "eGon100RE"]
577
578
    for scenario in scenarios:
579
        assert isclose(
580
            scenario_data(scenario=scenario).capacity.sum() * 1000,
581
            merge_df.loc[merge_df.scenario == scenario].capacity.sum(),
582
            rel_tol=1e-02,
583
        ), (
584
            f"{scenario_data(scenario=scenario).capacity.sum() * 1000} != "
585
            f"{merge_df.loc[merge_df.scenario == scenario].capacity.sum()}"
586
        )
587
588
589
def sanitycheck_emobility_mit():
590
    """Execute sanity checks for eMobility: motorized individual travel
591
592
    Checks data integrity for eGon2035, eGon2035_lowflex and eGon100RE scenario
593
    using assertions:
594
      1. Allocated EV numbers and EVs allocated to grid districts
595
      2. Trip data (original inout data from simBEV)
596
      3. Model data in eTraGo PF tables (grid.egon_etrago_*)
597
598
    Parameters
599
    ----------
600
    None
601
602
    Returns
603
    -------
604
    None
605
    """
606
607
    def check_ev_allocation():
608
        # Get target number for scenario
609
        ev_count_target = scenario_variation_parameters["ev_count"]
610
        print(f"  Target count: {str(ev_count_target)}")
611
612
        # Get allocated numbers
613
        ev_counts_dict = {}
614
        with db.session_scope() as session:
615
            for table, level in zip(
616
                [
617
                    EgonEvCountMvGridDistrict,
618
                    EgonEvCountMunicipality,
619
                    EgonEvCountRegistrationDistrict,
620
                ],
621
                ["Grid District", "Municipality", "Registration District"],
622
            ):
623
                query = session.query(
624
                    func.sum(
625
                        table.bev_mini
626
                        + table.bev_medium
627
                        + table.bev_luxury
628
                        + table.phev_mini
629
                        + table.phev_medium
630
                        + table.phev_luxury
631
                    ).label("ev_count")
632
                ).filter(
633
                    table.scenario == scenario_name,
634
                    table.scenario_variation == scenario_var_name,
635
                )
636
637
                ev_counts = pd.read_sql(
638
                    query.statement, query.session.bind, index_col=None
639
                )
640
                ev_counts_dict[level] = ev_counts.iloc[0].ev_count
641
                print(
642
                    f"    Count table: Total count for level {level} "
643
                    f"(table: {table.__table__}): "
644
                    f"{str(ev_counts_dict[level])}"
645
                )
646
647
        # Compare with scenario target (only if not in testmode)
648
        if TESTMODE_OFF:
649
            for level, count in ev_counts_dict.items():
650
                np.testing.assert_allclose(
651
                    count,
652
                    ev_count_target,
653
                    rtol=0.0001,
654
                    err_msg=f"EV numbers in {level} seems to be flawed.",
655
                )
656
        else:
657
            print("    Testmode is on, skipping sanity check...")
658
659
        # Get allocated EVs in grid districts
660
        with db.session_scope() as session:
661
            query = session.query(
662
                func.count(EgonEvMvGridDistrict.egon_ev_pool_ev_id).label(
663
                    "ev_count"
664
                ),
665
            ).filter(
666
                EgonEvMvGridDistrict.scenario == scenario_name,
667
                EgonEvMvGridDistrict.scenario_variation == scenario_var_name,
668
            )
669
        ev_count_alloc = (
670
            pd.read_sql(query.statement, query.session.bind, index_col=None)
671
            .iloc[0]
672
            .ev_count
673
        )
674
        print(
675
            f"    EVs allocated to Grid Districts "
676
            f"(table: {EgonEvMvGridDistrict.__table__}) total count: "
677
            f"{str(ev_count_alloc)}"
678
        )
679
680
        # Compare with scenario target (only if not in testmode)
681
        if TESTMODE_OFF:
682
            np.testing.assert_allclose(
683
                ev_count_alloc,
684
                ev_count_target,
685
                rtol=0.0001,
686
                err_msg=(
687
                    "EV numbers allocated to Grid Districts seems to be flawed."
688
                ),
689
            )
690
        else:
691
            print("    Testmode is on, skipping sanity check...")
692
693
        return ev_count_alloc
694
695
    def check_trip_data():
696
        # Check if trips start at timestep 0 and have a max. of 35040 steps
697
        # (8760h in 15min steps)
698
        print("  Checking timeranges...")
699
        with db.session_scope() as session:
700
            query = session.query(
701
                func.count(EgonEvTrip.event_id).label("cnt")
702
            ).filter(
703
                or_(
704
                    and_(
705
                        EgonEvTrip.park_start > 0,
706
                        EgonEvTrip.simbev_event_id == 0,
707
                    ),
708
                    EgonEvTrip.park_end
709
                    > (60 / int(meta_run_config.stepsize)) * 8760,
710
                ),
711
                EgonEvTrip.scenario == scenario_name,
712
            )
713
        invalid_trips = pd.read_sql(
714
            query.statement, query.session.bind, index_col=None
715
        )
716
        np.testing.assert_equal(
717
            invalid_trips.iloc[0].cnt,
718
            0,
719
            err_msg=(
720
                f"{str(invalid_trips.iloc[0].cnt)} trips in table "
721
                f"{EgonEvTrip.__table__} have invalid timesteps."
722
            ),
723
        )
724
725
        # Check if charging demand can be covered by available charging energy
726
        # while parking
727
        print("  Compare charging demand with available power...")
728
        with db.session_scope() as session:
729
            query = session.query(
730
                func.count(EgonEvTrip.event_id).label("cnt")
731
            ).filter(
732
                func.round(
733
                    cast(
734
                        (EgonEvTrip.park_end - EgonEvTrip.park_start + 1)
735
                        * EgonEvTrip.charging_capacity_nominal
736
                        * (int(meta_run_config.stepsize) / 60),
737
                        Numeric,
738
                    ),
739
                    3,
740
                )
741
                < cast(EgonEvTrip.charging_demand, Numeric),
742
                EgonEvTrip.scenario == scenario_name,
743
            )
744
        invalid_trips = pd.read_sql(
745
            query.statement, query.session.bind, index_col=None
746
        )
747
        np.testing.assert_equal(
748
            invalid_trips.iloc[0].cnt,
749
            0,
750
            err_msg=(
751
                f"In {str(invalid_trips.iloc[0].cnt)} trips (table: "
752
                f"{EgonEvTrip.__table__}) the charging demand cannot be "
753
                f"covered by available charging power."
754
            ),
755
        )
756
757
    def check_model_data():
758
        # Check if model components were fully created
759
        print("  Check if all model components were created...")
760
        # Get MVGDs which got EV allocated
761
        with db.session_scope() as session:
762
            query = (
763
                session.query(
764
                    EgonEvMvGridDistrict.bus_id,
765
                )
766
                .filter(
767
                    EgonEvMvGridDistrict.scenario == scenario_name,
768
                    EgonEvMvGridDistrict.scenario_variation
769
                    == scenario_var_name,
770
                )
771
                .group_by(EgonEvMvGridDistrict.bus_id)
772
            )
773
        mvgds_with_ev = (
774
            pd.read_sql(query.statement, query.session.bind, index_col=None)
775
            .bus_id.sort_values()
776
            .to_list()
777
        )
778
779
        # Load model components
780
        with db.session_scope() as session:
781
            query = (
782
                session.query(
783
                    EgonPfHvLink.bus0.label("mvgd_bus_id"),
784
                    EgonPfHvLoad.bus.label("emob_bus_id"),
785
                    EgonPfHvLoad.load_id.label("load_id"),
786
                    EgonPfHvStore.store_id.label("store_id"),
787
                )
788
                .select_from(EgonPfHvLoad, EgonPfHvStore)
789
                .join(
790
                    EgonPfHvLoadTimeseries,
791
                    EgonPfHvLoadTimeseries.load_id == EgonPfHvLoad.load_id,
792
                )
793
                .join(
794
                    EgonPfHvStoreTimeseries,
795
                    EgonPfHvStoreTimeseries.store_id == EgonPfHvStore.store_id,
796
                )
797
                .filter(
798
                    EgonPfHvLoad.carrier == "land transport EV",
799
                    EgonPfHvLoad.scn_name == scenario_name,
800
                    EgonPfHvLoadTimeseries.scn_name == scenario_name,
801
                    EgonPfHvStore.carrier == "battery storage",
802
                    EgonPfHvStore.scn_name == scenario_name,
803
                    EgonPfHvStoreTimeseries.scn_name == scenario_name,
804
                    EgonPfHvLink.scn_name == scenario_name,
805
                    EgonPfHvLink.bus1 == EgonPfHvLoad.bus,
806
                    EgonPfHvLink.bus1 == EgonPfHvStore.bus,
807
                )
808
            )
809
        model_components = pd.read_sql(
810
            query.statement, query.session.bind, index_col=None
811
        )
812
813
        # Check number of buses with model components connected
814
        mvgd_buses_with_ev = model_components.loc[
815
            model_components.mvgd_bus_id.isin(mvgds_with_ev)
816
        ]
817
        np.testing.assert_equal(
818
            len(mvgds_with_ev),
819
            len(mvgd_buses_with_ev),
820
            err_msg=(
821
                f"Number of Grid Districts with connected model components "
822
                f"({str(len(mvgd_buses_with_ev))} in tables egon_etrago_*) "
823
                f"differ from number of Grid Districts that got EVs "
824
                f"allocated ({len(mvgds_with_ev)} in table "
825
                f"{EgonEvMvGridDistrict.__table__})."
826
            ),
827
        )
828
829
        # Check if all required components exist (if no id is NaN)
830
        np.testing.assert_equal(
831
            model_components.drop_duplicates().isna().any().any(),
832
            False,
833
            err_msg=(
834
                f"Some components are missing (see True values): "
835
                f"{model_components.drop_duplicates().isna().any()}"
836
            ),
837
        )
838
839
        # Get all model timeseries
840
        print("  Loading model timeseries...")
841
        # Get all model timeseries
842
        model_ts_dict = {
843
            "Load": {
844
                "carrier": "land transport EV",
845
                "table": EgonPfHvLoad,
846
                "table_ts": EgonPfHvLoadTimeseries,
847
                "column_id": "load_id",
848
                "columns_ts": ["p_set"],
849
                "ts": None,
850
            },
851
            "Link": {
852
                "carrier": "BEV charger",
853
                "table": EgonPfHvLink,
854
                "table_ts": EgonPfHvLinkTimeseries,
855
                "column_id": "link_id",
856
                "columns_ts": ["p_max_pu"],
857
                "ts": None,
858
            },
859
            "Store": {
860
                "carrier": "battery storage",
861
                "table": EgonPfHvStore,
862
                "table_ts": EgonPfHvStoreTimeseries,
863
                "column_id": "store_id",
864
                "columns_ts": ["e_min_pu", "e_max_pu"],
865
                "ts": None,
866
            },
867
        }
868
869
        with db.session_scope() as session:
870
            for node, attrs in model_ts_dict.items():
871
                print(f"    Loading {node} timeseries...")
872
                subquery = (
873
                    session.query(getattr(attrs["table"], attrs["column_id"]))
874
                    .filter(attrs["table"].carrier == attrs["carrier"])
875
                    .filter(attrs["table"].scn_name == scenario_name)
876
                    .subquery()
877
                )
878
879
                cols = [
880
                    getattr(attrs["table_ts"], c) for c in attrs["columns_ts"]
881
                ]
882
                query = session.query(
883
                    getattr(attrs["table_ts"], attrs["column_id"]), *cols
884
                ).filter(
885
                    getattr(attrs["table_ts"], attrs["column_id"]).in_(
886
                        subquery
887
                    ),
888
                    attrs["table_ts"].scn_name == scenario_name,
889
                )
890
                attrs["ts"] = pd.read_sql(
891
                    query.statement,
892
                    query.session.bind,
893
                    index_col=attrs["column_id"],
894
                )
895
896
        # Check if all timeseries have 8760 steps
897
        print("    Checking timeranges...")
898
        for node, attrs in model_ts_dict.items():
899
            for col in attrs["columns_ts"]:
900
                ts = attrs["ts"]
901
                invalid_ts = ts.loc[ts[col].apply(lambda _: len(_)) != 8760][
902
                    col
903
                ].apply(len)
904
                np.testing.assert_equal(
905
                    len(invalid_ts),
906
                    0,
907
                    err_msg=(
908
                        f"{str(len(invalid_ts))} rows in timeseries do not "
909
                        f"have 8760 timesteps. Table: "
910
                        f"{attrs['table_ts'].__table__}, Column: {col}, IDs: "
911
                        f"{str(list(invalid_ts.index))}"
912
                    ),
913
                )
914
915
        # Compare total energy demand in model with some approximate values
916
        # (per EV: 14,000 km/a, 0.17 kWh/km)
917
        print("  Checking energy demand in model...")
918
        total_energy_model = (
919
            model_ts_dict["Load"]["ts"].p_set.apply(lambda _: sum(_)).sum()
920
            / 1e6
921
        )
922
        print(f"    Total energy amount in model: {total_energy_model} TWh")
923
        total_energy_scenario_approx = ev_count_alloc * 14000 * 0.17 / 1e9
924
        print(
925
            f"    Total approximated energy amount in scenario: "
926
            f"{total_energy_scenario_approx} TWh"
927
        )
928
        np.testing.assert_allclose(
929
            total_energy_model,
930
            total_energy_scenario_approx,
931
            rtol=0.1,
932
            err_msg=(
933
                "The total energy amount in the model deviates heavily "
934
                "from the approximated value for current scenario."
935
            ),
936
        )
937
938
        # Compare total storage capacity
939
        print("  Checking storage capacity...")
940
        # Load storage capacities from model
941
        with db.session_scope() as session:
942
            query = session.query(
943
                func.sum(EgonPfHvStore.e_nom).label("e_nom")
944
            ).filter(
945
                EgonPfHvStore.scn_name == scenario_name,
946
                EgonPfHvStore.carrier == "battery storage",
947
            )
948
        storage_capacity_model = (
949
            pd.read_sql(
950
                query.statement, query.session.bind, index_col=None
951
            ).e_nom.sum()
952
            / 1e3
953
        )
954
        print(
955
            f"    Total storage capacity ({EgonPfHvStore.__table__}): "
956
            f"{round(storage_capacity_model, 1)} GWh"
957
        )
958
959
        # Load occurences of each EV
960
        with db.session_scope() as session:
961
            query = (
962
                session.query(
963
                    EgonEvMvGridDistrict.bus_id,
964
                    EgonEvPool.type,
965
                    func.count(EgonEvMvGridDistrict.egon_ev_pool_ev_id).label(
966
                        "count"
967
                    ),
968
                )
969
                .join(
970
                    EgonEvPool,
971
                    EgonEvPool.ev_id
972
                    == EgonEvMvGridDistrict.egon_ev_pool_ev_id,
973
                )
974
                .filter(
975
                    EgonEvMvGridDistrict.scenario == scenario_name,
976
                    EgonEvMvGridDistrict.scenario_variation
977
                    == scenario_var_name,
978
                    EgonEvPool.scenario == scenario_name,
979
                )
980
                .group_by(EgonEvMvGridDistrict.bus_id, EgonEvPool.type)
981
            )
982
        count_per_ev_all = pd.read_sql(
983
            query.statement, query.session.bind, index_col="bus_id"
984
        )
985
        count_per_ev_all["bat_cap"] = count_per_ev_all.type.map(
986
            meta_tech_data.battery_capacity
987
        )
988
        count_per_ev_all["bat_cap_total_MWh"] = (
989
            count_per_ev_all["count"] * count_per_ev_all.bat_cap / 1e3
990
        )
991
        storage_capacity_simbev = count_per_ev_all.bat_cap_total_MWh.div(
992
            1e3
993
        ).sum()
994
        print(
995
            f"    Total storage capacity (simBEV): "
996
            f"{round(storage_capacity_simbev, 1)} GWh"
997
        )
998
999
        np.testing.assert_allclose(
1000
            storage_capacity_model,
1001
            storage_capacity_simbev,
1002
            rtol=0.01,
1003
            err_msg=(
1004
                "The total storage capacity in the model deviates heavily "
1005
                "from the input data provided by simBEV for current scenario."
1006
            ),
1007
        )
1008
1009
        # Check SoC storage constraint: e_min_pu < e_max_pu for all timesteps
1010
        print("  Validating SoC constraints...")
1011
        stores_with_invalid_soc = []
1012
        for idx, row in model_ts_dict["Store"]["ts"].iterrows():
1013
            ts = row[["e_min_pu", "e_max_pu"]]
1014
            x = np.array(ts.e_min_pu) > np.array(ts.e_max_pu)
1015
            if x.any():
1016
                stores_with_invalid_soc.append(idx)
1017
1018
        np.testing.assert_equal(
1019
            len(stores_with_invalid_soc),
1020
            0,
1021
            err_msg=(
1022
                f"The store constraint e_min_pu < e_max_pu does not apply "
1023
                f"for some storages in {EgonPfHvStoreTimeseries.__table__}. "
1024
                f"Invalid store_ids: {stores_with_invalid_soc}"
1025
            ),
1026
        )
1027
1028
    def check_model_data_lowflex_eGon2035():
1029
        # TODO: Add eGon100RE_lowflex
1030
        print("")
1031
        print("SCENARIO: eGon2035_lowflex")
1032
1033
        # Compare driving load and charging load
1034
        print("  Loading eGon2035 model timeseries: driving load...")
1035
        with db.session_scope() as session:
1036
            query = (
1037
                session.query(
1038
                    EgonPfHvLoad.load_id,
1039
                    EgonPfHvLoadTimeseries.p_set,
1040
                )
1041
                .join(
1042
                    EgonPfHvLoadTimeseries,
1043
                    EgonPfHvLoadTimeseries.load_id == EgonPfHvLoad.load_id,
1044
                )
1045
                .filter(
1046
                    EgonPfHvLoad.carrier == "land transport EV",
1047
                    EgonPfHvLoad.scn_name == "eGon2035",
1048
                    EgonPfHvLoadTimeseries.scn_name == "eGon2035",
1049
                )
1050
            )
1051
        model_driving_load = pd.read_sql(
1052
            query.statement, query.session.bind, index_col=None
1053
        )
1054
        driving_load = np.array(model_driving_load.p_set.to_list()).sum(axis=0)
1055
1056
        print(
1057
            "  Loading eGon2035_lowflex model timeseries: dumb charging "
1058
            "load..."
1059
        )
1060
        with db.session_scope() as session:
1061
            query = (
1062
                session.query(
1063
                    EgonPfHvLoad.load_id,
1064
                    EgonPfHvLoadTimeseries.p_set,
1065
                )
1066
                .join(
1067
                    EgonPfHvLoadTimeseries,
1068
                    EgonPfHvLoadTimeseries.load_id == EgonPfHvLoad.load_id,
1069
                )
1070
                .filter(
1071
                    EgonPfHvLoad.carrier == "land transport EV",
1072
                    EgonPfHvLoad.scn_name == "eGon2035_lowflex",
1073
                    EgonPfHvLoadTimeseries.scn_name == "eGon2035_lowflex",
1074
                )
1075
            )
1076
        model_charging_load_lowflex = pd.read_sql(
1077
            query.statement, query.session.bind, index_col=None
1078
        )
1079
        charging_load = np.array(
1080
            model_charging_load_lowflex.p_set.to_list()
1081
        ).sum(axis=0)
1082
1083
        # Ratio of driving and charging load should be 0.9 due to charging
1084
        # efficiency
1085
        print("  Compare cumulative loads...")
1086
        print(f"    Driving load (eGon2035): {driving_load.sum() / 1e6} TWh")
1087
        print(
1088
            f"    Dumb charging load (eGon2035_lowflex): "
1089
            f"{charging_load.sum() / 1e6} TWh"
1090
        )
1091
        driving_load_theoretical = (
1092
            float(meta_run_config.eta_cp) * charging_load.sum()
0 ignored issues
show
introduced by
The variable meta_run_config does not seem to be defined in case the for loop on line 1110 is not entered. Are you sure this can never be the case?
Loading history...
1093
        )
1094
        np.testing.assert_allclose(
1095
            driving_load.sum(),
1096
            driving_load_theoretical,
1097
            rtol=0.01,
1098
            err_msg=(
1099
                f"The driving load (eGon2035) deviates by more than 1% "
1100
                f"from the theoretical driving load calculated from charging "
1101
                f"load (eGon2035_lowflex) with an efficiency of "
1102
                f"{float(meta_run_config.eta_cp)}."
1103
            ),
1104
        )
1105
1106
    print("=====================================================")
1107
    print("=== SANITY CHECKS FOR MOTORIZED INDIVIDUAL TRAVEL ===")
1108
    print("=====================================================")
1109
1110
    for scenario_name in ["eGon2035", "eGon100RE"]:
1111
        scenario_var_name = DATASET_CFG["scenario"]["variation"][scenario_name]
1112
1113
        print("")
1114
        print(f"SCENARIO: {scenario_name}, VARIATION: {scenario_var_name}")
1115
1116
        # Load scenario params for scenario and scenario variation
1117
        scenario_variation_parameters = get_sector_parameters(
1118
            "mobility", scenario=scenario_name
1119
        )["motorized_individual_travel"][scenario_var_name]
1120
1121
        # Load simBEV run config and tech data
1122
        meta_run_config = read_simbev_metadata_file(
1123
            scenario_name, "config"
1124
        ).loc["basic"]
1125
        meta_tech_data = read_simbev_metadata_file(scenario_name, "tech_data")
1126
1127
        print("")
1128
        print("Checking EV counts...")
1129
        ev_count_alloc = check_ev_allocation()
1130
1131
        print("")
1132
        print("Checking trip data...")
1133
        check_trip_data()
1134
1135
        print("")
1136
        print("Checking model data...")
1137
        check_model_data()
1138
1139
    print("")
1140
    check_model_data_lowflex_eGon2035()
1141
1142
    print("=====================================================")
1143