openego /
eGon-data
| @@ 30-75 (lines=46) @@ | ||
| 27 | ||
| 28 | ||
| 29 | class ZensusPopulation(Dataset): |
|
| 30 | def __init__(self, dependencies): |
|
| 31 | super().__init__( |
|
| 32 | name="ZensusPopulation", |
|
| 33 | version="0.0.2", |
|
| 34 | dependencies=dependencies, |
|
| 35 | tasks=( |
|
| 36 | create_zensus_pop_table, |
|
| 37 | population_to_postgres, |
|
| 38 | ), |
|
| 39 | validation={ |
|
| 40 | "data-quality":[ |
|
| 41 | RowCountValidation( |
|
| 42 | table="society.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 43 | rule_id="TEST_ROW_COUNT.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 44 | expected_count={"Schleswig-Holstein": 145634, "Everything": 3206490} |
|
| 45 | ), |
|
| 46 | DataTypeValidation( |
|
| 47 | table="society.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 48 | rule_id="TEST_DATA_MULTIPLE_TYPES.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 49 | column_types={ |
|
| 50 | "grid_id": "character varying", "zensus_population_id": "integer", "building_count": "smallint", |
|
| 51 | "apartment_count": "smallint", "geom": "geometry", "geom_point": "geometry" |
|
| 52 | } |
|
| 53 | ), |
|
| 54 | NotNullAndNotNaNValidation( |
|
| 55 | table="society.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 56 | rule_id="TEST_NOT_NAN.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 57 | columns=["grid_id", "zensus_population_id", "building_count", "apartment_count", "geom", "geom_point"] |
|
| 58 | ), |
|
| 59 | WholeTableNotNullAndNotNaNValidation( |
|
| 60 | table="society.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 61 | rule_id="TEST_WHOLE_TABLE_NOT_NAN.egon_destatis_zensus_apartment_building_population_per_ha" |
|
| 62 | ), |
|
| 63 | SRIDUniqueNonZero( |
|
| 64 | table="society.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 65 | rule_id="SRIDUniqueNonZero.egon_destatis_zensus_apartment_building_population_per_ha.geom", |
|
| 66 | column="geom" |
|
| 67 | ), |
|
| 68 | SRIDUniqueNonZero( |
|
| 69 | table="society.egon_destatis_zensus_apartment_building_population_per_ha", |
|
| 70 | rule_id="SRIDUniqueNonZero.egon_destatis_zensus_apartment_building_population_per_ha.geom_point", |
|
| 71 | column="geom_point" |
|
| 72 | ), |
|
| 73 | ] |
|
| 74 | }, |
|
| 75 | on_validation_failure="continue" |
|
| 76 | ) |
|
| 77 | ||
| 78 | ||
| @@ 86-123 (lines=38) @@ | ||
| 83 | #: |
|
| 84 | version: str = "0.0.3" |
|
| 85 | ||
| 86 | def __init__(self, dependencies): |
|
| 87 | super().__init__( |
|
| 88 | name=self.name, |
|
| 89 | # version=self.target_files + "_0.0", |
|
| 90 | version=self.version, # maybe rethink the naming |
|
| 91 | dependencies=dependencies, |
|
| 92 | tasks=(create_tables, demarcation), |
|
| 93 | validation={ |
|
| 94 | "data_quality": [ |
|
| 95 | RowCountValidation( |
|
| 96 | table=" demand.egon_district_heating_areas", |
|
| 97 | rule_id="ROW_COUNT.egon_district_heating_areas", |
|
| 98 | expected_count={"Schleswig-Holstein": 100, "Everything": 6335} |
|
| 99 | ), |
|
| 100 | DataTypeValidation( |
|
| 101 | table="demand.egon_district_heating_areas", |
|
| 102 | rule_id="DATA_MULTIPLE_TYPES.egon_district_heating_areas", |
|
| 103 | column_types={"id": "integer", "area_id": "integer", "scenario": "character varying", |
|
| 104 | "geom_polygon": "geometry", "residential_and_service_demand": "double precision"} |
|
| 105 | ), |
|
| 106 | WholeTableNotNullAndNotNaNValidation( |
|
| 107 | table="demand.egon_district_heating_areas", |
|
| 108 | rule_id="WHOLE_TABLE_NOT_NAN.egon_district_heating_areas" |
|
| 109 | ), |
|
| 110 | ValueSetValidation( |
|
| 111 | table="demand.egon_district_heating_areas", |
|
| 112 | rule_id="VALUE_SET_VALIDATION_SCENARIO.egon_district_heating_areas", |
|
| 113 | column="scenario", |
|
| 114 | expected_values=["eGon2035", "eGon100RE"] |
|
| 115 | ), |
|
| 116 | SRIDUniqueNonZero( |
|
| 117 | table="demand.egon_district_heating_areas", |
|
| 118 | rule_id="SRIDUniqueNonZero.egon_district_heating_areas", |
|
| 119 | column="geom_polygon" |
|
| 120 | ), |
|
| 121 | ] |
|
| 122 | }, |
|
| 123 | on_validation_failure="continue" |
|
| 124 | ) |
|
| 125 | ||
| 126 | ||
