data.datasets.osm_buildings_streets.OsmBuildingsStreets.__init__() - Code Metrics - Inspection of "feature/validation_integration" - openego/eGon-data - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — dev (#1375)

unknown

created 2026-01-08 11:43 UTC

OsmBuildingsStreets.init() B

↳ Parent: data.datasets.osm_buildings_streets

Complexity

Conditions

Size

Total Lines	54
Code Lines	44

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	44
dl	0
loc	54
rs	8.824
c	0
b	0
f	0
cc	1
nop	2

How to fix Long Method

"""
Filtering and preprocessing of buildings, streets and amenities from OpenStreetMap

"""

import os

from egon.data import db
from egon.data.datasets import Dataset
from egon_validation import (
    RowCountValidation,
    DataTypeValidation,
    WholeTableNotNullAndNotNaNValidation
)


def execute_sql_script(script):
    """Execute SQL script

    Parameters
    ----------
    script : str
        Filename of script
    """
    db.execute_sql_script(os.path.join(os.path.dirname(__file__), script))


def preprocessing():
    print("Extracting buildings, amenities and shops...")
    sql_scripts = [
        "osm_amenities_shops_preprocessing.sql",
        "osm_buildings_extract.sql",
    ]
    for script in sql_scripts:
        execute_sql_script(script)


def filter_buildings():
    print("Filter buildings...")
    execute_sql_script("osm_buildings_filter.sql")


def filter_buildings_residential():
    print("Filter residential buildings...")
    execute_sql_script("osm_buildings_filter_residential.sql")


def extend_buildings_residential():
    print(
        "Extend residential buildings by commercial/retail/office/hotel "
        "buildings in cells with census population but without buildings..."
    )
    execute_sql_script("osm_buildings_extend_residential.sql")


def create_buildings_filtered_zensus_mapping():
    print(
        "Create census mapping table for filtered buildings in populated areas..."
    )
    execute_sql_script("osm_buildings_filtered_zensus_mapping.sql")


def create_buildings_filtered_all_zensus_mapping():
    print("Create census mapping table for all filtered buildings...")
    execute_sql_script("osm_buildings_filtered_all_zensus_mapping.sql")


def create_buildings_residential_zensus_mapping():
    print("Create census mapping table for residential buildings...")
    execute_sql_script("osm_buildings_residential_zensus_mapping.sql")


def create_buildings_temp_tables():
    print("Create temp tables for buildings...")
    execute_sql_script("osm_buildings_temp_tables.sql")


def extract_buildings_w_amenities():
    print("Extracting buildings with amenities...")
    execute_sql_script("osm_results_buildings_w_amenities.sql")


def extract_buildings_wo_amenities():
    print("Extracting buildings without amenities...")
    execute_sql_script("osm_results_buildings_wo_amenities.sql")


def extract_amenities():
    print("Extracting amenities...")
    execute_sql_script("osm_results_amenities.sql")


def extract_buildings_filtered_amenities():
    print("Extracting buildings filtered with and without amenities...")
    execute_sql_script("osm_buildings_filter_amenities.sql")


def extract_ways():
    print("Extracting ways...")
    execute_sql_script("osm_ways_preprocessing.sql")


def drop_temp_tables():
    print("Dropping temp tables...")
    execute_sql_script("drop_temp_tables.sql")


def add_metadata():
    pass


class OsmBuildingsStreets(Dataset):
    """
    Filter and preprocess buildings, streets and amenities from OpenStreetMap (OSM).

    This dataset on buildings and amenities is required by several tasks in the
    pipeline, such as the distribution of household demand profiles or PV home
    systems to buildings. This data is enriched by population and apartments from
    Zensus 2011. Those derived datasets and the data on streets will be used in the
    DIstribution Network Generat0r
    `ding0 <https://github.com/openego/ding0>`_ e.g. to cluster loads and create low
    voltage grids.

    *Dependencies*
      * :py:class:`OpenStreetMap <egon.data.datasets.osm.OpenStreetMap>`
      * :py:class:`ZensusMiscellaneous <egon.data.datasets.zensus.ZensusMiscellaneous>`

    *Resulting Tables*
      * openstreetmap.osm_buildings is created and filled (table has no associated python class)
      * openstreetmap.osm_buildings_filtered is created and filled (table has no associated python class)
      * openstreetmap.osm_buildings_residential is created and filled (table has no associated python class)
      * openstreetmap.osm_amenities_shops_filtered is created and filled (table has no associated python class)
      * openstreetmap.osm_buildings_with_amenities is created and filled (table has no associated python class)
      * openstreetmap.osm_buildings_without_amenities is created and filled (table has no associated python class)
      * openstreetmap.osm_amenities_not_in_buildings is created and filled (table has no associated python class)
      * openstreetmap.osm_ways_preprocessed is created and filled (table has no associated python class)
      * openstreetmap.osm_ways_with_segments is created and filled (table has no associated python class)
      * boundaries.egon_map_zensus_buildings_filtered is created and filled (table has no associated python class)
      * boundaries.egon_map_zensus_buildings_residential is created and filled (table has no associated python class)
      * openstreetmap.osm_buildings is created and filled (table has no associated python class)

    **Details and Steps**

    * Extract buildings and filter using relevant tags, e.g. residential and
      commercial, see script `osm_buildings_filter.sql` for the full list of tags.
      Resulting tables:
      * All buildings: `openstreetmap.osm_buildings`
      * Filtered buildings: `openstreetmap.osm_buildings_filtered`
      * Residential buildings: `openstreetmap.osm_buildings_residential`
        * 1st step: Filter by tags (see `osm_buildings_filter_residential.sql`)
        * 2nd step: Table is extended by finding census cells with population
          but no residential buildings and extended by commercial/retail/office/
          hotel buildings (see `osm_buildings_extend_residential.sql`) since they
          often include apartments as well.
    * Extract amenities and filter using relevant tags, e.g. shops and restaurants,
      see script `osm_amenities_shops_preprocessing.sql` for the full list of tags.
      Resulting table: `openstreetmap.osm_amenities_shops_filtered`
    * Create a mapping table for building's osm IDs to the Zensus cells the
      building's centroid is located in.
      Resulting tables:
      * `boundaries.egon_map_zensus_buildings_filtered` (filtered)
      * `boundaries.egon_map_zensus_buildings_residential` (residential only)
    * Enrich each building by number of apartments from Zensus table
      `society.egon_destatis_zensus_apartment_building_population_per_ha`
      by splitting up the cell's sum equally to the buildings. In some cases, a
      Zensus cell does not contain buildings but there's a building nearby which
      the no. of apartments is to be allocated to. To make sure apartments are
      allocated to at least one building, a radius of 77m is used to catch building
      geometries.
    * Split filtered buildings into 3 datasets using the amenities' locations:
      temporary tables are created in script `osm_buildings_temp_tables.sql` the
      final tables in `osm_buildings_amentities_results.sql`.
      Resulting tables:

      * Buildings w/ amenities: `openstreetmap.osm_buildings_with_amenities`
      * Buildings w/o amenities: `openstreetmap.osm_buildings_without_amenities`
      * Amenities not allocated to buildings:
        `openstreetmap.osm_amenities_not_in_buildings`
    * Extract streets (OSM ways) and filter using relevant tags, e.g.
      highway=secondary, see script `osm_ways_preprocessing.sql` for the full list
      of tags. Additionally, each way is split into its line segments and their
      lengths is retained.
      Resulting tables:
      * Filtered streets: `openstreetmap.osm_ways_preprocessed`
      * Filtered streets w/ segments: `openstreetmap.osm_ways_with_segments`

    """

    #:
    name: str = "OsmBuildingsStreets"
    #:
    version: str = "0.0.7"

    def __init__(self, dependencies):
        super().__init__(
            name=self.name,
            version=self.version,
            dependencies=dependencies,
            tasks=(
                preprocessing,
                {filter_buildings, filter_buildings_residential},
                extend_buildings_residential,
                extract_buildings_filtered_amenities,
                {
                    create_buildings_filtered_zensus_mapping,
                    create_buildings_residential_zensus_mapping,
                    create_buildings_filtered_all_zensus_mapping,
                },
                create_buildings_temp_tables,
                {
                    extract_buildings_w_amenities,
                    extract_buildings_wo_amenities,
                    extract_amenities,
                    extract_ways,
                },
                drop_temp_tables,
                add_metadata,
            ),
            validation={
                "data_quality": [
                    RowCountValidation(
                        table="boundaries.egon_map_zensus_buildings_filtered",
                        rule_id="TEST_ROW_COUNT",
                        expected_count=28070301
                    ),
                    DataTypeValidation(
                        table="boundaries.egon_map_zensus_buildings_filtered",
                        rule_id="TEST_DATA_MULTIPLE_TYPES",
                        column_types={"id": "integer", "cell_id": "integer"}
                    ),
                    WholeTableNotNullAndNotNaNValidation(
                        table="boundaries.egon_map_zensus_buildings_filtered",
                        rule_id="TEST_WHOLE_TABLE_NOT_NAN"
                    ),
                    RowCountValidation(
                        table="boundaries.egon_map_zensus_buildings_residential",
                        rule_id="TEST_ROW_COUNT",
                        expected_count=27477467
                    ),
                    DataTypeValidation(
                        table="boundaries.egon_map_zensus_buildings_residential",
                        rule_id="TEST_DATA_MULTIPLE_TYPES",
                        column_types={"id": "integer", "cell_id": "integer"}
                    ),
                    WholeTableNotNullAndNotNaNValidation(
                        table="boundaries.egon_map_zensus_buildings_residential",
                        rule_id="TEST_WHOLE_TABLE_NOT_NAN"
                    )
                ]
            }
        )


1			"""
2			Filtering and preprocessing of buildings, streets and amenities from OpenStreetMap
3
4			"""
5
6			import os
7
8			from egon.data import db
9			from egon.data.datasets import Dataset
10			from egon_validation import (
11			RowCountValidation,
12			DataTypeValidation,
13			WholeTableNotNullAndNotNaNValidation
14			)
15
16
17			def execute_sql_script(script):
18			"""Execute SQL script
19
20			Parameters
21			----------
22			script : str
23			Filename of script
24			"""
25			db.execute_sql_script(os.path.join(os.path.dirname(__file__), script))
26
27
28			def preprocessing():
29			print("Extracting buildings, amenities and shops...")
30			sql_scripts = [
31			"osm_amenities_shops_preprocessing.sql",
32			"osm_buildings_extract.sql",
33			]
34			for script in sql_scripts:
35			execute_sql_script(script)
36
37
38			def filter_buildings():
39			print("Filter buildings...")
40			execute_sql_script("osm_buildings_filter.sql")
41
42
43			def filter_buildings_residential():
44			print("Filter residential buildings...")
45			execute_sql_script("osm_buildings_filter_residential.sql")
46
47
48			def extend_buildings_residential():
49			print(
50			"Extend residential buildings by commercial/retail/office/hotel "
51			"buildings in cells with census population but without buildings..."
52			)
53			execute_sql_script("osm_buildings_extend_residential.sql")
54
55
56			def create_buildings_filtered_zensus_mapping():
57			print(
58			"Create census mapping table for filtered buildings in populated areas..."
59			)
60			execute_sql_script("osm_buildings_filtered_zensus_mapping.sql")
61
62
63			def create_buildings_filtered_all_zensus_mapping():
64			print("Create census mapping table for all filtered buildings...")
65			execute_sql_script("osm_buildings_filtered_all_zensus_mapping.sql")
66
67
68			def create_buildings_residential_zensus_mapping():
69			print("Create census mapping table for residential buildings...")
70			execute_sql_script("osm_buildings_residential_zensus_mapping.sql")
71
72
73			def create_buildings_temp_tables():
74			print("Create temp tables for buildings...")
75			execute_sql_script("osm_buildings_temp_tables.sql")
76
77
78			def extract_buildings_w_amenities():
79			print("Extracting buildings with amenities...")
80			execute_sql_script("osm_results_buildings_w_amenities.sql")
81
82
83			def extract_buildings_wo_amenities():
84			print("Extracting buildings without amenities...")
85			execute_sql_script("osm_results_buildings_wo_amenities.sql")
86
87
88			def extract_amenities():
89			print("Extracting amenities...")
90			execute_sql_script("osm_results_amenities.sql")
91
92
93			def extract_buildings_filtered_amenities():
94			print("Extracting buildings filtered with and without amenities...")
95			execute_sql_script("osm_buildings_filter_amenities.sql")
96
97
98			def extract_ways():
99			print("Extracting ways...")
100			execute_sql_script("osm_ways_preprocessing.sql")
101
102
103			def drop_temp_tables():
104			print("Dropping temp tables...")
105			execute_sql_script("drop_temp_tables.sql")
106
107
108			def add_metadata():
109			pass
110
111
112			class OsmBuildingsStreets(Dataset):
113			"""
114			Filter and preprocess buildings, streets and amenities from OpenStreetMap (OSM).
115
116			This dataset on buildings and amenities is required by several tasks in the
117			pipeline, such as the distribution of household demand profiles or PV home
118			systems to buildings. This data is enriched by population and apartments from
119			Zensus 2011. Those derived datasets and the data on streets will be used in the
120			DIstribution Network Generat0r
121			`ding0 <https://github.com/openego/ding0>`_ e.g. to cluster loads and create low
122			voltage grids.
123
124			Dependencies
125			* :py:class:`OpenStreetMap <egon.data.datasets.osm.OpenStreetMap>`
126			* :py:class:`ZensusMiscellaneous <egon.data.datasets.zensus.ZensusMiscellaneous>`
127
128			Resulting Tables
129			* openstreetmap.osm_buildings is created and filled (table has no associated python class)
130			* openstreetmap.osm_buildings_filtered is created and filled (table has no associated python class)
131			* openstreetmap.osm_buildings_residential is created and filled (table has no associated python class)
132			* openstreetmap.osm_amenities_shops_filtered is created and filled (table has no associated python class)
133			* openstreetmap.osm_buildings_with_amenities is created and filled (table has no associated python class)
134			* openstreetmap.osm_buildings_without_amenities is created and filled (table has no associated python class)
135			* openstreetmap.osm_amenities_not_in_buildings is created and filled (table has no associated python class)
136			* openstreetmap.osm_ways_preprocessed is created and filled (table has no associated python class)
137			* openstreetmap.osm_ways_with_segments is created and filled (table has no associated python class)
138			* boundaries.egon_map_zensus_buildings_filtered is created and filled (table has no associated python class)
139			* boundaries.egon_map_zensus_buildings_residential is created and filled (table has no associated python class)
140			* openstreetmap.osm_buildings is created and filled (table has no associated python class)
141
142			Details and Steps
143
144			* Extract buildings and filter using relevant tags, e.g. residential and
145			commercial, see script `osm_buildings_filter.sql` for the full list of tags.
146			Resulting tables:
147			* All buildings: `openstreetmap.osm_buildings`
148			* Filtered buildings: `openstreetmap.osm_buildings_filtered`
149			* Residential buildings: `openstreetmap.osm_buildings_residential`
150			* 1st step: Filter by tags (see `osm_buildings_filter_residential.sql`)
151			* 2nd step: Table is extended by finding census cells with population
152			but no residential buildings and extended by commercial/retail/office/
153			hotel buildings (see `osm_buildings_extend_residential.sql`) since they
154			often include apartments as well.
155			* Extract amenities and filter using relevant tags, e.g. shops and restaurants,
156			see script `osm_amenities_shops_preprocessing.sql` for the full list of tags.
157			Resulting table: `openstreetmap.osm_amenities_shops_filtered`
158			* Create a mapping table for building's osm IDs to the Zensus cells the
159			building's centroid is located in.
160			Resulting tables:
161			* `boundaries.egon_map_zensus_buildings_filtered` (filtered)
162			* `boundaries.egon_map_zensus_buildings_residential` (residential only)
163			* Enrich each building by number of apartments from Zensus table
164			`society.egon_destatis_zensus_apartment_building_population_per_ha`
165			by splitting up the cell's sum equally to the buildings. In some cases, a
166			Zensus cell does not contain buildings but there's a building nearby which
167			the no. of apartments is to be allocated to. To make sure apartments are
168			allocated to at least one building, a radius of 77m is used to catch building
169			geometries.
170			* Split filtered buildings into 3 datasets using the amenities' locations:
171			temporary tables are created in script `osm_buildings_temp_tables.sql` the
172			final tables in `osm_buildings_amentities_results.sql`.
173			Resulting tables:
174
175			* Buildings w/ amenities: `openstreetmap.osm_buildings_with_amenities`
176			* Buildings w/o amenities: `openstreetmap.osm_buildings_without_amenities`
177			* Amenities not allocated to buildings:
178			`openstreetmap.osm_amenities_not_in_buildings`
179			* Extract streets (OSM ways) and filter using relevant tags, e.g.
180			highway=secondary, see script `osm_ways_preprocessing.sql` for the full list
181			of tags. Additionally, each way is split into its line segments and their
182			lengths is retained.
183			Resulting tables:
184			* Filtered streets: `openstreetmap.osm_ways_preprocessed`
185			* Filtered streets w/ segments: `openstreetmap.osm_ways_with_segments`
186
187			"""
188
189			#:
190			name: str = "OsmBuildingsStreets"
191			#:
192			version: str = "0.0.7"
193
194			def __init__(self, dependencies):
195			super().__init__(
196			name=self.name,
197			version=self.version,
198			dependencies=dependencies,
199			tasks=(
200			preprocessing,
201			{filter_buildings, filter_buildings_residential},
202			extend_buildings_residential,
203			extract_buildings_filtered_amenities,
204			{
205			create_buildings_filtered_zensus_mapping,
206			create_buildings_residential_zensus_mapping,
207			create_buildings_filtered_all_zensus_mapping,
208			},
209			create_buildings_temp_tables,
210			{
211			extract_buildings_w_amenities,
212			extract_buildings_wo_amenities,
213			extract_amenities,
214			extract_ways,
215			},
216			drop_temp_tables,
217			add_metadata,
218			),
219			validation={
220			"data_quality": [
221			RowCountValidation(
222			table="boundaries.egon_map_zensus_buildings_filtered",
223			rule_id="TEST_ROW_COUNT",
224			expected_count=28070301
225			),
226			DataTypeValidation(
227			table="boundaries.egon_map_zensus_buildings_filtered",
228			rule_id="TEST_DATA_MULTIPLE_TYPES",
229			column_types={"id": "integer", "cell_id": "integer"}
230			),
231			WholeTableNotNullAndNotNaNValidation(
232			table="boundaries.egon_map_zensus_buildings_filtered",
233			rule_id="TEST_WHOLE_TABLE_NOT_NAN"
234			),
235			RowCountValidation(
236			table="boundaries.egon_map_zensus_buildings_residential",
237			rule_id="TEST_ROW_COUNT",
238			expected_count=27477467
239			),
240			DataTypeValidation(
241			table="boundaries.egon_map_zensus_buildings_residential",
242			rule_id="TEST_DATA_MULTIPLE_TYPES",
243			column_types={"id": "integer", "cell_id": "integer"}
244			),
245			WholeTableNotNullAndNotNaNValidation(
246			table="boundaries.egon_map_zensus_buildings_residential",
247			rule_id="TEST_WHOLE_TABLE_NOT_NAN"
248			)
249			]
250			}
251			)
252

openego / eGon-data

Pull Request — dev (#1375)

OsmBuildingsStreets.__init__() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like

OsmBuildingsStreets.init() B