| Conditions | 90 |
| Total Lines | 265 |
| Code Lines | 215 |
| Lines | 0 |
| Ratio | 0 % |
| Changes | 0 | ||
Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.
For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.
Commonly applied refactorings include:
If many parameters/temporary variables are present:
Complex classes like osm_poi_matchmaker.libs.file_output.generate_osm_xml() often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | # -*- coding: utf-8 -*- |
||
| 173 | def generate_osm_xml(df, session=None): |
||
| 174 | """Crete OpenStreetMap (OSM XML) file from passed Panda Dataframe |
||
| 175 | |||
| 176 | Args: |
||
| 177 | df ([type]): [description] |
||
| 178 | session ([type], optional): [description]. Defaults to None. |
||
| 179 | |||
| 180 | Returns: |
||
| 181 | [type]: [description] |
||
| 182 | """ |
||
| 183 | db = POIBase('{}://{}:{}@{}:{}/{}'.format(config.get_database_type(), config.get_database_writer_username(), |
||
| 184 | config.get_database_writer_password(), |
||
| 185 | config.get_database_writer_host(), |
||
| 186 | config.get_database_writer_port(), |
||
| 187 | config.get_database_poi_database())) |
||
| 188 | pgsql_pool = db.pool |
||
| 189 | session_factory = sessionmaker(pgsql_pool) |
||
| 190 | Session = scoped_session(session_factory) |
||
| 191 | session = Session() |
||
| 192 | osm_xml_data = etree.Element('osm', version='0.6', generator='JOSM') |
||
| 193 | default_osm_id = -1 |
||
| 194 | current_osm_id = default_osm_id |
||
| 195 | added_nodes = [] |
||
| 196 | try: |
||
| 197 | for index, row in df.iterrows(): |
||
| 198 | tags = {} |
||
| 199 | osm_live_tags = {} |
||
| 200 | main_data = {} |
||
| 201 | current_osm_id = default_osm_id if row.get('osm_id') is None else row.get('osm_id') |
||
| 202 | osm_version = '99999' if row.get('osm_version') is None else row.get('osm_version') |
||
| 203 | if row.get('osm_node') is None or row.get('osm_node') == OSM_object_type.node: |
||
| 204 | try: |
||
| 205 | josm_object = 'n{}'.format(current_osm_id) |
||
| 206 | main_data = etree.SubElement(osm_xml_data, 'node', add_osm_node(current_osm_id, row)) |
||
| 207 | except Exception as e: |
||
| 208 | logging.exception('Exception occurred') |
||
| 209 | elif row.get('osm_node') is not None and row.get('osm_node') == OSM_object_type.way: |
||
| 210 | try: |
||
| 211 | main_data = etree.SubElement(osm_xml_data, 'way', add_osm_way(current_osm_id, row)) |
||
| 212 | josm_object = 'w{}'.format(current_osm_id) |
||
| 213 | # Add way nodes without any modification) |
||
| 214 | node_data = [] |
||
| 215 | for n in row.get('osm_nodes'): |
||
| 216 | data = etree.SubElement(main_data, 'nd', ref=str(n)) |
||
| 217 | if session is not None: |
||
| 218 | # Go through the list except the last value (which is same as the first) |
||
| 219 | for n in row.get('osm_nodes'): |
||
| 220 | # Add nodes only when it is not already added. |
||
| 221 | if n not in added_nodes: |
||
| 222 | added_nodes.append(n) |
||
| 223 | way_node = db.query_from_cache(n, OSM_object_type.node) |
||
| 224 | if way_node is not None: |
||
| 225 | node_data = etree.SubElement(osm_xml_data, 'node', |
||
| 226 | list_osm_node(n, way_node, 'osm')) |
||
| 227 | if node_data.get('osm_live_tags') is not None and \ |
||
| 228 | node_data.get('osm_live_tags') != '': |
||
| 229 | node_osm_live_tags = node_data.get('osm_live_tags') |
||
| 230 | for k, v in sorted(node_osm_live_tags).items(): |
||
| 231 | xml_node_tags = etree.SubElement(node_data, 'tag', k=k, v='{}'.format(v)) |
||
| 232 | except TypeError as e: |
||
| 233 | logging.warning('Missing nodes on this way: %s.', row.get('osm_id')) |
||
| 234 | logging.exception('Exception occurred') |
||
| 235 | elif row.get('osm_node') is not None and row.get('osm_node') == OSM_object_type.relation: |
||
| 236 | try: |
||
| 237 | main_data = etree.SubElement(osm_xml_data, 'relation', add_osm_way(current_osm_id, row)) |
||
| 238 | josm_object = 'r{}'.format(current_osm_id) |
||
| 239 | relations = relationer(row.get('osm_nodes')) |
||
| 240 | for i in relations: |
||
| 241 | data = etree.SubElement(main_data, 'member', type=i.get('type'), ref=i.get('ref'), |
||
| 242 | role=i.get('role')) |
||
| 243 | except TypeError as e: |
||
| 244 | logging.warning('Missing nodes on this relation: %s.', row['osm_id']) |
||
| 245 | logging.exception('Exception occurred') |
||
| 246 | # Add already existing node, way, relation OpenStreetMap reference as comment |
||
| 247 | try: |
||
| 248 | if current_osm_id > 0: |
||
| 249 | osm_xml_data.append(etree.Comment(add_osm_link_comment(current_osm_id, row.get('osm_node')))) |
||
| 250 | # Add original POI coordinates as comment |
||
| 251 | comment = etree.Comment(' Original coordinates: {} '.format(row.get('poi_geom'))) |
||
| 252 | osm_xml_data.append(comment) |
||
| 253 | if 'poi_distance' in row: |
||
| 254 | if row.get('poi_distance') is not None: |
||
| 255 | comment = etree.Comment(' OSM <-> POI distance: {} m'.format(row.get('poi_distance'))) |
||
| 256 | else: |
||
| 257 | comment = etree.Comment(' OSM <-> POI distance: Non exist') |
||
| 258 | osm_xml_data.append(comment) |
||
| 259 | if 'poi_good' in row and 'poi_bad' in row: |
||
| 260 | comment = etree.Comment(' Checker good: {}; bad {}'.format(row.get('poi_good'), row.get('poi_bad'))) |
||
| 261 | osm_xml_data.append(comment) |
||
| 262 | # Using already definied OSM tags if exists |
||
| 263 | if row.get('osm_live_tags') is not None: |
||
| 264 | tags.update(row.get('osm_live_tags').copy()) |
||
| 265 | logging.critical(row.get('osm_live_tags')) |
||
| 266 | osm_live_tags.update(row.get('osm_live_tags').copy()) |
||
| 267 | # Adding POI common tags |
||
| 268 | if row.get('poi_tags') is not None: |
||
| 269 | tags.update(row.get('poi_tags')) |
||
| 270 | # Save live name tags if preserve name is enabled |
||
| 271 | except Exception as e: |
||
| 272 | logging.exception('Exception occurred') |
||
| 273 | try: |
||
| 274 | if row.get('preserve_original_name') is True: |
||
| 275 | preserved_name = tags.get('name') |
||
| 276 | except KeyError as e: |
||
| 277 | logging.debug('No name tag is specified to save in original OpenStreetMap data.') |
||
| 278 | try: |
||
| 279 | # Overwriting with data from data providers |
||
| 280 | for k, v in POI_TAGS.items(): |
||
| 281 | if row.get(k) is not None: |
||
| 282 | tags[v] = row.get(k) |
||
| 283 | except Exception as e: |
||
| 284 | logging.exception('Exception occurred') |
||
| 285 | try: |
||
| 286 | if config.get_geo_alternative_opening_hours(): |
||
| 287 | alternative_oh_tag = config.get_geo_alternative_opening_hours_tag() |
||
| 288 | # Alternative opening_hours handling for COVID-19 code path |
||
| 289 | if tags.get('opening_hours') is not None and tags.get('opening_hours') != '': |
||
| 290 | if row.get('poi_opening_hours') is not None and row.get('poi_opening_hours') != '': |
||
| 291 | if tags.get('opening_hours') == row.get('poi_opening_hours'): |
||
| 292 | tags[alternative_oh_tag] = 'same' |
||
| 293 | else: |
||
| 294 | tags[alternative_oh_tag] = row.get('poi_opening_hours') |
||
| 295 | else: |
||
| 296 | if row.get('poi_opening_hours') is not None and row.get('poi_opening_hours') != '': |
||
| 297 | tags['opening_hours'] = row.get('poi_opening_hours') |
||
| 298 | tags[alternative_oh_tag] = 'same' |
||
| 299 | else: |
||
| 300 | # Alternative opening_hours handling for NON COVID-19 code path: just simply add opening_hours to tags |
||
| 301 | if row.get('poi_opening_hours') is not None and row.get('poi_opening_hours') != '': |
||
| 302 | tags['opening_hours'] = row.get('poi_opening_hours') |
||
| 303 | except Exception as e: |
||
| 304 | logging.exception('Exception occurred') |
||
| 305 | try: |
||
| 306 | # If we got POI phone tag use it as OSM contact:phone tag |
||
| 307 | if row.get('poi_phone') is not None and row.get('poi_phone') != '': |
||
| 308 | tags['contact:phone'] = row.get('poi_phone') |
||
| 309 | except Exception as e: |
||
| 310 | logging.exception('Exception occurred') |
||
| 311 | try: |
||
| 312 | # If we got POI website tag use it as OSM contact:website tag |
||
| 313 | if row.get('poi_url_base') is not None and row.get('poi_website') is not None: |
||
| 314 | if row['poi_url_base'] in row.get('poi_website'): |
||
| 315 | # The POI website contains the base URL use the POI website field only |
||
| 316 | tags['contact:website'] = clean_url('{}'.format((row.get('poi_website')))) |
||
| 317 | else: |
||
| 318 | # The POI website does not contain the base URL use the merged base URL and POI website field |
||
| 319 | tags['contact:website'] = clean_url('{}/{}'.format(row.get('poi_url_base'), row.get('poi_website'))) |
||
| 320 | # If only the base URL is available |
||
| 321 | elif row.get('poi_url_base') is not None: |
||
| 322 | tags['contact:website'] = row.get('poi_url_base') |
||
| 323 | except Exception as e: |
||
| 324 | logging.exception('Exception occurred') |
||
| 325 | try: |
||
| 326 | # Short URL for source |
||
| 327 | if row['poi_url_base'] is not None: |
||
| 328 | source_url = 'source:{}:date'.format(row.get('poi_url_base').split('/')[2]) |
||
| 329 | else: |
||
| 330 | source_url = 'source:website:date' |
||
| 331 | tags[source_url] = '{:{dfmt}}'.format(datetime.datetime.now(), dfmt=DATE_FOTMAT) |
||
| 332 | except Exception as e: |
||
| 333 | logging.exception('Exception occurred') |
||
| 334 | try: |
||
| 335 | # Write back the saved name tag |
||
| 336 | if 'preserved_name' in locals(): |
||
| 337 | tags['name'] = preserved_name |
||
| 338 | # Rewrite old contact tags to contact:* tag form |
||
| 339 | tags_rewrite = ['website', 'phone', 'email', 'facebook', 'instagram', 'youtube', 'pinterest', 'fax'] |
||
| 340 | for tr in tags_rewrite: |
||
| 341 | if tr in tags: |
||
| 342 | # Never overwrite already existing contact:* tags |
||
| 343 | if 'contact:' + tr in tags: |
||
| 344 | # We already have this contact:* tag so remove the simple contact tag |
||
| 345 | tags.pop(tr, None) |
||
| 346 | else: |
||
| 347 | # Rewrite simple contact tag to contact:* tag |
||
| 348 | tags['contact:' + tr] = tags.pop(tr, None) |
||
| 349 | except Exception as e: |
||
| 350 | logging.exception('Exception occurred') |
||
| 351 | try: |
||
| 352 | if row.get('poi_description') is not None and row.get('poi_description') != '': |
||
| 353 | tags['description'] = row.get('poi_description') |
||
| 354 | except Exception as e: |
||
| 355 | logging.exception('Exception occurred') |
||
| 356 | try: |
||
| 357 | # Write tags with yes/no value |
||
| 358 | for k, v in POI_YESNO_TAGS.items(): |
||
| 359 | if row.get(k) is not None and row.get(k) != '': |
||
| 360 | tags[v] = 'yes' if row.get(k) is True else 'no' |
||
| 361 | for k, v in POI_EV_TAGS.items(): |
||
| 362 | if row.get(k) is not None and row.get(k) != '': |
||
| 363 | if isinstance(row.get(k), float): |
||
| 364 | if not math.isnan(row.get(k)): |
||
| 365 | tags[v] = int(row.get(k)) |
||
| 366 | else: |
||
| 367 | tags[v] = row.get(k) |
||
| 368 | except Exception as e: |
||
| 369 | logging.exception('Exception occurred') |
||
| 370 | try: |
||
| 371 | # This is a new POI - will add fix me tag to the new items. |
||
| 372 | if row.get('poi_new') is not None and row.get('poi_new') is True: |
||
| 373 | tags['fixme'] = 'verify import' |
||
| 374 | # Remove unwanted addr:country from file output as we discussed in Issue #33 |
||
| 375 | tags.pop('addr:country', None) |
||
| 376 | # tags['import'] = 'osm_poi_matchmaker' |
||
| 377 | # Rendering tags to the XML file and JOSM magic link |
||
| 378 | except Exception as e: |
||
| 379 | logging.exception('Exception occurred') |
||
| 380 | try: |
||
| 381 | josm_link = '' |
||
| 382 | comment = '\nKey\t\t\t\tStatus\t\tNew value\t\tOSM value\n' |
||
| 383 | for k, v in sorted(tags.items()): |
||
| 384 | xml_tags = etree.SubElement(main_data, 'tag', k=k, v='{}'.format(v)) |
||
| 385 | josm_link = '{}|{}={}'.format(josm_link, k, v) |
||
| 386 | # Add original POI tags as comment |
||
| 387 | try: |
||
| 388 | if isinstance(v, str): |
||
| 389 | v = v.replace('-', '\-').replace('\n', '') |
||
| 390 | w = osm_live_tags[k] |
||
| 391 | except KeyError: |
||
| 392 | comment += "{:32} N\t\t'{}'\n".format(k, v) |
||
| 393 | else: |
||
| 394 | if isinstance(w, str): |
||
| 395 | w = w.replace('-', '\-').replace('\n', '') |
||
| 396 | comment += "{:32} {}\t\t'{}'\t\t\t'{}'\n".format(k, compare_strings(v, w), v, w) |
||
| 397 | comment = etree.Comment(comment) |
||
| 398 | except Exception as e: |
||
| 399 | logging.exception('Exception occurred') |
||
| 400 | try: |
||
| 401 | osm_xml_data.append(comment) |
||
| 402 | except Exception as e: |
||
| 403 | logging.exception('Exception occurred') |
||
| 404 | try: |
||
| 405 | # URL encode link and '--' in comment |
||
| 406 | josm_link = quote(josm_link) |
||
| 407 | josm_link = josm_link.replace('--', '%2D%2D') |
||
| 408 | comment = etree.Comment(' JOSM magic link: {}?new_layer=false&objects={}&addtags={} '.format |
||
| 409 | ('http://localhost:8111/load_object', josm_object, josm_link)) |
||
| 410 | osm_xml_data.append(comment) |
||
| 411 | except Exception as e: |
||
| 412 | logging.exception('Exception occurred') |
||
| 413 | try: |
||
| 414 | test_case = {k: row.get(k, None) for k in TESTCASE_GEN_KEYS} |
||
| 415 | comment = etree.Comment("ˇ'original': '{t[original]}', 'postcode': '{t[poi_postcode]}', 'city': '{t[poi_city]}', 'street': '{t[poi_addr_street]}', 'housenumber': '{t[poi_addr_housenumber]}', 'conscriptionnumber': '{t[poi_conscriptionnumber]}'°".format(t=test_case)) |
||
| 416 | osm_xml_data.append(comment) |
||
| 417 | except Exception as e: |
||
| 418 | logging.exception('Exception occurred') |
||
| 419 | try: |
||
| 420 | osm_xml_data.append(xml_node_tags) |
||
| 421 | except UnboundLocalError as e: |
||
| 422 | logging.debug('Unbound local error extra node tags') |
||
| 423 | try: |
||
| 424 | osm_xml_data.append(main_data) |
||
| 425 | # Next deafult OSM id is one more less for non existing objects |
||
| 426 | default_osm_id -= 1 |
||
| 427 | except Exception as e: |
||
| 428 | logging.exception('Exception occurred') |
||
| 429 | except ValueError as e: |
||
| 430 | logging.error(e) |
||
| 431 | logging.exception('Exception occurred') |
||
| 432 | |||
| 433 | except Exception as e: |
||
| 434 | logging.error(e) |
||
| 435 | logging.exception('Exception occurred') |
||
| 436 | |||
| 437 | return lxml.etree.tostring(osm_xml_data, pretty_print=True, xml_declaration=True, encoding="UTF-8") |
||
| 438 |