Conditions | 90 |
Total Lines | 265 |
Code Lines | 215 |
Lines | 0 |
Ratio | 0 % |
Changes | 0 |
Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.
For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.
Commonly applied refactorings include:
If many parameters/temporary variables are present:
Complex classes like osm_poi_matchmaker.libs.file_output.generate_osm_xml() often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
1 | # -*- coding: utf-8 -*- |
||
173 | def generate_osm_xml(df, session=None): |
||
174 | """Crete OpenStreetMap (OSM XML) file from passed Panda Dataframe |
||
175 | |||
176 | Args: |
||
177 | df ([type]): [description] |
||
178 | session ([type], optional): [description]. Defaults to None. |
||
179 | |||
180 | Returns: |
||
181 | [type]: [description] |
||
182 | """ |
||
183 | db = POIBase('{}://{}:{}@{}:{}/{}'.format(config.get_database_type(), config.get_database_writer_username(), |
||
184 | config.get_database_writer_password(), |
||
185 | config.get_database_writer_host(), |
||
186 | config.get_database_writer_port(), |
||
187 | config.get_database_poi_database())) |
||
188 | pgsql_pool = db.pool |
||
189 | session_factory = sessionmaker(pgsql_pool) |
||
190 | Session = scoped_session(session_factory) |
||
191 | session = Session() |
||
192 | osm_xml_data = etree.Element('osm', version='0.6', generator='JOSM') |
||
193 | default_osm_id = -1 |
||
194 | current_osm_id = default_osm_id |
||
195 | added_nodes = [] |
||
196 | try: |
||
197 | for index, row in df.iterrows(): |
||
198 | tags = {} |
||
199 | osm_live_tags = {} |
||
200 | main_data = {} |
||
201 | current_osm_id = default_osm_id if row.get('osm_id') is None else row.get('osm_id') |
||
202 | osm_version = '99999' if row.get('osm_version') is None else row.get('osm_version') |
||
203 | if row.get('osm_node') is None or row.get('osm_node') == OSM_object_type.node: |
||
204 | try: |
||
205 | josm_object = 'n{}'.format(current_osm_id) |
||
206 | main_data = etree.SubElement(osm_xml_data, 'node', add_osm_node(current_osm_id, row)) |
||
207 | except Exception as e: |
||
208 | logging.exception('Exception occurred') |
||
209 | elif row.get('osm_node') is not None and row.get('osm_node') == OSM_object_type.way: |
||
210 | try: |
||
211 | main_data = etree.SubElement(osm_xml_data, 'way', add_osm_way(current_osm_id, row)) |
||
212 | josm_object = 'w{}'.format(current_osm_id) |
||
213 | # Add way nodes without any modification) |
||
214 | node_data = [] |
||
215 | for n in row.get('osm_nodes'): |
||
216 | data = etree.SubElement(main_data, 'nd', ref=str(n)) |
||
217 | if session is not None: |
||
218 | # Go through the list except the last value (which is same as the first) |
||
219 | for n in row.get('osm_nodes'): |
||
220 | # Add nodes only when it is not already added. |
||
221 | if n not in added_nodes: |
||
222 | added_nodes.append(n) |
||
223 | way_node = db.query_from_cache(n, OSM_object_type.node) |
||
224 | if way_node is not None: |
||
225 | node_data = etree.SubElement(osm_xml_data, 'node', |
||
226 | list_osm_node(n, way_node, 'osm')) |
||
227 | if node_data.get('osm_live_tags') is not None and \ |
||
228 | node_data.get('osm_live_tags') != '': |
||
229 | node_osm_live_tags = node_data.get('osm_live_tags') |
||
230 | for k, v in sorted(node_osm_live_tags).items(): |
||
231 | xml_node_tags = etree.SubElement(node_data, 'tag', k=k, v='{}'.format(v)) |
||
232 | except TypeError as e: |
||
233 | logging.warning('Missing nodes on this way: %s.', row.get('osm_id')) |
||
234 | logging.exception('Exception occurred') |
||
235 | elif row.get('osm_node') is not None and row.get('osm_node') == OSM_object_type.relation: |
||
236 | try: |
||
237 | main_data = etree.SubElement(osm_xml_data, 'relation', add_osm_way(current_osm_id, row)) |
||
238 | josm_object = 'r{}'.format(current_osm_id) |
||
239 | relations = relationer(row.get('osm_nodes')) |
||
240 | for i in relations: |
||
241 | data = etree.SubElement(main_data, 'member', type=i.get('type'), ref=i.get('ref'), |
||
242 | role=i.get('role')) |
||
243 | except TypeError as e: |
||
244 | logging.warning('Missing nodes on this relation: %s.', row['osm_id']) |
||
245 | logging.exception('Exception occurred') |
||
246 | # Add already existing node, way, relation OpenStreetMap reference as comment |
||
247 | try: |
||
248 | if current_osm_id > 0: |
||
249 | osm_xml_data.append(etree.Comment(add_osm_link_comment(current_osm_id, row.get('osm_node')))) |
||
250 | # Add original POI coordinates as comment |
||
251 | comment = etree.Comment(' Original coordinates: {} '.format(row.get('poi_geom'))) |
||
252 | osm_xml_data.append(comment) |
||
253 | if 'poi_distance' in row: |
||
254 | if row.get('poi_distance') is not None: |
||
255 | comment = etree.Comment(' OSM <-> POI distance: {} m'.format(row.get('poi_distance'))) |
||
256 | else: |
||
257 | comment = etree.Comment(' OSM <-> POI distance: Non exist') |
||
258 | osm_xml_data.append(comment) |
||
259 | if 'poi_good' in row and 'poi_bad' in row: |
||
260 | comment = etree.Comment(' Checker good: {}; bad {}'.format(row.get('poi_good'), row.get('poi_bad'))) |
||
261 | osm_xml_data.append(comment) |
||
262 | # Using already definied OSM tags if exists |
||
263 | if row.get('osm_live_tags') is not None: |
||
264 | tags.update(row.get('osm_live_tags').copy()) |
||
265 | logging.critical(row.get('osm_live_tags')) |
||
266 | osm_live_tags.update(row.get('osm_live_tags').copy()) |
||
267 | # Adding POI common tags |
||
268 | if row.get('poi_tags') is not None: |
||
269 | tags.update(row.get('poi_tags')) |
||
270 | # Save live name tags if preserve name is enabled |
||
271 | except Exception as e: |
||
272 | logging.exception('Exception occurred') |
||
273 | try: |
||
274 | if row.get('preserve_original_name') is True: |
||
275 | preserved_name = tags.get('name') |
||
276 | except KeyError as e: |
||
277 | logging.debug('No name tag is specified to save in original OpenStreetMap data.') |
||
278 | try: |
||
279 | # Overwriting with data from data providers |
||
280 | for k, v in POI_TAGS.items(): |
||
281 | if row.get(k) is not None: |
||
282 | tags[v] = row.get(k) |
||
283 | except Exception as e: |
||
284 | logging.exception('Exception occurred') |
||
285 | try: |
||
286 | if config.get_geo_alternative_opening_hours(): |
||
287 | alternative_oh_tag = config.get_geo_alternative_opening_hours_tag() |
||
288 | # Alternative opening_hours handling for COVID-19 code path |
||
289 | if tags.get('opening_hours') is not None and tags.get('opening_hours') != '': |
||
290 | if row.get('poi_opening_hours') is not None and row.get('poi_opening_hours') != '': |
||
291 | if tags.get('opening_hours') == row.get('poi_opening_hours'): |
||
292 | tags[alternative_oh_tag] = 'same' |
||
293 | else: |
||
294 | tags[alternative_oh_tag] = row.get('poi_opening_hours') |
||
295 | else: |
||
296 | if row.get('poi_opening_hours') is not None and row.get('poi_opening_hours') != '': |
||
297 | tags['opening_hours'] = row.get('poi_opening_hours') |
||
298 | tags[alternative_oh_tag] = 'same' |
||
299 | else: |
||
300 | # Alternative opening_hours handling for NON COVID-19 code path: just simply add opening_hours to tags |
||
301 | if row.get('poi_opening_hours') is not None and row.get('poi_opening_hours') != '': |
||
302 | tags['opening_hours'] = row.get('poi_opening_hours') |
||
303 | except Exception as e: |
||
304 | logging.exception('Exception occurred') |
||
305 | try: |
||
306 | # If we got POI phone tag use it as OSM contact:phone tag |
||
307 | if row.get('poi_phone') is not None and row.get('poi_phone') != '': |
||
308 | tags['contact:phone'] = row.get('poi_phone') |
||
309 | except Exception as e: |
||
310 | logging.exception('Exception occurred') |
||
311 | try: |
||
312 | # If we got POI website tag use it as OSM contact:website tag |
||
313 | if row.get('poi_url_base') is not None and row.get('poi_website') is not None: |
||
314 | if row['poi_url_base'] in row.get('poi_website'): |
||
315 | # The POI website contains the base URL use the POI website field only |
||
316 | tags['contact:website'] = clean_url('{}'.format((row.get('poi_website')))) |
||
317 | else: |
||
318 | # The POI website does not contain the base URL use the merged base URL and POI website field |
||
319 | tags['contact:website'] = clean_url('{}/{}'.format(row.get('poi_url_base'), row.get('poi_website'))) |
||
320 | # If only the base URL is available |
||
321 | elif row.get('poi_url_base') is not None: |
||
322 | tags['contact:website'] = row.get('poi_url_base') |
||
323 | except Exception as e: |
||
324 | logging.exception('Exception occurred') |
||
325 | try: |
||
326 | # Short URL for source |
||
327 | if row['poi_url_base'] is not None: |
||
328 | source_url = 'source:{}:date'.format(row.get('poi_url_base').split('/')[2]) |
||
329 | else: |
||
330 | source_url = 'source:website:date' |
||
331 | tags[source_url] = '{:{dfmt}}'.format(datetime.datetime.now(), dfmt=DATE_FOTMAT) |
||
332 | except Exception as e: |
||
333 | logging.exception('Exception occurred') |
||
334 | try: |
||
335 | # Write back the saved name tag |
||
336 | if 'preserved_name' in locals(): |
||
337 | tags['name'] = preserved_name |
||
338 | # Rewrite old contact tags to contact:* tag form |
||
339 | tags_rewrite = ['website', 'phone', 'email', 'facebook', 'instagram', 'youtube', 'pinterest', 'fax'] |
||
340 | for tr in tags_rewrite: |
||
341 | if tr in tags: |
||
342 | # Never overwrite already existing contact:* tags |
||
343 | if 'contact:' + tr in tags: |
||
344 | # We already have this contact:* tag so remove the simple contact tag |
||
345 | tags.pop(tr, None) |
||
346 | else: |
||
347 | # Rewrite simple contact tag to contact:* tag |
||
348 | tags['contact:' + tr] = tags.pop(tr, None) |
||
349 | except Exception as e: |
||
350 | logging.exception('Exception occurred') |
||
351 | try: |
||
352 | if row.get('poi_description') is not None and row.get('poi_description') != '': |
||
353 | tags['description'] = row.get('poi_description') |
||
354 | except Exception as e: |
||
355 | logging.exception('Exception occurred') |
||
356 | try: |
||
357 | # Write tags with yes/no value |
||
358 | for k, v in POI_YESNO_TAGS.items(): |
||
359 | if row.get(k) is not None and row.get(k) != '': |
||
360 | tags[v] = 'yes' if row.get(k) is True else 'no' |
||
361 | for k, v in POI_EV_TAGS.items(): |
||
362 | if row.get(k) is not None and row.get(k) != '': |
||
363 | if isinstance(row.get(k), float): |
||
364 | if not math.isnan(row.get(k)): |
||
365 | tags[v] = int(row.get(k)) |
||
366 | else: |
||
367 | tags[v] = row.get(k) |
||
368 | except Exception as e: |
||
369 | logging.exception('Exception occurred') |
||
370 | try: |
||
371 | # This is a new POI - will add fix me tag to the new items. |
||
372 | if row.get('poi_new') is not None and row.get('poi_new') is True: |
||
373 | tags['fixme'] = 'verify import' |
||
374 | # Remove unwanted addr:country from file output as we discussed in Issue #33 |
||
375 | tags.pop('addr:country', None) |
||
376 | # tags['import'] = 'osm_poi_matchmaker' |
||
377 | # Rendering tags to the XML file and JOSM magic link |
||
378 | except Exception as e: |
||
379 | logging.exception('Exception occurred') |
||
380 | try: |
||
381 | josm_link = '' |
||
382 | comment = '\nKey\t\t\t\tStatus\t\tNew value\t\tOSM value\n' |
||
383 | for k, v in sorted(tags.items()): |
||
384 | xml_tags = etree.SubElement(main_data, 'tag', k=k, v='{}'.format(v)) |
||
385 | josm_link = '{}|{}={}'.format(josm_link, k, v) |
||
386 | # Add original POI tags as comment |
||
387 | try: |
||
388 | if isinstance(v, str): |
||
389 | v = v.replace('-', '\-').replace('\n', '') |
||
390 | w = osm_live_tags[k] |
||
391 | except KeyError: |
||
392 | comment += "{:32} N\t\t'{}'\n".format(k, v) |
||
393 | else: |
||
394 | if isinstance(w, str): |
||
395 | w = w.replace('-', '\-').replace('\n', '') |
||
396 | comment += "{:32} {}\t\t'{}'\t\t\t'{}'\n".format(k, compare_strings(v, w), v, w) |
||
397 | comment = etree.Comment(comment) |
||
398 | except Exception as e: |
||
399 | logging.exception('Exception occurred') |
||
400 | try: |
||
401 | osm_xml_data.append(comment) |
||
402 | except Exception as e: |
||
403 | logging.exception('Exception occurred') |
||
404 | try: |
||
405 | # URL encode link and '--' in comment |
||
406 | josm_link = quote(josm_link) |
||
407 | josm_link = josm_link.replace('--', '%2D%2D') |
||
408 | comment = etree.Comment(' JOSM magic link: {}?new_layer=false&objects={}&addtags={} '.format |
||
409 | ('http://localhost:8111/load_object', josm_object, josm_link)) |
||
410 | osm_xml_data.append(comment) |
||
411 | except Exception as e: |
||
412 | logging.exception('Exception occurred') |
||
413 | try: |
||
414 | test_case = {k: row.get(k, None) for k in TESTCASE_GEN_KEYS} |
||
415 | comment = etree.Comment("ˇ'original': '{t[original]}', 'postcode': '{t[poi_postcode]}', 'city': '{t[poi_city]}', 'street': '{t[poi_addr_street]}', 'housenumber': '{t[poi_addr_housenumber]}', 'conscriptionnumber': '{t[poi_conscriptionnumber]}'°".format(t=test_case)) |
||
416 | osm_xml_data.append(comment) |
||
417 | except Exception as e: |
||
418 | logging.exception('Exception occurred') |
||
419 | try: |
||
420 | osm_xml_data.append(xml_node_tags) |
||
421 | except UnboundLocalError as e: |
||
422 | logging.debug('Unbound local error extra node tags') |
||
423 | try: |
||
424 | osm_xml_data.append(main_data) |
||
425 | # Next deafult OSM id is one more less for non existing objects |
||
426 | default_osm_id -= 1 |
||
427 | except Exception as e: |
||
428 | logging.exception('Exception occurred') |
||
429 | except ValueError as e: |
||
430 | logging.error(e) |
||
431 | logging.exception('Exception occurred') |
||
432 | |||
433 | except Exception as e: |
||
434 | logging.error(e) |
||
435 | logging.exception('Exception occurred') |
||
436 | |||
437 | return lxml.etree.tostring(osm_xml_data, pretty_print=True, xml_declaration=True, encoding="UTF-8") |
||
438 |