KAMI911 /
osm_poi_matchmaker
| Total Complexity | 52 |
| Total Lines | 229 |
| Duplicated Lines | 20.96 % |
| Changes | 0 | ||
Duplicate code is one of the most pungent code smells. A rule that is often used is to re-structure code once it is duplicated in three or more places.
Common duplication problems, and corresponding solutions are:
Complex classes like osm_poi_matchmaker.dao.data_handlers often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | # -*- coding: utf-8 -*- |
||
|
introduced
by
|
|||
| 2 | |||
| 3 | try: |
||
| 4 | import logging |
||
| 5 | import sys |
||
| 6 | import hashlib |
||
| 7 | from osm_poi_matchmaker.dao.data_structure import City, POI_common, POI_address, Street_type |
||
| 8 | from osm_poi_matchmaker.libs import address |
||
| 9 | from osm_poi_matchmaker.dao import poi_array_structure |
||
| 10 | except ImportError as err: |
||
| 11 | logging.error('Error %s import module: %s', __name__, err) |
||
| 12 | logging.exception('Exception occurred') |
||
| 13 | |||
| 14 | sys.exit(128) |
||
| 15 | |||
| 16 | POI_COLS = poi_array_structure.POI_COLS |
||
| 17 | |||
| 18 | |||
| 19 | def get_or_create(session, model, **kwargs): |
||
| 20 | instance = session.query(model).filter_by(**kwargs).first() |
||
| 21 | View Code Duplication | if instance: |
|
| 22 | logging.debug('Already added: %s', instance) |
||
| 23 | return instance |
||
| 24 | else: |
||
| 25 | try: |
||
| 26 | instance = model(**kwargs) |
||
| 27 | session.add(instance) |
||
| 28 | return instance |
||
| 29 | except Exception as e: |
||
| 30 | logging.error('Cannot add to the database. (%s)', e) |
||
| 31 | logging.exception('Exception occurred') |
||
| 32 | raise e |
||
| 33 | |||
| 34 | |||
| 35 | def get_or_create_poi(session, model, **kwargs): |
||
| 36 | if kwargs['poi_common_id'] is not None: |
||
| 37 | if kwargs['poi_common_id'] is not None and kwargs['poi_addr_city'] is not None and ( |
||
| 38 | (kwargs['poi_addr_street'] and kwargs['poi_addr_housenumber'] is not None) or ( |
||
| 39 | kwargs['poi_conscriptionnumber'] is not None)): |
||
| 40 | logging.debug('Fully filled basic data record') |
||
| 41 | else: |
||
| 42 | logging.warning('Missing record data: %s', kwargs) |
||
| 43 | instance = session.query(model) \ |
||
| 44 | .filter_by(poi_common_id=kwargs['poi_common_id']) \ |
||
| 45 | .filter_by(poi_addr_city=kwargs['poi_addr_city']) \ |
||
| 46 | .filter_by(poi_addr_street=kwargs['poi_addr_street']) \ |
||
| 47 | .filter_by(poi_addr_housenumber=kwargs['poi_addr_housenumber']) \ |
||
| 48 | .filter_by(poi_conscriptionnumber=kwargs['poi_conscriptionnumber']) \ |
||
| 49 | .filter_by(poi_branch=kwargs['poi_branch']) \ |
||
| 50 | .first() |
||
| 51 | View Code Duplication | if instance: |
|
| 52 | logging.debug('Already added: %s', instance) |
||
| 53 | return instance |
||
| 54 | else: |
||
| 55 | try: |
||
| 56 | instance = model(**kwargs) |
||
| 57 | session.add(instance) |
||
| 58 | return instance |
||
| 59 | except Exception as e: |
||
| 60 | logging.error('Cannot add to the database. (%s)', e) |
||
| 61 | logging.exception('Exception occurred') |
||
| 62 | raise e |
||
| 63 | |||
| 64 | |||
| 65 | def get_or_create_cache(session, model, **kwargs): |
||
| 66 | if kwargs.get('osm_id') is not None and kwargs.get('osm_object_type'): |
||
| 67 | instance = session.query(model) \ |
||
| 68 | .filter_by(osm_id=kwargs.get('osm_id')).filter_by(osm_object_type=kwargs.get('poi_addr_city')).first() |
||
| 69 | View Code Duplication | if instance: |
|
| 70 | logging.debug('Already added: %s', instance) |
||
| 71 | return instance |
||
| 72 | else: |
||
| 73 | try: |
||
| 74 | instance = model(**kwargs) |
||
| 75 | session.add(instance) |
||
| 76 | return instance |
||
| 77 | except Exception as e: |
||
| 78 | logging.error('Cannot add to the database. (%s)', e) |
||
| 79 | logging.exception('Exception occurred') |
||
| 80 | raise e |
||
| 81 | |||
| 82 | def get_or_create_common(session, model, **kwargs): |
||
| 83 | if kwargs['poi_code'] is not None and kwargs['poi_code'] != '': |
||
| 84 | instance = session.query(model).filter_by(poi_code=kwargs['poi_code']).first() |
||
| 85 | View Code Duplication | if instance: |
|
| 86 | logging.debug('Already added: %s', instance) |
||
| 87 | return instance |
||
| 88 | else: |
||
| 89 | try: |
||
| 90 | instance = model(**kwargs) |
||
| 91 | session.add(instance) |
||
| 92 | return instance |
||
| 93 | except Exception as e: |
||
| 94 | logging.error('Cannot add to the database. (%s)', e) |
||
| 95 | logging.exception('Exception occurred') |
||
| 96 | raise e |
||
| 97 | |||
| 98 | def insert_city_dataframe(session, city_df): |
||
| 99 | city_df.columns = ['city_post_code', 'city_name'] |
||
| 100 | try: |
||
| 101 | for index, city_data in city_df.iterrows(): |
||
| 102 | get_or_create(session, City, city_post_code=city_data['city_post_code'], |
||
| 103 | city_name=address.clean_city(city_data['city_name'])) |
||
| 104 | except Exception as e: |
||
| 105 | |||
| 106 | logging.error('Rolled back: %s.', e) |
||
| 107 | logging.error(city_data) |
||
| 108 | logging.exception('Exception occurred') |
||
| 109 | |||
| 110 | session.rollback() |
||
| 111 | else: |
||
| 112 | logging.info('Successfully added %s city items to the dataset.', len(city_df)) |
||
| 113 | session.commit() |
||
| 114 | |||
| 115 | |||
| 116 | def insert_street_type_dataframe(session, city_df): |
||
| 117 | city_df.columns = ['street_type'] |
||
| 118 | try: |
||
| 119 | for index, city_data in city_df.iterrows(): |
||
| 120 | get_or_create(session, Street_type, street_type=city_data['street_type']) |
||
| 121 | except Exception as e: |
||
| 122 | logging.error('Rolled back: %s.', e) |
||
| 123 | logging.error(city_data) |
||
| 124 | logging.exception('Exception occurred') |
||
| 125 | |||
| 126 | session.rollback() |
||
| 127 | else: |
||
| 128 | logging.info('Successfully added %s street type items to the dataset.', len(city_df)) |
||
| 129 | session.commit() |
||
| 130 | |||
| 131 | |||
| 132 | def insert_common_dataframe(session, common_df): |
||
| 133 | common_df.columns = ['poi_name', 'poi_tags', 'poi_url_base', 'poi_code'] |
||
| 134 | try: |
||
| 135 | for index, poi_common_data in common_df.iterrows(): |
||
| 136 | get_or_create_common(session, POI_common, **poi_common_data) |
||
| 137 | except Exception as e: |
||
| 138 | logging.error('Rolled back: %s.', e) |
||
| 139 | logging.error(poi_common_data) |
||
| 140 | logging.exception('Exception occurred') |
||
| 141 | |||
| 142 | session.rollback() |
||
| 143 | else: |
||
| 144 | logging.info('Successfully added %s common items to the dataset.', len(common_df)) |
||
| 145 | session.commit() |
||
| 146 | |||
| 147 | |||
| 148 | def search_for_postcode(session, city_name): |
||
| 149 | city_col = session.query(City.city_post_code).filter(City.city_name == city_name).all() |
||
| 150 | if len(city_col) == 1: |
||
| 151 | return city_col |
||
| 152 | else: |
||
| 153 | logging.info('Cannot determine the post code from city name (%s).', city_name) |
||
| 154 | return None |
||
| 155 | |||
| 156 | |||
| 157 | def insert_poi_dataframe(session, poi_df): |
||
| 158 | poi_df.columns = POI_COLS |
||
| 159 | poi_df[['poi_postcode']] = poi_df[['poi_postcode']].fillna('0000') |
||
| 160 | poi_df[['poi_postcode']] = poi_df[['poi_postcode']].astype(int) |
||
| 161 | poi_dict = poi_df.to_dict('records') |
||
| 162 | try: |
||
| 163 | for poi_data in poi_dict: |
||
| 164 | city_col = session.query(City.city_id).filter(City.city_name == poi_data['poi_city']).filter( |
||
| 165 | City.city_post_code == poi_data['poi_postcode']).first() |
||
| 166 | common_col = session.query(POI_common.pc_id).filter(POI_common.poi_code == poi_data['poi_code']).first() |
||
| 167 | poi_data['poi_addr_city'] = city_col |
||
| 168 | poi_data['poi_common_id'] = common_col |
||
| 169 | if 'poi_name' in poi_data: del poi_data['poi_name'] |
||
| 170 | if 'poi_code' in poi_data: del poi_data['poi_code'] |
||
| 171 | get_or_create_poi(session, POI_address, **poi_data) |
||
| 172 | except Exception as e: |
||
| 173 | logging.error('Rolled back: %s.', e) |
||
| 174 | logging.error(poi_data) |
||
| 175 | logging.exception('Exception occurred') |
||
| 176 | |||
| 177 | session.rollback() |
||
| 178 | raise e |
||
| 179 | else: |
||
| 180 | try: |
||
| 181 | session.commit() |
||
| 182 | logging.info('Successfully added %s POI items to the dataset.', len(poi_dict)) |
||
| 183 | except Exception as e: |
||
| 184 | logging.error('Unsuccessfull commit: %s.', e) |
||
| 185 | logging.exception('Exception occurred') |
||
| 186 | |||
| 187 | |||
| 188 | def insert_type(session, type_data): |
||
| 189 | try: |
||
| 190 | for i in type_data: |
||
| 191 | get_or_create_common(session, POI_common, **i) |
||
| 192 | except Exception as e: |
||
| 193 | logging.error('Rolled back: %s.', e) |
||
| 194 | logging.error(i) |
||
| 195 | logging.exception('Exception occurred') |
||
| 196 | |||
| 197 | session.rollback() |
||
| 198 | else: |
||
| 199 | logging.info('Successfully added %s type items to the dataset.', len(type_data)) |
||
| 200 | session.commit() |
||
| 201 | |||
| 202 | |||
| 203 | def insert(session, **kwargs): |
||
| 204 | try: |
||
| 205 | city_col = session.query(City.city_id).filter(City.city_name == kwargs['poi_city']).filter( |
||
| 206 | City.city_post_code == kwargs['poi_postcode']).first() |
||
| 207 | common_col = session.query(POI_common.pc_id).filter(POI_common.poi_code == kwargs['poi_code']).first() |
||
| 208 | kwargs['poi_addr_city'] = city_col |
||
| 209 | kwargs['poi_common_id'] = common_col |
||
| 210 | kwargs['poi_hash'] = hashlib.sha512( |
||
| 211 | '{}{}{}{}{}{}'.format(kwargs['poi_code'], kwargs['poi_postcode'], kwargs['poi_city'], |
||
| 212 | kwargs['poi_addr_street'], kwargs['poi_addr_housenumber'], |
||
| 213 | kwargs['poi_conscriptionnumber']).lower().replace(' ', '').encode( |
||
| 214 | 'utf-8')).hexdigest() |
||
| 215 | if 'poi_name' in kwargs: del kwargs['poi_name'] |
||
| 216 | if 'poi_code' in kwargs: del kwargs['poi_code'] |
||
| 217 | get_or_create_poi(session, POI_address, **kwargs) |
||
| 218 | except Exception as e: |
||
| 219 | logging.error('Rolled back: %s.', e) |
||
| 220 | logging.error(kwargs) |
||
| 221 | logging.exception('Exception occurred') |
||
| 222 | |||
| 223 | session.rollback() |
||
| 224 | else: |
||
| 225 | logging.debug('Successfully added the item to the dataset.') |
||
| 226 | session.commit() |
||
| 227 | finally: |
||
| 228 | session.close() |
||
| 229 |
Loading history...