osm_poi_matchmaker.dao.data_handlers.insert() - Code Metrics - Inspection of "Improve Excepion handling using built-in solution" - KAMI911/osm_poi_matchmaker - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( ea0d3c...cc0a45 )

by KAMI

created 2020-10-13 09:55 UTC

osm_poi_matchmaker.dao.data_handlers.insert() B

↳ Parent: osm_poi_matchmaker.dao.data_handlers

Complexity

Conditions

Size

Total Lines	25
Code Lines	23

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	5
eloc	23
nop	2
dl	0
loc	25
rs	8.8613
c	0
b	0
f	0

# -*- coding: utf-8 -*-


try:
    import logging
    import sys
    import hashlib
    from osm_poi_matchmaker.dao.data_structure import City, POI_common, POI_address, Street_type
    from osm_poi_matchmaker.libs import address
    from osm_poi_matchmaker.dao import poi_array_structure
except ImportError as err:
    logging.error('Error %s import module: %s', __name__, err)
    logging.exception("Exception occurred")
    sys.exit(128)

POI_COLS = poi_array_structure.POI_COLS


def get_or_create(session, model, **kwargs):

    instance = session.query(model).filter_by(**kwargs).first()
    if instance:

        logging.debug('Already added: %s' ,instance)

        return instance
    else:
        try:
            instance = model(**kwargs)
            session.add(instance)
            return instance
        except Exception as e:

            logging.error('Cannot add to the database. (%s)', e)
            logging.exception("Exception occurred")
            raise (e)



def get_or_create_poi(session, model, **kwargs):

    if kwargs['poi_common_id'] is not None:
        if kwargs['poi_common_id'] is not None and kwargs['poi_addr_city'] is not None and (
                (kwargs['poi_addr_street'] and kwargs['poi_addr_housenumber'] is not None) or (
                kwargs['poi_conscriptionnumber'] is not None)):

            logging.debug('Fully filled basic data record')
        else:
            logging.warning('Missing record data: %s', kwargs)
    instance = session.query(model)\
        .filter_by(poi_common_id=kwargs['poi_common_id'])\
        .filter_by(poi_addr_city=kwargs['poi_addr_city'])\
        .filter_by(poi_addr_street=kwargs['poi_addr_street'])\
        .filter_by(poi_addr_housenumber=kwargs['poi_addr_housenumber'])\
        .filter_by(poi_conscriptionnumber=kwargs['poi_conscriptionnumber'])\
        .filter_by(poi_branch=kwargs['poi_branch'])\
        .first()
    if instance:

        logging.debug('Already added: %s', instance)
        return instance
    else:
        try:
            instance = model(**kwargs)
            session.add(instance)
            return instance
        except Exception as e:

            logging.error('Cannot add to the database. (%s)', e)
            logging.exception("Exception occurred")
            raise (e)



def insert_city_dataframe(session, city_df):

    city_df.columns = ['city_post_code', 'city_name']
    try:
        for index, city_data in city_df.iterrows():

            get_or_create(session, City, city_post_code=city_data['city_post_code'],
                          city_name=address.clean_city(city_data['city_name']))
    except Exception as e:


        logging.error('Rolled back: %s.', e)
        logging.error(city_data)

        logging.exception("Exception occurred")
        session.rollback()
    else:
        logging.info('Successfully added %s city items to the dataset.', len(city_df))
        session.commit()


def insert_street_type_dataframe(session, city_df):

    city_df.columns = ['street_type']
    try:
        for index, city_data in city_df.iterrows():

            get_or_create(session, Street_type, street_type=city_data['street_type'])
    except Exception as e:

        logging.error('Rolled back: %s.', e)
        logging.error(city_data)

        logging.exception("Exception occurred")
        session.rollback()
    else:
        logging.info('Successfully added %s street type items to the dataset.', len(city_df))
        session.commit()


def insert_common_dataframe(session, common_df):

    common_df.columns = ['poi_name', 'poi_tags', 'poi_url_base', 'poi_code']
    try:
        for index, poi_common_data in common_df.iterrows():

            get_or_create(session, POI_common, **poi_common_data)
    except Exception as e:

        logging.error('Rolled back: %s.', e)
        logging.error(poi_common_data)

        logging.exception("Exception occurred")
        session.rollback()
    else:
        logging.info('Successfully added %s common items to the dataset.', len(common_df))
        session.commit()


def search_for_postcode(session, city_name):

    city_col = session.query(City.city_post_code).filter(City.city_name == city_name).all()
    if len(city_col) == 1:

        return city_col
    else:
        logging.info('Cannot determine the post code from city name (%s).', city_name)
        return None


def insert_poi_dataframe(session, poi_df):

    poi_df.columns = POI_COLS
    poi_df[['poi_postcode']] = poi_df[['poi_postcode']].fillna('0000')
    poi_df[['poi_postcode']] = poi_df[['poi_postcode']].astype(int)
    poi_dict = poi_df.to_dict('records')
    try:
        for poi_data in poi_dict:
            city_col = session.query(City.city_id).filter(City.city_name == poi_data['poi_city']).filter(

                City.city_post_code == poi_data['poi_postcode']).first()
            common_col = session.query(POI_common.pc_id).filter(POI_common.poi_code == poi_data['poi_code']).first()

            poi_data['poi_addr_city'] = city_col
            poi_data['poi_common_id'] = common_col
            if 'poi_name' in poi_data: del poi_data['poi_name']

            if 'poi_code' in poi_data: del poi_data['poi_code']

            get_or_create_poi(session, POI_address, **poi_data)
    except Exception as e:

        logging.error('Rolled back: %s.', e)
        logging.error(poi_data)
        logging.exception("Exception occurred")
        session.rollback()
        raise (e)

    else:
        try:
            session.commit()
            logging.info('Successfully added %s POI items to the dataset.', len(poi_dict))
        except Exception as e:

            logging.error('Unsuccessfull commit: %s.', e)
            logging.exception("Exception occurred")


def insert_type(session, type_data):

    try:
        for i in type_data:
            get_or_create(session, POI_common, **i)
    except Exception as e:

        logging.error('Rolled back: %s.', e)
        logging.error(i)
        logging.exception("Exception occurred")
        session.rollback()
    else:
        logging.info('Successfully added %s type items to the dataset.', len(type_data))
        session.commit()


def insert(session, **kwargs):

    try:
        city_col = session.query(City.city_id).filter(City.city_name == kwargs['poi_city']).filter(
            City.city_post_code == kwargs['poi_postcode']).first()
        common_col = session.query(POI_common.pc_id).filter(POI_common.poi_code == kwargs['poi_code']).first()

        kwargs['poi_addr_city'] = city_col
        kwargs['poi_common_id'] = common_col
        kwargs['poi_hash'] = hashlib.sha512(
            '{}{}{}{}{}{}'.format(kwargs['poi_code'], kwargs['poi_postcode'], kwargs['poi_city'],
                                  kwargs['poi_addr_street'], kwargs['poi_addr_housenumber'],
                                  kwargs['poi_conscriptionnumber']).lower().replace(' ', '').encode(
                'utf-8')).hexdigest()

        if 'poi_name' in kwargs: del kwargs['poi_name']

        if 'poi_code' in kwargs: del kwargs['poi_code']

        get_or_create(session, POI_address, **kwargs)
    except Exception as e:

        logging.error('Rolled back: %s.', e)
        logging.error(kwargs)
        logging.exception("Exception occurred")
        session.rollback()
    else:
        logging.debug('Successfully added the item to the dataset.')
        session.commit()
    finally:
        session.close()


1		# -- coding: utf-8 --
		0 ignored issues – show introduced 2020-10-11 06:10 UTC by Report Bug Copy Issue Report Missing module docstring Loading history...
2
3		try:
4		import logging
5		import sys
6		import hashlib
7		from osm_poi_matchmaker.dao.data_structure import City, POI_common, POI_address, Street_type
8		from osm_poi_matchmaker.libs import address
9		from osm_poi_matchmaker.dao import poi_array_structure
10		except ImportError as err:
11		logging.error('Error %s import module: %s', __name__, err)
12		logging.exception("Exception occurred")
13		sys.exit(128)
14
15		POI_COLS = poi_array_structure.POI_COLS
16
17
18		def get_or_create(session, model, **kwargs):
		0 ignored issues – show introduced 2020-10-11 06:10 UTC by Report Bug Copy Issue Report Missing function or method docstring Loading history...
19		instance = session.query(model).filter_by(**kwargs).first()
20	View Code Duplication	if instance:
		0 ignored issues – show Duplication introduced 2020-10-11 06:10 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history... unused-code introduced 2020-10-11 06:10 UTC by Report Bug Copy Issue Report Unnecessary "else" after "return" Loading history...
21		logging.debug('Already added: %s' ,instance)
		0 ignored issues – show Coding Style introduced 2020-10-13 09:58 UTC by Report Bug Copy Issue Report No space allowed before comma Loading history... Coding Style introduced 2020-10-13 09:58 UTC by Report Bug Copy Issue Report Exactly one space required after comma Loading history...
22		return instance
23		else:
24		try:
25		instance = model(**kwargs)
26		session.add(instance)
27		return instance
28		except Exception as e:
		0 ignored issues – show Coding Style Naming introduced 2020-10-11 06:10 UTC by Report Bug Copy Issue Report Variable name "e" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]*\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
29		logging.error('Cannot add to the database. (%s)', e)
30		logging.exception("Exception occurred")
31		raise (e)
		0 ignored issues – show Unused Code Coding Style introduced 2020-10-11 06:10 UTC by Report Bug Copy Issue Report There is an unnecessary parenthesis after `raise`. Loading history...
32
33
34		def get_or_create_poi(session, model, **kwargs):

KAMI911 / osm_poi_matchmaker

Push — master ( ea0d3c...cc0a45 )

osm_poi_matchmaker.dao.data_handlers.insert() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like