osm_poi_matchmaker.libs.soup

Complexity

Total Complexity

28

Size/Duplication

Total Lines	104
Duplicated Lines	0 %

Importance

Changes

0

3 Functions

Rating	Name	Duplication	Size	Complexity
B	readfile()	0	16	8
A	download_content()	0	20	5
F	save_downloaded_soup()	0	46	15

# -*- coding: utf-8 -*-

Missing module docstring

try:
    import logging
    import sys
    import requests
    import os
    from bs4 import BeautifulSoup
    from osm_poi_matchmaker.utils import config
    from osm_poi_matchmaker.utils.enums import FileType
except ImportError as err:
    logging.error('Error %s import module: %s', __name__, err)
    logging.exception('Exception occurred')

    sys.exit(128)


def download_content(link, verify_link=config.get_download_verify_link(), post_parm=None, headers=None,

This line is too long as per the coding-style (103/100).

Missing function or method docstring

                     encoding='utf-8'):
    try:
        if post_parm is None:
            logging.debug('Downloading without post parameters.')
            page = requests.get(link, verify=verify_link, headers=headers)
            page.encoding = encoding
        else:
            logging.debug('Downloading with post parameters.')
            headers_static = {"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"}
            if headers is not None:
                headers.update(headers_static)
            else:
                headers = headers_static
            page = requests.post(link, verify=verify_link, data=post_parm, headers=headers)
            page.encoding = encoding
    except requests.exceptions.ConnectionError as e:

Variable name "e" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,|_[^\\WA-Z]*|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern)

        logging.warning('Unable to open connection. (%s)', e)
        return None
    return page.text if page.status_code == 200 else None


def save_downloaded_soup(link, file, filetype, post_data=None, verify=config.get_download_verify_link(), headers=None):

This line is too long as per the coding-style (119/100).

Missing function or method docstring

Too many arguments (6/5)

    if config.get_download_use_cached_data() is True and os.path.isfile(file):
        soup = readfile(file, filetype)
    else:
        if link is not None:
            soup = download_content(link, verify, post_data, headers)
            if soup is not None:
                logging.info('We got content, write to file.')
                if not os.path.exists(config.get_directory_cache_url()):
                    os.makedirs(config.get_directory_cache_url())
                with open(file, mode='w', encoding='utf-8') as code:
                    if filetype == FileType.html:
                        soup = BeautifulSoup(soup, 'html.parser')
                        code.write(str(soup.prettify()))
                    elif filetype == FileType.xml:
                        soup = BeautifulSoup(soup, 'lxml', from_encoding='utf-8')
                        logging.debug('original encoding: %s', soup.original_encoding)
                        code.write(str(soup.prettify()))
                    elif filetype == FileType.csv or filetype == FileType.json:

Consider merging these comparisons with "in" to 'filetype in (FileType.csv, FileType.json)'

                        code.write(str(soup))
                    else:
                        logging.error('Unexpected type to write: %s', filetype)
            else:
                if os.path.exists(file):
                    logging.info(
                        'The %s link returned error code other than 200 but there is an already downloaded file. Try to open it.',

This line is too long as per the coding-style (130/100).

                        link)
                    soup = readfile(file, filetype)
                else:
                    logging.warning(
                        'Skipping dataset: %s. There is not downloadable URL, nor already downbloaded file.', link)

This line is too long as per the coding-style (115/100).

        else:
            if os.path.exists(file):
                soup = readfile(file, filetype)
                if filetype == FileType.html:
                    soup = BeautifulSoup(soup, 'html.parser')
                elif filetype == FileType.xml:
                    soup = BeautifulSoup(soup, 'lxml')
                logging.info(
                    'Using file only: %s. There is not downloadable URL only just the file. Do not forget to update file manually!',

This line is too long as per the coding-style (132/100).

                    file)
            else:
                logging.warning(
                    'Cannot use download and file: %s. There is not downloadable URL, nor already downbloaded file.',

This line is too long as per the coding-style (117/100).

                    file)
    return soup

The variable soup does not seem to be defined for all execution paths.

def readfile(r_filename, r_filetype):

Missing function or method docstring

    try:
        if os.path.exists(r_filename):

Unnecessary "else" after "return"

            with open(r_filename, mode='r', encoding='utf-8') as code:
                if r_filetype == FileType.html:
                    soup = BeautifulSoup(code.read(), 'html.parser')
                elif r_filetype == FileType.csv or r_filetype == FileType.json or r_filetype == FileType.xml:

This line is too long as per the coding-style (109/100).

Consider merging these comparisons with "in" to 'r_filetype in (FileType.csv, FileType.json, FileType.xml)'

                    soup = code.read()
                else:
                    logging.error('Unexpected type to read: %s', r_filetype)
            return soup

The variable soup does not seem to be defined for all execution paths.

        else:
            return None
    except Exception as e:

Catching very general exceptions such as Exception is usually not recommended.

Variable name "e" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,|_[^\\WA-Z]*|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern)

        logging.error(e)
        logging.exception('Exception occurred')