klib.utils._diff_report() - Code Metrics - Inspection of "refactor data_cleaning() / _diff_report" - akanz1/klib - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 99c19c...2e344c )

by Andreas

created 2020-04-25 10:39 UTC

klib.utils._diff_report() B

↳ Parent: klib.utils

Complexity

Conditions

Size

Total Lines	65
Code Lines	32

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	5
eloc	32
nop	5
dl	0
loc	65
rs	8.6453
c	0
b	0
f	0

How to fix Long Method

'''
Utilities and auxiliary functions.

:author: Andreas Kanz

'''

# Imports
import numpy as np
import pandas as pd


def _corr_selector(corr, split=None, threshold=0):
    '''
    Select correlations based on the provided parameters.

    Parameters
    ----------
    corr: List or matrix of correlations.

    split: {None, 'pos', 'neg', 'high', 'low'}, default None
        Type of split to be performed.

    threshold: float, default 0
        Value between 0 <= threshold <= 1

    Returns:
    -------
    corr: List or matrix of (filtered) correlations.
    '''

    if split == 'pos':
        corr = corr.where((corr >= threshold) & (corr > 0))
        print('Displaying positive correlations. Use "threshold" to further limit the results.')
    elif split == 'neg':
        corr = corr.where((corr <= threshold) & (corr < 0))
        print('Displaying negative correlations. Use "threshold" to further limit the results.')
    elif split == 'high':
        corr = corr.where(np.abs(corr) >= threshold)
        print(f'Displaying absolute correlations above the threshold ({threshold}).')
    elif split == 'low':
        corr = corr.where(np.abs(corr) <= threshold)
        print(f'Displaying absolute correlations below the threshold ({threshold}).')
    else:
        corr = corr

    return corr


def _diff_report(data, data_cleaned, dupl_rows=None, single_val_cols=None, show='changes'):
    '''
    Perform initial data cleaning tasks on a dataset, such as dropping single valued and empty rows, empty \
        columns as well as optimizing the datatypes.

    Parameters
    ----------
    data: 2D dataset that can be coerced into Pandas DataFrame.
        Input the initial dataset here.

    data_cleaned: 2D dataset that can be coerced into Pandas DataFrame.
        Input the cleaned / updated dataset here.

    dupl_rows: list, default None
        List of duplicate row indices.

    single_val_cols: list, default None
        List of single-valued column indices. I.e. columns where all cells contain the same value. \
        NaNs count as a separate value.

    show: {'all', 'changes', None} default 'all'
        Specify verbosity of the output.
        * 'all': Print information about the data before and after cleaning as well as information about changes.
        * 'changes': Print out differences in the data before and after cleaning.
        * None: No information about the data and the data cleaning is printed.

    Returns:
    -------
    Print statement highlighting the datasets or changes between the two datasets.

    '''

    if show in ['changes', 'all']:
        dupl_rows = [] if dupl_rows is None else dupl_rows.copy()
        single_val_cols = [] if single_val_cols is None else single_val_cols.copy()
        data_mem = _memory_usage(data)
        data_cl_mem = _memory_usage(data_cleaned)
        data_mv_tot = _missing_vals(data)['mv_total']
        data_cl_mv_tot = _missing_vals(data_cleaned)['mv_total']

        if show == 'all':
            print('Before data cleaning:\n')
            print(f'dtypes:\n{data.dtypes.value_counts()}')
            print(f'\nNumber of rows: {data.shape[0]}')
            print(f'Number of cols: {data.shape[1]}')
            print(f"Missing values: {data_mv_tot}")
            print(f'Memory usage: {data_mem} KB')
            print('_______________________________________________________\n')
            print('After data cleaning:\n')
            print(f'dtypes:\n{data_cleaned.dtypes.value_counts()}')
            print(f'\nNumber of rows: {data_cleaned.shape[0]}')
            print(f'Number of cols: {data_cleaned.shape[1]}')
            print(f"Missing values: {data_cl_mv_tot}")
            print(f'Memory usage: {data_cl_mem} KB')
            print('_______________________________________________________\n')

        print(f'Shape of cleaned data: {data_cleaned.shape} - Remaining NAs: {data_cl_mv_tot}')
        print(f'\nChanges:')
        print(f'Dropped rows: {data.shape[0]-data_cleaned.shape[0]}')
        print(f'     of which {len(dupl_rows)} duplicates. (Rows: {dupl_rows})')
        print(f'Dropped columns: {data.shape[1]-data_cleaned.shape[1]}')
        print(f'     of which {len(single_val_cols)} single valued. (Columns: {single_val_cols})')
        print(f"Dropped missing values: {data_mv_tot-data_cl_mv_tot}")
        mem_change = data_mem-data_cl_mem
        print(f'Reduced memory by: {round(mem_change,2)} KB (-{round(100*mem_change/data_mem,1)}%)')


def _drop_duplicates(data):
    '''
    Provides information and drops duplicate rows.

    Parameters
    ----------
    data: 2D dataset that can be coerced into Pandas DataFrame.

    Returns
    -------
    data: Deduplicated Pandas DataFrame
    rows_dropped: Index Object of rows dropped.
    '''

    data = pd.DataFrame(data).copy()
    dupl_rows = data[data.duplicated()].index.tolist()
    data = data.drop(dupl_rows, axis='index')

    return data, dupl_rows


def _memory_usage(data):
    '''
    Gives the total memory usage in kilobytes.

    Parameters
    ----------
    data: 2D dataset that can be coerced into Pandas DataFrame.

    Returns
    -------
    memory_usage: float
    '''

    data = pd.DataFrame(data).copy()
    memory_usage = round(data.memory_usage(index=True, deep=True).sum()/1024, 2)

    return memory_usage


def _missing_vals(data):
    '''
    Gives metrics of missing values in the dataset.

    Parameters
    ----------
    data: 2D dataset that can be coerced into Pandas DataFrame.

    Returns
    -------
    mv_total: float, number of missing values in the entire dataset
    mv_rows: float, number of missing values in each row
    mv_cols: float, number of missing values in each column
    mv_rows_ratio: float, ratio of missing values for each row
    mv_cols_ratio: float, ratio of missing values for each column
    '''

    data = pd.DataFrame(data).copy()
    mv_rows = data.isna().sum(axis=1)
    mv_cols = data.isna().sum(axis=0)
    mv_total = data.isna().sum().sum()
    mv_rows_ratio = mv_rows/data.shape[1]
    mv_cols_ratio = mv_cols/data.shape[0]

    return {'mv_total': mv_total,
            'mv_rows': mv_rows,
            'mv_cols': mv_cols,
            'mv_rows_ratio': mv_rows_ratio,
            'mv_cols_ratio': mv_cols_ratio}


def _validate_input_range(value, desc, lower, upper):
    if value < lower or value > upper:
        raise ValueError(
            f'Input value for {desc} is {value} but should be in the range {lower} <= {desc} <= {upper}.')


def _validate_input_int(value, desc):
    if type(value) != int:
        raise TypeError(f'Input value for {desc} is {type(value)} but should be an integer.')


def _validate_input_bool(value, desc):
    if not(isinstance(value, bool)):
        raise TypeError(f'Input value for {desc} is {type(value)} but should be a boolean.')


1			'''
2			Utilities and auxiliary functions.
3
4			:author: Andreas Kanz
5
6			'''
7
8			# Imports
9			import numpy as np
10			import pandas as pd
11
12
13			def _corr_selector(corr, split=None, threshold=0):
14			'''
15			Select correlations based on the provided parameters.
16
17			Parameters
18			----------
19			corr: List or matrix of correlations.
20
21			split: {None, 'pos', 'neg', 'high', 'low'}, default None
22			Type of split to be performed.
23
24			threshold: float, default 0
25			Value between 0 <= threshold <= 1
26
27			Returns:
28			-------
29			corr: List or matrix of (filtered) correlations.
30			'''
31
32			if split == 'pos':
33			corr = corr.where((corr >= threshold) & (corr > 0))
34			print('Displaying positive correlations. Use "threshold" to further limit the results.')
35			elif split == 'neg':
36			corr = corr.where((corr <= threshold) & (corr < 0))
37			print('Displaying negative correlations. Use "threshold" to further limit the results.')
38			elif split == 'high':
39			corr = corr.where(np.abs(corr) >= threshold)
40			print(f'Displaying absolute correlations above the threshold ({threshold}).')
41			elif split == 'low':
42			corr = corr.where(np.abs(corr) <= threshold)
43			print(f'Displaying absolute correlations below the threshold ({threshold}).')
44			else:
45			corr = corr
46
47			return corr
48
49
50			def _diff_report(data, data_cleaned, dupl_rows=None, single_val_cols=None, show='changes'):
51			'''
52			Perform initial data cleaning tasks on a dataset, such as dropping single valued and empty rows, empty \
53			columns as well as optimizing the datatypes.
54
55			Parameters
56			----------
57			data: 2D dataset that can be coerced into Pandas DataFrame.
58			Input the initial dataset here.
59
60			data_cleaned: 2D dataset that can be coerced into Pandas DataFrame.
61			Input the cleaned / updated dataset here.
62
63			dupl_rows: list, default None
64			List of duplicate row indices.
65
66			single_val_cols: list, default None
67			List of single-valued column indices. I.e. columns where all cells contain the same value. \
68			NaNs count as a separate value.
69
70			show: {'all', 'changes', None} default 'all'
71			Specify verbosity of the output.
72			* 'all': Print information about the data before and after cleaning as well as information about changes.
73			* 'changes': Print out differences in the data before and after cleaning.
74			* None: No information about the data and the data cleaning is printed.
75
76			Returns:
77			-------
78			Print statement highlighting the datasets or changes between the two datasets.
79
80			'''
81
82			if show in ['changes', 'all']:
83			dupl_rows = [] if dupl_rows is None else dupl_rows.copy()
84			single_val_cols = [] if single_val_cols is None else single_val_cols.copy()
85			data_mem = _memory_usage(data)
86			data_cl_mem = _memory_usage(data_cleaned)
87			data_mv_tot = _missing_vals(data)['mv_total']
88			data_cl_mv_tot = _missing_vals(data_cleaned)['mv_total']
89
90			if show == 'all':
91			print('Before data cleaning:\n')
92			print(f'dtypes:\n{data.dtypes.value_counts()}')
93			print(f'\nNumber of rows: {data.shape[0]}')
94			print(f'Number of cols: {data.shape[1]}')
95			print(f"Missing values: {data_mv_tot}")
96			print(f'Memory usage: {data_mem} KB')
97			print('_______________________________________________________\n')
98			print('After data cleaning:\n')
99			print(f'dtypes:\n{data_cleaned.dtypes.value_counts()}')
100			print(f'\nNumber of rows: {data_cleaned.shape[0]}')
101			print(f'Number of cols: {data_cleaned.shape[1]}')
102			print(f"Missing values: {data_cl_mv_tot}")
103			print(f'Memory usage: {data_cl_mem} KB')
104			print('_______________________________________________________\n')
105
106			print(f'Shape of cleaned data: {data_cleaned.shape} - Remaining NAs: {data_cl_mv_tot}')
107			print(f'\nChanges:')
108			print(f'Dropped rows: {data.shape[0]-data_cleaned.shape[0]}')
109			print(f' of which {len(dupl_rows)} duplicates. (Rows: {dupl_rows})')
110			print(f'Dropped columns: {data.shape[1]-data_cleaned.shape[1]}')
111			print(f' of which {len(single_val_cols)} single valued. (Columns: {single_val_cols})')
112			print(f"Dropped missing values: {data_mv_tot-data_cl_mv_tot}")
113			mem_change = data_mem-data_cl_mem
114			print(f'Reduced memory by: {round(mem_change,2)} KB (-{round(100*mem_change/data_mem,1)}%)')
115
116
117			def _drop_duplicates(data):
118			'''
119			Provides information and drops duplicate rows.
120
121			Parameters
122			----------
123			data: 2D dataset that can be coerced into Pandas DataFrame.
124
125			Returns
126			-------
127			data: Deduplicated Pandas DataFrame
128			rows_dropped: Index Object of rows dropped.
129			'''
130
131			data = pd.DataFrame(data).copy()
132			dupl_rows = data[data.duplicated()].index.tolist()
133			data = data.drop(dupl_rows, axis='index')
134
135			return data, dupl_rows
136
137
138			def _memory_usage(data):
139			'''
140			Gives the total memory usage in kilobytes.
141
142			Parameters
143			----------
144			data: 2D dataset that can be coerced into Pandas DataFrame.
145
146			Returns
147			-------
148			memory_usage: float
149			'''
150
151			data = pd.DataFrame(data).copy()
152			memory_usage = round(data.memory_usage(index=True, deep=True).sum()/1024, 2)
153
154			return memory_usage
155
156
157			def _missing_vals(data):
158			'''
159			Gives metrics of missing values in the dataset.
160
161			Parameters
162			----------
163			data: 2D dataset that can be coerced into Pandas DataFrame.
164
165			Returns
166			-------
167			mv_total: float, number of missing values in the entire dataset
168			mv_rows: float, number of missing values in each row
169			mv_cols: float, number of missing values in each column
170			mv_rows_ratio: float, ratio of missing values for each row
171			mv_cols_ratio: float, ratio of missing values for each column
172			'''
173
174			data = pd.DataFrame(data).copy()
175			mv_rows = data.isna().sum(axis=1)
176			mv_cols = data.isna().sum(axis=0)
177			mv_total = data.isna().sum().sum()
178			mv_rows_ratio = mv_rows/data.shape[1]
179			mv_cols_ratio = mv_cols/data.shape[0]
180
181			return {'mv_total': mv_total,
182			'mv_rows': mv_rows,
183			'mv_cols': mv_cols,
184			'mv_rows_ratio': mv_rows_ratio,
185			'mv_cols_ratio': mv_cols_ratio}
186
187
188			def _validate_input_range(value, desc, lower, upper):
189			if value < lower or value > upper:
190			raise ValueError(
191			f'Input value for {desc} is {value} but should be in the range {lower} <= {desc} <= {upper}.')
192
193
194			def _validate_input_int(value, desc):
195			if type(value) != int:
196			raise TypeError(f'Input value for {desc} is {type(value)} but should be an integer.')
197
198
199			def _validate_input_bool(value, desc):
200			if not(isinstance(value, bool)):
201			raise TypeError(f'Input value for {desc} is {type(value)} but should be a boolean.')
202

akanz1 / klib

GitHub Access Token became invalid

Push — master ( 99c19c...2e344c )

klib.utils._diff_report() B

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like