akanz1 /
klib
| Conditions | 3 |
| Total Lines | 77 |
| Code Lines | 28 |
| Lines | 0 |
| Ratio | 0 % |
| Changes | 0 | ||
Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.
For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.
Commonly applied refactorings include:
If many parameters/temporary variables are present:
| 1 | ''' |
||
| 77 | def data_cleaning(data, drop_threshold_cols=0.9, drop_threshold_rows=0.9, category=True, cat_threshold=0.05, cat_exclude=[], show='all'): |
||
| 78 | ''' |
||
| 79 | Perform initial data cleaning tasks on a dataset, such as dropping empty rows and columns and optimizing the datatypes. |
||
| 80 | |||
| 81 | Parameters |
||
| 82 | ---------- |
||
| 83 | data: 2D dataset that can be coerced into Pandas DataFrame. If a Pandas DataFrame is provided, the index/column information is used to label the plots. |
||
| 84 | |||
| 85 | drop_threshold_cols: float, default 1 |
||
| 86 | Drop columns with NA-ratio above the specified threshold. |
||
| 87 | |||
| 88 | drop_threshold_rows: float, default 1 |
||
| 89 | Drop rows with NA-ratio above the specified threshold. |
||
| 90 | |||
| 91 | category: bool, default True |
||
| 92 | Change dtypes of columns to "category". Set threshold using cat_threshold. |
||
| 93 | |||
| 94 | cat_threshold: float, default 0.05 |
||
| 95 | Ratio of unique values below which categories are inferred and column dtype is changed to categorical. |
||
| 96 | |||
| 97 | cat_exclude: default [] (empty list) |
||
| 98 | List of columns to exclude from categorical conversion. |
||
| 99 | |||
| 100 | show: {'all', 'changes', None} default 'all' |
||
| 101 | Specify verbosity of the output. |
||
| 102 | * 'all': Print information about the data before and after cleaning as well as information about changes. |
||
| 103 | * 'changes': Print out differences in the data before and after cleaning. |
||
| 104 | * None: no information about the data is printed. |
||
| 105 | |||
| 106 | Returns |
||
| 107 | ------- |
||
| 108 | Pandas DataFrame. |
||
| 109 | |||
| 110 | See Also |
||
| 111 | -------- |
||
| 112 | convert_datatypes: Converts columns to best possible dtypes. |
||
| 113 | drop_missing : Flexibly drops columns and rows. |
||
| 114 | _memory_usage: Gives the total memory usage in kilobytes. |
||
| 115 | _missing_vals: Metrics about missing values in the dataset. |
||
| 116 | |||
| 117 | |||
| 118 | Notes |
||
| 119 | ----- |
||
| 120 | The category dtype is not grouped in the summary, unless it contains exactly the same categories. |
||
| 121 | |||
| 122 | ''' |
||
| 123 | |||
| 124 | data = pd.DataFrame(data) |
||
| 125 | data_cleaned = drop_missing(data, drop_threshold_cols, drop_threshold_rows) |
||
| 126 | data_cleaned = convert_datatypes(data_cleaned, category=True, cat_threshold=0.05, cat_exclude=cat_exclude) |
||
| 127 | |||
| 128 | if show in ['changes', 'all']: |
||
| 129 | if show == 'all': |
||
| 130 | print('Before data cleaning:\n') |
||
| 131 | print(f'dtypes:\n{data.dtypes.value_counts()}') |
||
| 132 | print(f'\nNumber of rows: {data.shape[0]}') |
||
| 133 | print(f'Number of cols: {data.shape[1]}') |
||
| 134 | print(f'Missing values: {_missing_vals(data)[0]}') |
||
| 135 | print(f'Memory usage: {_memory_usage(data)} KB') |
||
| 136 | print('_______________________________________________________\n') |
||
| 137 | print('After data cleaning:\n') |
||
| 138 | print(f'dtypes:\n{data_cleaned.dtypes.value_counts()}') |
||
| 139 | print(f'\nNumber of rows: {data_cleaned.shape[0]}') |
||
| 140 | print(f'Number of cols: {data_cleaned.shape[1]}') |
||
| 141 | print(f'Missing values: {_missing_vals(data_cleaned)[0]}') |
||
| 142 | print(f'Memory usage: {_memory_usage(data_cleaned)} KB') |
||
| 143 | print('_______________________________________________________\n') |
||
| 144 | |||
| 145 | print(f'Shape of cleaned dataset: {data_cleaned.shape} - Remaining NAs: {_missing_vals(data_cleaned)[0]}') |
||
| 146 | print(f'\nChanges:') |
||
| 147 | print(f'Dropped rows: {data.shape[0]-data_cleaned.shape[0]}') |
||
| 148 | print(f'Dropped columns: {data.shape[1]-data_cleaned.shape[1]}') |
||
| 149 | print(f'Dropped missing values: {_missing_vals(data)[0]-_missing_vals(data_cleaned)[0]}') |
||
| 150 | mem_change = _memory_usage(data)-_memory_usage(data_cleaned) |
||
| 151 | print(f'Reduced memory by: {mem_change} KB (-{round(100*mem_change/_memory_usage(data),1)}%)') |
||
| 152 | |||
| 153 | return data_cleaned |
||
| 154 |
