klib.clean.pool_duplicate_subsets()

Complexity

Conditions

9

Size

Total Lines	87
Code Lines	38

Duplication

Lines	0
Ratio	0 %

Importance

Changes

0

'''
def pool_duplicate_subsets(data, col_dupl_thresh=0.2, subset_thresh=0.2, min_col_pool=3, exclude=None,
                           return_details=False):
    '''
    Checks for duplicates in subsets of columns and pools them. This reduced the number of columns in the data without \
    loosing any information. Suitable columns are combined to subsets and tested for duplicates. In case sufficient \
    duplicates can be found, the respective columns are aggregated into a 'pooled_var' column. Identical numbers in \
    the 'pooled_var' column indicate identical information in the respective rows.

    Parameters
    ----------
    data: 2D dataset that can be coerced into Pandas DataFrame.

    col_dupl_thresh: float, default 0.2
        Columns with a ratio of duplicates higher than 'col_dupl_thresh' are considered in the further analysis. \
        Columns with a lower ratio are not considered for pooling.

    subset_thresh: float, default 0.2
        The first subset with a duplicate threshold higher than 'subset_thresh' is chosen and aggregated. If no subset \
        reaches the threshold, the algorithm continues with continuously smaller subsets until 'min_col_pool' is \
        reached.

    min_col_pool: integer, default 3
        Minimum number of columns to pool. The algorithm attempts to combine as many columns as possible to suitable \
        subsets and stops when 'min_col_pool' is reached.

    exclude. list, default None
        List of column names to be excluded from the analysis. These columns are passed through without modification.

    return_details: bool, default False
        Provdies flexibility to return intermediary results.

    Returns:
    -------
    data: pd.DataFrame

    optional:
    subset_cols: List of columns used as subset.
    '''

    # Input validation
    _validate_input_range(col_dupl_thresh, 'col_dupl_thresh', 0, 1)
    _validate_input_range(subset_thresh, 'subset_thresh', 0, 1)
    _validate_input_range(min_col_pool, 'min_col_pool', 0, data.shape[1])

    excluded_cols = []
    if exclude is not None:
        excluded_cols = data[exclude]
        data = data.drop(columns=exclude)

    subset_cols = []
    for i in range(data.shape[1]+1-min_col_pool):
        check_list = []
        for col in data.columns:
            cdr = data.duplicated(subset=col).mean()
            if cdr > col_dupl_thresh:
                check_list.append(col)

        if len(check_list) > 0:
            combinations = itertools.combinations(check_list, len(check_list)-i)
        else:
            continue

        ratios = []
        for comb in combinations:
            ratios.append(data[list(comb)].duplicated().mean())

        max_ratio = max(ratios)
        max_idx = np.argmax(ratios)

        if max_ratio > subset_thresh:
            best_subset = itertools.islice(itertools.combinations(
                check_list, len(check_list)-i), max_idx, max_idx+1)
            best_subset = data[list(list(best_subset)[0])]
            subset_cols = best_subset.columns.tolist()

            unique_subset = best_subset.drop_duplicates().reset_index().rename(columns={'index': 'pooled_vars'})
            data = data.merge(unique_subset, how='left', on=best_subset.columns.tolist()
                              ).drop(columns=best_subset.columns.tolist())
            data.index = pd.RangeIndex(len(data))
            break

    data = pd.concat([data, pd.DataFrame(excluded_cols)], axis=1)

    if return_details:
        return data, subset_cols

    return data