topik.fileio.read_document_folder()

Complexity

Conditions

7

Size

Total Lines

39

Duplication

Lines	0
Ratio	0 %

import os
@register_input
def read_document_folder(folder, content_field='text'):
    """Iterate over the files in a folder to retrieve the content to process and tokenize.

    Parameters
    ----------
    folder : str
        The folder containing the files you want to analyze.

    content_field : str
        The usage of 'content_field' in this source is different from most other sources.  The 

Trailing whitespace

        assumption in this source is that each file contains raw text, NOT dictionaries of 

Trailing whitespace

        categorized data.  The content_field argument here specifies what key to store the raw
        text under in the returned dictionary for each document.

    Examples
    --------
    >>> documents = read_document_folder(
    ...     '{}/test_data_folder_files'.format(test_data_path))
    >>> next(documents)['text'] == (
    ...     u"'Interstellar' was incredible. The visuals, the score, " +
    ...     u"the acting, were all amazing. The plot is definitely one " +
    ...     u"of the most original I've seen in a while.")
    True
    """

    if not os.path.exists(folder):
        raise IOError("Folder not found!")

    for directory, subdirectories, files in os.walk(folder):

The variable subdirectories seems to be unused.

        for n, file in enumerate(sorted(files)):

The name n does not conform to the variable naming conventions ([a-z_][a-z0-9_]{2,30}$).

The variable n seems to be unused.

            _open = gzip.open if file.endswith('.gz') else open
            try:
                fullpath = os.path.join(directory, file)
                with _open(fullpath, 'rb') as f:

The name f does not conform to the variable naming conventions ([a-z_][a-z0-9_]{2,30}$).

                    yield {content_field: f.read().decode('utf-8'),
                           'filename': fullpath}
            except (ValueError, UnicodeDecodeError) as err:
                logging.warning("Unable to process file: {}, error: {}".format(fullpath, err))