GreedyDict

Complexity

Total Complexity

5

Size/Duplication

Total Lines	9
Duplicated Lines	0 %

2 Methods

Rating	Name	Duplication	Size	Complexity
A	__setitem__()	0	4	3
A	__iter__()	0	3	2

from six.moves import UserDict

This module should have a docstring.

The import six.moves could not be resolved.

import types

from ._registry import register_output
from .base_output import OutputInterface


class GreedyDict(UserDict, object):

This class should have a docstring.

    def __setitem__(self, key, value):
        if isinstance(value, types.GeneratorType):
            value = [val for val in value]
        super(GreedyDict, self).__setitem__(key, value)

    def __iter__(self):
        for val in self.data.values():

The Instance of GreedyDict does not seem to have a member named data.

            yield val


@register_output

This class should have a docstring.

This interface does not seem to be used anywhere.

class InMemoryOutput(OutputInterface):
    def __init__(self, iterable=None, hash_field=None,
                 tokenized_corpora=None,
                 vectorized_corpora=None, modeled_corpora=None):
        super(InMemoryOutput, self).__init__()

        self.corpus = GreedyDict()

        if iterable:
            self.import_from_iterable(iterable, hash_field)

        self.tokenized_corpora = tokenized_corpora if tokenized_corpora else GreedyDict()
        self.vectorized_corpora = vectorized_corpora if vectorized_corpora else GreedyDict()
        self.modeled_corpora = modeled_corpora if modeled_corpora else GreedyDict()

    def import_from_iterable(self, iterable, field_to_hash):
        """
        iterable: generally a list of dicts, but possibly a list of strings
            This is your data.  Your dictionary structure defines the schema
            of the elasticsearch index.
        """
        self.hash_field=field_to_hash

Exactly one space required around assignment
self.hash_field=field_to_hash
^

        for item in iterable:
            if isinstance(item, basestring):

Undefined variable 'basestring'

                item = {field_to_hash: item}
            elif field_to_hash not in item and field_to_hash in item.values()[0]:
                item = item.values()[0]
            id = hash(item[field_to_hash])

This seems to re-define the built-in id.

The name id does not conform to the variable naming conventions ([a-z_][a-z0-9_]{2,30}$).

            self.corpus[id] = item

    # TODO: generalize for datetimes

TODO and FIXME comments should generally be avoided.

    # TODO: validate input data to ensure that it has valid year data

TODO and FIXME comments should generally be avoided.

    def get_date_filtered_data(self, field_to_get, start, end, filter_field="year"):

This method should have a docstring.

        return self.get_filtered_data(field_to_get,
                                      "{}<=int({}['{}'])<={}".format(start, "{}",
                                                                     filter_field, end))

    def get_filtered_data(self, field_to_get, filter=""):

This seems to re-define the built-in filter.

        if not filter:
            for doc_id, doc in self.corpus.items():

The Instance of GreedyDict does not seem to have a member named items.

                yield doc_id, doc[field_to_get]
        else:
            for doc_id, doc in self.corpus.items():

The Instance of GreedyDict does not seem to have a member named items.

                if eval(filter.format(doc)):

eval should generally only be used if absolutely necessary.

                    yield doc_id, doc[field_to_get]

    def save(self, filename):

Arguments number differs from overridden 'save' method

        saved_data = {"iterable": self.corpus,
                      "hash_field": self.hash_field,
                      "modeled_corpora": self.modeled_corpora,
                      "vectorized_corpora": self.vectorized_corpora,
                      "tokenized_corpora": self.tokenized_corpora}
        return super(InMemoryOutput, self).save(filename, saved_data)