ModeledElasticCorpora

Complexity

Total Complexity

8

Size/Duplication

Total Lines	26
Duplicated Lines	100 %

3 Methods

Rating	Name	Duplication	Size	Complexity
B	__getitem__()	15	15	6
A	__setitem__()	6	6	1
A	__lt__()	2	2	1

from six.moves import UserDict

This module should have a docstring.

The import six.moves could not be resolved.

import logging
import time

from elasticsearch import Elasticsearch, helpers

from ._registry import register_output
from .base_output import OutputInterface
from topik.vectorizers.vectorizer_output import VectorizerOutput
from topik.models.base_model_output import ModelOutput

def es_setitem(key, value, doc_type, instance, index, batch_size=1000):
    """load an iterable of (id, value) pairs to the specified new or
           new or existing field within existing documents."""
    batch = []
    for id, val in value:

This seems to re-define the built-in id.

The name id does not conform to the variable naming conventions ([a-z_][a-z0-9_]{2,30}$).

        action = {'_op_type': 'update',
                  '_index': index,
                  '_type': doc_type,
                  '_id': id,
                  'doc': {key: val},
                  'doc_as_upsert': "true",
                  }
        batch.append(action)
        if len(batch) >= batch_size:
            helpers.bulk(client=instance, actions=batch,
                         index=index)
            batch = []
    if batch:
        helpers.bulk(client=instance, actions=batch, index=index)
    instance.indices.refresh(index)

def es_getitem(key, doc_type, instance, index, query=None):

This function should have a docstring.

    results = helpers.scan(instance, index=index,
                               query=query, doc_type=doc_type)
    for result in results:
        try:
            id = int(result["_id"])

This seems to re-define the built-in id.

The name id does not conform to the variable naming conventions ([a-z_][a-z0-9_]{2,30}$).

        except ValueError:

The variable ValueError does not seem to be defined.

            id = result["_id"]

The name id does not conform to the variable naming conventions ([a-z_][a-z0-9_]{2,30}$).

        yield id, result['_source'][key]

class BaseElasticCorpora(UserDict):

This class should have a docstring.

The variable UserDict does not seem to be defined.

    def __init__(self, instance, index, corpus_type, query=None,
                 batch_size=1000):
        self.instance = instance

The variable instance does not seem to be defined.

        self.index = index

The variable index does not seem to be defined.

        self.corpus_type = corpus_type

The variable corpus_type does not seem to be defined.

        self.query = query

The variable query does not seem to be defined.

        self.batch_size = batch_size

The variable batch_size does not seem to be defined.

        pass

Unnecessary pass statement

    def __setitem__(self, key, value):
        es_setitem(key, value, self.corpus_type, self.instance, self.index)

The variable self does not seem to be defined.

The variable key does not seem to be defined.

The variable value does not seem to be defined.

    def __getitem__(self, key):
        return es_getitem(key,self.corpus_type,self.instance,self.index,

Exactly one space required after comma
return es_getitem(key,self.corpus_type,self.instance,self.index,
^

Exactly one space required after comma
return es_getitem(key,self.corpus_type,self.instance,self.index,
^

Exactly one space required after comma
return es_getitem(key,self.corpus_type,self.instance,self.index,
^

The variable key does not seem to be defined.

The variable self does not seem to be defined.

                          self.query)

class VectorizedElasticCorpora(BaseElasticCorpora):

This class should have a docstring.

This code seems to be duplicated in your project.

The variable BaseElasticCorpora does not seem to be defined.

    def __setitem__(self, key, value):
        #id_term_map
        es_setitem(key,value.id_term_map.items(),"term",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.id_term_map.items(),"term",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.id_term_map.items(),"term",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.id_term_map.items(),"term",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.id_term_map.items(),"term",self.instance,self.index)
^

The variable key does not seem to be defined.

The variable self does not seem to be defined.

        #document_term_counts
        es_setitem(key,value.document_term_counts.items(),"document_term_count",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.document_term_counts.items(),"document_term_count",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.document_term_counts.items(),"document_term_count",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.document_term_counts.items(),"document_term_count",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.document_term_counts.items(),"document_term_count",self.instance,self.index)
^

        #doc_lengths
        es_setitem(key,value.doc_lengths.items(),"document_length",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.doc_lengths.items(),"document_length",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_lengths.items(),"document_length",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_lengths.items(),"document_length",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_lengths.items(),"document_length",self.instance,self.index)
^

        #global term_frequency
        es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)
^

        #vectors
        es_setitem(key,value.vectors.items(),"vector",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.vectors.items(),"vector",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.vectors.items(),"vector",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.vectors.items(),"vector",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.vectors.items(),"vector",self.instance,self.index)
^

        # could either upload vectors explicitly here (above) or using Super (below)
        #super(VectorizedElasticCorpora, self).__setitem__(key, value)

    def __getitem__(self, key):
        # TODO: each of these should be retrieved from a query.  Populate the VectorizerOutput object

TODO and FIXME comments should generally be avoided.

        # and return it.  These things can be iterators instead of dicts; VectorizerOutput should
        # not care.
        # TODO: this is the id->term map for the full set of unique terms across all docs

TODO and FIXME comments should generally be avoided.

        id_term_map = {int(term_id): term for term_id, term in es_getitem(key,"term",self.instance,self.index,self.query)}

Exactly one space required after comma
id_term_map = int(term_id): term for term_id, term in es_getitem(key,"term",self.instance,self.index,self.query)
^

Exactly one space required after comma
id_term_map = int(term_id): term for term_id, term in es_getitem(key,"term",self.instance,self.index,self.query)
^

Exactly one space required after comma
id_term_map = int(term_id): term for term_id, term in es_getitem(key,"term",self.instance,self.index,self.query)
^

Exactly one space required after comma
id_term_map = int(term_id): term for term_id, term in es_getitem(key,"term",self.instance,self.index,self.query)
^

The variable term does not seem to be defined.

The variable term_id does not seem to be defined.

The variable self does not seem to be defined.

The variable key does not seem to be defined.

        # 15
        # TODO: this is the count of terms associated with each document

TODO and FIXME comments should generally be avoided.

        document_term_count = {int(doc_id): doc_term_count for doc_id, doc_term_count in es_getitem(key,"document_term_count",self.instance,self.index,self.query)}

Exactly one space required after comma
document_term_count = int(doc_id): doc_term_count for doc_id, doc_term_count in es_getitem(key,"document_term_count",self.instance,self.index,self.query)
^

Exactly one space required after comma
document_term_count = int(doc_id): doc_term_count for doc_id, doc_term_count in es_getitem(key,"document_term_count",self.instance,self.index,self.query)
^

Exactly one space required after comma
document_term_count = int(doc_id): doc_term_count for doc_id, doc_term_count in es_getitem(key,"document_term_count",self.instance,self.index,self.query)
^

Exactly one space required after comma
document_term_count = int(doc_id): doc_term_count for doc_id, doc_term_count in es_getitem(key,"document_term_count",self.instance,self.index,self.query)
^

The variable doc_id does not seem to be defined.

The variable doc_term_count does not seem to be defined.

        # {"doc1": 3, "doc2": 5}
        doc_lengths = {int(doc_id): doc_length for doc_id, doc_length in es_getitem(key,"document_length",self.instance,self.index,self.query)}

Exactly one space required after comma
doc_lengths = int(doc_id): doc_length for doc_id, doc_length in es_getitem(key,"document_length",self.instance,self.index,self.query)
^

Exactly one space required after comma
doc_lengths = int(doc_id): doc_length for doc_id, doc_length in es_getitem(key,"document_length",self.instance,self.index,self.query)
^

Exactly one space required after comma
doc_lengths = int(doc_id): doc_length for doc_id, doc_length in es_getitem(key,"document_length",self.instance,self.index,self.query)
^

Exactly one space required after comma
doc_lengths = int(doc_id): doc_length for doc_id, doc_length in es_getitem(key,"document_length",self.instance,self.index,self.query)
^

The variable doc_length does not seem to be defined.

        term_frequency = {int(term_id): global_frequency for term_id, global_frequency in es_getitem(key,"term_frequency",self.instance,self.index,self.query)}

Exactly one space required after comma
term_frequency = int(term_id): global_frequency for term_id, global_frequency in es_getitem(key,"term_frequency",self.instance,self.index,self.query)
^

Exactly one space required after comma
term_frequency = int(term_id): global_frequency for term_id, global_frequency in es_getitem(key,"term_frequency",self.instance,self.index,self.query)
^

Exactly one space required after comma
term_frequency = int(term_id): global_frequency for term_id, global_frequency in es_getitem(key,"term_frequency",self.instance,self.index,self.query)
^

Exactly one space required after comma
term_frequency = int(term_id): global_frequency for term_id, global_frequency in es_getitem(key,"term_frequency",self.instance,self.index,self.query)
^

The variable global_frequency does not seem to be defined.

        # TODO: this is the vectorized representation of each document

TODO and FIXME comments should generally be avoided.

        vectors = {int(doc_id): {int(term_id): term_weight for term_id, term_weight in doc_term_weights.items()} for doc_id, doc_term_weights in es_getitem(key,"vector",self.instance,self.index,self.query)}

Exactly one space required after comma
vectors = int(doc_id): {int(term_id): term_weight for term_id, term_weight in doc_term_weights.items() for doc_id, doc_term_weights in es_getitem(key,"vector",self.instance,self.index,self.query)}
^

Exactly one space required after comma
vectors = int(doc_id): {int(term_id): term_weight for term_id, term_weight in doc_term_weights.items() for doc_id, doc_term_weights in es_getitem(key,"vector",self.instance,self.index,self.query)}
^

Exactly one space required after comma
vectors = int(doc_id): {int(term_id): term_weight for term_id, term_weight in doc_term_weights.items() for doc_id, doc_term_weights in es_getitem(key,"vector",self.instance,self.index,self.query)}
^

Exactly one space required after comma
vectors = int(doc_id): {int(term_id): term_weight for term_id, term_weight in doc_term_weights.items() for doc_id, doc_term_weights in es_getitem(key,"vector",self.instance,self.index,self.query)}
^

The variable doc_term_weights does not seem to be defined.

The variable term_weight does not seem to be defined.

        #vectors = {int(doc_id): {doc_term_weights for doc_id, doc_term_weights in es_getitem(key,"vector",self.instance,self.index,self.query)}
        #vectors = list(es_getitem(key,"vector",self.instance,self.index,self.query))
        #  {"doc1": {1: 3, 2: 1}  # word id is key, word count is value (for bag of words model)
        return VectorizerOutput(id_term_map=id_term_map,

The variable id_term_map does not seem to be defined.

                                document_term_counts=document_term_count,

The variable document_term_count does not seem to be defined.

                                doc_lengths=doc_lengths,

The variable doc_lengths does not seem to be defined.

                                term_frequency=term_frequency,

The variable term_frequency does not seem to be defined.

                                vectors=vectors)

The variable vectors does not seem to be defined.

class ModeledElasticCorpora(BaseElasticCorpora):

This class should have a docstring.

This code seems to be duplicated in your project.

The variable BaseElasticCorpora does not seem to be defined.

    def __setitem__(self, key, value):
        es_setitem(key,value.vocab.items(),"term",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.vocab.items(),"term",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.vocab.items(),"term",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.vocab.items(),"term",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.vocab.items(),"term",self.instance,self.index)
^

The variable self does not seem to be defined.

The variable key does not seem to be defined.

        es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.term_frequency.items(),"term_frequency",self.instance,self.index)
^

        es_setitem(key,value.topic_term_matrix.items(),"topic_term_dist",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.topic_term_matrix.items(),"topic_term_dist",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.topic_term_matrix.items(),"topic_term_dist",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.topic_term_matrix.items(),"topic_term_dist",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.topic_term_matrix.items(),"topic_term_dist",self.instance,self.index)
^

        es_setitem(key,value.doc_lengths.items(),"doc_length",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.doc_lengths.items(),"doc_length",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_lengths.items(),"doc_length",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_lengths.items(),"doc_length",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_lengths.items(),"doc_length",self.instance,self.index)
^

        es_setitem(key,value.doc_topic_matrix.items(),"doc_topic_dist",self.instance,self.index)

Exactly one space required after comma
es_setitem(key,value.doc_topic_matrix.items(),"doc_topic_dist",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_topic_matrix.items(),"doc_topic_dist",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_topic_matrix.items(),"doc_topic_dist",self.instance,self.index)
^

Exactly one space required after comma
es_setitem(key,value.doc_topic_matrix.items(),"doc_topic_dist",self.instance,self.index)
^

    def __lt__(self, y):

The name y does not conform to the argument naming conventions ([a-z_][a-z0-9_]{2,30}$).

        return super(ModeledElasticCorpora, self).__lt__(y)

The variable y does not seem to be defined.

    def __getitem__(self, key):
        vocab = {int(term_id): term for term_id, term in \

The variable term does not seem to be defined.

The variable term_id does not seem to be defined.

                 es_getitem(key,"term",self.instance,self.index,self.query)}

Exactly one space required after comma
es_getitem(key,"term",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"term",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"term",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"term",self.instance,self.index,self.query)}
^

The variable self does not seem to be defined.

The variable key does not seem to be defined.

        term_frequency = {int(term_id): tf for term_id, tf in \

The variable tf does not seem to be defined.

                          es_getitem(key,"term_frequency",self.instance,self.index,self.query)}

Exactly one space required after comma
es_getitem(key,"term_frequency",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"term_frequency",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"term_frequency",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"term_frequency",self.instance,self.index,self.query)}
^

        topic_term_matrix = {topic_id: topic_term_dist for topic_id, topic_term_dist in \

The variable topic_term_dist does not seem to be defined.

The variable topic_id does not seem to be defined.

                             es_getitem(key,"topic_term_dist",self.instance,self.index,self.query)}

Exactly one space required after comma
es_getitem(key,"topic_term_dist",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"topic_term_dist",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"topic_term_dist",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"topic_term_dist",self.instance,self.index,self.query)}
^

        doc_lengths = {topic_id: doc_length for topic_id, doc_length in \

The variable doc_length does not seem to be defined.

                       es_getitem(key,"doc_length",self.instance,self.index,self.query)}

Exactly one space required after comma
es_getitem(key,"doc_length",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"doc_length",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"doc_length",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"doc_length",self.instance,self.index,self.query)}
^

        doc_topic_matrix = {int(doc_id): doc_topic_dist for doc_id, doc_topic_dist in \

The variable doc_id does not seem to be defined.

The variable doc_topic_dist does not seem to be defined.

                             es_getitem(key,"doc_topic_dist",self.instance,self.index,self.query)}

Exactly one space required after comma
es_getitem(key,"doc_topic_dist",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"doc_topic_dist",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"doc_topic_dist",self.instance,self.index,self.query)}
^

Exactly one space required after comma
es_getitem(key,"doc_topic_dist",self.instance,self.index,self.query)}
^

        return ModelOutput(vocab=vocab, term_frequency=term_frequency,

The variable term_frequency does not seem to be defined.

The variable vocab does not seem to be defined.

                           topic_term_matrix=topic_term_matrix,

The variable topic_term_matrix does not seem to be defined.

                           doc_lengths=doc_lengths,

The variable doc_lengths does not seem to be defined.

                           doc_topic_matrix=doc_topic_matrix)

The variable doc_topic_matrix does not seem to be defined.

@register_output

This class should have a docstring.

This interface does not seem to be used anywhere.

The variable register_output does not seem to be defined.

class ElasticSearchOutput(OutputInterface):

The variable OutputInterface does not seem to be defined.

    def __init__(self, source, index, hash_field=None, doc_type='continuum',
                 query=None, iterable=None, filter_expression="",
                 vectorized_corpora=None, tokenized_corpora=None, modeled_corpora=None,
                 **kwargs):
        super(ElasticSearchOutput, self).__init__()
        self.hosts = source

The variable source does not seem to be defined.

        self.instance = Elasticsearch(hosts=source, **kwargs)

The variable kwargs does not seem to be defined.

        self.index = index

The variable index does not seem to be defined.

        self.doc_type = doc_type

The variable doc_type does not seem to be defined.

        self.query = query

The variable query does not seem to be defined.

        self.hash_field = hash_field

The variable hash_field does not seem to be defined.

        if iterable:

The variable iterable does not seem to be defined.

            self.import_from_iterable(iterable, hash_field)
        self.filter_expression = filter_expression

The variable filter_expression does not seem to be defined.

        self.tokenized_corpora = tokenized_corpora if tokenized_corpora else \

The variable tokenized_corpora does not seem to be defined.

            BaseElasticCorpora(self.instance, self.index, 'tokenized', self.query)

The variable self does not seem to be defined.

        self.vectorized_corpora = vectorized_corpora if vectorized_corpora else \

The variable vectorized_corpora does not seem to be defined.

            VectorizedElasticCorpora(self.instance, self.index, 'vectorized', self.query)
        self.modeled_corpora = modeled_corpora if modeled_corpora else \

The variable modeled_corpora does not seem to be defined.

            ModeledElasticCorpora(self.instance, self.index, "models", self.query)


    @property

The variable property does not seem to be defined.

    def filter_string(self):

This method should have a docstring.

        return self.filter_expression

The variable self does not seem to be defined.

    def import_from_iterable(self, iterable, field_to_hash='text', batch_size=500):
        """Load data into Elasticsearch from iterable.

        iterable: generally a list of dicts, but possibly a list of strings
            This is your data.  Your dictionary structure defines the schema
            of the elasticsearch index.
        field_to_hash: string identifier of field to hash for content ID.  For
            list of dicts, a valid key value in the dictionary is required. For
            list of strings, a dictionary with one key, "text" is created and
            used.
        """
        if field_to_hash:

The variable field_to_hash does not seem to be defined.

            self.hash_field = field_to_hash
            batch = []
            for item in iterable:

The variable iterable does not seem to be defined.

                if isinstance(item, basestring):

Undefined variable 'basestring'

The variable item does not seem to be defined.

The variable basestring does not seem to be defined.

                    item = {field_to_hash: item}
                id = hash(item[field_to_hash])

This seems to re-define the built-in id.

The name id does not conform to the variable naming conventions ([a-z_][a-z0-9_]{2,30}$).

                action = {'_op_type': 'update',
                          '_index': self.index,

The variable self does not seem to be defined.

                          '_type': self.doc_type,
                          '_id': id,

The variable id does not seem to be defined.

                          'doc': item,
                          'doc_as_upsert': "true",
                          }
                batch.append(action)

The variable action does not seem to be defined.

                if len(batch) >= batch_size:

The variable batch does not seem to be defined.

                    helpers.bulk(client=self.instance, actions=batch, index=self.index)
                    batch = []
            if batch:
                helpers.bulk(client=self.instance, actions=batch, index=self.index)
            self.instance.indices.refresh(self.index)
        else:
            raise ValueError("A field_to_hash is required for import_from_iterable")

    def convert_date_field_and_reindex(self, field):

This method should have a docstring.

        index = self.index

The variable self does not seem to be defined.

        if self.instance.indices.get_field_mapping(fields=[field],

The variable field does not seem to be defined.

                                           index=index,

The variable index does not seem to be defined.

                                           doc_type=self.doc_type) != 'date':
            index = self.index+"_{}_alias_date".format(field)
            if not self.instance.indices.exists(index) or self.instance.indices.get_field_mapping(field=field,
                                           index=index,
                                           doc_type=self.doc_type) != 'date':
                mapping = self.instance.indices.get_mapping(index=self.index,
                                                            doc_type=self.doc_type)
                mapping[self.index]["mappings"][self.doc_type]["properties"][field] = {"type": "date"}
                self.instance.indices.put_alias(index=self.index,
                                                name=index,
                                                body=mapping)

The variable mapping does not seem to be defined.

                self.instance.indices.refresh(index)
                while self.instance.count(index=self.index) != self.instance.count(index=index):
                    logging.info("Waiting for date indexed data to be indexed...")
                    time.sleep(1)
        return index

    # TODO: validate input data to ensure that it has valid year data

TODO and FIXME comments should generally be avoided.

    def get_date_filtered_data(self, field_to_get, start, end, filter_field="date"):

This method should have a docstring.

        converted_index = self.convert_date_field_and_reindex(field=filter_field)

The variable filter_field does not seem to be defined.

        results = helpers.scan(self.instance, index=converted_index,

The variable converted_index does not seem to be defined.

The variable self does not seem to be defined.

                               doc_type=self.doc_type, query={
            "query": {"filtered": {"filter": {"range": {filter_field: {
                "gte": start,"lte": end}}}}}})

Exactly one space required after comma
"gte": start,"lte": end}}}}}})
^

The variable start does not seem to be defined.

The variable end does not seem to be defined.

        for result in results:

The variable results does not seem to be defined.

            yield result["_id"], result['_source'][field_to_get]

The variable field_to_get does not seem to be defined.

The variable result does not seem to be defined.

    def get_filtered_data(self, field_to_get, filter=""):

This seems to re-define the built-in filter.

        results = helpers.scan(self.instance, index=self.index,

The variable self does not seem to be defined.

                               query=self.query, doc_type=self.doc_type)
        for result in results:

The variable results does not seem to be defined.

            yield result["_id"], result['_source'][field_to_get]

The variable result does not seem to be defined.

The variable field_to_get does not seem to be defined.

    def save(self, filename, saved_data=None):
        if saved_data is None:

The variable saved_data does not seem to be defined.

            saved_data = {"source": self.hosts, "index": self.index, "hash_field": self.hash_field,

The variable self does not seem to be defined.

                          "doc_type": self.doc_type, "query": self.query}
        return super(ElasticSearchOutput, self).save(filename, saved_data)

The variable filename does not seem to be defined.

    def synchronize(self, max_wait, field):
        # TODO: change this to a more general condition for wider use, including read_input

TODO and FIXME comments should generally be avoided.

        # could just pass in a string condition and then 'while not eval(condition)'
        count_not_yet_updated = -1
        while count_not_yet_updated != 0:

The variable count_not_yet_updated does not seem to be defined.

            count_not_yet_updated = self.instance.count(index=self.index,

The variable self does not seem to be defined.

                                             doc_type=self.doc_type,
                                             body={"query": {
                                                        "constant_score" : {
                                                            "filter" : {
                                                                "missing" : {
                                                                    "field" : field}}}}})['count']

The variable field does not seem to be defined.

            logging.debug("Count not yet updated: {}".format(count_not_yet_updated))
            time.sleep(0.01)
        pass

Unnecessary pass statement