e2edutch.predict

Complexity

Total Complexity

17

Size/Duplication

Total Lines	132
Duplicated Lines	0 %

Importance

Changes

0

3 Functions

Rating	Name	Duplication	Size	Complexity
A	get_parser()	0	15	1
A	read_jsonlines()	0	4	2
F	main()	0	86	14

import sys

Missing module docstring

import json
import os
import io
import collections
import argparse
import logging

from e2edutch import conll
from e2edutch import minimize
from e2edutch import util
from e2edutch import coref_model as cm
from e2edutch import naf

import tensorflow.compat.v1 as tf

Unable to import 'tensorflow.compat.v1'

third party import "import tensorflow.compat.v1 as tf" should be placed before "from e2edutch import conll"

tf.disable_v2_behavior()


def get_parser():

Missing function or method docstring

    parser = argparse.ArgumentParser()
    parser.add_argument('config')
    parser.add_argument('input_filename')
    parser.add_argument('-o', '--output_file',
                        type=argparse.FileType('w'), default=sys.stdout)
    parser.add_argument('-f', '--format_out', default='conll',
                        choices=['conll', 'jsonlines', 'naf'])
    parser.add_argument('-c', '--word_col', type=int, default=2)
    parser.add_argument('--cfg_file',
                        type=str,
                        default=None,
                        help="config file")
    parser.add_argument('-v', '--verbose', action='store_true')
    return parser


def read_jsonlines(input_filename):

Missing function or method docstring

    for line in open(input_filename).readlines():
        example = json.loads(line)
        yield example


def main(args=None):

Missing function or method docstring

This function exceeds the maximum number of variables (30/15).

    parser = get_parser()
    args = parser.parse_args()
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    config = util.initialize_from_env(args.config, args.cfg_file)

    # Input file in .jsonlines format or .conll.
    input_filename = args.input_filename

    ext_input = os.path.splitext(input_filename)[-1]
    if ext_input not in ['.conll', '.jsonlines', '.txt', '.naf']:
        raise Exception(
            'Input file should be .naf, .conll, .txt or .jsonlines, but is {}.'
            .format(ext_input))

    if ext_input == '.conll':
        labels = collections.defaultdict(set)
        stats = collections.defaultdict(int)
        docs = minimize.minimize_partition(
            input_filename, labels, stats, args.word_col)
    elif ext_input == '.jsonlines':
        docs = read_jsonlines(input_filename)
    elif ext_input == '.naf':
        naf_obj = naf.get_naf(input_filename)
        jsonlines_obj, term_ids, tok_ids = naf.get_jsonlines(naf_obj)

The variable tok_ids seems to be unused.

        docs = [jsonlines_obj]
    else:
        text = open(input_filename).read()
        docs = [util.create_example(text)]

    output_file = args.output_file
    model = cm.CorefModel(config)
    sentences = {}
    predictions = {}
    with tf.Session() as session:
        model.restore(session)
        for example_num, example in enumerate(docs):
            # logging.info(example['doc_key'])
            tensorized_example = model.tensorize_example(
                example, is_training=False)
            feed_dict = {i: t for i, t in zip(

Unnecessary use of a comprehension

                model.input_tensors, tensorized_example)}
            _, _, _, top_span_starts, top_span_ends, top_antecedents, top_antecedent_scores = session.run(

This line is too long as per the coding-style (106/100).

                model.predictions, feed_dict=feed_dict)
            predicted_antecedents = model.get_predicted_antecedents(
                top_antecedents, top_antecedent_scores)
            example["predicted_clusters"], _ = model.get_predicted_clusters(
                top_span_starts, top_span_ends, predicted_antecedents)
            if args.format_out == 'jsonlines':
                output_file.write(json.dumps(example))
                output_file.write("\n")
            else:
                predictions[example['doc_key']] = example["predicted_clusters"]
                sentences[example['doc_key']] = example["sentences"]
            if example_num % 100 == 0:
                logging.info("Decoded {} examples.".format(example_num + 1))

Use lazy % formatting in logging functions

        if args.format_out == 'conll':
            conll.output_conll(output_file, sentences, predictions)
        elif args.format_out == 'naf':
            # Check number of docs - what to do if multiple?
            # Create naf obj if input format was not naf
            if ext_input != '.naf':
                # To do: add linguistic processing layers for terms and tokens
                logging.warn(

Using deprecated method warn()

Use lazy % formatting in logging functions

                    'Outputting NAF when input was not naf,'
                    + 'no dependency information available')
                for doc_key in sentences:
                    naf_obj, term_ids = naf.get_naf_from_sentences(
                        sentences[doc_key])
                    naf_obj = naf.create_coref_layer(
                        naf_obj, predictions[doc_key], term_ids)
                    naf_obj = naf.add_linguistic_processors(naf_obj)
                    buffer = io.BytesIO()
                    naf_obj.dump(buffer)
                    output_file.write(buffer.getvalue().decode('utf-8'))
                    # To do, make sepearate outputs?
                    # TO do, use dependency information from conll?
            else:
                # We only have one input doc
                naf_obj = naf.create_coref_layer(
                    naf_obj, example["predicted_clusters"], term_ids)

The variable naf_obj does not seem to be defined for all execution paths.

The variable term_ids does not seem to be defined for all execution paths.

The variable example does not seem to be defined in case the for loop on line 79 is not entered. Are you sure this can never be the case?

The loop variable example might not be defined here.

                naf_obj = naf.add_linguistic_processors(naf_obj)
                buffer = io.BytesIO()
                naf_obj.dump(buffer)
                output_file.write(buffer.getvalue().decode('utf-8'))


if __name__ == "__main__":
    main()