Filter-Bubble /
e2e-Dutch
| Total Complexity | 41 |
| Total Lines | 226 |
| Duplicated Lines | 0 % |
| Changes | 0 | ||
Complex classes like e2edutch.minimize often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | import re |
||
|
introduced
by
|
|||
| 2 | import os |
||
| 3 | import sys |
||
| 4 | import json |
||
| 5 | import collections |
||
| 6 | import logging |
||
| 7 | import argparse |
||
| 8 | |||
| 9 | from e2edutch import util |
||
| 10 | from e2edutch import conll |
||
| 11 | |||
| 12 | logger = logging.getLogger('e2edutch') |
||
| 13 | |||
| 14 | |||
| 15 | class DocumentState(object): |
||
| 16 | def __init__(self): |
||
| 17 | self.doc_key = None |
||
| 18 | self.text = [] |
||
| 19 | self.sentences = [] |
||
| 20 | self.constituents = {} |
||
| 21 | self.const_stack = [] |
||
| 22 | self.ner = {} |
||
| 23 | self.ner_stack = [] |
||
| 24 | self.clusters = collections.defaultdict(list) |
||
| 25 | self.coref_stacks = collections.defaultdict(list) |
||
| 26 | |||
| 27 | def assert_empty(self): |
||
| 28 | assert self.doc_key is None |
||
| 29 | assert len(self.text) == 0 |
||
| 30 | assert len(self.sentences) == 0 |
||
| 31 | assert len(self.constituents) == 0 |
||
| 32 | assert len(self.const_stack) == 0 |
||
| 33 | assert len(self.ner) == 0 |
||
| 34 | assert len(self.ner_stack) == 0 |
||
| 35 | assert len(self.coref_stacks) == 0 |
||
| 36 | assert len(self.clusters) == 0 |
||
| 37 | |||
| 38 | def assert_finalizable(self): |
||
| 39 | assert self.doc_key is not None |
||
| 40 | assert len(self.text) == 0 |
||
| 41 | assert len(self.sentences) > 0 |
||
| 42 | assert len(self.const_stack) == 0 |
||
| 43 | assert len(self.ner_stack) == 0 |
||
| 44 | assert all(len(s) == 0 for s in self.coref_stacks.values()) |
||
| 45 | |||
| 46 | def span_dict_to_list(self, span_dict): |
||
| 47 | return [(s, e, l) for (s, e), l in span_dict.items()] |
||
| 48 | |||
| 49 | def finalize(self): |
||
| 50 | merged_clusters = [] |
||
| 51 | for c1 in self.clusters.values(): |
||
| 52 | existing = None |
||
| 53 | for m in c1: |
||
| 54 | for c2 in merged_clusters: |
||
| 55 | if m in c2: |
||
| 56 | existing = c2 |
||
| 57 | break |
||
| 58 | if existing is not None: |
||
| 59 | break |
||
| 60 | if existing is not None: |
||
| 61 | print("Merging clusters (shouldn't happen very often.)") |
||
| 62 | print(self.doc_key, m) |
||
| 63 | existing.update(c1) |
||
| 64 | else: |
||
| 65 | merged_clusters.append(set(c1)) |
||
| 66 | merged_clusters = [list(c) for c in merged_clusters] |
||
| 67 | all_mentions = util.flatten(merged_clusters) |
||
| 68 | assert len(all_mentions) == len(set(all_mentions)) |
||
| 69 | |||
| 70 | return { |
||
| 71 | "doc_key": self.doc_key, |
||
| 72 | "sentences": self.sentences, |
||
| 73 | "clusters": merged_clusters |
||
| 74 | } |
||
| 75 | |||
| 76 | |||
| 77 | def normalize_word(word): |
||
| 78 | if word == "/." or word == "/?": |
||
| 79 | return word[1:] |
||
| 80 | else: |
||
| 81 | return word |
||
| 82 | |||
| 83 | |||
| 84 | def handle_bit(word_index, bit, stack, spans): |
||
| 85 | asterisk_idx = bit.find("*") |
||
| 86 | if asterisk_idx >= 0: |
||
| 87 | open_parens = bit[:asterisk_idx] |
||
| 88 | close_parens = bit[asterisk_idx + 1:] |
||
| 89 | else: |
||
| 90 | open_parens = bit[:-1] |
||
| 91 | close_parens = bit[-1] |
||
| 92 | |||
| 93 | current_idx = open_parens.find("(") |
||
| 94 | while current_idx >= 0: |
||
| 95 | next_idx = open_parens.find("(", current_idx + 1) |
||
| 96 | if next_idx >= 0: |
||
| 97 | label = open_parens[current_idx + 1:next_idx] |
||
| 98 | else: |
||
| 99 | label = open_parens[current_idx + 1:] |
||
| 100 | stack.append((word_index, label)) |
||
| 101 | current_idx = next_idx |
||
| 102 | |||
| 103 | for c in close_parens: |
||
| 104 | assert c == ")" |
||
| 105 | open_index, label = stack.pop() |
||
| 106 | current_span = (open_index, word_index) |
||
| 107 | """ |
||
| 108 | if current_span in spans: |
||
| 109 | spans[current_span] += "_" + label |
||
| 110 | else: |
||
| 111 | spans[current_span] = label |
||
| 112 | """ |
||
| 113 | spans[current_span] = label |
||
| 114 | |||
| 115 | |||
| 116 | def handle_line(line, document_state, labels, stats, word_col): |
||
| 117 | begin_document_match = re.match(conll.BEGIN_DOCUMENT_REGEX, line) |
||
| 118 | if begin_document_match: |
||
| 119 | document_state.assert_empty() |
||
| 120 | document_state.doc_key = conll.get_doc_key( |
||
| 121 | *begin_document_match.groups()) |
||
| 122 | return None |
||
| 123 | elif line.startswith("#end document"): |
||
| 124 | if len(document_state.text) > 0: # no newline before end document |
||
| 125 | stats["max_sent_len"] = max( |
||
| 126 | len(document_state.text), stats["max_sent_len"]) |
||
| 127 | stats["num_sents"] += 1 |
||
| 128 | document_state.sentences.append(tuple(document_state.text)) |
||
| 129 | del document_state.text[:] |
||
| 130 | document_state.assert_finalizable() |
||
| 131 | finalized_state = document_state.finalize() |
||
| 132 | stats["num_clusters"] += len(finalized_state["clusters"]) |
||
| 133 | stats["num_mentions"] += sum(len(c) |
||
| 134 | for c in finalized_state["clusters"]) |
||
| 135 | # labels["const_labels"].update( |
||
| 136 | # l for _, _, l in finalized_state["constituents"]) |
||
| 137 | # labels["ner"].update(l for _, _, l in finalized_state["ner"]) |
||
| 138 | return finalized_state |
||
| 139 | else: |
||
| 140 | row = line.split() |
||
| 141 | if len(row) == 0 and len(document_state.text) > 0: |
||
| 142 | stats["max_sent_len"] = max( |
||
| 143 | len(document_state.text), stats["max_sent_len"]) |
||
| 144 | stats["num_sents"] += 1 |
||
| 145 | document_state.sentences.append(tuple(document_state.text)) |
||
| 146 | del document_state.text[:] |
||
| 147 | return None |
||
| 148 | elif len(row) == 0 and len(document_state.text) == 0: |
||
| 149 | return None |
||
| 150 | assert len(row) >= 4 |
||
| 151 | |||
| 152 | word = normalize_word(row[word_col]) |
||
| 153 | coref = row[-1] |
||
| 154 | |||
| 155 | word_index = (len(document_state.text) |
||
| 156 | + sum(len(s) for s in document_state.sentences)) |
||
| 157 | document_state.text.append(word) |
||
| 158 | |||
| 159 | if coref != "-" and coref != '_': |
||
| 160 | for segment in coref.split("|"): |
||
| 161 | if segment[0] == "(": |
||
| 162 | if segment[-1] == ")": |
||
| 163 | cluster_id = int(segment[1:-1]) |
||
| 164 | document_state.clusters[cluster_id].append( |
||
| 165 | (word_index, word_index)) |
||
| 166 | else: |
||
| 167 | cluster_id = int(segment[1:]) |
||
| 168 | document_state.coref_stacks[cluster_id].append( |
||
| 169 | word_index) |
||
| 170 | elif segment[-1] == ")": |
||
| 171 | cluster_id = int(segment[:-1]) |
||
| 172 | start = document_state.coref_stacks[cluster_id].pop() |
||
| 173 | document_state.clusters[cluster_id].append( |
||
| 174 | (start, word_index)) |
||
| 175 | return None |
||
| 176 | |||
| 177 | |||
| 178 | def minimize_partition(input_path, labels, stats, word_col): |
||
| 179 | with open(input_path, "r") as input_file: |
||
| 180 | document_state = DocumentState() |
||
| 181 | for line in input_file.readlines(): |
||
| 182 | document = handle_line(line, document_state, |
||
| 183 | labels, stats, word_col) |
||
| 184 | if document is not None: |
||
| 185 | yield document |
||
| 186 | document_state = DocumentState() |
||
| 187 | |||
| 188 | |||
| 189 | def minimize_partition_file( |
||
| 190 | input_path, labels, stats, word_col, output_file=None): |
||
| 191 | if output_file is None: |
||
| 192 | output_path = "{}.jsonlines".format(os.path.splitext(input_path)[0]) |
||
| 193 | output_file = open(output_path, "w") |
||
| 194 | count = 0 |
||
| 195 | logger.info("Minimizing {}".format(input_path)) |
||
| 196 | for document in minimize_partition(input_path, labels, stats, word_col): |
||
| 197 | output_file.write(json.dumps(document)) |
||
| 198 | output_file.write("\n") |
||
| 199 | count += 1 |
||
| 200 | logger.info("Wrote {} documents to {}".format(count, output_path)) |
||
| 201 | |||
| 202 | |||
| 203 | def get_parser(): |
||
| 204 | parser = argparse.ArgumentParser() |
||
| 205 | parser.add_argument('input_filename') |
||
| 206 | parser.add_argument('-o', '--output_file', |
||
| 207 | type=argparse.FileType('w'), default=sys.stdout) |
||
| 208 | parser.add_argument('-v', '--verbose', action='store_true') |
||
| 209 | return parser |
||
| 210 | |||
| 211 | |||
| 212 | if __name__ == "__main__": |
||
| 213 | parser = get_parser() |
||
| 214 | args = parser.parse_args() |
||
| 215 | input_path = args.input_filename |
||
| 216 | output_file = args.output_file |
||
| 217 | word_col = 2 |
||
| 218 | labels = collections.defaultdict(set) |
||
| 219 | stats = collections.defaultdict(int) |
||
| 220 | minimize_partition_file(input_path, labels, stats, word_col, output_file) |
||
| 221 | for k, v in labels.items(): |
||
| 222 | print("{} = [{}]".format(k, ", ".join( |
||
| 223 | "\"{}\"".format(label) for label in v))) |
||
| 224 | for k, v in stats.items(): |
||
| 225 | print("{} = {}".format(k, v)) |
||
| 226 |
Loading history...