CrowdTruth /
CrowdTruth-core
| Conditions | 23 |
| Total Lines | 211 |
| Code Lines | 156 |
| Lines | 0 |
| Ratio | 0 % |
| Tests | 126 |
| CRAP Score | 23 |
| Changes | 0 | ||
Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.
For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.
Commonly applied refactorings include:
If many parameters/temporary variables are present:
Complex classes like crowdtruth.models.metrics.Metrics.run() often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.
Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.
| 1 | 1 | import logging |
|
| 230 | 1 | @staticmethod |
|
| 231 | 1 | def run(results, config, max_delta=0.001): |
|
| 232 | ''' |
||
| 233 | iteratively run the CrowdTruth metrics |
||
| 234 | ''' |
||
| 235 | |||
| 236 | 1 | judgments = results['judgments'].copy() |
|
| 237 | 1 | units = results['units'].copy() |
|
| 238 | |||
| 239 | # sent_work_rel_dict, work_sent_rel_dict, sent_rel_dict |
||
| 240 | # to be done: change to use all vectors in one unit |
||
| 241 | 1 | col = list(config.output.values())[0] |
|
| 242 | 1 | sent_rel_dict = dict(units.copy()[col]) |
|
| 243 | |||
| 244 | 1 | def expanded_vector(worker, unit): |
|
| 245 | ''' |
||
| 246 | expand the vector of a worker on a given unit |
||
| 247 | ''' |
||
| 248 | 1 | vector = Counter() |
|
| 249 | 1 | for rel in unit: |
|
| 250 | 1 | if rel in worker: |
|
| 251 | 1 | vector[rel] = worker[rel] |
|
| 252 | else: |
||
| 253 | 1 | vector[rel] = 0 |
|
| 254 | 1 | return vector |
|
| 255 | |||
| 256 | # fill judgment vectors with unit keys |
||
| 257 | 1 | for index, row in judgments.iterrows(): |
|
| 258 | # judgments.set_value(index, col, expandedVector(row[col], units.at[row['unit'], col])) |
||
| 259 | 1 | judgments.at[index, col] = expanded_vector(row[col], units.at[row['unit'], col]) |
|
| 260 | |||
| 261 | 1 | sent_work_rel_dict = judgments[['unit', 'worker', col]].copy().groupby('unit') |
|
| 262 | 1 | sent_work_rel_dict = {name : group.set_index('worker')[col].to_dict() \ |
|
| 263 | for name, group in sent_work_rel_dict} |
||
| 264 | |||
| 265 | 1 | work_sent_rel_dict = judgments[['worker', 'unit', col]].copy().groupby('worker') |
|
| 266 | 1 | work_sent_rel_dict = {name : group.set_index('unit')[col].to_dict() \ |
|
| 267 | for name, group in work_sent_rel_dict} |
||
| 268 | |||
| 269 | #initialize data structures |
||
| 270 | 1 | sqs_list = list() |
|
| 271 | 1 | wqs_list = list() |
|
| 272 | 1 | wwa_list = list() |
|
| 273 | 1 | wsa_list = list() |
|
| 274 | 1 | rqs_list = list() |
|
| 275 | |||
| 276 | 1 | sqs = dict((sentence_id, 1.0) for sentence_id in sent_work_rel_dict) |
|
| 277 | 1 | wqs = dict((worker_id, 1.0) for worker_id in work_sent_rel_dict) |
|
| 278 | 1 | wwa = dict((worker_id, 1.0) for worker_id in work_sent_rel_dict) |
|
| 279 | 1 | wsa = dict((worker_id, 1.0) for worker_id in work_sent_rel_dict) |
|
| 280 | |||
| 281 | 1 | sqs_list.append(sqs.copy()) |
|
| 282 | 1 | wqs_list.append(wqs.copy()) |
|
| 283 | 1 | wwa_list.append(wwa.copy()) |
|
| 284 | 1 | wsa_list.append(wsa.copy()) |
|
| 285 | |||
| 286 | # initialize RQS depending on whether or not it is an open ended task |
||
| 287 | 1 | rqs = dict() |
|
| 288 | 1 | if not config.open_ended_task: |
|
| 289 | 1 | rqs_keys = list(sent_rel_dict[list(sent_rel_dict.keys())[0]].keys()) |
|
| 290 | 1 | for relation in rqs_keys: |
|
| 291 | 1 | rqs[relation] = 1.0 |
|
| 292 | else: |
||
| 293 | 1 | for sentence_id in sent_rel_dict: |
|
| 294 | 1 | for relation in sent_rel_dict[sentence_id]: |
|
| 295 | 1 | rqs[relation] = 1.0 |
|
| 296 | 1 | rqs_list.append(rqs.copy()) |
|
| 297 | |||
| 298 | 1 | sqs_len = len(list(sqs.keys())) * 1.0 |
|
| 299 | 1 | wqs_len = len(list(wqs.keys())) * 1.0 |
|
| 300 | 1 | rqs_len = len(list(rqs.keys())) * 1.0 |
|
| 301 | |||
| 302 | # compute metrics until stable values |
||
| 303 | 1 | iterations = 0 |
|
| 304 | 1 | while max_delta >= 0.001: |
|
| 305 | 1 | sqs_new = dict() |
|
| 306 | 1 | wqs_new = dict() |
|
| 307 | 1 | wwa_new = dict() |
|
| 308 | 1 | wsa_new = dict() |
|
| 309 | |||
| 310 | 1 | avg_sqs_delta = 0.0 |
|
| 311 | 1 | avg_wqs_delta = 0.0 |
|
| 312 | 1 | avg_rqs_delta = 0.0 |
|
| 313 | 1 | max_delta = 0.0 |
|
| 314 | |||
| 315 | # pdb.set_trace() |
||
| 316 | |||
| 317 | 1 | def compute_wqs(wwa_new, wsa_new, wqs_new, work_sent_rel_dict, sent_rel_dict, \ |
|
| 318 | sent_work_rel_dict, wqs_list, sqs_list, rqs_list, wqs_len, \ |
||
| 319 | max_delta, avg_wqs_delta): |
||
| 320 | """ compute worker quality score (WQS) """ |
||
| 321 | 1 | for worker_id, _ in work_sent_rel_dict.items(): |
|
| 322 | 1 | wwa_new[worker_id] = Metrics.worker_worker_agreement( \ |
|
| 323 | worker_id, work_sent_rel_dict, \ |
||
| 324 | sent_work_rel_dict, \ |
||
| 325 | wqs_list[len(wqs_list) - 1], \ |
||
| 326 | sqs_list[len(sqs_list) - 1], \ |
||
| 327 | rqs_list[len(rqs_list) - 1]) |
||
| 328 | 1 | wsa_new[worker_id] = Metrics.worker_sentence_agreement( \ |
|
| 329 | worker_id, \ |
||
| 330 | sent_rel_dict, \ |
||
| 331 | work_sent_rel_dict, \ |
||
| 332 | sqs_list[len(sqs_list) - 1], \ |
||
| 333 | rqs_list[len(rqs_list) - 1], \ |
||
| 334 | wqs_list[len(rqs_list) - 1][worker_id]) |
||
| 335 | 1 | wqs_new[worker_id] = wwa_new[worker_id] * wsa_new[worker_id] |
|
| 336 | 1 | max_delta = max(max_delta, \ |
|
| 337 | abs(wqs_new[worker_id] - wqs_list[len(wqs_list) - 1][worker_id])) |
||
| 338 | 1 | avg_wqs_delta += abs(wqs_new[worker_id] - wqs_list[len(wqs_list) - 1][worker_id]) |
|
| 339 | 1 | avg_wqs_delta /= wqs_len |
|
| 340 | |||
| 341 | 1 | return wwa_new, wsa_new, wqs_new, max_delta, avg_wqs_delta |
|
| 342 | |||
| 343 | 1 | def reconstruct_sent_rel_dict(sent_rel_dict, work_sent_rel_dict, wqs_new): |
|
| 344 | """ reconstruct sent_rel_dict with worker scores """ |
||
| 345 | 1 | new_sent_rel_dict = dict() |
|
| 346 | 1 | for sent_id, rel_dict in sent_rel_dict.items(): |
|
| 347 | 1 | new_sent_rel_dict[sent_id] = dict() |
|
| 348 | 1 | for relation, _ in rel_dict.items(): |
|
| 349 | 1 | new_sent_rel_dict[sent_id][relation] = 0.0 |
|
| 350 | 1 | for work_id, srd in work_sent_rel_dict.items(): |
|
| 351 | 1 | wqs_work_id = wqs_new[work_id] |
|
| 352 | 1 | for sent_id, rel_dict in srd.items(): |
|
| 353 | 1 | for relation, score in rel_dict.items(): |
|
| 354 | 1 | new_sent_rel_dict[sent_id][relation] += score * wqs_work_id |
|
| 355 | |||
| 356 | 1 | return new_sent_rel_dict |
|
| 357 | |||
| 358 | 1 | def save_unit_rel_score(sent_rel_dict, sent_work_rel_dict, iteration_value): |
|
| 359 | """ save the unit relation score for print """ |
||
| 360 | 1 | srs = Counter() |
|
| 361 | 1 | for sentence_id in sent_rel_dict: |
|
| 362 | 1 | srs[sentence_id] = Counter() |
|
| 363 | 1 | for relation in sent_rel_dict[sentence_id]: |
|
| 364 | 1 | srs[sentence_id][relation] = Metrics.sentence_relation_score(sentence_id, \ |
|
| 365 | relation, sent_work_rel_dict, \ |
||
| 366 | iteration_value) |
||
| 367 | 1 | return srs |
|
| 368 | |||
| 369 | 1 | def compute_rqs(rqs, work_sent_rel_dict, sqs_list, wqs_list, rqs_list, rqs_len, max_delta, avg_rqs_delta): |
|
| 370 | """ compute relation quality score (RQS) """ |
||
| 371 | 1 | rqs_new = Metrics.relation_quality_score(list(rqs.keys()), work_sent_rel_dict, \ |
|
| 372 | sqs_list[len(sqs_list) - 1], \ |
||
| 373 | wqs_list[len(wqs_list) - 1]) |
||
| 374 | 1 | for rel, _ in rqs_new.items(): |
|
| 375 | 1 | max_delta = max(max_delta, abs(rqs_new[rel] - rqs_list[len(rqs_list) - 1][rel])) |
|
| 376 | 1 | avg_rqs_delta += abs(rqs_new[rel] - rqs_list[len(rqs_list) - 1][rel]) |
|
| 377 | 1 | avg_rqs_delta /= rqs_len |
|
| 378 | 1 | return rqs_new, max_delta, avg_rqs_delta |
|
| 379 | |||
| 380 | 1 | def compute_sqs(sqs_new, sent_work_rel_dict, wqs_list, rqs_list, sqs_list, sqs_len, max_delta, avg_sqs_delta): |
|
| 381 | """ compute sentence quality score (SQS) """ |
||
| 382 | 1 | for sent_id, _ in sent_work_rel_dict.items(): |
|
| 383 | 1 | sqs_new[sent_id] = Metrics.sentence_quality_score(sent_id, sent_work_rel_dict, \ |
|
| 384 | wqs_list[len(wqs_list) - 1], \ |
||
| 385 | rqs_list[len(rqs_list) - 1]) |
||
| 386 | 1 | max_delta = max(max_delta, \ |
|
| 387 | abs(sqs_new[sent_id] - sqs_list[len(sqs_list) - 1][sent_id])) |
||
| 388 | 1 | avg_sqs_delta += abs(sqs_new[sent_id] - sqs_list[len(sqs_list) - 1][sent_id]) |
|
| 389 | 1 | avg_sqs_delta /= sqs_len |
|
| 390 | 1 | return sqs_new, max_delta, avg_sqs_delta |
|
| 391 | |||
| 392 | 1 | if not config.open_ended_task: |
|
| 393 | # compute relation quality score (RQS) |
||
| 394 | 1 | rqs_new, max_delta, avg_rqs_delta = compute_rqs(rqs, work_sent_rel_dict, \ |
|
| 395 | sqs_list, wqs_list, rqs_list, rqs_len, max_delta, avg_rqs_delta) |
||
| 396 | |||
| 397 | # compute sentence quality score (SQS) |
||
| 398 | 1 | sqs_new, max_delta, avg_sqs_delta = compute_sqs(sqs_new, sent_work_rel_dict, \ |
|
| 399 | wqs_list, rqs_list, sqs_list, sqs_len, max_delta, avg_sqs_delta) |
||
| 400 | |||
| 401 | # compute worker quality score (WQS) |
||
| 402 | 1 | wwa_new, wsa_new, wqs_new, max_delta, avg_wqs_delta = compute_wqs(\ |
|
| 403 | wwa_new, wsa_new, wqs_new, \ |
||
| 404 | work_sent_rel_dict, sent_rel_dict, sent_work_rel_dict, wqs_list, \ |
||
| 405 | sqs_list, rqs_list, wqs_len, max_delta, avg_wqs_delta) |
||
| 406 | |||
| 407 | # save results for current iteration |
||
| 408 | 1 | sqs_list.append(sqs_new.copy()) |
|
| 409 | 1 | wqs_list.append(wqs_new.copy()) |
|
| 410 | 1 | wwa_list.append(wwa_new.copy()) |
|
| 411 | 1 | wsa_list.append(wsa_new.copy()) |
|
| 412 | 1 | if not config.open_ended_task: |
|
| 413 | 1 | rqs_list.append(rqs_new.copy()) |
|
|
introduced
by
|
|||
| 414 | 1 | iterations += 1 |
|
| 415 | |||
| 416 | 1 | sent_rel_dict = reconstruct_sent_rel_dict(sent_rel_dict, work_sent_rel_dict, wqs_new) |
|
| 417 | |||
| 418 | 1 | logging.info(str(iterations) + " iterations; max d= " + str(max_delta) + \ |
|
| 419 | " ; wqs d= " + str(avg_wqs_delta) + "; sqs d= " + str(avg_sqs_delta) + \ |
||
| 420 | "; rqs d= " + str(avg_rqs_delta)) |
||
| 421 | |||
| 422 | 1 | srs = save_unit_rel_score(sent_rel_dict, sent_work_rel_dict, wqs_list[len(wqs_list) - 1]) |
|
| 423 | 1 | srs_initial = save_unit_rel_score(sent_rel_dict, sent_work_rel_dict, wqs_list[0]) |
|
| 424 | |||
| 425 | 1 | results['units']['uqs'] = pd.Series(sqs_list[-1]) |
|
| 426 | 1 | results['units']['unit_annotation_score'] = pd.Series(srs) |
|
| 427 | 1 | results['workers']['wqs'] = pd.Series(wqs_list[-1]) |
|
| 428 | 1 | results['workers']['wwa'] = pd.Series(wwa_list[-1]) |
|
| 429 | 1 | results['workers']['wsa'] = pd.Series(wsa_list[-1]) |
|
| 430 | 1 | if not config.open_ended_task: |
|
| 431 | 1 | results['annotations']['aqs'] = pd.Series(rqs_list[-1]) |
|
| 432 | |||
| 433 | 1 | results['units']['uqs_initial'] = pd.Series(sqs_list[1]) |
|
| 434 | 1 | results['units']['unit_annotation_score_initial'] = pd.Series(srs_initial) |
|
| 435 | 1 | results['workers']['wqs_initial'] = pd.Series(wqs_list[1]) |
|
| 436 | 1 | results['workers']['wwa_initial'] = pd.Series(wwa_list[1]) |
|
| 437 | 1 | results['workers']['wsa_initial'] = pd.Series(wsa_list[1]) |
|
| 438 | 1 | if not config.open_ended_task: |
|
| 439 | 1 | results['annotations']['aqs_initial'] = pd.Series(rqs_list[1]) |
|
| 440 | return results |
||
| 441 |
Loading history...
