|
@@ 73-86 (lines=14) @@
|
| 70 |
|
if not config.open_ended_task: |
| 71 |
|
self.assertAlmostEqual(results["annotations"]["aqs"]["A"], 1.0) |
| 72 |
|
|
| 73 |
|
def test_all_workers_disagree(self): |
| 74 |
|
for w in range(2, 11): |
| 75 |
|
test_config = self.test_conf_const.__class__ |
| 76 |
|
data, config = crowdtruth.load( |
| 77 |
|
file=TEST_FILE_PREF + str(w) + "work_disagr.csv", |
| 78 |
|
config=test_config()) |
| 79 |
|
results = crowdtruth.run(data, config) |
| 80 |
|
self.assertAlmostEqual(results["units"]["uqs"].at[1], 0.0) |
| 81 |
|
for wid in range(w): |
| 82 |
|
self.assertAlmostEqual(results["workers"]["wqs"].at["W" + str(wid + 1)], 0.0) |
| 83 |
|
if not config.open_ended_task: |
| 84 |
|
self.assertAlmostEqual( |
| 85 |
|
results["annotations"]["aqs"][list(string.ascii_uppercase)[w]], |
| 86 |
|
0.0) |
| 87 |
|
|
| 88 |
|
def test_outlier_worker(self): |
| 89 |
|
for w in range(3, 11): |
|
@@ 60-71 (lines=12) @@
|
| 57 |
|
class TestAgreementClosed(unittest.TestCase): |
| 58 |
|
test_conf_const = TestConfigClosed() |
| 59 |
|
|
| 60 |
|
def test_all_workers_agree(self): |
| 61 |
|
for w in range(2,11): |
| 62 |
|
test_config = self.test_conf_const.__class__ |
| 63 |
|
data, config = crowdtruth.load( |
| 64 |
|
file=TEST_FILE_PREF + str(w) + "work_agr.csv", |
| 65 |
|
config=test_config()) |
| 66 |
|
results = crowdtruth.run(data, config) |
| 67 |
|
self.assertAlmostEqual(results["units"]["uqs"].at[1], 1.0) |
| 68 |
|
for wid in range(w): |
| 69 |
|
self.assertAlmostEqual(results["workers"]["wqs"].at["W" + str(wid + 1)], 1.0) |
| 70 |
|
if not config.open_ended_task: |
| 71 |
|
self.assertAlmostEqual(results["annotations"]["aqs"]["A"], 1.0) |
| 72 |
|
|
| 73 |
|
def test_all_workers_disagree(self): |
| 74 |
|
for w in range(2, 11): |