stats.py (3074B)
1 from typing import Any, Dict, List, Tuple 2 3 import numpy 4 import sklearn.metrics 5 import statsmodels.stats.inter_rater 6 7 import annotations.annotator 8 import mmsrl.utils 9 10 11 def show_scores(annotator: str, data: Dict[str, Any], answers: List[Tuple[str, str, int]]): 12 truth = [] 13 predictions = [] 14 assert(len(answers) == 100) 15 for image, entity, answer in answers: 16 sample = data[image] 17 truth.append(sample["labels"][sample["entities_name"].index(entity)].item()) 18 predictions.append(answer) 19 20 f1 = sklearn.metrics.f1_score(truth, predictions, average="macro") 21 class_precision, class_recall, class_f1, _ = sklearn.metrics.precision_recall_fscore_support(truth, predictions) 22 class_scores = ", ".join(f"{label}[F{class_f1[i]:4.2f}/P{class_precision[i]:4.2f}/R{class_recall[i]:4.2f}]" for i, label in enumerate(mmsrl.utils.LABELS)) 23 print(f"Annotator {annotator}, Macro-F1: {f1:5.3f}, {class_scores}") 24 return f1 25 26 27 def get_annotations(annotator: str) -> List[Tuple[str, str, int]]: 28 answers = [] 29 with open(f"annotations/annotations of {annotator}", "r") as file: 30 for line in file: 31 image, entity, answer = line.rstrip('\n').split('\t') 32 answers.append((image, entity, int(answer))) 33 return answers 34 35 36 def main(annotators: List[str]): 37 data, _ = annotations.annotator.get_data() 38 answers = {} 39 x = numpy.empty((100, len(annotators)), dtype=numpy.int32) 40 m = numpy.zeros((100, 4), dtype=numpy.int32) 41 scores = [] 42 for i, annotator in enumerate(annotators): 43 answers[annotator] = get_annotations(annotator) 44 f1 = show_scores(annotator, data, answers[annotator]) 45 scores.append(f1) 46 for j, (_, _, answer) in enumerate(answers[annotator]): 47 x[j, i] = answer 48 m[j, answer] += 1 49 print(f"Average Macro-F1: {numpy.mean(scores)*100:.1f}") 50 print(f"Std Macro-F1: {numpy.std(scores)*100:.1f}") 51 y = numpy.array([data[image]["labels"][data[image]["entities_name"].index(entity)].item() for image, entity, _ in answers[annotator]], dtype=numpy.int32) 52 print("") 53 54 print(f"Fleiss kappa: {statsmodels.stats.inter_rater.fleiss_kappa(m):.2f}") 55 kappas = [] 56 for i, annotatorA in enumerate(annotators): 57 for jmi, annotatorB in enumerate(annotators[i+1:]): 58 j = i + 1 + jmi 59 kappa = sklearn.metrics.cohen_kappa_score(x[:, i], x[:, j]) 60 kappas.append(kappa) 61 print(f"Cohen kappa between annotators {annotatorA} and {annotatorB}: {kappa:.2f}") 62 avg_kappa = numpy.mean(kappas) 63 print(f"Average Cohen kappa: {avg_kappa:.2f}") 64 65 66 print("") 67 print(f"Percent of samples where all annotators agree: {numpy.equal(x[:, [0]], x[:, 1:]).all(axis=1).sum()}%") 68 for i, annotatorA in enumerate(annotators): 69 for jmi, annotatorB in enumerate(annotators[i+1:]): 70 j = i + 1 + jmi 71 print(f"Percent agreement between {annotatorA} and {annotatorB}: {(x[:, i] == x[:, j]).sum()}%") 72 73 74 if __name__ == "__main__": 75 main(["A", "É", "S", "D", "U"])