hplt quality.py (1919B)
1 from typing import Any 2 import argparse 3 import collections 4 import pathlib 5 import json 6 7 8 DOMAINS: list[str] = ["actor", "date", "location", "deaths"] 9 10 11 def main() -> None: 12 annotations_path: pathlib.Path = pathlib.Path(__file__).parent / ".." / "hplt_align" / "align_annotations.jsonl" 13 total: int = 0 14 similars: int = 0 15 partial_error: int = 0 16 counts: dict[str, int] = { domain: 0 for domain in DOMAINS } 17 with annotations_path.open("r") as annotations_file: 18 for line in annotations_file: 19 annotation: dict[str, Any] = json.loads(line) 20 total += 1 21 if annotation["similar"]: 22 similars += 1 23 perfect = True 24 for domain in DOMAINS: 25 if annotation[f"wrong_{domain}"]: 26 counts[domain] += 1 27 perfect = False 28 if not perfect: 29 partial_error += 1 30 else: 31 partial_error += 1 32 33 perfect: int = total - partial_error 34 completly_wrong: int = total - similars 35 print(f"Total documents annotated: {total}") 36 print(f"Perfects: {perfect:4} / {100*perfect/total:5.1f}%") 37 print(f"Similars: {similars:4} / {100*similars/total:5.1f}%") 38 print(f"Partial error: {partial_error:4} / {100*partial_error/total:5.1f}%") 39 print(f"Completly wrong: {completly_wrong:4} / {100*completly_wrong/total:5.1f}%") 40 for domain, count in counts.items(): 41 print(f"Domain {domain:8}: {count:4} / {100*count/total:5.1f}%") 42 43 44 def parse_command_argument() -> argparse.Namespace: 45 """ Parse command line arguments. """ 46 parser = argparse.ArgumentParser(description="Generate statistics from HPLT align annotations.") 47 args: argparse.Namespace = parser.parse_args() 48 return args 49 50 51 if __name__ == "__main__": 52 args: argparse.Namespace = parse_command_argument() 53 main()