write_results.py (1790B)
1 from typing import List, Dict 2 import json 3 import jsonlines 4 import numpy as np 5 6 7 def get_acc(objects) -> float: 8 predictions: List[float] = [] 9 for obj in objects: 10 predictions.append(float(obj["pred"] == obj["label"])) 11 return np.mean(predictions) 12 13 14 def get_scores_normal(model_name: str, seeds: List[int] = [42, 101, 987, 64, 923]) -> Dict[str, float]: 15 runs = [] 16 for seed in seeds: 17 with jsonlines.open(f"./results/{model_name}_seed_{seed}.jsonl", 'r') as f: 18 samples = [s for s in f] 19 runs.append(samples) 20 scores = [get_acc(r) for r in runs] 21 return {"mean": round(np.mean(scores)*100, 2), "std": round(np.std(scores)*100, 2)} 22 23 24 def get_scores_system(model_name: str, hop) -> Dict[str, float]: 25 runs = [] 26 print(model_name) 27 for fold in [0, 1, 2, 3]: 28 with jsonlines.open(f"./results/{model_name}_super_system_round_{rr}_hop_{hop}_fold_{fold}_seed_42.jsonl", 'r') as f: 29 samples = [s for s in f] 30 runs.append(samples) 31 scores = [get_acc(r) for r in runs] 32 print(scores) 33 return {"mean": round(np.mean(scores)*100, 2), "std": round(np.std(scores)*100, 2)} 34 35 36 if __name__ == "__main__": 37 print("Write results") 38 models = ["disjoint", "grounded", "gnn", "unidirectional", "bidirectional"] 39 tests = ["subst_2_hop", "subst_3_hop", "subst_4_hop", "prod_T2a3_V4", "prod_T2a4_V3", "prod_T3a4_V2"] 40 results = {} 41 for model in models: 42 for test in tests: 43 test_path = f"{model}_{test}" 44 results[test_path] = get_scores_normal(test_path) 45 for hop in [2, 3, 4]: 46 results[f"{model}_system_{hop}_hop"] = get_scores_system(model, hop) 47 print(results) 48 with open("results.json", "w") as f: 49 json.dump(results, f, indent=4)