epsilon-compositionality

commit 5b41abe6e5efdcfd13c6cf20407422dae40fdd96
parent c40575d579cb4cc46119c52a5d8eb1d381c78796
Author: Sondre Wold <sondrewold1@gmail.com>
Date:   Mon, 18 Aug 2025 14:31:09 +0200

calculate epsilon for all result files

Diffstat:
A
get_epsilon.py
 | 
92
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

1 file changed, 92 insertions(+), 0 deletions(-)
diff --git a/get_epsilon.py b/get_epsilon.py
@@ -0,0 +1,92 @@
+import sys
+import pickle
+import numpy
+from torch import histogram
+from experiment import Point
+import matplotlib.pyplot as plt
+
+from typing import Any
+
+import pandas
+
+import argparse
+import pathlib
+
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Gather epsilon values")
+    parser.add_argument("-points_path", type=pathlib.Path)
+    parser.add_argument("-result_path", type=pathlib.Path)
+    return parser.parse_args()
+
+
+def get_dataset_and_model_name(filename: str) -> tuple[str, str]:
+    result_arguments: list[str] = filename.split("_")
+    dataset_name: str = result_arguments[0]
+    model_name: str = result_arguments[1]
+    return dataset_name, model_name
+
+
+def get_epsilons(
+    word_sims: numpy.ndarray, context_sims: numpy.ndarray, percentile: int = 80
+) -> dict[str, float]:
+    # Epsilons
+    original: list[float] = []
+    division: list[float] = []
+    additive: list[float] = []
+    for i in range(len(word_sims)):
+        if word_sims[i] > 0:
+            original.append(1 - context_sims[i] / word_sims[i])
+            division.append(context_sims[i] / word_sims[i])
+            additive.append(context_sims[i] - word_sims[i])
+    final_original: float = float(numpy.percentile(original, percentile))
+    final_division: float = float(numpy.percentile(division, percentile))
+    final_additive: float = float(numpy.percentile(additive, percentile))
+
+    return {
+        "original": final_original,
+        "divisive": final_division,
+        "additive": final_additive,
+    }
+
+
+def main() -> None:
+    args = parse_args()
+    points_folder: str = args.points_path
+    raw_files = list(pathlib.Path(points_folder).glob("*.pkl"))
+
+    final_result: dict[str, Any] = {
+        "dataset": [],
+        "model": [],
+        "original": [],
+        "divisive": [],
+        "additive": [],
+    }
+    for raw_file in raw_files:
+        with open(raw_file, "rb") as file:
+            points: list[Point] = pickle.load(file)
+            dataset_name, model_name = get_dataset_and_model_name(raw_file.name)
+            n: int = len(points)
+            word_sims: numpy.ndarray = numpy.array(
+                [point.phrase_similarity for point in points]
+            )
+            context_sims: numpy.ndarray = numpy.array(
+                [point.contextualized_similarity for point in points]
+            )
+            tmp_result: dict[str, float] = get_epsilons(word_sims, context_sims)
+            final_result["dataset"].append(dataset_name)
+            final_result["model"].append(model_name)
+            final_result["original"].append(tmp_result["original"])
+            final_result["divisive"].append(tmp_result["divisive"])
+            final_result["additive"].append(tmp_result["additive"])
+
+    df: pandas.DataFrame = pandas.DataFrame.from_dict(final_result)
+    df.to_csv(args.result_path, sep=",")
+
+
+if __name__ == "__main__":
+    main()