epsilon-compositionality

commit 670984d25213a03afccea8bee91896f989485c85
parent 19f2f7818d3862b9d9092c1252e67bf5129b8bf6
Author: Sondre Wold <sondrewold1@gmail.com>
Date:   Mon, 18 Aug 2025 13:01:09 +0200

make folder on save

Diffstat:
M
experiment.py
 | 
26
++++++++++++++++++++++++--
D
load_and_plot.py
 | 
38
--------------------------------------

2 files changed, 24 insertions(+), 40 deletions(-)
diff --git a/experiment.py b/experiment.py
@@ -93,10 +93,31 @@ class ModelFactory:
         return cls._factory[name](*args, **kwargs)
 
 
+@ModelFactory("BERT_pooler")
+class BERT_pooler(Model):
+
+    def __init__(self) -> None:
+        self.model_name:str = "bert-base-uncased"
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+        self.encoder = AutoModel.from_pretrained(self.model_name)
+        self.encoder.eval()
+
+    @torch.no_grad()
+    def embed(self, string: str) -> torch.Tensor:
+        return self.encoder(
+            self.tokenizer(string.lower(), return_tensors="pt")["input_ids"]
+        ).pooler_output 
+
+    def calculate_similarity(self, a: torch.Tensor, b: torch.Tensor) -> float:
+        return torch.nn.functional.cosine_similarity(
+            a, b
+        ).item()  # dist(\mu(a), \mu(b))
+
+
 @ModelFactory("BERT_cls")
 class BERT_cls(Model):
 
-    def __init__(self):
+    def __init__(self) -> None:
         self.model_name:str = "bert-base-uncased"
         self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
         self.encoder = AutoModel.from_pretrained(self.model_name)
@@ -106,7 +127,7 @@ class BERT_cls(Model):
     def embed(self, string: str) -> torch.Tensor:
         return self.encoder(
             self.tokenizer(string.lower(), return_tensors="pt")["input_ids"]
-        ).pooler_output  # the phrase in a that is not in b
+        ).last_hidden_state[:, 0, :]
 
     def calculate_similarity(self, a: torch.Tensor, b: torch.Tensor) -> float:
         return torch.nn.functional.cosine_similarity(
@@ -346,6 +367,7 @@ class Experiment:
             )
 
     def save_results(self):
+        pathlib.Path(self.result_path).parent.mkdir(parents=True, exist_ok=True)
         with open(self.result_path.with_suffix(".pkl"), "wb") as file:
             pickle.dump(self.points, file)
 
diff --git a/load_and_plot.py b/load_and_plot.py
@@ -1,38 +0,0 @@
-import sys
-import pickle
-import numpy
-from torch import histogram
-from experiment import Point
-import matplotlib.pyplot as plt
-
-if __name__ == "__main__":
-    points_path: str = sys.argv[1]
-    with open(points_path, "rb") as file:
-        points: list[Point] = pickle.load(file)
-        n: int = len(points)
-        word_sims: numpy.ndarray  = numpy.array([point.phrase_similarity for point in points])
-        context_sims: numpy.ndarray = numpy.array(
-            [point.contextualized_similarity for point in points]
-        )
-
-        epsilons: list[float] = []
-        for i in range(len(word_sims)):
-            if word_sims[i] > 0:
-                epsilons.append(1 - context_sims[i] / word_sims[i])
-        final_epsilon: float = float(numpy.percentile(epsilons, 80))
-
-        plt.scatter((1 - final_epsilon) * word_sims, context_sims)
-        xpoints = ypoints = plt.xlim()
-        plt.plot(
-            xpoints,
-            ypoints,
-            linestyle="--",
-            color="k",
-            lw=3,
-            scalex=False,
-            scaley=False,
-        )
-
-        plt.xlabel("Word similarity")
-        plt.ylabel("Contextualized similarity")
-        plt.show()