epsilon-compositionality

commit 9212ccc92b52c0b9d06181648ebc8e5a878eb8a8
parent c32a69fb7640aa7b3f74dc2a0d1202faa674840b
Author: Sondre Wold <sondrewold1@gmail.com>
Date:   Tue, 26 Aug 2025 08:31:06 +0200

changes to experiment class, save embeddings

Diffstat:
M
experiment.py
 | 
59
++++++++++++++++++++++++++++++-----------------------------
M
get_epsilon.py
 | 
2
+-

2 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/experiment.py b/experiment.py
@@ -35,12 +35,16 @@ class EmbedError(Exception):
 
 
 class Point(NamedTuple):
-    phrase_1: str
-    phrase_2: str
-    context_1: str
-    context_2: str
+    phrase_a: str
+    phrase_b: str
+    context_ac: str
+    context_bc: str
     phrase_similarity: float
     contextualized_similarity: float
+    phrase_a_embedded: torch.Tensor
+    phrase_b_embedded: torch.Tensor
+    context_ac_embedded: torch.Tensor
+    context_bc_embedded: torch.Tensor
 
 
 class SentencePairs(NamedTuple):
@@ -93,7 +97,7 @@ class ModelFactory:
         return cls._factory[name](*args, **kwargs)
 
 
-@ModelFactory("BERT_pooler")
+@ModelFactory("BERT-pooler")
 class BERT_pooler(Model):
 
     def __init__(self) -> None:
@@ -114,7 +118,7 @@ class BERT_pooler(Model):
         ).item()  # dist(\mu(a), \mu(b))
 
 
-@ModelFactory("BERT_cls")
+@ModelFactory("BERT-cls")
 class BERT_cls(Model):
 
     def __init__(self) -> None:
@@ -135,7 +139,7 @@ class BERT_cls(Model):
         ).item()  # dist(\mu(a), \mu(b))
 
 
-@ModelFactory("BERT_mean")
+@ModelFactory("BERT-mean")
 class BERT_mean(Model):
 
     def __init__(self) -> None:
@@ -173,7 +177,7 @@ class BERT_mean(Model):
         ).item()  # dist(\mu(a), \mu(b))
 
 
-@ModelFactory("S_BERT")
+@ModelFactory("S-BERT")
 class S_BERT(Model):
 
     def __init__(self):
@@ -186,7 +190,7 @@ class S_BERT(Model):
         return self.encoder.similarity(a, b)
 
 
-@ModelFactory("S_GPT")
+@ModelFactory("S-GPT")
 class S_GPT(Model):
 
     def __init__(self):
@@ -357,12 +361,16 @@ class Experiment:
 
             self.points.append(
                 Point(
-                    phrase_1=phrase_a,
-                    phrase_2=phrase_b,
-                    context_1=sentence_a,
-                    context_2=b_in_a,
+                    phrase_a=phrase_a,
+                    phrase_b=phrase_b,
+                    context_ac=sentence_a,
+                    context_bc=b_in_a,
                     phrase_similarity=phrase_similarity,
                     contextualized_similarity=context_similarity,
+                    phrase_a_embedded=phrase_a_embedded,
+                    phrase_b_embedded=phrase_b_embedded,
+                    context_ac_embedded=a_in_context_embedded,
+                    context_bc_embedded=b_in_context_embedded
                 )
             )
 
@@ -381,26 +389,19 @@ def main() -> None:
     df: pandas.DataFrame = pandas.read_csv(args.data_path, sep=sep)
 
     if args.experiment_type == "ncip":
-        original_sentence: list[str] = df["original sentence"].values.tolist()
-        synonym_sentence: list[str] = df["synonym both"].values.tolist()
-        synonym_sentence2: list[str] = df["both synonym alt2_1"].values.tolist()
-        synonym_sentence3: list[str] = df["both synonym alt3_1"].values.tolist()
-        synonym_sentence4: list[str] = df["both synonym alt4_1"].values.tolist()
+        # Here we measure the difference between the idiomatic phrase and a synonym.
+        original_sentence: list[str] = df["original sentence"].values.tolist()  # Original idiomatic noun compound in context.
+        synonym_sentence: list[str] = df["synonym both"].values.tolist()  # Both the modifier and the noun are swapped with synonyms from word net.
 
-        sentence_A: list[str] = original_sentence * 4
+        sentence_A: list[str] = original_sentence
         sentence_B: list[str] = (
-            synonym_sentence + synonym_sentence2 + synonym_sentence3 + synonym_sentence4
+            synonym_sentence
         )
     elif args.experiment_type == "ncip-non":
-        A: list[str] = df["synonym both"].values.tolist()
-        B: list[str]= df["both synonym alt2_1"].values.tolist()
-        C: list[str] = df["both synonym alt3_1"].values.tolist()
-        D: list[str] = df["both synonym alt4_1"].values.tolist()
+        # Here we measure the difference between two non-idiomatic phrases in context.
+        sentence_A = df["synonym both"].values.tolist()
+        sentence_B = df["both synonym alt2_1"].values.tolist()
 
-        assert (len(A) + len(B) + len(C) + len(D)) / 4 == len(A)
-
-        sentence_A = A + B + C + D
-        sentence_B = B + C + D + A
     elif args.experiment_type == "sick":
         lexA = df.loc[df["sentence_A_expRule"] == "S2_lex"]
         lexB = df.loc[df["sentence_B_expRule"] == "S2_lex"]
@@ -409,7 +410,7 @@ def main() -> None:
         sentence_B = lex["sentence_B"].values.tolist()
 
     else:
-        raise ValueError("Undefined experiment type...")
+        raise ValueError("Undefined dataset type...")
 
     sentence_pairs = SentencePairs(sentence_A=sentence_A, sentence_B=sentence_B)
 
diff --git a/get_epsilon.py b/get_epsilon.py
@@ -40,7 +40,7 @@ def get_epsilons(
     additive: list[float] = []
     for i in range(len(word_sims)):
         if word_sims[i] > 0:
-            original.append(1 - context_sims[i] / word_sims[i])
+            original.append(1 - context_sims[i] / word_sims[i])  # Equivalent to our formal relaxation
             division.append(context_sims[i] / word_sims[i])
             additive.append(context_sims[i] - word_sims[i])
     final_original: float = float(numpy.percentile(original, percentile))