epsilon-compositionality

commit 3edfc2c20bd174c7a0aacc19744a8a878da03685
parent 774cb9989b048edf55ea4ec5973d240f6472b21c
Author: Sondre Wold <sondrewold1@gmail.com>
Date:   Thu, 11 Sep 2025 11:38:17 +0200

add deberta and modernbert + a small effort validation of the diff

Diffstat:
M
experiment.py
 | 
60
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--

1 file changed, 58 insertions(+), 2 deletions(-)
diff --git a/experiment.py b/experiment.py
@@ -30,7 +30,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "-model",
         type=str,
-        choices=["BERT-cls", "BERT-mean", "S-BERT", "S-GPT", "E5", "QWEN"],
+        choices=list(ModelFactory._factory.keys())
     )
     return parser.parse_args()
 
@@ -146,6 +146,48 @@ class BERT_cls(Model):
         ).item()  # dist(\mu(a), \mu(b))
 
 
+@ModelFactory("ModernBERT-cls")
+class ModernBERT_cls(Model):
+
+    def __init__(self) -> None:
+        self.model_name: str = "answerdotai/ModernBERT-base"
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+        self.encoder = AutoModel.from_pretrained(self.model_name)
+        self.encoder.eval()
+
+    @torch.no_grad()
+    def embed(self, string: str) -> torch.Tensor:
+        return self.encoder(
+            self.tokenizer(string.lower(), return_tensors="pt")["input_ids"]
+        ).last_hidden_state[:, 0, :]
+
+    def calculate_similarity(self, a: torch.Tensor, b: torch.Tensor) -> float:
+        return torch.nn.functional.cosine_similarity(
+            a, b
+        ).item()  # dist(\mu(a), \mu(b))
+
+@ModelFactory("DeBERTa-cls")
+class DeBERTa_cls(Model):
+
+    def __init__(self) -> None:
+        self.model_name: str = "microsoft/deberta-v3-base"
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+        self.encoder = AutoModel.from_pretrained(self.model_name)
+        self.encoder.eval()
+
+    @torch.no_grad()
+    def embed(self, string: str) -> torch.Tensor:
+        return self.encoder(
+            self.tokenizer(string.lower(), return_tensors="pt")["input_ids"]
+        ).last_hidden_state[:, 0, :]
+
+    def calculate_similarity(self, a: torch.Tensor, b: torch.Tensor) -> float:
+        return torch.nn.functional.cosine_similarity(
+            a, b
+        ).item()  # dist(\mu(a), \mu(b))
+
+
+
 @ModelFactory("BERT-mean")
 class BERT_mean(Model):
 
@@ -325,10 +367,11 @@ class Experiment:
         self.phrase_b_length: list[int] = []
 
     def run_experiment(self) -> None:
-        for sentence_a, sentence_b in tqdm.tqdm(zip(*self.sentence_pairs)):
+        for i, (sentence_a, sentence_b) in enumerate(tqdm.tqdm(zip(*self.sentence_pairs))):
             diffs: list[str] | None = find_diff(sentence_a, sentence_b)
             if not diffs:
                 self.diff_errors += 1
+                print(f"Error at index {i}, sentence: {sentence_a}")
                 continue
 
             phrase_a: str
@@ -414,12 +457,25 @@ def main() -> None:
         original_sentence: list[str] = df[
             "original sentence"
         ].values.tolist()  # Original idiomatic noun compound in context.
+        original_idiom: list[str] = df[
+            "compound noun"
+        ].values.tolist()  # Original idiomatic noun compound isolated
         synonym_sentence: list[str] = df[
             "synonym both"
         ].values.tolist()  # Both the modifier and the noun are swapped with synonyms from word net.
 
         sentence_A: list[str] = original_sentence
         sentence_B: list[str] = synonym_sentence
+
+        # Validate the find_diff function somewhat
+        for i, (sA, sB, goldA) in enumerate(zip(sentence_A, sentence_B, original_idiom)):
+            res = find_diff(sA, sB)
+            if res is not None:
+                diffa, diffb = res
+                if diffa != goldA:
+                    print(i, sA, sB, goldA, diffa, diffb)
+                    exit()
+
     elif args.experiment_type == "ncip-non":
         # Here we measure the difference between two non-idiomatic phrases in context.
         sentence_A = df["synonym both"].values.tolist()