commit 6f26c1c43b327708c744a8c8b4b30ee896b6f3a4
parent 4c5cc9e300012ca3bdcb175a17b9791985191479
Author: Sondre Wold <sondrewold1@gmail.com>
Date: Thu, 3 Jul 2025 15:49:04 +0200
minimal common substring on SICK
Diffstat:
| A | extract_sick_pairs.py | | | 113 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
1 file changed, 113 insertions(+), 0 deletions(-)
diff --git a/extract_sick_pairs.py b/extract_sick_pairs.py
@@ -0,0 +1,113 @@
+from typing import NamedTuple
+import sys
+import pandas
+
+import torch
+import pickle
+import tqdm
+
+from transformers import AutoTokenizer, AutoModel
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+class Point(NamedTuple):
+ word_similarity: float
+ contextualized_similarity: float
+ weight: float
+
+
+def find_diff(a: str, b: str) -> list[str] | None:
+ """
+ Finds the longest continuous substring where A differs from B and B differs from A.
+ """
+ if a == b: # Should not happen for SICK, but you never know.
+ return None
+
+ words_a = a.split()
+ words_b = b.split()
+
+ prefix_len = 0
+ min_len = min(len(words_a), len(words_b))
+ for i in range(min_len):
+ if words_a[i] == words_b[i]:
+ prefix_len += 1
+ else:
+ break
+
+ suffix_len = 0
+ for i in range(1, min_len - prefix_len + 1):
+ if words_a[-i] == words_b[-i]:
+ suffix_len += 1
+ else:
+ break
+
+ diff_a = words_a[
+ prefix_len : len(words_a) - suffix_len if suffix_len > 0 else len(words_a)
+ ]
+ diff_b = words_b[
+ prefix_len : len(words_b) - suffix_len if suffix_len > 0 else len(words_b)
+ ]
+
+ if not diff_a or not diff_b:
+ return None
+ else:
+ diff_str_a = " ".join(diff_a) if diff_a else ""
+ diff_str_b = " ".join(diff_b) if diff_b else ""
+ return [diff_str_a, diff_str_b]
+
+
+def main() -> None:
+ data_path: str = sys.argv[1]
+ output_path: str = sys.argv[2]
+ df: pandas.DataFrame = pandas.read_csv(data_path, delimiter="\t")
+ sentence_as: list[str] = df["sentence_A"].values.tolist()
+ sentence_bs: list[str] = df["sentence_B"].values.tolist()
+
+ model_name: str = "bert-base-uncased"
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ model = AutoModel.from_pretrained(model_name)
+
+ points: list[Point] = []
+ for sentence_a, sentence_b in tqdm.tqdm(zip(sentence_as, sentence_bs)):
+ diffs: list[str] | None = find_diff(sentence_a, sentence_b)
+ if not diffs:
+ continue
+
+ phrase_a_embed: torch.Tensor = model(
+ tokenizer(diffs[0], return_tensors="pt")["input_ids"]
+ ).pooler_output # the phrase in a that is not in b
+ phrase_b_embed: torch.Tensor = model(
+ tokenizer(diffs[1], return_tensors="pt")["input_ids"]
+ ).pooler_output # the phrase in b that is not in a
+
+ b_in_a = sentence_a.replace(diffs[0], diffs[1]) # put b phrase in sentence a
+ b_in_a_embed: torch.Tensor = model(
+ tokenizer(b_in_a, return_tensors="pt")["input_ids"]
+ ).pooler_output
+ sentence_a_embed: torch.Tensor = model(
+ tokenizer(sentence_a, return_tensors="pt")["input_ids"]
+ ).pooler_output
+
+ # a bit confusing notation to match the formalization, but sentence_a is the context which we test the two phrases in, so c from the formalization.
+ phrase_sim = torch.nn.functional.cosine_similarity(
+ phrase_a_embed, phrase_b_embed
+ ).item() # dist(\mu(a), \mu(b))
+ context_sim = torch.nn.functional.cosine_similarity(
+ b_in_a_embed, sentence_a_embed
+ ).item() # dist(\mu(b, c), \mu(a,c))
+
+ points.append(
+ Point(
+ word_similarity=phrase_sim,
+ contextualized_similarity=context_sim,
+ weight=1,
+ )
+ )
+
+ with open(f"{output_path}.pkl", "wb") as file:
+ pickle.dump(points, file)
+
+
+if __name__ == "__main__":
+ main()
