commit d5509b1a6efa4fd84eef40c2b61a19fd46eab8a1
parent 63a153636d64e89c73b03ed3919275b5af51913b
Author: Sondre Wold <sondrewold1@gmail.com>
Date: Sat, 12 Jul 2025 20:15:11 +0200
remove old main
Diffstat:
| D | extract_sick_pairs.py | | | 113 | ------------------------------------------------------------------------------- |
| D | main.py | | | 114 | ------------------------------------------------------------------------------- |
2 files changed, 0 insertions(+), 227 deletions(-)
diff --git a/extract_sick_pairs.py b/extract_sick_pairs.py
@@ -1,113 +0,0 @@
-from typing import NamedTuple
-import sys
-import pandas
-
-import torch
-import pickle
-import tqdm
-
-from transformers import AutoTokenizer, AutoModel
-
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-
-class Point(NamedTuple):
- word_similarity: float
- contextualized_similarity: float
- weight: float
-
-
-def find_diff(a: str, b: str) -> list[str] | None:
- """
- Finds the longest continuous substring where A differs from B and B differs from A.
- """
- if a == b: # Should not happen for SICK, but you never know.
- return None
-
- words_a = a.split()
- words_b = b.split()
-
- prefix_len = 0
- min_len = min(len(words_a), len(words_b))
- for i in range(min_len):
- if words_a[i] == words_b[i]:
- prefix_len += 1
- else:
- break
-
- suffix_len = 0
- for i in range(1, min_len - prefix_len + 1):
- if words_a[-i] == words_b[-i]:
- suffix_len += 1
- else:
- break
-
- diff_a = words_a[
- prefix_len : len(words_a) - suffix_len if suffix_len > 0 else len(words_a)
- ]
- diff_b = words_b[
- prefix_len : len(words_b) - suffix_len if suffix_len > 0 else len(words_b)
- ]
-
- if not diff_a or not diff_b:
- return None
- else:
- diff_str_a = " ".join(diff_a) if diff_a else ""
- diff_str_b = " ".join(diff_b) if diff_b else ""
- return [diff_str_a, diff_str_b]
-
-
-def main() -> None:
- data_path: str = sys.argv[1]
- output_path: str = sys.argv[2]
- df: pandas.DataFrame = pandas.read_csv(data_path, delimiter="\t")
- sentence_as: list[str] = df["sentence_A"].values.tolist()
- sentence_bs: list[str] = df["sentence_B"].values.tolist()
-
- model_name: str = "bert-base-uncased"
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- model = AutoModel.from_pretrained(model_name)
-
- points: list[Point] = []
- for sentence_a, sentence_b in tqdm.tqdm(zip(sentence_as, sentence_bs)):
- diffs: list[str] | None = find_diff(sentence_a, sentence_b)
- if not diffs:
- continue
-
- phrase_a_embed: torch.Tensor = model(
- tokenizer(diffs[0], return_tensors="pt")["input_ids"]
- ).pooler_output # the phrase in a that is not in b
- phrase_b_embed: torch.Tensor = model(
- tokenizer(diffs[1], return_tensors="pt")["input_ids"]
- ).pooler_output # the phrase in b that is not in a
-
- b_in_a = sentence_a.replace(diffs[0], diffs[1]) # put b phrase in sentence a
- b_in_a_embed: torch.Tensor = model(
- tokenizer(b_in_a, return_tensors="pt")["input_ids"]
- ).pooler_output
- sentence_a_embed: torch.Tensor = model(
- tokenizer(sentence_a, return_tensors="pt")["input_ids"]
- ).pooler_output
-
- # a bit confusing notation to match the formalization, but sentence_a is the context which we test the two phrases in, so c from the formalization.
- phrase_sim = torch.nn.functional.cosine_similarity(
- phrase_a_embed, phrase_b_embed
- ).item() # dist(\mu(a), \mu(b))
- context_sim = torch.nn.functional.cosine_similarity(
- b_in_a_embed, sentence_a_embed
- ).item() # dist(\mu(b, c), \mu(a,c))
-
- points.append(
- Point(
- word_similarity=phrase_sim,
- contextualized_similarity=context_sim,
- weight=1,
- )
- )
-
- with open(f"{output_path}.pkl", "wb") as file:
- pickle.dump(points, file)
-
-
-if __name__ == "__main__":
- main()
diff --git a/main.py b/main.py
@@ -1,114 +0,0 @@
-from __future__ import annotations
-from typing import Iterable, NamedTuple
-import collections
-import math
-import pickle
-import re
-
-import datasets
-import numpy
-import matplotlib.pyplot
-import tqdm
-import torch
-import transformers
-
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-
-class Point(NamedTuple):
- word_similarity: float
- contextualized_similarity: float
- weight: float
-
-
-class Experiment:
- def __init__(self: Experiment, model: str, dataset: dict[str, str], ngram: int, grams_are_tokens: bool, minimal_occurences: int, maximum_occurences: int) -> None:
- self.ngram = ngram
- self.dataset_name = dataset["name"]
- self.grams_are_tokens = grams_are_tokens
- self.minimal_occurences = minimal_occurences
- self.maximum_occurences = maximum_occurences
- self.dataset = datasets.load_dataset(**dataset)
- self.model = transformers.AutoModel.from_pretrained(model).to(device)
- self.tokenizer = transformers.AutoTokenizer.from_pretrained(model)
-
- def join_tokens(self: Experiment, tokens: list[str]) -> str:
- if self.grams_are_tokens:
- raise NotImplementedError("TODO tokenizer")
- else:
- return " ".join(tokens)
-
- def loop_strings_to_ngrams(self: Experiment, input: list[str]) -> dict[str, collections.Counter]:
- counters = collections.defaultdict(collections.Counter)
- for i in range(len(input) - self.ngram - 1):
- context: str = self.join_tokens(input[i+1:i + 1 + self.ngram])
- if self.filter(input[i]) and self.filter(context):
- counters[context][input[i]] += 1
- return counters
-
- def get_ngrams(self: Experiment) -> dict[str, dict[str, float]]:
- counters = collections.defaultdict(collections.Counter)
- for sample in tqdm.tqdm(self.dataset):
- if self.grams_are_tokens:
- raise NotImplementedError("TODO tokenizer")
- else:
- for key, value in self.loop_strings_to_ngrams(sample["text"].split()).items():
- counters[key] += value
- return collections.defaultdict(collections.Counter, {
- key: value for key, value in counters.items()
- if sum(value.values()) >= self.minimal_occurences and sum(value.values()) <= self.maximum_occurences and len(value) >= 2
- })
-
- def embed(self: Experiment, text: str) -> torch.Tensor:
- return self.model(self.tokenizer(text, return_tensors="pt")["input_ids"].to(device)).pooler_output
-
- def similarity(self: Experiment, lhs: str, rhs: str) -> float:
- # TODO cpu is needed?
- return torch.nn.functional.cosine_similarity(self.embed(lhs), self.embed(rhs))[0].cpu().item()
-
- def filter(self: Experiment, text: str) -> bool:
- return bool(re.fullmatch(r"[a-zA-Z ]+", text))
-
- def all_similarities(self: Experiment) -> Iterable[Point]:
- ngrams = self.get_ngrams()
- print(f"Number of {self.ngram}-grams after filtering: {len(ngrams)}")
-
- with tqdm.tqdm(ngrams.items()) as ctqdm:
- for context, counter in ctqdm:
- ctqdm.set_description(context)
- total_count: int = sum(counter.values())
- for first, first_count in tqdm.tqdm(counter.items()):
- cfirst = self.join_tokens([first, context])
- for second, second_count in counter.items():
- csecond = self.join_tokens([second, context])
- if cfirst == csecond:
- pass
- yield Point(word_similarity=self.similarity(first, second),
- contextualized_similarity=self.similarity(cfirst, csecond),
- weight=first_count/sum(counter.values()))
-
- def __call__(self: Experiment):
- points: list[Point] = list(self.all_similarities())
- with open(f"points_output_{self.dataset_name}.pkl", "wb") as file:
- pickle.dump(points, file)
- data, xedges, yedges = numpy.histogram2d([point.word_similarity for point in points],
- [point.contextualized_similarity for point in points],
- weights=[point.weight for point in points],
- bins=100)
- extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]]
- matplotlib.pyplot.clf()
- matplotlib.pyplot.imshow(data.T, extent=extent, origin="lower")
- matplotlib.pyplot.savefig("figure.svg", format="svg")
- matplotlib.pyplot.show()
-
-
-if __name__ == "__main__":
- Experiment(
- model="bert-base-uncased",
- dataset={"path":"Salesforce/wikitext", "name":"wikitext-2-v1", "split":"train"},
- #dataset={"path":"sedthh/gutenberg_english", "name": None, "split":"train"},
- ngram=4,
- grams_are_tokens=False,
- minimal_occurences=10,
- maximum_occurences=20,
- )()
