commit 63a153636d64e89c73b03ed3919275b5af51913b
parent 686525fcec3aacf31d60ec9a1c312d14cf3a7bbd
Author: Sondre Wold <sondrewold1@gmail.com>
Date: Sat, 12 Jul 2025 20:13:59 +0200
linting
Diffstat:
| M | experiment.py | | | 111 | ++++++++++++++++++++++++++++++++++++++++++++++--------------------------------- |
1 file changed, 65 insertions(+), 46 deletions(-)
diff --git a/experiment.py b/experiment.py
@@ -1,7 +1,6 @@
from __future__ import annotations
-from typing import Callable, NamedTuple, Any
+from typing import NamedTuple, Any
-import sys
import pandas
import pathlib
import argparse
@@ -16,9 +15,7 @@ from sentence_transformers import SentenceTransformer
def parse_args() -> argparse.Namespace:
- parser = argparse.ArgumentParser(
- description="Experiment"
- )
+ parser = argparse.ArgumentParser(description="Experiment")
parser.add_argument("-data_path", type=pathlib.Path)
parser.add_argument("-result_path", type=pathlib.Path)
parser.add_argument("-normalization_value", type=float)
@@ -46,7 +43,12 @@ class Model(ABC):
def embed(self, string: str) -> torch.Tensor:
pass
- def normalize(self, composed: torch.Tensor, original_phrase_removed: torch.Tensor, lambda_value: float):
+ def normalize(
+ self,
+ composed: torch.Tensor,
+ original_phrase_removed: torch.Tensor,
+ lambda_value: float,
+ ):
return composed - (original_phrase_removed * lambda_value)
@abstractmethod
@@ -61,7 +63,6 @@ class ModelFactory:
def __init__(self, name: str):
self._name = name
-
def __call__(self, model: type[Model]) -> type[Model]:
self._factory[self._name] = model
return model
@@ -81,12 +82,14 @@ class BERT_cls(Model):
def embed(self, string: str) -> torch.Tensor:
return self.encoder(
- self.tokenizer(string, return_tensors="pt")["input_ids"]
- ).pooler_output # the phrase in a that is not in b
+ self.tokenizer(string, return_tensors="pt")["input_ids"]
+ ).pooler_output # the phrase in a that is not in b
-
def calculate_similarity(self, a: torch.Tensor, b: torch.Tensor) -> float:
- return torch.nn.functional.cosine_similarity(a, b).item() # dist(\mu(a), \mu(b))
+ return torch.nn.functional.cosine_similarity(
+ a, b
+ ).item() # dist(\mu(a), \mu(b))
+
@ModelFactory("S_BERT")
class S_BERT(Model):
@@ -104,8 +107,9 @@ class S_BERT(Model):
def find_diff(a: str, b: str) -> list[str] | None:
"""
Finds the longest continuous substring where A differs from B and B differs from A.
+ Subtracts the longest common prefix and suffix and takes whats left. Not the smartest algorithm, perhaps.
"""
- if a == b: # Should not happen for SICK, but you never know.
+ if a == b: # Should not happen for SICK or NCIP, but you never know.
return None
words_a = a.split()
@@ -127,10 +131,10 @@ def find_diff(a: str, b: str) -> list[str] | None:
break
diff_a = words_a[
- prefix_len : len(words_a) - suffix_len if suffix_len > 0 else len(words_a)
+ prefix_len:len(words_a) - suffix_len if suffix_len > 0 else len(words_a)
]
diff_b = words_b[
- prefix_len : len(words_b) - suffix_len if suffix_len > 0 else len(words_b)
+ prefix_len:len(words_b) - suffix_len if suffix_len > 0 else len(words_b)
]
if not diff_a or not diff_b:
@@ -144,7 +148,13 @@ def find_diff(a: str, b: str) -> list[str] | None:
class Experiment:
- def __init__(self, sentence_pairs: SentencePairs, model: Model, result_path: pathlib.Path, normalization_value: float):
+ def __init__(
+ self,
+ sentence_pairs: SentencePairs,
+ model: Model,
+ result_path: pathlib.Path,
+ normalization_value: float,
+ ):
self.model: Model = model
self.sentence_pairs = sentence_pairs
self.normalization_value: float = normalization_value
@@ -152,8 +162,7 @@ class Experiment:
self.points: list[Point] = []
self.result_path: pathlib.Path = result_path
-
- def run_experiment(self):
+ def run_experiment(self) -> None:
for sentence_a, sentence_b in tqdm.tqdm(zip(*self.sentence_pairs)):
diffs: list[str] | None = find_diff(sentence_a, sentence_b)
if not diffs:
@@ -166,37 +175,44 @@ class Experiment:
b_in_a: str = sentence_a.replace(phrase_a, phrase_b)
phrase_a_embedded: torch.Tensor = self.model.embed(phrase_a)
- phrase_b_embedded: torch.Tensor = self.model.embed(phrase_b)
+ phrase_b_embedded: torch.Tensor = self.model.embed(phrase_b)
- phrase_similarity: float = self.model.calculate_similarity(phrase_a_embedded, phrase_b_embedded)
+ phrase_similarity: float = self.model.calculate_similarity(
+ phrase_a_embedded, phrase_b_embedded
+ )
a_in_context_embedded = self.model.embed(sentence_a)
b_in_context_embedded = self.model.embed(b_in_a)
if self.normalization_value > 0.0:
a_removed: str = sentence_a.replace(phrase_a, "")
- b_removed: str = sentence_b.replace(phrase_b, "")
+ b_removed: str = sentence_b.replace(phrase_b, "")
a_removed_embedded = self.model.embed(a_removed)
b_removed_embedded = self.model.embed(b_removed)
- a_in_context_embedded = self.model.normalize(a_in_context_embedded, a_removed_embedded, self.normalization_value)
- b_in_context_embedded = self.model.normalize(b_in_context_embedded, b_removed_embedded, self.normalization_value)
+ a_in_context_embedded = self.model.normalize(
+ a_in_context_embedded, a_removed_embedded, self.normalization_value
+ )
+ b_in_context_embedded = self.model.normalize(
+ b_in_context_embedded, b_removed_embedded, self.normalization_value
+ )
- context_similarity: float = self.model.calculate_similarity(a_in_context_embedded, b_in_context_embedded)
+ context_similarity: float = self.model.calculate_similarity(
+ a_in_context_embedded, b_in_context_embedded
+ )
self.points.append(
- Point(
- phrase_1=phrase_a,
- phrase_2=phrase_b,
- context_1=sentence_a,
- context_2=b_in_a,
- phrase_similarity=phrase_similarity,
- contextualized_similarity=context_similarity
- )
+ Point(
+ phrase_1=phrase_a,
+ phrase_2=phrase_b,
+ context_1=sentence_a,
+ context_2=b_in_a,
+ phrase_similarity=phrase_similarity,
+ contextualized_similarity=context_similarity,
+ )
)
def save_results(self):
- pathlib.Path(self.result_path).mkdir(parents=True, exist_ok=True)
with open(self.result_path.with_suffix(".pkl"), "wb") as file:
pickle.dump(self.points, file)
@@ -206,7 +222,7 @@ def main() -> None:
sep = "\t"
if args.experiment_type == "ncip":
sep = ","
-
+
df: pandas.DataFrame = pandas.read_csv(args.data_path, sep=sep)
if args.experiment_type == "ncip":
@@ -216,29 +232,33 @@ def main() -> None:
synonym_sentence3: list[str] = df["both synonym alt3_1"].values.tolist()
synonym_sentence4: list[str] = df["both synonym alt4_1"].values.tolist()
- sentence_A = original_sentence * 4
- sentence_B = synonym_sentence + synonym_sentence2 + synonym_sentence3 + synonym_sentence4
- sentence_pairs = SentencePairs(sentence_A=sentence_A, sentence_B=sentence_B)
+ sentence_A: list[str] = original_sentence * 4
+ sentence_B: list[str] = (
+ synonym_sentence + synonym_sentence2 + synonym_sentence3 + synonym_sentence4
+ )
elif args.experiment_type == "sick":
lexA = df.loc[df["sentence_A_expRule"] == "S2_lex"]
lexB = df.loc[df["sentence_B_expRule"] == "S2_lex"]
lex = pandas.concat([lexA, lexB])
- sentence_A: list[str] = lex["sentence_A"].values.tolist()
- sentence_B: list[str] = lex["sentence_B"].values.tolist()
- sentence_pairs = SentencePairs(sentence_A=sentence_A, sentence_B=sentence_B)
+ sentence_A = lex["sentence_A"].values.tolist()
+ sentence_B = lex["sentence_B"].values.tolist()
else:
- assert ValueError("Undefined experiment type...")
+ raise ValueError("Undefined experiment type...")
+ sentence_pairs = SentencePairs(sentence_A=sentence_A, sentence_B=sentence_B)
for model_type in ModelFactory._factory.keys():
model = ModelFactory.get(model_type)
- result_path = args.result_path / f"{args.experiment_type}_{model_type}_lambda{int(args.normalization_value * 10)}" # shitty logging of lambda parameter, need to fix
+ result_path = (
+ args.result_path
+ / f"{args.experiment_type}-{model_type}-lambda{int(args.normalization_value * 10)}"
+ ) # shitty logging of lambda parameter, need to fix
experiment = Experiment(
- sentence_pairs=sentence_pairs,
- model=model,
- result_path = result_path,
- normalization_value=args.normalization_value,
+ sentence_pairs=sentence_pairs,
+ model=model,
+ result_path=result_path,
+ normalization_value=args.normalization_value,
)
print(f"Running experiment for model: {model_type}.")
experiment.run_experiment()
@@ -248,4 +268,3 @@ def main() -> None:
if __name__ == "__main__":
main()
-
