commit ba126b546e384dedfaa083c1b48f227bae104fa3
parent 446e8c0ef8dd8162fa76826893e43c4db52ffe42
Author: Sondre Wold <sondrewold1@gmail.com>
Date: Wed, 27 Aug 2025 14:31:56 +0200
format
Diffstat:
| M | experiment.py | | | 87 | +++++++++++++++++++++++++++++++++++++++++++++++++------------------------------ |
| M | get_epsilon.py | | | 32 | +++++++++++++++++++++++--------- |
2 files changed, 77 insertions(+), 42 deletions(-)
diff --git a/experiment.py b/experiment.py
@@ -24,13 +24,20 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("-data_path", type=pathlib.Path)
parser.add_argument("-result_path", type=pathlib.Path)
parser.add_argument("-normalization_value", type=float)
- parser.add_argument("-experiment_type", type=str, choices=["sick", "ncip", "ncip-non"])
- parser.add_argument("-model", type=str, choices=["BERT-cls", "BERT-mean", "S-BERT", "S-GPT", "E5", "QWEN"])
+ parser.add_argument(
+ "-experiment_type", type=str, choices=["sick", "ncip", "ncip-non"]
+ )
+ parser.add_argument(
+ "-model",
+ type=str,
+ choices=["BERT-cls", "BERT-mean", "S-BERT", "S-GPT", "E5", "QWEN"],
+ )
return parser.parse_args()
class EmbedError(Exception):
"""If the model is undefined for a subword of the MultiWordExpression, for example, then I should skip this"""
+
pass
@@ -56,7 +63,7 @@ def normalize_v2():
"""
For a context c and a substring x (to be replaced), replace x with n candidates, embed them, calculate the mean the std.dev.
Subtract the mean and divide by the std.dev (gaussian assumption) (mu(ac) - (mean(c with x)) / std.dev
-
+
Sample x from a list of random words.
"""
pass
@@ -101,7 +108,7 @@ class ModelFactory:
class BERT_pooler(Model):
def __init__(self) -> None:
- self.model_name:str = "bert-base-uncased"
+ self.model_name: str = "bert-base-uncased"
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
self.encoder = AutoModel.from_pretrained(self.model_name)
self.encoder.eval()
@@ -110,7 +117,7 @@ class BERT_pooler(Model):
def embed(self, string: str) -> torch.Tensor:
return self.encoder(
self.tokenizer(string.lower(), return_tensors="pt")["input_ids"]
- ).pooler_output
+ ).pooler_output
def calculate_similarity(self, a: torch.Tensor, b: torch.Tensor) -> float:
return torch.nn.functional.cosine_similarity(
@@ -122,7 +129,7 @@ class BERT_pooler(Model):
class BERT_cls(Model):
def __init__(self) -> None:
- self.model_name:str = "bert-base-uncased"
+ self.model_name: str = "bert-base-uncased"
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
self.encoder = AutoModel.from_pretrained(self.model_name)
self.encoder.eval()
@@ -147,30 +154,36 @@ class BERT_mean(Model):
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
self.encoder = AutoModel.from_pretrained(self.model_name)
self.encoder.eval()
-
+
@torch.no_grad()
def embed(self, string: str) -> torch.Tensor:
inputs = self.tokenizer(string.lower(), return_tensors="pt")
outputs = self.encoder(**inputs)
-
+
token_embeddings = outputs.last_hidden_state
attention_mask = inputs["attention_mask"]
-
+
# Expand attention mask to match token embeddings dimensions
- attention_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
-
+ attention_mask_expanded = (
+ attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+ )
+
# Apply mask to token embeddings (set padding tokens to 0)
masked_embeddings = token_embeddings * attention_mask_expanded
-
+
# Sum embeddings and divide by number of non-padding tokens
- summed_embeddings = torch.sum(masked_embeddings, dim=1) # Shape: (batch_size, hidden_size)
- summed_mask = torch.clamp(attention_mask_expanded.sum(dim=1), min=1e-9) # Avoid division by zero
-
+ summed_embeddings = torch.sum(
+ masked_embeddings, dim=1
+ ) # Shape: (batch_size, hidden_size)
+ summed_mask = torch.clamp(
+ attention_mask_expanded.sum(dim=1), min=1e-9
+ ) # Avoid division by zero
+
# Create sentence representation by mean pooling
mean_pooled = summed_embeddings / summed_mask
-
+
return mean_pooled
-
+
def calculate_similarity(self, a: torch.Tensor, b: torch.Tensor) -> float:
return torch.nn.functional.cosine_similarity(
a, b
@@ -194,7 +207,9 @@ class S_BERT(Model):
class S_GPT(Model):
def __init__(self):
- self.encoder = SentenceTransformer("Muennighoff/SGPT-125M-weightedmean-nli-bitfit", device=DEVICE)
+ self.encoder = SentenceTransformer(
+ "Muennighoff/SGPT-125M-weightedmean-nli-bitfit", device=DEVICE
+ )
def embed(self, string: str) -> torch.Tensor:
return self.encoder.encode([string])
@@ -207,11 +222,13 @@ class S_GPT(Model):
class E5(Model):
def __init__(self):
- self.encoder = SentenceTransformer("intfloat/multilingual-e5-large-instruct", device=DEVICE)
+ self.encoder = SentenceTransformer(
+ "intfloat/multilingual-e5-large-instruct", device=DEVICE
+ )
self.task = "Given a sentence, encode its semantic properties"
-
+
def get_detailed_instruct(self, query: str) -> str:
- return f'Instruct: {self.task}\nQuery: {query}'
+ return f"Instruct: {self.task}\nQuery: {query}"
def embed(self, string: str) -> torch.Tensor:
prepared_string = self.get_detailed_instruct(string)
@@ -229,7 +246,7 @@ class QWEN(Model):
self.task = "Given a sentence, encode its semantic properties"
def get_detailed_instruct(self, query: str) -> str:
- return f'Instruct: {self.task}\nQuery: {query}'
+ return f"Instruct: {self.task}\nQuery: {query}"
def embed(self, string: str) -> torch.Tensor:
prepared_string = self.get_detailed_instruct(string)
@@ -247,7 +264,7 @@ def find_diff(a: str, b: str) -> list[str] | None:
if a == b: # Should not happen for SICK or NCIP, but you never know.
return None
- #print(f"A={a}\nB={b}\n##############")
+ # print(f"A={a}\nB={b}\n##############")
try:
words_a = a.split()
@@ -255,7 +272,6 @@ def find_diff(a: str, b: str) -> list[str] | None:
except:
raise Exception(f"Parse error. A={a}, B={b}")
-
prefix_len = 0
min_len = min(len(words_a), len(words_b))
for i in range(min_len):
@@ -336,7 +352,6 @@ class Experiment:
phrase_a_embedded, phrase_b_embedded
)
-
try:
a_in_context_embedded = self.model.embed(sentence_a)
b_in_context_embedded = self.model.embed(b_in_a)
@@ -376,7 +391,7 @@ class Experiment:
phrase_a_embedded=phrase_a_embedded,
phrase_b_embedded=phrase_b_embedded,
context_ac_embedded=a_in_context_embedded,
- context_bc_embedded=b_in_context_embedded
+ context_bc_embedded=b_in_context_embedded,
)
)
@@ -396,13 +411,15 @@ def main() -> None:
if args.experiment_type == "ncip":
# Here we measure the difference between the idiomatic phrase and a synonym.
- original_sentence: list[str] = df["original sentence"].values.tolist() # Original idiomatic noun compound in context.
- synonym_sentence: list[str] = df["synonym both"].values.tolist() # Both the modifier and the noun are swapped with synonyms from word net.
+ original_sentence: list[str] = df[
+ "original sentence"
+ ].values.tolist() # Original idiomatic noun compound in context.
+ synonym_sentence: list[str] = df[
+ "synonym both"
+ ].values.tolist() # Both the modifier and the noun are swapped with synonyms from word net.
sentence_A: list[str] = original_sentence
- sentence_B: list[str] = (
- synonym_sentence
- )
+ sentence_B: list[str] = synonym_sentence
elif args.experiment_type == "ncip-non":
# Here we measure the difference between two non-idiomatic phrases in context.
sentence_A = df["synonym both"].values.tolist()
@@ -420,7 +437,9 @@ def main() -> None:
sentence_pairs = SentencePairs(sentence_A=sentence_A, sentence_B=sentence_B)
- models = [args.model] if args.model is not None else list(ModelFactory._factory.keys())
+ models = (
+ [args.model] if args.model is not None else list(ModelFactory._factory.keys())
+ )
for model_type in models:
model = ModelFactory.get(model_type)
@@ -436,7 +455,9 @@ def main() -> None:
)
print(f"Running experiment for model: {model_type}.")
experiment.run_experiment()
- print(f"Diff errors: {experiment.diff_errors}, Embedding errors: {experiment.embed_errors}")
+ print(
+ f"Diff errors: {experiment.diff_errors}, Embedding errors: {experiment.embed_errors}"
+ )
print(f"Saving experiment to: {str(result_path)}")
sent_a_mean = numpy.mean(experiment.sentence_a_length)
diff --git a/get_epsilon.py b/get_epsilon.py
@@ -22,7 +22,12 @@ def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Gather epsilon values")
parser.add_argument("-points_path", type=pathlib.Path)
parser.add_argument("-result_path", type=pathlib.Path)
- parser.add_argument("-percentile", type=float, default=None, help="The percentile for the epsilon value")
+ parser.add_argument(
+ "-percentile",
+ type=float,
+ default=None,
+ help="The percentile for the epsilon value",
+ )
return parser.parse_args()
@@ -42,7 +47,9 @@ def get_epsilons(
additive: list[float] = []
for i in range(len(word_sims)):
if word_sims[i] > 0:
- original.append(float(1 - context_sims[i] / word_sims[i])) # Equivalent to our formal relaxation)
+ original.append(
+ float(1 - context_sims[i] / word_sims[i])
+ ) # Equivalent to our formal relaxation)
division.append(float(context_sims[i] / word_sims[i]))
additive.append(float(context_sims[i] - word_sims[i]))
@@ -58,7 +65,6 @@ def main() -> None:
points_folder: str = args.points_path
raw_files = list(pathlib.Path(points_folder).glob("*.pkl"))
-
final_result: dict[str, Any] = {
"dataset": [],
"model": [],
@@ -80,10 +86,18 @@ def main() -> None:
tmp_result: dict[str, list[float]] = get_epsilons(word_sims, context_sims)
final_result["dataset"].append(dataset_name)
final_result["model"].append(model_name)
- if args.percentile is not None: # Save the percentile epsilons to disk as a csv
- final_original: Any = float(numpy.percentile(tmp_result["original"], args.percentile))
- final_division: Any = float(numpy.percentile(tmp_result["divisive"], args.percentile))
- final_additive: Any = float(numpy.percentile(tmp_result["additive"], args.percentile))
+ if (
+ args.percentile is not None
+ ): # Save the percentile epsilons to disk as a csv
+ final_original: Any = float(
+ numpy.percentile(tmp_result["original"], args.percentile)
+ )
+ final_division: Any = float(
+ numpy.percentile(tmp_result["divisive"], args.percentile)
+ )
+ final_additive: Any = float(
+ numpy.percentile(tmp_result["additive"], args.percentile)
+ )
else:
final_original = tmp_result["original"]
final_division = tmp_result["divisive"]
@@ -93,8 +107,8 @@ def main() -> None:
final_result["divisive"].append(final_division)
final_result["additive"].append(final_additive)
- #df: pandas.DataFrame = pandas.DataFrame.from_dict(final_result)
- #df.to_csv(args.result_path, sep=",", index=False)
+ # df: pandas.DataFrame = pandas.DataFrame.from_dict(final_result)
+ # df.to_csv(args.result_path, sep=",", index=False)
with open(args.result_path, "w") as f:
json.dump(final_result, f)
