commit 9a230d17128ab1459afa66e3381b29102ee3170b
parent 5eff8e8f5cc73ca14364ae980b875f0c43d43d51
Author: Sondre Wold <sondrewold1@gmail.com>
Date: Wed, 3 Sep 2025 08:43:29 +0200
plotting, random experiments, t-test
Diffstat:
| M | debug/plot_sims.py | | | 99 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- |
| A | random_experiment.py | | | 89 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | t_test.py | | | 69 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
3 files changed, 255 insertions(+), 2 deletions(-)
diff --git a/debug/plot_sims.py b/debug/plot_sims.py
@@ -1,3 +1,7 @@
+import sys
+
+parent_dir = ".."
+sys.path.append(parent_dir)
import argparse
import pathlib
import pickle
@@ -20,10 +24,25 @@ from get_epsilon import get_dataset_and_model_name
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Gather epsilon values")
parser.add_argument("-points_path", type=pathlib.Path)
+ parser.add_argument("-random_path", type=pathlib.Path)
parser.add_argument("-result_path", type=pathlib.Path)
return parser.parse_args()
+def annotate_means(ax, data, positions):
+ for i, (d, pos) in enumerate(zip(data, positions)):
+ mean_val = numpy.mean(d)
+ ax.text(
+ pos,
+ mean_val + 0.02 * (ax.get_ylim()[1] - ax.get_ylim()[0]), # Check this LLM spagetthi
+ f"{mean_val:.3f}",
+ ha="center",
+ va="bottom",
+ fontsize=8,
+ bbox=dict(facecolor="white", alpha=0.7, edgecolor="none")
+ )
+
+
def main() -> None:
args = parse_args()
points_folder: str = args.points_path
@@ -36,10 +55,15 @@ def main() -> None:
"context_sim": [],
}
+ chosen_dataset = "ncip-non"
+
+
for raw_file in raw_files:
with open(raw_file, "rb") as file:
points: list[Point] = pickle.load(file)
dataset_name, model_name = get_dataset_and_model_name(raw_file.name)
+ if dataset_name != chosen_dataset:
+ continue
word_sims = [
(
@@ -58,11 +82,27 @@ def main() -> None:
for point in points
]
- final_result["dataset"].append(dataset_name)
+ print(model_name, dataset_name)
+ #final_result["dataset"].append(dataset_name)
final_result["model"].append(model_name)
final_result["phrase_sim"].append(word_sims)
final_result["context_sim"].append(context_sims)
+
+
+
+
+ # Sort the "model" list alphabetically
+ sorted_models = sorted(final_result["model"])
+
+ # Create a mapping from model name to its sorted index
+ model_order = {model: idx for idx, model in enumerate(sorted_models)}
+
+ # Sort all other lists in the dict based on the order of "model"
+ final_result = {
+ key: sorted(value, key=lambda x: model_order[final_result["model"][value.index(x)]])
+ for key, value in final_result.items()
+ }
fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(10, 4))
axs[0, 0].violinplot(
final_result["phrase_sim"],
@@ -72,6 +112,11 @@ def main() -> None:
[y + 1 for y in range(len(final_result["phrase_sim"]))],
labels=final_result["model"],
)
+ annotate_means(
+ axs[0, 0],
+ final_result["phrase_sim"],
+ [y + 1 for y in range(len(final_result["phrase_sim"]))]
+ )
axs[1, 0].violinplot(
final_result["context_sim"],
[y + 1 for y in range(len(final_result["context_sim"]))],
@@ -80,8 +125,58 @@ def main() -> None:
[y + 1 for y in range(len(final_result["context_sim"]))],
labels=final_result["model"],
)
- plt.show()
+ annotate_means(
+ axs[1, 0],
+ final_result["context_sim"],
+ [y + 1 for y in range(len(final_result["context_sim"]))]
+ )
+ axs[0, 0].set_title(f"{chosen_dataset} phrase similarity")
+ axs[0, 1].set_title(f"Random phrase similarity")
+ axs[1, 0].set_title(f"{chosen_dataset} context similarity")
+ axs[1, 1].set_title(f"Random context similarity")
+
+ if args.random_path is not None:
+ with open(args.random_path, "rb") as f:
+ random_result = pickle.load(f)
+ sorted_models = sorted(random_result["model"])
+
+ # Create a mapping from model name to its sorted index
+ model_order = {model: idx for idx, model in enumerate(sorted_models)}
+
+ # Sort all other lists in the dict based on the order of "model"
+ random_result = {
+ key: sorted(value, key=lambda x: model_order[random_result["model"][value.index(x)]])
+ for key, value in random_result.items()
+ }
+
+ axs[0, 1].violinplot(
+ random_result["phrase_sim"],
+ [y + 1 for y in range(len(random_result["phrase_sim"]))],
+ )
+ axs[0, 1].set_xticks(
+ [y + 1 for y in range(len(random_result["phrase_sim"]))],
+ labels=random_result["model"],
+ )
+ annotate_means(
+ axs[0, 1],
+ random_result["phrase_sim"],
+ [y + 1 for y in range(len(random_result["phrase_sim"]))]
+ )
+ axs[1, 1].violinplot(
+ random_result["context_sim"],
+ [y + 1 for y in range(len(random_result["context_sim"]))],
+ )
+ axs[1, 1].set_xticks(
+ [y + 1 for y in range(len(random_result["context_sim"]))],
+ labels=random_result["model"],
+ )
+ annotate_means(
+ axs[1, 1],
+ random_result["context_sim"],
+ [y + 1 for y in range(len(random_result["context_sim"]))]
+ )
+ plt.show()
if __name__ == "__main__":
main()
diff --git a/random_experiment.py b/random_experiment.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+from typing import NamedTuple, Any
+
+import pandas
+import pathlib
+import argparse
+
+import random
+
+import numpy
+
+from abc import ABC, abstractmethod
+import torch
+import pickle
+import tqdm
+
+from experiment import ModelFactory
+
+from transformers import AutoTokenizer, AutoModel
+from sentence_transformers import SentenceTransformer
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Experiment")
+ parser.add_argument("-data_path", type=pathlib.Path)
+ parser.add_argument("-result_path", type=pathlib.Path)
+ parser.add_argument("-n", type=int, default=100, help="The number of samples to calculate for")
+ parser.add_argument(
+ "-experiment_type", type=str, choices=["sick", "ncip", "ncip-non"]
+ )
+ parser.add_argument(
+ "-model",
+ type=str,
+ choices=["BERT-cls", "BERT-mean", "S-BERT", "S-GPT", "E5", "QWEN"],
+ )
+ return parser.parse_args()
+
+
+def main() -> None:
+ args = parse_args()
+ sep = ","
+ df: pandas.DataFrame = pandas.read_csv(args.data_path, sep=sep)
+ print(f"Similarity between random items, n_samples={args.n}")
+
+ sentences: list[str] = df["original sentence"].values.tolist()
+ phrases: list[str] = df["synonym both"].values.tolist()
+
+ choices = ["BERT-cls", "BERT-mean", "S-BERT", "E5", "QWEN"]
+ models = [args.model] if args.model is not None else choices
+ result = {
+ "model": [],
+ "phrase_sim": [],
+ "context_sim": []
+ }
+
+ for model_type in models:
+ model = ModelFactory.get(model_type)
+ context_sims: list[float] = []
+ phrase_sims: list[float] = []
+ for _ in tqdm.tqdm(range(0, args.n)):
+ phrase_a = random.choice(phrases)
+ phrase_b = random.choice(phrases)
+ context_a = random.choice(sentences)
+ context_b = random.choice(sentences)
+
+ if (phrase_a == phrase_b) or (context_a == context_b):
+ continue
+
+ phrase_a_embedded = model.embed(phrase_a)
+ phrase_b_embedded = model.embed(phrase_b)
+
+ context_a_embedded = model.embed(context_a)
+ context_b_embedded = model.embed(context_b)
+
+ context_sims.append(model.calculate_similarity(context_a_embedded, context_b_embedded))
+ phrase_sims.append(model.calculate_similarity(phrase_a_embedded, phrase_b_embedded))
+ result["model"].append(model_type)
+ result["phrase_sim"].append(phrase_sims)
+ result["context_sim"].append(context_sims)
+
+ print(f"Model: {model}")
+ print("Random context sims", numpy.mean(context_sims))
+ print("Random phrase cosine similarity", numpy.mean(phrase_sims))
+
+ with open(args.result_path, "wb") as f:
+ pickle.dump(result, f)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/t_test.py b/t_test.py
@@ -0,0 +1,69 @@
+import sys
+import pickle
+import numpy
+from torch import equal, histogram
+from experiment import Point
+import matplotlib.pyplot as plt
+
+import scipy.stats as stats
+
+from typing import Any
+import json
+
+import pandas
+
+import argparse
+import pathlib
+
+import warnings
+
+from typing import NamedTuple
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Calculate the Welsch T-Test")
+ parser.add_argument("-epsilons_path", type=pathlib.Path)
+ parser.add_argument("-result_path", type=pathlib.Path)
+ return parser.parse_args()
+
+
+class ResultPoint():
+ def __init__(self, model_name: str = "", idiomatic_result: list[float] = [], non_idiomatic_result: list[float] = []):
+ self.model_name: str = model_name
+ self.idiomatic_result: list[float] = idiomatic_result
+ self.non_idiomatic_result: list[float] = non_idiomatic_result
+
+
+def main() -> None:
+ args = parse_args()
+ with open(args.epsilons_path, "r") as f:
+ loaded_dict = json.load(f)
+
+ model_results: dict[str, ResultPoint] = {}
+ for i in range(0, len(loaded_dict["dataset"])):
+ dataset_name = loaded_dict["dataset"][i]
+ model_name = loaded_dict["model"][i]
+ is_idiomatic = True if dataset_name == "ncip" else False
+ if model_name not in model_results:
+ model_results[model_name] = ResultPoint(model_name=model_name)
+ if is_idiomatic:
+ model_results[model_name].idiomatic_result = loaded_dict["original"][i]
+ else:
+ model_results[model_name].non_idiomatic_result = loaded_dict["original"][i]
+
+ for model, res in model_results.items():
+ idiom_variance = numpy.var(res.idiomatic_result)
+ non_idiom_variance = numpy.var(res.non_idiomatic_result)
+ # null hypothesis: the two groups are from the same means, p_value < 0.05 -> the two groups are different.
+ t_test_statistics = stats.ttest_ind(res.idiomatic_result, res.non_idiomatic_result, equal_var=False, alternative="greater").pvalue
+ t_rel_statistics = stats.ttest_rel(res.idiomatic_result[:250], res.non_idiomatic_result[:250]).pvalue
+ is_signficant: bool = t_test_statistics < 0.05
+ #print(f"Model name: {model}, Idiom Variance: {idiom_variance}, Non-Idiom Variance: {non_idiom_variance}, p_value={t_test_statistics}, is_significant: {is_signficant}")
+ print(f"Model name: {model}, Idiom Variance: {idiom_variance}, Non-Idiom Variance: {non_idiom_variance}, p_value={t_rel_statistics}, is_significant: {is_signficant}")
+
+
+
+if __name__ == "__main__":
+ main()
