commit cbf9c61e2d82a75dd4e851d3ca40f420488245a3
parent 5593640d6ab0e33f157352ad6fd72e1a99bc3fe4
Author: Étienne Simon <esimon@esimon.eu>
Date: Mon, 6 Oct 2025 16:11:09 +0200
model display name and factorization
Diffstat:
1 file changed, 52 insertions(+), 43 deletions(-)
diff --git a/epsilon_compositionality/compute_statistics.py b/epsilon_compositionality/compute_statistics.py
@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Callable, IO
import collections
import itertools
import json
@@ -15,6 +15,24 @@ MODELS_ORDER: list[str] = ["BERT", "ModernBERT", "DeBERTa", "S-BERT", "S-GPT", "
MODELS_HAS_SUB: set[str] = set(["BERT", "ModernBERT", "DeBERTa"])
MODELS_SUB_ORDER: list[str] = ["cls", "clssep", "mean"]
+MODEL_DISPLAY_NAME: dict[str, str] = {
+ "BERT": r"\textsc{bert}",
+ "ModernBERT": r"Modern\textsc{bert}",
+ "DeBERTa": r"De\textsc{bert}a",
+ "S-BERT": r"\textsc{s-bert}",
+ "S-GPT": r"\textsc{s-gpt}",
+ "E5": r"\textsc{e5}",
+ "UAE": r"\textsc{uae}",
+ "QWEN": r"\textsc{qwen}",
+}
+
+SUBMODEL_DISPLAY_NAME: dict[str, str] = {
+ "cls": r"\textsc{cls}",
+ "clssep": r"\(\textsc{cls}+\textsc{sep}\)",
+ "mean": r"mean",
+ "": "", # no submodels for sentence transformers
+}
+
def get_epsilons(sentences: list[dict[str, Any]], method: str, offset: int = 0) -> tuple[float, float]:
epsilons: dict[str, list[float]] = {"positives": [], "negatives": []}
@@ -89,18 +107,22 @@ def table_cell(positives: list[float], negatives: list[float], method: str) -> s
return f"{wilcoxon.pvalue:<1.3f}"
-def latex_main_statistics_table(output_path: pathlib.Path, models: dict[str, Any]) -> None:
+def latex_models_table(output_path: pathlib.Path, models: dict[str, Any], model_line: Callable[[pathlib.Path, dict[str, Any], str], None]) -> None:
with output_path.open("w") as output:
for model_name in MODELS_ORDER:
for isubmodel, submodel in enumerate(MODELS_SUB_ORDER if model_name in MODELS_HAS_SUB else [""]):
- print(f"{model_name if isubmodel==0 else '':<10}", end=" & ", file=output)
- print(f"{submodel:<6}", end=" & ", file=output)
+ print(f"{MODEL_DISPLAY_NAME[model_name] if isubmodel==0 else '':<20}", end=" & ", file=output)
+ print(f"{SUBMODEL_DISPLAY_NAME[submodel]:<30}", end=" & ", file=output)
model_key: str = f"{model_name}-{submodel}" if submodel else model_name
- for imethod, method in enumerate(["p-value", "rank biserial"]):
- for isubset, subset in enumerate(CLASSES_ORDER):
- positives = list(itertools.chain(*(models[model_key][subset][side]["positives"] for side in ["head", "modifier"])))
- negatives = list(itertools.chain(*(models[model_key][subset][side]["negatives"] for side in ["head", "modifier"])))
- print(table_cell(positives, negatives, method), end=" & " if isubset<len(CLASSES_ORDER)-1 or imethod==0 else " \\\\\n", file=output)
+ model_line(output, models, model_key)
+
+
+def main_statistics(output: IO[str], models: dict[str, Any], model_key: str) -> None:
+ for imethod, method in enumerate(["p-value", "rank biserial"]):
+ for isubset, subset in enumerate(CLASSES_ORDER):
+ positives = list(itertools.chain(*(models[model_key][subset][side]["positives"] for side in ["head", "modifier"])))
+ negatives = list(itertools.chain(*(models[model_key][subset][side]["negatives"] for side in ["head", "modifier"])))
+ print(table_cell(positives, negatives, method), end=" & " if isubset<len(CLASSES_ORDER)-1 or imethod==0 else " \\\\\n", file=output)
def get_mu_sigma(data: list[float]) -> tuple[float, float]:
@@ -109,24 +131,17 @@ def get_mu_sigma(data: list[float]) -> tuple[float, float]:
return mu, sigma
-def latex_distances_statistics(output_path: pathlib.Path) -> None:
- models = read_similarities("pairs")
- with output_path.open("w") as output:
- for model_name in MODELS_ORDER:
- for isubmodel, submodel in enumerate(MODELS_SUB_ORDER if model_name in MODELS_HAS_SUB else [""]):
- print(f"{model_name if isubmodel==0 else '':<10}", end=" & ", file=output)
- print(f"{submodel:<6}", end=" & ", file=output)
- model_key: str = f"{model_name}-{submodel}" if submodel else model_name
- contextual: list[float] = []
- phrasal: list[float] = []
- for subset in CLASSES_ORDER:
- for side in ["head", "modifier"]:
- for polarity in ["positives", "negatives"]:
- contextual.extend(x for x, _ in models[model_key][subset][side][polarity])
- phrasal.extend(x for _, x in models[model_key][subset][side][polarity])
- pmu, psigma = get_mu_sigma(phrasal)
- cmu, csigma = get_mu_sigma(contextual)
- print(f"{pmu:5.3f} & {psigma:5.3f} & {cmu:5.3f} & {csigma:5.3f} \\\\", file=output)
+def distances_statistics(output: IO[str], models: dict[str, Any], model_key: str) -> None:
+ contextual: list[float] = []
+ phrasal: list[float] = []
+ for subset in CLASSES_ORDER:
+ for side in ["head", "modifier"]:
+ for polarity in ["positives", "negatives"]:
+ contextual.extend(x for x, _ in models[model_key][subset][side][polarity])
+ phrasal.extend(x for _, x in models[model_key][subset][side][polarity])
+ pmu, psigma = get_mu_sigma(phrasal)
+ cmu, csigma = get_mu_sigma(contextual)
+ print(f"{pmu:5.3f} & {psigma:5.3f} & {cmu:5.3f} & {csigma:5.3f} \\\\", file=output)
def write_violins(output_path: pathlib.Path) -> None:
@@ -142,27 +157,21 @@ def write_violins(output_path: pathlib.Path) -> None:
print("", file=output)
-def latex_extended_statistics_table(output_path: pathlib.Path, models: dict[str, Any]) -> None:
- with output_path.open("w") as output:
- for model_name in MODELS_ORDER:
- for isubmodel, submodel in enumerate(MODELS_SUB_ORDER if model_name in MODELS_HAS_SUB else [""]):
- print(f"{model_name if isubmodel==0 else '':<10}", end=" & ", file=output)
- print(f"{submodel:<6}", end=" & ", file=output)
- model_key: str = f"{model_name}-{submodel}" if submodel else model_name
- for isubset, subset in enumerate(CLASSES_ORDER):
- for iside, side in enumerate(["modifier", "head"]):
- positives = models[model_key][subset][side]["positives"]
- negatives = models[model_key][subset][side]["negatives"]
- print(table_cell(positives, negatives, "rank biserial"), end=" & " if isubset<len(CLASSES_ORDER)-1 or iside==0 else " \\\\\n", file=output)
+def extended_statistics(output: IO[str], models: dict[str, Any], model_key: str) -> None:
+ for isubset, subset in enumerate(CLASSES_ORDER):
+ for iside, side in enumerate(["modifier", "head"]):
+ positives = models[model_key][subset][side]["positives"]
+ negatives = models[model_key][subset][side]["negatives"]
+ print(table_cell(positives, negatives, "rank biserial"), end=" & " if isubset<len(CLASSES_ORDER)-1 or iside==0 else " \\\\\n", file=output)
def main() -> None:
- models = read_similarities("multiplicative")
RESULTS_PATH.mkdir(exist_ok=True)
+ models = read_similarities("multiplicative")
latex_data_statistics(RESULTS_PATH / "data statistics.tex")
- latex_main_statistics_table(RESULTS_PATH / "main statistics.tex", models)
- latex_extended_statistics_table(RESULTS_PATH / "extended statistics.tex", models)
- latex_distances_statistics(RESULTS_PATH / "distances statistics.tex")
+ latex_models_table(RESULTS_PATH / "main statistics.tex", models, main_statistics)
+ latex_models_table(RESULTS_PATH / "extended statistics.tex", models, extended_statistics)
+ latex_models_table(RESULTS_PATH / "distances statistics.tex", read_similarities("pairs"), distances_statistics)
write_violins(RESULTS_PATH / "violins.dat")
