commit d0edaaf471da72428199eaadd2fe4460feed0e0e
parent 91fb4c2e6710a54517cc5f58ebba5b7883eb3f51
Author: Étienne Simon <esimon@esimon.eu>
Date: Mon, 6 Oct 2025 06:28:52 +0200
use distance epsilon and output latex
Diffstat:
6 files changed, 216 insertions(+), 206 deletions(-)
diff --git a/README.md b/README.md
@@ -1,13 +1,15 @@
# Evaluating Idiomaticity in Sentence Embeddings with 𝜀-compositionality
-Reproduce everything:
+To reproduce everything:
```sh
+git clone --recurse-submodules THIS_REPOSITORY
python -m epsilon_compositionality
```
+The data is included as a git submodule, from there the python module `epsilon_compositionality` run all the necessary code and output a `results` directory containing the LaTeX code we included in the article.
This is equivalent to:
```sh
python -m epsilon_compositionality.build_dataset
-python -m epsilon_compositionality.extract_embeddings
+python -m epsilon_compositionality.extract_similarities
python -m epsilon_compositionality.compute_statistics
```
diff --git a/epsilon_compositionality/__main__.py b/epsilon_compositionality/__main__.py
@@ -1,9 +1,9 @@
import epsilon_compositionality.build_dataset
-import epsilon_compositionality.extract_embeddings
+import epsilon_compositionality.extract_similarities
import epsilon_compositionality.compute_statistics
if __name__ == "__main__":
epsilon_compositionality.build_dataset.main()
- epsilon_compositionality.extract_embeddings.main()
+ epsilon_compositionality.extract_similarities.main()
epsilon_compositionality.compute_statistics.main()
diff --git a/epsilon_compositionality/compute_statistics.py b/epsilon_compositionality/compute_statistics.py
@@ -4,111 +4,122 @@ import itertools
import json
import pathlib
-import matplotlib.pyplot
import scipy.stats
INPUT_PATH = pathlib.Path(__file__).parent / "data" / "similarities"
-CLASSES: list[str] = ["C", "PC", "NC"]
+RESULTS_PATH = pathlib.Path() / "results"
+CLASSES_ORDER: list[str] = ["C", "PC", "NC"]
+MODELS_ORDER: list[str] = ["BERT", "ModernBERT", "DeBERTa", "S-BERT", "S-GPT", "E5", "UAE", "QWEN"]
+MODELS_HAS_SUB: set[str] = set(["BERT", "ModernBERT", "DeBERTa"])
+MODELS_SUB_ORDER: list[str] = ["cls", "clssep", "mean"]
-def get_epsilons(sentences: list[dict[str, Any]], method: str, parameter: float) -> tuple[float, float]:
+
+def get_epsilons(sentences: list[dict[str, Any]], method: str, offset: int = 0) -> tuple[float, float]:
epsilons: dict[str, list[float]] = {"positives": [], "negatives": []}
for sentence in sentences:
for polarity in ["positives", "negatives"]:
for alternative in sentence[polarity]:
if method == "multiplicative":
- epsilons[polarity].append((alternative["contextual_similarity"] - parameter) / (alternative["phrase_similarity"] - parameter))
+ epsilons[polarity].append((1 - alternative["contextual_similarity"] + offset) / (1 - alternative["phrase_similarity"] + offset) - 1)
elif method == "context":
- epsilons[polarity].append(alternative["contextual_similarity"])
+ epsilons[polarity].append(1 - alternative["contextual_similarity"])
+ elif method == "pairs":
+ epsilons[polarity].append((1 - alternative["contextual_similarity"], 1 - alternative["phrase_similarity"]))
assert len(epsilons["positives"]) == len(epsilons["negatives"])
return epsilons["positives"], epsilons["negatives"]
-def read_similarities(epsilon_method: str, epsilon_parameter: float) -> dict[str, Any]:
+def read_similarities(epsilon_method: str, offset: int = 0) -> dict[str, Any]:
models: dict[str, Any] = {}
for model_path in INPUT_PATH.iterdir():
models[model_path.stem] = {
subset: {
"head": {"positives": [], "negatives": []},
"modifier": {"positives": [], "negatives": []},
- } for subset in CLASSES
+ } for subset in CLASSES_ORDER
}
with model_path.open("r") as model_file:
for line in model_file:
compound = json.loads(line)
for side in ["head", "modifier"]:
- positives, negatives = get_epsilons(compound[side], epsilon_method, epsilon_parameter)
+ positives, negatives = get_epsilons(compound[side], epsilon_method, offset)
models[model_path.stem][compound["comp_class"]][side]["positives"].extend(positives)
models[model_path.stem][compound["comp_class"]][side]["negatives"].extend(negatives)
return models
-def plot_violins(models: dict[str, Any]) -> None:
- fig, axs = matplotlib.pyplot.subplots(nrows=len(models))
- for model_index, (model_name, model_data) in enumerate(models.items()):
- positives = []
- negatives = []
- labels = []
- for subset in CLASSES:
- labels.append(subset)
- positives.append([])
- negatives.append([])
-
- for isubset, subset in enumerate(CLASSES):
- for side in ["head", "modifier"]:
- labels.append(f"{subset}.{side[0]}")
- positives.append(model_data[subset][side]["positives"])
- negatives.append(model_data[subset][side]["negatives"])
- positives[isubset].extend(model_data[subset][side]["positives"])
- negatives[isubset].extend(model_data[subset][side]["negatives"])
- axs[model_index].violinplot(positives, side="low", showextrema=False, showmeans=True)
- axs[model_index].violinplot(negatives, side="high", showextrema=False, showmeans=True)
- axs[model_index].set_xticks(range(len(labels)+1), labels=[""]+labels)
- axs[model_index].set_ylabel(model_name)
- matplotlib.pyplot.show()
-
-
-def print_table_cell(positives: list[float], negatives: list[float], method: str) -> None:
+def table_cell(positives: list[float], negatives: list[float], method: str) -> str:
wilcoxon = scipy.stats.wilcoxon(positives, negatives, alternative="greater")
if method == "rank biserial":
n: int = len(positives)
- rank_biserial = wilcoxon.statistic / (n * (n + 1) // 2)
- print(f" {rank_biserial:<1.3f}", end="")
+ rank_biserial = 100 * wilcoxon.statistic / (n * (n + 1) // 2)
+ return f"{rank_biserial:<4.1f}"
elif method == "p-value":
- print(f" {wilcoxon.pvalue:<1.3f}", end="")
-
-
-def print_table(models: dict[str, Any], method: str) -> None:
- print(f"{'Model':>20}", end="")
- for subset in CLASSES:
- print(f" {subset:>5}", end="")
- for subset in CLASSES:
- for side in ["head", "modifier"]:
- print(f" {subset:>3}.{side[0]}", end="")
- print("")
-
- for model_name, model_data in models.items():
- print(f"{model_name:>20}", end ="")
- for subset in CLASSES:
- positives = list(itertools.chain(*(model_data[subset][side]["positives"] for side in ["head", "modifier"])))
- negatives = list(itertools.chain(*(model_data[subset][side]["negatives"] for side in ["head", "modifier"])))
- print_table_cell(positives, negatives, method)
-
- for subset in CLASSES:
- for side in ["head", "modifier"]:
- print_table_cell(model_data[subset][side]["positives"], model_data[subset][side]["negatives"], method)
- print("")
+ return f"{wilcoxon.pvalue:<1.3f}"
+
+
+def latex_main_statistics_table(output_path: pathlib.Path, models: dict[str, Any]) -> None:
+ with output_path.open("w") as output:
+ for model_name in MODELS_ORDER:
+ for isubmodel, submodel in enumerate(MODELS_SUB_ORDER if model_name in MODELS_HAS_SUB else [""]):
+ print(f"{model_name if isubmodel==0 else '':<10}", end=" & ", file=output)
+ print(f"{submodel:<6}", end=" & ", file=output)
+ model_key: str = f"{model_name}-{submodel}" if submodel else model_name
+ for imethod, method in enumerate(["p-value", "rank biserial"]):
+ for isubset, subset in enumerate(CLASSES_ORDER):
+ positives = list(itertools.chain(*(models[model_key][subset][side]["positives"] for side in ["head", "modifier"])))
+ negatives = list(itertools.chain(*(models[model_key][subset][side]["negatives"] for side in ["head", "modifier"])))
+ print(table_cell(positives, negatives, method), end=" & " if isubset<len(CLASSES_ORDER)-1 or imethod==0 else " \\\\\n", file=output)
+
+
+def get_mu_sigma(data: list[float]) -> tuple[float, float]:
+ mu = sum(data) / len(data)
+ sigma = (sum((x - mu)**2 for x in data) / (len(data) - 1))**0.5
+ return mu, sigma
+
+
+def latex_distances_statistics(output_path: pathlib.Path) -> None:
+ models = read_similarities("pairs")
+ with output_path.open("w") as output:
+ for model_name in MODELS_ORDER:
+ for isubmodel, submodel in enumerate(MODELS_SUB_ORDER if model_name in MODELS_HAS_SUB else [""]):
+ print(f"{model_name if isubmodel==0 else '':<10}", end=" & ", file=output)
+ print(f"{submodel:<6}", end=" & ", file=output)
+ model_key: str = f"{model_name}-{submodel}" if submodel else model_name
+ contextual: list[float] = []
+ phrasal: list[float] = []
+ for subset in CLASSES_ORDER:
+ for side in ["head", "modifier"]:
+ for polarity in ["positives", "negatives"]:
+ contextual.extend(x for x, _ in models[model_key][subset][side][polarity])
+ phrasal.extend(x for _, x in models[model_key][subset][side][polarity])
+ pmu, psigma = get_mu_sigma(phrasal)
+ cmu, csigma = get_mu_sigma(contextual)
+ print(f"{pmu:5.3f} & {psigma:5.3f} & {cmu:5.3f} & {csigma:5.3f} \\\\", file=output)
+
+
+def write_violins(output_path: pathlib.Path) -> None:
+ models = read_similarities("multiplicative", 0.05)
+ with output_path.open("w") as output:
+ for model, data in models.items():
+ for polarity in ["positives", "negatives"]:
+ print(f"{model}\x1e{polarity}", end="", file=output)
+ for subset in CLASSES_ORDER:
+ for side in ["head", "modifier"]:
+ for sample in models[model][subset][side][polarity]:
+ print(f"\t{sample}", end="", file=output)
+ print("", file=output)
def main() -> None:
- models = read_similarities("multiplicative", 1)
- print("P-VALUE")
- print_table(models, "p-value")
- print("\n\nRANK BISERIAL")
- print_table(models, "rank biserial")
- #plot_violins(models)
+ models = read_similarities("multiplicative")
+ RESULTS_PATH.mkdir(exist_ok=True)
+ latex_main_statistics_table(RESULTS_PATH / "main statistics.tex", models)
+ latex_distances_statistics(RESULTS_PATH / "distances statistics.tex")
+ write_violins(RESULTS_PATH / "violins.dat")
if __name__ == "__main__":
diff --git a/epsilon_compositionality/extract_embeddings.py b/epsilon_compositionality/extract_embeddings.py
@@ -1,136 +0,0 @@
-from typing import Any, NamedTuple, Iterator
-import copy
-import json
-import pathlib
-
-import torch
-import tqdm
-
-from epsilon_compositionality.models import ModelFactory, Model
-
-
-INPUT_PATH = pathlib.Path(__file__).parent / "data" / "ncimp.processed.jsonl"
-OUTPUT_PATH = pathlib.Path(__file__).parent / "data" / "similarities"
-
-
-def process_sample(input_sample: dict[str, Any]) -> dict[str, Any] | None:
- """Process a NCIMP sample into set of (a, b, ac, bc).
-
- A processed sample has the following shape:
- {
- "compound": "acid test",
- "comp_class": "NC",
- "head": [
- { # sentence 1
- "positives": [ # compare "acid test" to alternatives
- {
- "phrase_a": "acid test",
- "phrase_b": "acid trial",
- "context_ac": "sentence1 with acid test…",
- "context_bc": "sentence1 with acid trial…"
- },
- {
- "phrase_a": "acid test",
- "phrase_b": "acid analysis",
- "context_ac": "sentence1 with acid test…",
- "context_bc": "sentence1 with acid analysis…"
- },
- # …
- ],
- "negatives": [ # compare alternatives between themselves
- {
- "phrase_a": "acid analysis",
- "phrase_b": "acid trial",
- "context_ac": "sentence1 with acid analysis…",
- "context_bc": "sentence1 with acid trial…"
- },
- {
- "phrase_a": "acid tryout",
- "phrase_b": "acid analysis",
- "context_ac": "sentence1 with acid tryout…",
- "context_bc": "sentence1 with acid analysis…"
- },
- ]
- },
- # … more sentences
- ],
- "modifier": [
- # similarly set of sentences where acid is replaced instead of test (e.g. sour test)
- ]
- }
- """
- output_sample: dict[str, Any] = {
- "compound": input_sample["compound"],
- "comp_class": input_sample["general class"],
- }
- for side_index, side in enumerate(["modifier", "head"]):
- output_sample[side] = []
- for sentence in input_sample["sentences"]:
- if len(sentence[f"{side} synonym"]) <= 1:
- continue
-
- alternatives = list(sentence[f"{side} synonym"].items())
- second_alternatives = alternatives[1:] + [alternatives[0]]
- if len(alternatives) == 2:
- alternatives, second_alternatives = [alternatives[0]], [alternatives[1]]
-
- positives: list[Any] = []
- negatives: list[Any] = []
- for (alt_phrase, alt_sentence), (second_alt_phrase, second_alt_sentence) in zip(alternatives, second_alternatives):
- positives.append({
- "phrase_a": input_sample["compound"].split()[side_index],
- "phrase_b": alt_phrase.split()[side_index],
- "context_ac": sentence["original"],
- "context_bc": alt_sentence,
- })
- negatives.append({
- "phrase_a": second_alt_phrase.split()[side_index],
- "phrase_b": alt_phrase.split()[side_index],
- "context_ac": second_alt_sentence,
- "context_bc": alt_sentence,
- })
- output_sample[side].append({"positives": positives, "negatives": negatives})
- if not output_sample[side]:
- return None
- return output_sample
-
-
-def load_dataset() -> list[dict[str, Any]]:
- dataset: list[dict[str, Any]] = []
- with INPUT_PATH.open("r") as input_file:
- for line in input_file:
- sample: dict[str, Any] | None = process_sample(json.loads(line))
- if sample:
- dataset.append(sample)
- return dataset
-
-
-def main() -> None:
- dataset: list[dict[str, Any]] = load_dataset()
- OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
- for model_name in ModelFactory._factory:
- model_path: pathlib.Path = OUTPUT_PATH / f"{model_name}.jsonl"
- if "DeBERTa" in model_name: # FIXME random bug I don't want to investigate
- continue
-
- model: Model = ModelFactory.get(model_name)
- output: list[dict[str, Any]] = []
-
- with model_path.open("w") as output_file:
- for input_sample in tqdm.tqdm(dataset, f"Embedding dataset with {model_name}"):
- output_sample: dict[str, Any] = copy.deepcopy(input_sample)
- for side_index, side in enumerate(["modifier", "head"]):
- for sentence in output_sample[side]:
- for polarity in ["positives", "negatives"]:
- for subsample in sentence[polarity]:
- phrase_a: torch.Tensor = model.embed(subsample["phrase_a"])
- phrase_b: torch.Tensor = model.embed(subsample["phrase_b"])
- context_ac: torch.Tensor = model.embed(subsample["context_ac"])
- context_bc: torch.Tensor = model.embed(subsample["context_bc"])
- subsample["phrase_similarity"] = model.calculate_similarity(phrase_a, phrase_b)
- subsample["contextual_similarity"] = model.calculate_similarity(context_ac, context_bc)
- print(json.dumps(output_sample), file=output_file)
-
-
-if __name__ == "__main__":
- main()
diff --git a/epsilon_compositionality/extract_similarities.py b/epsilon_compositionality/extract_similarities.py
@@ -0,0 +1,134 @@
+from typing import Any, NamedTuple, Iterator
+import copy
+import json
+import pathlib
+
+import torch
+import tqdm
+
+from epsilon_compositionality.models import ModelFactory, Model
+
+
+INPUT_PATH = pathlib.Path(__file__).parent / "data" / "ncimp.processed.jsonl"
+OUTPUT_PATH = pathlib.Path(__file__).parent / "data" / "similarities"
+
+
+def process_sample(input_sample: dict[str, Any]) -> dict[str, Any] | None:
+ """Process a NCIMP sample into set of (a, b, ac, bc).
+
+ A processed sample has the following shape:
+ {
+ "compound": "acid test",
+ "comp_class": "NC",
+ "head": [
+ { # sentence 1
+ "positives": [ # compare "acid test" to alternatives
+ {
+ "phrase_a": "acid test",
+ "phrase_b": "acid trial",
+ "context_ac": "sentence1 with acid test…",
+ "context_bc": "sentence1 with acid trial…"
+ },
+ {
+ "phrase_a": "acid test",
+ "phrase_b": "acid analysis",
+ "context_ac": "sentence1 with acid test…",
+ "context_bc": "sentence1 with acid analysis…"
+ },
+ # …
+ ],
+ "negatives": [ # compare alternatives between themselves
+ {
+ "phrase_a": "acid analysis",
+ "phrase_b": "acid trial",
+ "context_ac": "sentence1 with acid analysis…",
+ "context_bc": "sentence1 with acid trial…"
+ },
+ {
+ "phrase_a": "acid tryout",
+ "phrase_b": "acid analysis",
+ "context_ac": "sentence1 with acid tryout…",
+ "context_bc": "sentence1 with acid analysis…"
+ },
+ ]
+ },
+ # … more sentences
+ ],
+ "modifier": [
+ # similarly set of sentences where acid is replaced instead of test (e.g. sour test)
+ ]
+ }
+ """
+ output_sample: dict[str, Any] = {
+ "compound": input_sample["compound"],
+ "comp_class": input_sample["general class"],
+ }
+ for side_index, side in enumerate(["modifier", "head"]):
+ output_sample[side] = []
+ for sentence in input_sample["sentences"]:
+ if len(sentence[f"{side} synonym"]) <= 1:
+ continue
+
+ alternatives = list(sentence[f"{side} synonym"].items())
+ second_alternatives = alternatives[1:] + [alternatives[0]]
+ if len(alternatives) == 2:
+ alternatives, second_alternatives = [alternatives[0]], [alternatives[1]]
+
+ positives: list[Any] = []
+ negatives: list[Any] = []
+ for (alt_phrase, alt_sentence), (second_alt_phrase, second_alt_sentence) in zip(alternatives, second_alternatives):
+ positives.append({
+ "phrase_a": input_sample["compound"].split()[side_index],
+ "phrase_b": alt_phrase.split()[side_index],
+ "context_ac": sentence["original"],
+ "context_bc": alt_sentence,
+ })
+ negatives.append({
+ "phrase_a": second_alt_phrase.split()[side_index],
+ "phrase_b": alt_phrase.split()[side_index],
+ "context_ac": second_alt_sentence,
+ "context_bc": alt_sentence,
+ })
+ output_sample[side].append({"positives": positives, "negatives": negatives})
+ if not output_sample[side]:
+ return None
+ return output_sample
+
+
+def load_dataset() -> list[dict[str, Any]]:
+ dataset: list[dict[str, Any]] = []
+ with INPUT_PATH.open("r") as input_file:
+ for line in input_file:
+ sample: dict[str, Any] | None = process_sample(json.loads(line))
+ if sample:
+ dataset.append(sample)
+ return dataset
+
+
+def main() -> None:
+ dataset: list[dict[str, Any]] = load_dataset()
+ OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
+ for model_name in ModelFactory._factory:
+ model_path: pathlib.Path = OUTPUT_PATH / f"{model_name}.jsonl"
+
+ model: Model = ModelFactory.get(model_name)
+ output: list[dict[str, Any]] = []
+
+ with model_path.open("w") as output_file:
+ for input_sample in tqdm.tqdm(dataset, f"Embedding dataset with {model_name}"):
+ output_sample: dict[str, Any] = copy.deepcopy(input_sample)
+ for side_index, side in enumerate(["modifier", "head"]):
+ for sentence in output_sample[side]:
+ for polarity in ["positives", "negatives"]:
+ for subsample in sentence[polarity]:
+ phrase_a: torch.Tensor = model.embed(subsample["phrase_a"])
+ phrase_b: torch.Tensor = model.embed(subsample["phrase_b"])
+ context_ac: torch.Tensor = model.embed(subsample["context_ac"])
+ context_bc: torch.Tensor = model.embed(subsample["context_bc"])
+ subsample["phrase_similarity"] = model.calculate_similarity(phrase_a, phrase_b)
+ subsample["contextual_similarity"] = model.calculate_similarity(context_ac, context_bc)
+ print(json.dumps(output_sample), file=output_file)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/shell.nix b/shell.nix
@@ -4,7 +4,6 @@ pkgs.mkShellNoCC {
(python3.withPackages (ps: with ps; [
datasets
ipdb
- matplotlib
openpyxl
pandas
pytorch
