commit f78433a84a8bf806b66fdb5a9d1122b8ef28bb82
parent 5c05b9ac51c42c9f654566d7d01778784d12959b
Author: Étienne Simon <esimon@esimon.eu>
Date: Mon, 29 Sep 2025 06:13:28 +0200
rewrite everything
Diffstat:
6 files changed, 401 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
@@ -1 +1,13 @@
-# WIP: Compositionality: Definitions and its Relationship with Distributional Semantics
+# Evaluating Idiomaticity in Sentence Embeddings with 𝜀-compositionality
+
+Reproduce everything:
+```sh
+python -m epsilon_compositionality
+```
+
+This is equivalent to:
+```sh
+python -m epsilon_compositionality.build_dataset
+python -m epsilon_compositionality.extract_embeddings
+python -m epsilon_compositionality.compute_statistics
+```
diff --git a/epsilon_compositionality/__main__.py b/epsilon_compositionality/__main__.py
@@ -0,0 +1,9 @@
+import epsilon_compositionality.build_dataset
+import epsilon_compositionality.extract_embeddings
+import epsilon_compositionality.compute_statistics
+
+
+if __name__ == "__main__":
+ epsilon_compositionality.build_dataset.main()
+ epsilon_compositionality.extract_embeddings.main()
+ epsilon_compositionality.compute_statistics.main()
diff --git a/epsilon_compositionality/build_dataset.py b/epsilon_compositionality/build_dataset.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+from typing import Any
+import json
+import pathlib
+
+import pandas as pd
+
+
+INPUT_PATH = pathlib.Path(__file__).parent / "data" / "Finding-Idiomaticity-in-Word-Representations" / "dataset"
+OUTPUT_PATH = pathlib.Path(__file__).parent / "data" / "ncimp.processed.jsonl"
+
+
+def sample_filter(frame: pd.DataFrame) -> pd.DataFrame:
+ return frame[(frame["language"] == "en") & (frame["experiment_type"] == "Naturalistic")]
+
+
+def extract_tagged(sentence: str, tag: str) -> str:
+ return " ".join(word for word, annotation in zip(sentence.split(), eval(tag)) if annotation)
+
+
+def fix_data(compound: str, keep_modifier: True, keep_head: True, alternatives: dict[str, str]) -> dict[str, str]:
+ fixed: dict[str, str] = {}
+
+ process = lambda x: " ".join(x.lower().split()[1 - keep_modifier:1 + keep_head])
+
+ uniques: set[str] = { process(compound), f"{process(compound)}s" }
+ for key, value in alternatives.items():
+ if process(key) not in uniques and f"{process(key)}s" not in uniques:
+ uniques.add(process(key))
+ uniques.add(f"{process(key)}s")
+ fixed[key] = value
+
+ return fixed
+
+
+def resolve_sentence(compound: str, input_sentence: pd.Series, support_annotations: dict[str, pd.DataFrame]) -> tuple[dict[str, Any], dict[str, Any]]:
+ filename: str = pathlib.Path(input_sentence["file"]).name
+ file_id: int = int(filename[len("naturalistics_examplesent"):-len(".csv")])
+ support_annotation: pd.Series = support_annotations[filename].loc[compound]
+
+ common: dict[str, Any] = {
+ "compound": compound,
+ "general class": input_sentence["ClassType"],
+ "general human compositionality": input_sentence["CompositionalityType"],
+ }
+
+ sentence: dict[str, Any] = {
+ "compound": extract_tagged(support_annotation["original sentence"], support_annotation["original sentence_tag"]),
+ "modifier": support_annotation[f"modifier_{file_id}"], # acid
+ "head": support_annotation[f"head_{file_id}"], # test
+ "contextual class": input_sentence["ClassToken"],
+ "contextual human compositionality": input_sentence["CompositionalityToken"],
+ "original": support_annotation["original sentence"], # acid test
+ "modifier synonym": fix_data(compound, True, False, { # sour test
+ extract_tagged(support_annotation[key], support_annotation[f"{key}_tag"]): support_annotation[key]
+ for key in [
+ f"synonym modifier_{file_id}",
+ f"modifier synonym alt1_{file_id}",
+ f"modifier synonym alt2_{file_id}",
+ f"modifier synonym alt3_{file_id}",
+ f"modifier synonym alt4_{file_id}",
+ ]
+ }),
+ "head synonym": fix_data(compound, False, True, { # acid analysis
+ extract_tagged(support_annotation[key], support_annotation[f"{key}_tag"]): support_annotation[key]
+ for key in [
+ f"synonym head_{file_id}",
+ f"head synonym alt1_{file_id}",
+ f"head synonym alt2_{file_id}",
+ f"head synonym alt3_{file_id}",
+ f"head synonym alt4_{file_id}",
+ ]
+ }),
+ "both synonym": fix_data(compound, True, True, { # sour analysis
+ extract_tagged(support_annotation[key], support_annotation[f"{key}_tag"]): support_annotation[key]
+ for key in [
+ f"synonym both",
+ f"both synonym alt1_{file_id}",
+ f"both synonym alt2_{file_id}",
+ f"both synonym alt3_{file_id}",
+ f"both synonym alt4_{file_id}",
+ ]
+ }),
+ }
+
+ return common, sentence
+
+
+def main() -> None:
+ input_annotations: pd.DataFrame = pd.read_excel(INPUT_PATH / "human_compositionality scores.xlsx")
+ input_annotations = sample_filter(input_annotations)
+ support_annotations: dict[str, pd.DataFrame] = {
+ path.name: pd.read_csv(path).set_index("compound")
+ for path in (INPUT_PATH / "EN").iterdir()
+ if path.name.startswith("naturalistics_examplesent")
+ }
+
+ with OUTPUT_PATH.open("w") as output_file:
+ for compound, input_sample in input_annotations.groupby("compound"):
+ sentences: list[dict[str, Any]] = []
+ sample: dict[str, Any] = {"sentences": sentences}
+ for input_sentence in input_sample.iloc:
+ sample_data, sentence = resolve_sentence(compound, input_sentence, support_annotations)
+ sentences.append(sentence)
+ sample.update(sample_data)
+ print(json.dumps(sample), file=output_file)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/epsilon_compositionality/compute_statistics.py b/epsilon_compositionality/compute_statistics.py
@@ -0,0 +1,115 @@
+from typing import Any
+import collections
+import itertools
+import json
+import pathlib
+
+import matplotlib.pyplot
+import scipy.stats
+
+
+INPUT_PATH = pathlib.Path(__file__).parent / "data" / "similarities"
+CLASSES: list[str] = ["C", "PC", "NC"]
+
+
+def get_epsilons(sentences: list[dict[str, Any]], method: str, parameter: float) -> tuple[float, float]:
+ epsilons: dict[str, list[float]] = {"positives": [], "negatives": []}
+ for sentence in sentences:
+ for polarity in ["positives", "negatives"]:
+ for alternative in sentence[polarity]:
+ if method == "multiplicative":
+ epsilons[polarity].append((alternative["contextual_similarity"] - parameter) / (alternative["phrase_similarity"] - parameter))
+ elif method == "context":
+ epsilons[polarity].append(alternative["contextual_similarity"])
+ assert len(epsilons["positives"]) == len(epsilons["negatives"])
+ return epsilons["positives"], epsilons["negatives"]
+
+
+def read_similarities(epsilon_method: str, epsilon_parameter: float) -> dict[str, Any]:
+ models: dict[str, Any] = {}
+ for model_path in INPUT_PATH.iterdir():
+ models[model_path.stem] = {
+ subset: {
+ "head": {"positives": [], "negatives": []},
+ "modifier": {"positives": [], "negatives": []},
+ } for subset in CLASSES
+ }
+
+ with model_path.open("r") as model_file:
+ for line in model_file:
+ compound = json.loads(line)
+ for side in ["head", "modifier"]:
+ positives, negatives = get_epsilons(compound[side], epsilon_method, epsilon_parameter)
+ models[model_path.stem][compound["comp_class"]][side]["positives"].extend(positives)
+ models[model_path.stem][compound["comp_class"]][side]["negatives"].extend(negatives)
+ return models
+
+
+def plot_violins(models: dict[str, Any]) -> None:
+ fig, axs = matplotlib.pyplot.subplots(nrows=len(models))
+ for model_index, (model_name, model_data) in enumerate(models.items()):
+ positives = []
+ negatives = []
+ labels = []
+ for subset in CLASSES:
+ labels.append(subset)
+ positives.append([])
+ negatives.append([])
+
+ for isubset, subset in enumerate(CLASSES):
+ for side in ["head", "modifier"]:
+ labels.append(f"{subset}.{side[0]}")
+ positives.append(model_data[subset][side]["positives"])
+ negatives.append(model_data[subset][side]["negatives"])
+ positives[isubset].extend(model_data[subset][side]["positives"])
+ negatives[isubset].extend(model_data[subset][side]["negatives"])
+ axs[model_index].violinplot(positives, side="low", showextrema=False, showmeans=True)
+ axs[model_index].violinplot(negatives, side="high", showextrema=False, showmeans=True)
+ axs[model_index].set_xticks(range(len(labels)+1), labels=[""]+labels)
+ axs[model_index].set_ylabel(model_name)
+ matplotlib.pyplot.show()
+
+
+def print_table_cell(positives: list[float], negatives: list[float], method: str) -> None:
+ wilcoxon = scipy.stats.wilcoxon(positives, negatives, alternative="greater")
+ if method == "rank biserial":
+ n: int = len(positives)
+ rank_biserial = wilcoxon.statistic / (n * (n + 1) // 2)
+ print(f" {rank_biserial:<1.3f}", end="")
+ elif method == "p-value":
+ print(f" {wilcoxon.pvalue:<1.3f}", end="")
+
+
+def print_table(models: dict[str, Any], method: str) -> None:
+ print(f"{'Model':>20}", end="")
+ for subset in CLASSES:
+ print(f" {subset:>5}", end="")
+ for subset in CLASSES:
+ for side in ["head", "modifier"]:
+ print(f" {subset:>3}.{side[0]}", end="")
+ print("")
+
+ for model_name, model_data in models.items():
+ print(f"{model_name:>20}", end ="")
+ for subset in CLASSES:
+ positives = list(itertools.chain(*(model_data[subset][side]["positives"] for side in ["head", "modifier"])))
+ negatives = list(itertools.chain(*(model_data[subset][side]["negatives"] for side in ["head", "modifier"])))
+ print_table_cell(positives, negatives, method)
+
+ for subset in CLASSES:
+ for side in ["head", "modifier"]:
+ print_table_cell(model_data[subset][side]["positives"], model_data[subset][side]["negatives"], method)
+ print("")
+
+
+def main() -> None:
+ models = read_similarities("multiplicative", 1)
+ print("P-VALUE")
+ print_table(models, "p-value")
+ print("\n\nRANK BISERIAL")
+ print_table(models, "rank biserial")
+ #plot_violins(models)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/epsilon_compositionality/extract_embeddings.py b/epsilon_compositionality/extract_embeddings.py
@@ -0,0 +1,136 @@
+from typing import Any, NamedTuple, Iterator
+import copy
+import json
+import pathlib
+
+import torch
+import tqdm
+
+from epsilon_compositionality.models import ModelFactory, Model
+
+
+INPUT_PATH = pathlib.Path(__file__).parent / "data" / "ncimp.processed.jsonl"
+OUTPUT_PATH = pathlib.Path(__file__).parent / "data" / "similarities"
+
+
+def process_sample(input_sample: dict[str, Any]) -> dict[str, Any] | None:
+ """Process a NCIMP sample into set of (a, b, ac, bc).
+
+ A processed sample has the following shape:
+ {
+ "compound": "acid test",
+ "comp_class": "NC",
+ "head": [
+ { # sentence 1
+ "positives": [ # compare "acid test" to alternatives
+ {
+ "phrase_a": "acid test",
+ "phrase_b": "acid trial",
+ "context_ac": "sentence1 with acid test…",
+ "context_bc": "sentence1 with acid trial…"
+ },
+ {
+ "phrase_a": "acid test",
+ "phrase_b": "acid analysis",
+ "context_ac": "sentence1 with acid test…",
+ "context_bc": "sentence1 with acid analysis…"
+ },
+ # …
+ ],
+ "negatives": [ # compare alternatives between themselves
+ {
+ "phrase_a": "acid analysis",
+ "phrase_b": "acid trial",
+ "context_ac": "sentence1 with acid analysis…",
+ "context_bc": "sentence1 with acid trial…"
+ },
+ {
+ "phrase_a": "acid tryout",
+ "phrase_b": "acid analysis",
+ "context_ac": "sentence1 with acid tryout…",
+ "context_bc": "sentence1 with acid analysis…"
+ },
+ ]
+ },
+ # … more sentences
+ ],
+ "modifier": [
+ # similarly set of sentences where acid is replaced instead of test (e.g. sour test)
+ ]
+ }
+ """
+ output_sample: dict[str, Any] = {
+ "compound": input_sample["compound"],
+ "comp_class": input_sample["general class"],
+ }
+ for side_index, side in enumerate(["modifier", "head"]):
+ output_sample[side] = []
+ for sentence in input_sample["sentences"]:
+ if len(sentence[f"{side} synonym"]) <= 1:
+ continue
+
+ alternatives = list(sentence[f"{side} synonym"].items())
+ second_alternatives = alternatives[1:] + [alternatives[0]]
+ if len(alternatives) == 2:
+ alternatives, second_alternatives = [alternatives[0]], [alternatives[1]]
+
+ positives: list[Any] = []
+ negatives: list[Any] = []
+ for (alt_phrase, alt_sentence), (second_alt_phrase, second_alt_sentence) in zip(alternatives, second_alternatives):
+ positives.append({
+ "phrase_a": input_sample["compound"].split()[side_index],
+ "phrase_b": alt_phrase.split()[side_index],
+ "context_ac": sentence["original"],
+ "context_bc": alt_sentence,
+ })
+ negatives.append({
+ "phrase_a": second_alt_phrase.split()[side_index],
+ "phrase_b": alt_phrase.split()[side_index],
+ "context_ac": second_alt_sentence,
+ "context_bc": alt_sentence,
+ })
+ output_sample[side].append({"positives": positives, "negatives": negatives})
+ if not output_sample[side]:
+ return None
+ return output_sample
+
+
+def load_dataset() -> list[dict[str, Any]]:
+ dataset: list[dict[str, Any]] = []
+ with INPUT_PATH.open("r") as input_file:
+ for line in input_file:
+ sample: dict[str, Any] | None = process_sample(json.loads(line))
+ if sample:
+ dataset.append(sample)
+ return dataset
+
+
+def main() -> None:
+ dataset: list[dict[str, Any]] = load_dataset()
+ OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
+ for model_name in ModelFactory._factory:
+ model_path: pathlib.Path = OUTPUT_PATH / f"{model_name}.jsonl"
+ if "DeBERTa" in model_name: # FIXME random bug I don't want to investigate
+ continue
+
+ model: Model = ModelFactory.get(model_name)
+ output: list[dict[str, Any]] = []
+
+ with model_path.open("w") as output_file:
+ for input_sample in tqdm.tqdm(dataset, f"Embedding dataset with {model_name}"):
+ output_sample: dict[str, Any] = copy.deepcopy(input_sample)
+ for side_index, side in enumerate(["modifier", "head"]):
+ for sentence in output_sample[side]:
+ for polarity in ["positives", "negatives"]:
+ for subsample in sentence[polarity]:
+ phrase_a: torch.Tensor = model.embed(subsample["phrase_a"])
+ phrase_b: torch.Tensor = model.embed(subsample["phrase_b"])
+ context_ac: torch.Tensor = model.embed(subsample["context_ac"])
+ context_bc: torch.Tensor = model.embed(subsample["context_bc"])
+ subsample["phrase_similarity"] = model.calculate_similarity(phrase_a, phrase_b)
+ subsample["contextual_similarity"] = model.calculate_similarity(context_ac, context_bc)
+ print(json.dumps(output_sample), file=output_file)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/shell.nix b/shell.nix
@@ -0,0 +1,17 @@
+{ pkgs ? import <nixpkgs> {} }:
+pkgs.mkShellNoCC {
+ nativeBuildInputs = with pkgs.buildPackages; [
+ (python3.withPackages (ps: with ps; [
+ datasets
+ ipdb
+ matplotlib
+ openpyxl
+ pandas
+ pytorch
+ sentence-transformers
+ tiktoken
+ tqdm
+ transformers
+ ]))
+ ];
+}
