compute_statistics.py (8392B)
1 from typing import Any, Callable, IO 2 import collections 3 import itertools 4 import json 5 import pathlib 6 7 import scipy.stats 8 9 10 INPUT_PATH = pathlib.Path(__file__).parent / "data" / "similarities" 11 RESULTS_PATH = pathlib.Path() / "results" 12 13 CLASSES_ORDER: list[str] = ["C", "PC", "NC"] 14 MODELS_ORDER: list[str] = ["BERT", "ModernBERT", "DeBERTa", "S-BERT", "S-GPT", "E5", "UAE", "QWEN"] 15 MODELS_HAS_SUB: set[str] = set(["BERT", "ModernBERT", "DeBERTa"]) 16 MODELS_SUB_ORDER: list[str] = ["cls", "clssep", "mean"] 17 18 MODEL_DISPLAY_NAME: dict[str, str] = { 19 "BERT": r"\textsc{bert}", 20 "ModernBERT": r"Modern\textsc{bert}", 21 "DeBERTa": r"De\textsc{bert}a", 22 "S-BERT": r"\textsc{s-bert}", 23 "S-GPT": r"\textsc{s-gpt}", 24 "E5": r"\textsc{e5}", 25 "UAE": r"\textsc{uae}", 26 "QWEN": r"\textsc{qwen}", 27 } 28 29 SUBMODEL_DISPLAY_NAME: dict[str, str] = { 30 "cls": r"\textsc{cls}", 31 "clssep": r"\(\textsc{cls}+\textsc{sep}\)", 32 "mean": r"mean", 33 "": "", # no submodels for sentence transformers 34 } 35 36 37 def get_epsilons(sentences: list[dict[str, Any]], method: str, offset: int = 0) -> tuple[float, float]: 38 epsilons: dict[str, list[float]] = {"positives": [], "negatives": []} 39 for sentence in sentences: 40 for polarity in ["positives", "negatives"]: 41 for alternative in sentence[polarity]: 42 if method == "multiplicative": 43 epsilons[polarity].append((1 - alternative["contextual_similarity"] + offset) / (1 - alternative["phrase_similarity"] + offset) - 1) 44 elif method == "context": 45 epsilons[polarity].append(1 - alternative["contextual_similarity"]) 46 elif method == "pairs": 47 epsilons[polarity].append((1 - alternative["contextual_similarity"], 1 - alternative["phrase_similarity"])) 48 assert len(epsilons["positives"]) == len(epsilons["negatives"]) 49 return epsilons["positives"], epsilons["negatives"] 50 51 52 def read_similarities(epsilon_method: str, offset: int = 0) -> dict[str, Any]: 53 models: dict[str, Any] = {} 54 for model_path in INPUT_PATH.iterdir(): 55 models[model_path.stem] = { 56 subset: { 57 "head": {"positives": [], "negatives": []}, 58 "modifier": {"positives": [], "negatives": []}, 59 } for subset in CLASSES_ORDER 60 } 61 62 with model_path.open("r") as model_file: 63 for line in model_file: 64 compound = json.loads(line) 65 for side in ["head", "modifier"]: 66 positives, negatives = get_epsilons(compound[side], epsilon_method, offset) 67 models[model_path.stem][compound["comp_class"]][side]["positives"].extend(positives) 68 models[model_path.stem][compound["comp_class"]][side]["negatives"].extend(negatives) 69 return models 70 71 72 def latex_data_statistics(output_path: pathlib.Path) -> None: 73 num_compounds: int = 0 74 num_sentences: int = 0 75 num_words: int = 0 76 num_epsilons: int = 0 77 78 model_path: pathlib.Path = next(INPUT_PATH.iterdir()) 79 with model_path.open("r") as model_file: 80 for line in model_file: 81 compound = json.loads(line) 82 num_compounds += 1 83 for side in ["head", "modifier"]: 84 for sentence in compound[side]: 85 num_sentences += 1 86 num_words += len(sentence["positives"][0]["context_ac"].split()) 87 for polarity in ["positives", "negatives"]: 88 num_epsilons += len(sentence[polarity]) 89 90 num_sentences //= 2 91 num_words //= 2 92 with output_path.open("w") as output: 93 print(r"\NewDocumentCommand\dataNumCompounds{}{" + str(num_compounds) + "}", file=output) 94 print(r"\NewDocumentCommand\dataNumSentences{}{" + str(num_sentences) + "}", file=output) 95 print(r"\NewDocumentCommand\dataNumEpsilons{}{" + str(num_epsilons) + "}", file=output) 96 print(r"\NewDocumentCommand\dataAvgSentenceLength{}{" + f"{num_words / num_sentences:4.1f}" + "}", file=output) 97 print(r"\NewDocumentCommand\dataAvgAlternatives{}{" + f"{num_epsilons / num_sentences / 2:3.1f}" + "}", file=output) 98 99 100 def table_cell(positives: list[float], negatives: list[float], method: str) -> str: 101 wilcoxon = scipy.stats.wilcoxon(positives, negatives, alternative="greater") 102 if method == "rank biserial": 103 n: int = len(positives) 104 rank_biserial = 100 * wilcoxon.statistic / (n * (n + 1) // 2) 105 return f"{rank_biserial:<4.1f}" 106 elif method == "p-value": 107 return f"{wilcoxon.pvalue:<1.3f}" 108 109 110 def latex_models_table(output_path: pathlib.Path, models: dict[str, Any], model_line: Callable[[pathlib.Path, dict[str, Any], str], None]) -> None: 111 with output_path.open("w") as output: 112 for model_name in MODELS_ORDER: 113 for isubmodel, submodel in enumerate(MODELS_SUB_ORDER if model_name in MODELS_HAS_SUB else [""]): 114 print(f"{MODEL_DISPLAY_NAME[model_name] if isubmodel==0 else '':<20}", end=" & ", file=output) 115 print(f"{SUBMODEL_DISPLAY_NAME[submodel]:<30}", end=" & ", file=output) 116 model_key: str = f"{model_name}-{submodel}" if submodel else model_name 117 model_line(output, models, model_key) 118 119 120 def main_statistics(output: IO[str], models: dict[str, Any], model_key: str) -> None: 121 for imethod, method in enumerate(["p-value", "rank biserial"]): 122 for isubset, subset in enumerate(CLASSES_ORDER): 123 positives = list(itertools.chain(*(models[model_key][subset][side]["positives"] for side in ["head", "modifier"]))) 124 negatives = list(itertools.chain(*(models[model_key][subset][side]["negatives"] for side in ["head", "modifier"]))) 125 print(table_cell(positives, negatives, method), end=" & " if isubset<len(CLASSES_ORDER)-1 or imethod==0 else " \\\\\n", file=output) 126 127 128 def get_mu_sigma(data: list[float]) -> tuple[float, float]: 129 mu = sum(data) / len(data) 130 sigma = (sum((x - mu)**2 for x in data) / (len(data) - 1))**0.5 131 return mu, sigma 132 133 134 def distances_statistics(output: IO[str], models: dict[str, Any], model_key: str) -> None: 135 contextual: list[float] = [] 136 phrasal: list[float] = [] 137 for subset in CLASSES_ORDER: 138 for side in ["head", "modifier"]: 139 for polarity in ["positives", "negatives"]: 140 contextual.extend(x for x, _ in models[model_key][subset][side][polarity]) 141 phrasal.extend(x for _, x in models[model_key][subset][side][polarity]) 142 pmu, psigma = get_mu_sigma(phrasal) 143 cmu, csigma = get_mu_sigma(contextual) 144 print(f"{pmu:5.3f} & {psigma:5.3f} & {cmu:5.3f} & {csigma:5.3f} \\\\", file=output) 145 146 147 def write_violins(output_path: pathlib.Path) -> None: 148 models = read_similarities("multiplicative", 0.05) 149 with output_path.open("w") as output: 150 for model, data in models.items(): 151 for polarity in ["positives", "negatives"]: 152 print(f"{model}\x1e{polarity}", end="", file=output) 153 for subset in CLASSES_ORDER: 154 for side in ["head", "modifier"]: 155 for sample in models[model][subset][side][polarity]: 156 print(f"\t{sample}", end="", file=output) 157 print("", file=output) 158 159 160 def extended_statistics(output: IO[str], models: dict[str, Any], model_key: str) -> None: 161 for isubset, subset in enumerate(CLASSES_ORDER): 162 for iside, side in enumerate(["modifier", "head"]): 163 positives = models[model_key][subset][side]["positives"] 164 negatives = models[model_key][subset][side]["negatives"] 165 print(table_cell(positives, negatives, "rank biserial"), end=" & " if isubset<len(CLASSES_ORDER)-1 or iside==0 else " \\\\\n", file=output) 166 167 168 def main() -> None: 169 RESULTS_PATH.mkdir(exist_ok=True) 170 models = read_similarities("multiplicative") 171 latex_data_statistics(RESULTS_PATH / "data statistics.tex") 172 latex_models_table(RESULTS_PATH / "main statistics.tex", models, main_statistics) 173 latex_models_table(RESULTS_PATH / "extended statistics.tex", models, extended_statistics) 174 latex_models_table(RESULTS_PATH / "distances statistics.tex", read_similarities("pairs"), distances_statistics) 175 write_violins(RESULTS_PATH / "violins.dat") 176 177 178 if __name__ == "__main__": 179 main()