epsilon-compositionality

commit 3621fa57a5de72257f42a3778b88d3976bcbb88f
parent 3eb2e5902a6113cac154f634499a6b055ef96dc0
Author: Sondre Wold <sondrewold1@gmail.com>
Date:   Thu, 11 Sep 2025 11:39:17 +0200

Add script for end-to-end reproduction

Diffstat:
A
reproduce.sh
 | 
16
++++++++++++++++
M
statistical_testing.py
 | 
9
+++++----

2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/reproduce.sh b/reproduce.sh
@@ -0,0 +1,16 @@
+results_folder=./reproduction_model_results
+epsilon_file=./distributions.json
+final_result_file=./result.txt
+
+
+echo "Running all the embedding models on NCIMP"
+for model in "BERT-cls" "BERT-mean" "S-BERT" "S-GPT" "E5" "ModernBERT-cls" "DeBERTa-cls" "QWEN"
+do
+		python experiment.py -data_path ncimp/ncip_clean.csv -experiment_type ncip -normalization_value 0.0 -result_path $results_folder -model $model
+		python experiment.py -data_path ncimp/ncip_clean.csv -experiment_type ncip-non -normalization_value 0.0 -result_path $results_folder -model $model
+done
+
+echo "Calculating epsilon values..."
+python get_epsilon.py -points_path $results_folder -result_path $epsilon_file
+echo "Running statistical testing..."
+python statistical_testing.py -epsilons_path $epsilon_file > $final_result_file
diff --git a/statistical_testing.py b/statistical_testing.py
@@ -59,6 +59,7 @@ def main() -> None:
     for i in range(0, len(loaded_dict["dataset"])):
         dataset_name = loaded_dict["dataset"][i]
         model_name = loaded_dict["model"][i]
+        print(f"{model_name}, {dataset_name}, {len(loaded_dict["original"][i])}")
         is_idiomatic = True if dataset_name == "ncip" else False
         if model_name not in model_results:
             model_results[model_name] = ResultPoint(model_name=model_name)
@@ -72,14 +73,14 @@ def main() -> None:
         non_idiom_variance = numpy.var(res.non_idiomatic_result)
         idiom_is_normal = stats.normaltest(res.idiomatic_result).pvalue > 0.05
         non_idiom_is_normal = stats.normaltest(res.non_idiomatic_result).pvalue > 0.05
-        mannwhitney = stats.mannwhitneyu(res.idiomatic_result, res.non_idiomatic_result) # Independence, not normally distributed, unequal sample size
-        U1 = mannwhitney.statistic
+        coxon = stats.wilcoxon(res.idiomatic_result, res.non_idiomatic_result) # Dependence, not normally distributed, unequal sample size
+        U1 = coxon.statistic
         U2 = len(res.idiomatic_result) * len(res.non_idiomatic_result) - U1  # as per the documentaiton notes on the scipy function
         cl_effect_size = U1 / (len(res.idiomatic_result) * len(res.non_idiomatic_result))  # as per Wikipedia on Mann Whitney U
         r_effect_size = ((2 * U1) / (len(res.idiomatic_result) * len(res.non_idiomatic_result))) - 1  # as per Wikipedia on Mann Whitney U
-        is_signficant: bool = mannwhitney.pvalue < 0.05
+        is_signficant: bool = coxon.pvalue < 0.05
         hodges_lehman = hodges_lehmann_2sample(res.idiomatic_result, res.non_idiomatic_result)  # To gauge the practical difference between the two groups, as suggested by the Wikipedia page on Mann Whitney U
-        print(f"######### \n Model name: {model} \n Idiom Variance (n={len(res.idiomatic_result)}): {idiom_variance}, Idiom is normal: {idiom_is_normal}, Non-Idiom Variance (n={len(res.non_idiomatic_result)}): {non_idiom_variance}, Non-idiom is normal: {non_idiom_is_normal} \n Hodges-Lehman Estimator: {hodges_lehman} \n Two-sided Mann-whitney U: p_value={mannwhitney.pvalue}, is_significant: {is_signficant} \n Effect size (Common Language F-test): {cl_effect_size} \n Effect size (Rank-biserial correlation): {r_effect_size}")
+        print(f"######### \n Model name: {model} \n Idiom Variance (n={len(res.idiomatic_result)}): {idiom_variance}, Idiom is normally distributed: {idiom_is_normal}, Non-Idiom Variance (n={len(res.non_idiomatic_result)}): {non_idiom_variance}, Non-idiom is normal distributed: {non_idiom_is_normal} \n Hodges-Lehman Estimator: {hodges_lehman} \n Two-sided Wilcoxon signed-rank test: p_value={coxon.pvalue}, is_significant: {is_signficant} \n Effect size (Common Language F-test): {cl_effect_size} \n Effect size (Rank-biserial correlation): {r_effect_size}")
 
 
 if __name__ == "__main__":