commit 774cb9989b048edf55ea4ec5973d240f6472b21c
parent b36336c342151068d1824430685ac012b0d95709
Author: Sondre Wold <sondrewold1@gmail.com>
Date: Wed, 3 Sep 2025 12:34:04 +0200
rename test file
Diffstat:
| A | statistical_testing.py | | | 86 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| D | t_test.py | | | 86 | ------------------------------------------------------------------------------- |
2 files changed, 86 insertions(+), 86 deletions(-)
diff --git a/statistical_testing.py b/statistical_testing.py
@@ -0,0 +1,86 @@
+import sys
+import pickle
+import numpy
+from torch import equal, histogram
+from experiment import Point
+import matplotlib.pyplot as plt
+
+import scipy.stats as stats
+
+from typing import Any
+import json
+
+import pandas
+
+import argparse
+import pathlib
+
+import warnings
+
+from typing import NamedTuple
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Calculate the Welsch T-Test")
+ parser.add_argument("-epsilons_path", type=pathlib.Path)
+ parser.add_argument("-result_path", type=pathlib.Path)
+ return parser.parse_args()
+
+
+class ResultPoint():
+ def __init__(self, model_name: str = "", idiomatic_result: list[float] = [], non_idiomatic_result: list[float] = []):
+ self.model_name: str = model_name
+ self.idiomatic_result: list[float] = idiomatic_result
+ self.non_idiomatic_result: list[float] = non_idiomatic_result
+
+
+def hodges_lehmann_2sample(x: list[float], y: list[float]) -> float:
+ """
+ https://en.wikipedia.org/wiki/Hodges%E2%80%93Lehmann_estimator
+ The median of all the pairwise differences estimates the true effect on the outcomes.
+ I calculated it as the median difference between pairs, as specified in:
+ https://link.springer.com/content/pdf/10.1007/978-3-662-69359-9.pdf at page 1145.
+ """
+ differences: list[float] = []
+ for x_i in x:
+ for y_i in y:
+ differences.append(y_i - x_i)
+ return float(numpy.median(differences))
+
+
+def main() -> None:
+ args = parse_args()
+ with open(args.epsilons_path, "r") as f:
+ loaded_dict = json.load(f)
+
+ model_results: dict[str, ResultPoint] = {}
+ for i in range(0, len(loaded_dict["dataset"])):
+ dataset_name = loaded_dict["dataset"][i]
+ model_name = loaded_dict["model"][i]
+ is_idiomatic = True if dataset_name == "ncip" else False
+ if model_name not in model_results:
+ model_results[model_name] = ResultPoint(model_name=model_name)
+ if is_idiomatic:
+ model_results[model_name].idiomatic_result = loaded_dict["original"][i]
+ else:
+ model_results[model_name].non_idiomatic_result = loaded_dict["original"][i]
+
+ for model, res in model_results.items():
+ idiom_variance = numpy.var(res.idiomatic_result)
+ non_idiom_variance = numpy.var(res.non_idiomatic_result)
+ idiom_is_normal = stats.normaltest(res.idiomatic_result).pvalue > 0.05
+ non_idiom_is_normal = stats.normaltest(res.non_idiomatic_result).pvalue > 0.05
+ mannwhitney = stats.mannwhitneyu(res.idiomatic_result, res.non_idiomatic_result) # Independence, not normally distributed, unequal sample size
+ U1 = mannwhitney.statistic
+ U2 = len(res.idiomatic_result) * len(res.non_idiomatic_result) - U1 # as per the documentaiton notes on the scipy function
+ cl_effect_size = U1 / (len(res.idiomatic_result) * len(res.non_idiomatic_result)) # as per Wikipedia on Mann Whitney U
+ r_effect_size = ((2 * U1) / (len(res.idiomatic_result) * len(res.non_idiomatic_result))) - 1 # as per Wikipedia on Mann Whitney U
+ is_signficant: bool = mannwhitney.pvalue < 0.05
+ hodges_lehman = hodges_lehmann_2sample(res.idiomatic_result, res.non_idiomatic_result) # To gauge the practical difference between the two groups, as suggested by the Wikipedia page on Mann Whitney U
+ print(f"######### \n Model name: {model} \n Idiom Variance (n={len(res.idiomatic_result)}): {idiom_variance}, Idiom is normal: {idiom_is_normal}, Non-Idiom Variance (n={len(res.non_idiomatic_result)}): {non_idiom_variance}, Non-idiom is normal: {non_idiom_is_normal} \n Hodges-Lehman Estimator: {hodges_lehman} \n Two-sided Mann-whitney U: p_value={mannwhitney.pvalue}, is_significant: {is_signficant} \n Effect size (Common Language F-test): {cl_effect_size} \n Effect size (Rank-biserial correlation): {r_effect_size}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/t_test.py b/t_test.py
@@ -1,86 +0,0 @@
-import sys
-import pickle
-import numpy
-from torch import equal, histogram
-from experiment import Point
-import matplotlib.pyplot as plt
-
-import scipy.stats as stats
-
-from typing import Any
-import json
-
-import pandas
-
-import argparse
-import pathlib
-
-import warnings
-
-from typing import NamedTuple
-
-warnings.filterwarnings("ignore")
-
-
-def parse_args() -> argparse.Namespace:
- parser = argparse.ArgumentParser(description="Calculate the Welsch T-Test")
- parser.add_argument("-epsilons_path", type=pathlib.Path)
- parser.add_argument("-result_path", type=pathlib.Path)
- return parser.parse_args()
-
-
-class ResultPoint():
- def __init__(self, model_name: str = "", idiomatic_result: list[float] = [], non_idiomatic_result: list[float] = []):
- self.model_name: str = model_name
- self.idiomatic_result: list[float] = idiomatic_result
- self.non_idiomatic_result: list[float] = non_idiomatic_result
-
-
-def hodges_lehmann_2sample(x: list[float], y: list[float]) -> float:
- """
- https://en.wikipedia.org/wiki/Hodges%E2%80%93Lehmann_estimator
- The median of all the pairwise differences estimates the true effect on the outcomes.
- I calculated it as the median difference between pairs, as specified in:
- https://link.springer.com/content/pdf/10.1007/978-3-662-69359-9.pdf at page 1145.
- """
- differences: list[float] = []
- for x_i in x:
- for y_i in y:
- differences.append(y_i - x_i)
- return float(numpy.median(differences))
-
-
-def main() -> None:
- args = parse_args()
- with open(args.epsilons_path, "r") as f:
- loaded_dict = json.load(f)
-
- model_results: dict[str, ResultPoint] = {}
- for i in range(0, len(loaded_dict["dataset"])):
- dataset_name = loaded_dict["dataset"][i]
- model_name = loaded_dict["model"][i]
- is_idiomatic = True if dataset_name == "ncip" else False
- if model_name not in model_results:
- model_results[model_name] = ResultPoint(model_name=model_name)
- if is_idiomatic:
- model_results[model_name].idiomatic_result = loaded_dict["original"][i]
- else:
- model_results[model_name].non_idiomatic_result = loaded_dict["original"][i]
-
- for model, res in model_results.items():
- idiom_variance = numpy.var(res.idiomatic_result)
- non_idiom_variance = numpy.var(res.non_idiomatic_result)
- idiom_is_normal = stats.normaltest(res.idiomatic_result).pvalue > 0.05
- non_idiom_is_normal = stats.normaltest(res.non_idiomatic_result).pvalue > 0.05
- mannwhitney = stats.mannwhitneyu(res.idiomatic_result, res.non_idiomatic_result) # Independence
- U1 = mannwhitney.statistic
- U2 = len(res.idiomatic_result) * len(res.non_idiomatic_result) - U1 # as per the documentaiton notes on the scipy function
- cl_effect_size = U1 / (len(res.idiomatic_result) * len(res.non_idiomatic_result))
- r_effect_size = ((2 * U1) / (len(res.idiomatic_result) * len(res.non_idiomatic_result))) - 1
- is_signficant: bool = mannwhitney.pvalue < 0.05
- hodges_lehman = hodges_lehmann_2sample(res.idiomatic_result, res.non_idiomatic_result) # To gauge the practical difference between the two groups.
- print(f"######### \n Model name: {model} \n Idiom Variance (n={len(res.idiomatic_result)}): {idiom_variance}, Idiom is normal: {idiom_is_normal}, Non-Idiom Variance (n={len(res.non_idiomatic_result)}): {non_idiom_variance}, Non-idiom is normal: {non_idiom_is_normal} \n Hodges-Lehman Estimator: {hodges_lehman} \n Two-sided Mann-whitney U: p_value={mannwhitney.pvalue}, is_significant: {is_signficant} \n Effect size (Common Language F-test): {cl_effect_size} \n Effect size (Rank-biserial correlation): {r_effect_size}")
-
-
-if __name__ == "__main__":
- main()
