commit dcf3ee48620f3a513161b14b1781b317f8fe8a83
parent 6d58f954e9f508dd227e50eae74a68434791ba56
Author: Sondre Wold <sondrewold1@gmail.com>
Date: Thu, 25 Sep 2025 09:44:25 +0200
add comments on normaltest, fix formatting
Diffstat:
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/statistical_testing.py b/statistical_testing.py
@@ -21,9 +21,10 @@ from typing import NamedTuple
warnings.filterwarnings("ignore")
+SIGNIFICANCE_LEVEL: float = 0.05
def parse_args() -> argparse.Namespace:
- parser = argparse.ArgumentParser(description="Calculate the Welsch T-Test")
+ parser = argparse.ArgumentParser(description="Calculate statistical tests between the two groups.")
parser.add_argument("-epsilons_path", type=pathlib.Path)
parser.add_argument("-result_path", type=pathlib.Path)
parser.add_argument(
@@ -78,7 +79,6 @@ def main() -> None:
for i in range(0, len(loaded_dict["dataset"])):
dataset_name = loaded_dict["dataset"][i]
model_name = loaded_dict["model"][i]
- print(f"{model_name}, {dataset_name}, {len(loaded_dict["original"][i])}")
is_idiomatic = True if dataset_name == "ncip" else False
key = args.epsilon_type
if model_name not in model_results:
@@ -98,8 +98,16 @@ def main() -> None:
for model, res in model_results.items():
idiom_variance = numpy.var(res.idiomatic_result)
non_idiom_variance = numpy.var(res.non_idiomatic_result)
- idiom_is_normal = stats.normaltest(res.idiomatic_result).pvalue > 0.05
- non_idiom_is_normal = stats.normaltest(res.non_idiomatic_result).pvalue > 0.05
+
+ # Test whether we can reject the 0-hypothesis that the samples are normally distributed.
+ # If the p-value is over the sig level, then we cannot reject it, meaning that they are normal.
+ # If the p-value is under the sig level, then we reject it, meaning that they are NOT normal.
+ idiom_normal_p: float = stats.normaltest(res.idiomatic_result).pvalue
+ idiom_is_normal: bool = idiom_normal_p > SIGNIFICANCE_LEVEL
+
+ non_idiom_normal_p: float = stats.normaltest(res.non_idiomatic_result).pvalue
+ non_idiom_is_normal: bool = non_idiom_normal_p > SIGNIFICANCE_LEVEL
+
coxon = stats.wilcoxon(
res.idiomatic_result, res.non_idiomatic_result, alternative="greater"
)
@@ -107,15 +115,14 @@ def main() -> None:
# As per the Wikipedia page of the Wilxocon signed-rank test. T divided by the sum of ranks.
rank_sums = sum([i for i in range(1, len(res.idiomatic_result) + 1)])
r_effect_size = T / rank_sums
- is_signficant: bool = coxon.pvalue < 0.05
+ is_signficant: bool = coxon.pvalue < SIGNIFICANCE_LEVEL
hodges_lehman = hodges_lehmann_2sample(
res.idiomatic_result, res.non_idiomatic_result
- ) # To gauge the practical difference between the two groups, as suggested by the Wikipedia page on Mann Whitney U
+ )
print(
- f"######### \n Model name: {model} \n Idiom Variance (n={len(res.idiomatic_result)}): {idiom_variance}, Idiom is normally distributed: {idiom_is_normal}, Non-Idiom Variance (n={len(res.non_idiomatic_result)}): {non_idiom_variance}, Non-idiom is normal distributed: {non_idiom_is_normal} \n Hodges-Lehman Estimator: {hodges_lehman} \n Two-sided Wilcoxon signed-rank test: p_value={coxon.pvalue}, is_significant: {is_signficant} \n Effect size (Rank-biserial correlation): {r_effect_size}"
+ f"######### \n Model name: {model} \n Idiom Variance (n={len(res.idiomatic_result)}): {round(idiom_variance, 5)}, Idiom is normally distributed: {idiom_is_normal} (p-value={round(idiom_normal_p, 5)}), Non-Idiom Variance (n={len(res.non_idiomatic_result)}): {round(non_idiom_variance, 5)}, Non-idiom is normal distributed: {non_idiom_is_normal} (p-value={round(non_idiom_normal_p, 5)}) \n Hodges-Lehman Estimator: {round(hodges_lehman, 5)} \n Two-sided Wilcoxon signed-rank test: p_value={round(coxon.pvalue, 5)}, is_significant: {is_signficant} \n Effect size (Rank-biserial correlation): {round(r_effect_size, 5)}"
)
-
if __name__ == "__main__":
main()
