commit b36336c342151068d1824430685ac012b0d95709
parent 9a230d17128ab1459afa66e3381b29102ee3170b
Author: Sondre Wold <sondrewold1@gmail.com>
Date: Wed, 3 Sep 2025 11:26:31 +0200
more statistics
Diffstat:
| M | t_test.py | | | 31 | ++++++++++++++++++++++++------- |
1 file changed, 24 insertions(+), 7 deletions(-)
diff --git a/t_test.py b/t_test.py
@@ -36,6 +36,20 @@ class ResultPoint():
self.non_idiomatic_result: list[float] = non_idiomatic_result
+def hodges_lehmann_2sample(x: list[float], y: list[float]) -> float:
+ """
+ https://en.wikipedia.org/wiki/Hodges%E2%80%93Lehmann_estimator
+ The median of all the pairwise differences estimates the true effect on the outcomes.
+ I calculated it as the median difference between pairs, as specified in:
+ https://link.springer.com/content/pdf/10.1007/978-3-662-69359-9.pdf at page 1145.
+ """
+ differences: list[float] = []
+ for x_i in x:
+ for y_i in y:
+ differences.append(y_i - x_i)
+ return float(numpy.median(differences))
+
+
def main() -> None:
args = parse_args()
with open(args.epsilons_path, "r") as f:
@@ -56,13 +70,16 @@ def main() -> None:
for model, res in model_results.items():
idiom_variance = numpy.var(res.idiomatic_result)
non_idiom_variance = numpy.var(res.non_idiomatic_result)
- # null hypothesis: the two groups are from the same means, p_value < 0.05 -> the two groups are different.
- t_test_statistics = stats.ttest_ind(res.idiomatic_result, res.non_idiomatic_result, equal_var=False, alternative="greater").pvalue
- t_rel_statistics = stats.ttest_rel(res.idiomatic_result[:250], res.non_idiomatic_result[:250]).pvalue
- is_signficant: bool = t_test_statistics < 0.05
- #print(f"Model name: {model}, Idiom Variance: {idiom_variance}, Non-Idiom Variance: {non_idiom_variance}, p_value={t_test_statistics}, is_significant: {is_signficant}")
- print(f"Model name: {model}, Idiom Variance: {idiom_variance}, Non-Idiom Variance: {non_idiom_variance}, p_value={t_rel_statistics}, is_significant: {is_signficant}")
-
+ idiom_is_normal = stats.normaltest(res.idiomatic_result).pvalue > 0.05
+ non_idiom_is_normal = stats.normaltest(res.non_idiomatic_result).pvalue > 0.05
+ mannwhitney = stats.mannwhitneyu(res.idiomatic_result, res.non_idiomatic_result) # Independence
+ U1 = mannwhitney.statistic
+ U2 = len(res.idiomatic_result) * len(res.non_idiomatic_result) - U1 # as per the documentaiton notes on the scipy function
+ cl_effect_size = U1 / (len(res.idiomatic_result) * len(res.non_idiomatic_result))
+ r_effect_size = ((2 * U1) / (len(res.idiomatic_result) * len(res.non_idiomatic_result))) - 1
+ is_signficant: bool = mannwhitney.pvalue < 0.05
+ hodges_lehman = hodges_lehmann_2sample(res.idiomatic_result, res.non_idiomatic_result) # To gauge the practical difference between the two groups.
+ print(f"######### \n Model name: {model} \n Idiom Variance (n={len(res.idiomatic_result)}): {idiom_variance}, Idiom is normal: {idiom_is_normal}, Non-Idiom Variance (n={len(res.non_idiomatic_result)}): {non_idiom_variance}, Non-idiom is normal: {non_idiom_is_normal} \n Hodges-Lehman Estimator: {hodges_lehman} \n Two-sided Mann-whitney U: p_value={mannwhitney.pvalue}, is_significant: {is_signficant} \n Effect size (Common Language F-test): {cl_effect_size} \n Effect size (Rank-biserial correlation): {r_effect_size}")
if __name__ == "__main__":
