extract_results_table.py (19910B)
1 from typing import Any, Dict, List, Optional, Tuple 2 import argparse 3 import collections 4 import functools 5 import math 6 import os 7 import pathlib 8 import re 9 10 import numpy 11 import scipy.stats 12 import statsmodels.stats.contingency_tables 13 14 import mmsrl.ensembling 15 import mmsrl.utils 16 17 RESULT_REGEX = re.compile(r"^Epoch\s+(\d+) (val|test), Macro-F1: (\d\.\d{3}), hero\[F(\d\.\d{2})/P(\d\.\d{2})/R(\d\.\d{2})\], villain\[F(\d\.\d{2})/P(\d\.\d{2})/R(\d\.\d{2})\], victim\[F(\d\.\d{2})/P(\d\.\d{2})/R(\d\.\d{2})\], other\[F(\d\.\d{2})/P(\d\.\d{2})/R(\d\.\d{2})\]$") 18 RESULT_KEYS = ["epoch", "split", "macro-F1", "hero F1", "hero P", "hero R", "villain F1", "villain P", "villain R", "victim F1", "victim P", "victim R", "other F1", "other P", "other R"] 19 CONFIG_REGEX = re.compile(r"^([^:]*): (.*)$") 20 CONFIG_UNIQUE_KEYS = ["output_test", "output_val"] 21 22 23 def all_equals(l: List[str]) -> bool: 24 return all(x == l[0] for x in l[1:]) 25 26 27 def config_to_run(config: Dict[str, str]) -> Any: 28 key = config.copy() 29 for unique in CONFIG_UNIQUE_KEYS: 30 key.pop(unique, None) 31 return tuple(sorted(key.items())) 32 33 34 def run_to_config(run: Any) -> Dict[str, str]: 35 return dict(list(run)) 36 37 38 @functools.cache 39 def levenshtein(lhs: str, rhs: str) -> int: 40 if not lhs or not rhs: 41 return len(lhs) + len(rhs) 42 if lhs[0] == rhs[0]: 43 return levenshtein(lhs[1:], rhs[1:]) 44 return min(2+levenshtein(lhs[1:], rhs), 2+levenshtein(lhs, rhs[1:]), 1+levenshtein(lhs[1:], rhs[1:])) 45 46 47 def get_experiments(args: argparse.Namespace, path: Optional[pathlib.Path] = None) -> Dict[str, List[pathlib.Path]]: 48 if path is None: 49 path = args.path 50 experiments: Dict[str, List[pathlib.Path]] = {} 51 sub_experiments: Dict[str, List[pathlib.Path]] = {} 52 for filename in sorted(os.listdir(path)): 53 if (path / filename).is_dir(): 54 if filename not in args.exclude: 55 sub_experiments.update(get_experiments(args, path / filename)) 56 continue 57 if not (filename.startswith(args.prefix) and filename.endswith(args.suffix)): 58 continue 59 60 matched: bool = False 61 for candidate, filenames in experiments.items(): 62 if levenshtein(filename, candidate) < args.group_threshold: 63 filenames.append(path / filename) 64 matched = True 65 break 66 67 if not matched: 68 experiments[filename] = [path / filename] 69 70 return experiments | sub_experiments 71 72 73 def get_result(args: argparse.Namespace, data: List[str]) -> Dict[str, float]: 74 last = {} 75 for line in data: 76 match = RESULT_REGEX.fullmatch(line) 77 if match: 78 last[match[2]] = match 79 80 if args.split not in last or int(last[args.split][1]) < args.min_epoch: 81 return {} 82 83 result = dict(zip(RESULT_KEYS, last[args.split].groups())) 84 result.pop("split") 85 return {key: float(value) for key, value in result.items()} 86 87 88 def get_config(data: List[str]) -> Dict[str, str]: 89 if not data or data[0] != "\033[33mConfiguration:\033[0m": 90 return {} 91 92 config: Dict[str, str] = {} 93 for line in data[1:]: 94 if "\033" in line or line.startswith("Epoch "): 95 break 96 97 match = CONFIG_REGEX.fullmatch(line) 98 if match: 99 config[match[1]] = match[2] 100 else: 101 break 102 return config 103 104 105 def merge_configs(configs: Dict[pathlib.Path, Dict[str, str]]) -> Tuple[Dict[str, str], Dict[pathlib.Path, Dict[str, str]]]: 106 transpose: Dict[str, List[str]] = collections.defaultdict(list) 107 for config in configs.values(): 108 for key, value in config.items(): 109 transpose[key].append(value) 110 111 base_config: Dict[str, str] = {} 112 for key, values in transpose.items(): 113 if all_equals(values) and len(values) == len(configs): 114 base_config[key] = values[0] 115 116 delta_configs: Dict[pathlib.Path, Dict[str, str]] = {} 117 for path, config in configs.items(): 118 delta_configs[path] = {key: value for key, value in config.items() if key not in base_config} 119 120 return base_config, delta_configs 121 122 123 def merge_runs(configs: Dict[pathlib.Path, Dict[str, str]]) -> Dict[Any, List[pathlib.Path]]: 124 runs: Dict[List[Tuple[str, str]], List[pathlib.Path]] = collections.defaultdict(list) 125 for path, config in configs.items(): 126 runs[config_to_run(config)].append(path) 127 return runs 128 129 130 def aggregate_results(results: List[Dict[str, float]]): 131 aggregated: Dict[str, Dict[str, Any]] = {} 132 for key in results[0].keys(): 133 values = numpy.array([result[key] for result in results]) 134 aggregated[key] = { 135 "mean": values.mean(), 136 "std": values.std(), 137 "min": values.min(), 138 "max": values.max(), 139 "values": values 140 } 141 return aggregated 142 143 144 def merge_results(runs: Dict[Any, List[pathlib.Path]], results: Dict[pathlib.Path, Dict[str, float]]) -> Dict[Any, Dict[str, Any]]: 145 merged: Dict[Any, Dict[str, Any]] = {} 146 for run, paths in runs.items(): 147 merged[run] = {} 148 merged[run]["runs"] = [results[path] for path in paths], 149 merged[run].update(aggregate_results([results[path] for path in paths])) 150 return merged 151 152 153 def mark_best(results: Dict[Any, Dict[str, Any]]) -> None: 154 best = collections.defaultdict(lambda: 0) 155 for run in results.values(): 156 for key, values in run.items(): 157 if isinstance(values, dict) and isinstance(values.get("mean"), float): 158 best[key] = max(best[key], values["mean"]) 159 160 for run in results.values(): 161 for key, values in run.items(): 162 if isinstance(values, dict) and isinstance(values.get("mean"), float): 163 values["best"] = (values["mean"] >= best[key]) 164 165 166 def get_predictions(args: argparse.Namespace, runs: Dict[Any, List[pathlib.Path]], delta_configs: Dict[pathlib.Path, Dict[str, str]]) -> Tuple[Dict[Any, numpy.ndarray], numpy.ndarray]: 167 predictions: Dict[Tuple[str, str], Dict[Any, List[int]]] = collections.defaultdict(lambda: collections.defaultdict(list)) 168 gold = {sample["image"]: sample for sample in mmsrl.ensembling.read_data(args.split)} 169 170 for run, paths in runs.items(): 171 for stdout_path in paths: 172 base_path = stdout_path.parent 173 prediction_path = pathlib.Path(delta_configs[stdout_path][f"output_{args.split}"]) 174 prediction = mmsrl.ensembling.read_prediction_file(base_path / prediction_path.parts[-1]) 175 for image, entities in prediction.items(): 176 for entity, probabilities in entities.items(): 177 predictions[(image, entity)][run].append(probabilities.argmax()) 178 179 dataset_size: int = len(predictions) 180 num_seeds: int = len(paths) 181 prediction_array = {run: numpy.empty((dataset_size, num_seeds), dtype=numpy.int32) for run in runs} 182 gold_array = numpy.empty((dataset_size,), dtype=numpy.int32) 183 184 for i, ((image, entity), run_predictions) in enumerate(predictions.items()): 185 assigned = False 186 for ilabel, label in enumerate(mmsrl.utils.LABELS): 187 if entity in gold[image][label]: 188 gold_array[i] = ilabel 189 assigned = True 190 assert(assigned) 191 192 for run, seed_predictions in run_predictions.items(): 193 prediction_array[run][i] = seed_predictions 194 return prediction_array, gold_array 195 196 197 def compute_contingency_matrix(aggregate: str, lhs: numpy.ndarray, rhs: numpy.ndarray) -> numpy.ndarray: 198 contingency = numpy.zeros((4, 4), dtype=numpy.int32) 199 if aggregate == "first": 200 for y, x in zip(lhs[:, 0], rhs[:, 0]): 201 contingency[y, x] += 1 202 elif aggregate == "flatten": 203 for y, x in zip(lhs.flatten(), rhs.flatten()): 204 contingency[y, x] += 1 205 elif aggregate == "cartesian mean": 206 for ys, xs in zip(lhs, rhs): 207 for y in ys: 208 for x in xs: 209 contingency[y, x] += 1 210 contingency = contingency // (len(ys) * len(xs)) 211 return contingency 212 213 214 def compute_contingency_matrices(aggregate: str, predictions: Dict[Any, numpy.ndarray]) -> Dict[Tuple[Any, Any], numpy.ndarray]: 215 contingencies = {} 216 for run_lhs, lhs in predictions.items(): 217 for run_rhs, rhs in predictions.items(): 218 contingencies[(run_lhs, run_rhs)] = compute_contingency_matrix(aggregate, lhs, rhs) 219 return contingencies 220 221 222 def compute_mcnemar_bowker(contingency: numpy.ndarray) -> float: 223 return statsmodels.stats.contingency_tables.mcnemar(contingency, exact=False, correction=True).pvalue 224 225 226 def compute_macro_f1_t_test(lhs: Dict[str, Any], rhs: Dict[str, Any], equal_var: bool) -> float: 227 return scipy.stats.ttest_ind(lhs["macro-F1"]["values"], rhs["macro-F1"]["values"], equal_var=equal_var, alternative="greater").pvalue 228 229 230 def compute_accuracies_t_test(lhs: numpy.ndarray, rhs: numpy.ndarray, gold: numpy.ndarray, equal_var: bool) -> float: 231 lhs = (lhs == gold[:, None]).flatten() 232 rhs = (rhs == gold[:, None]).flatten() 233 return scipy.stats.ttest_ind(lhs, rhs, equal_var=equal_var, alternative="greater").pvalue 234 235 236 def compute_pvalues(args: argparse.Namespace, runs: Dict[Any, List[pathlib.Path]], delta_configs: Dict[pathlib.Path, Dict[str, str]], run_results: Dict[Any, Dict[str, Any]]) -> Dict[str, Dict[Tuple[Any, Any], float]]: 237 if args.all_test: 238 predictions, gold = get_predictions(args, runs, delta_configs) 239 contingencies: Dict[str, Dict[Tuple[Any, Any], numpy.ndarray]] = { 240 aggregation: compute_contingency_matrices(aggregation, predictions) 241 for aggregation in ["first", "flatten", "cartesian mean"] 242 } 243 244 pvalues: Dict[str, Dict[Tuple[Any, Any], float]] = collections.defaultdict(dict) 245 for run_lhs in runs: 246 for run_rhs in runs: 247 nseed = len(runs[run_lhs]) 248 pvalues[f'one-sided t-test of macro-F1'][(run_lhs, run_rhs)] = compute_macro_f1_t_test(run_results[run_lhs], run_results[run_rhs], equal_var=True) 249 pvalues[f'one-sided Welch\'s unequal variances t-test of macro-F1'][(run_lhs, run_rhs)] = compute_macro_f1_t_test(run_results[run_lhs], run_results[run_rhs], equal_var=False) 250 251 if args.all_test: 252 pvalues[f'one-sided t-test of accuracies ({nseed} seed{"s" if nseed>1 else ""})'][(run_lhs, run_rhs)] = compute_accuracies_t_test(predictions[run_lhs], predictions[run_rhs], gold, equal_var=True) 253 pvalues[f'one-sided Welch\'s unequal variances t-test of accuracies ({nseed} seed{"s" if nseed>1 else ""})'][(run_lhs, run_rhs)] = compute_accuracies_t_test(predictions[run_lhs], predictions[run_rhs], gold, equal_var=False) 254 255 for aggregation, contingency in contingencies.items(): 256 pvalues[f"McNemar-Bowker ({aggregation})"][(run_lhs, run_rhs)] = compute_mcnemar_bowker(contingency[(run_lhs, run_rhs)]) 257 return pvalues 258 259 260 def pre_html() -> None: 261 print("""<html xmlns="http://www.w3.org/1999/xhtml" lang="en"> 262 <head> 263 <title>MMSRL results</title> 264 <meta charset="UTF-8" /> 265 <style type="text/css"> 266 /*<![CDATA[*/ 267 .pvalue { 268 paint-order: stroke fill; 269 -webkit-text-stroke: 10px rgba(255, 255, 255, .3); 270 } 271 table { 272 border-collapse: collapse; 273 } 274 table, td, th { 275 border: 1px solid; 276 } 277 td, th { 278 max-width: 600px; 279 padding: 4px; 280 } 281 dl.base_config > dt:before { 282 content: ""; 283 display: block; 284 } 285 .best_true { 286 font-weight: bold; 287 color: #800; 288 } 289 dt { font-weight: bold; } 290 dt:after { content: "="; } 291 dt, dd { 292 display: inline; 293 margin: 0; 294 } 295 /*]]>*/ 296 </style> 297 </head> 298 <body>""") 299 300 301 def output_html_config(config: Dict[str, str], style: str) -> None: 302 print(f'<dl class="{style}">') 303 for key, value in config.items(): 304 print(f'<dt>{key}</dt><dd>{value}</dd>') 305 print('</dl>') 306 307 308 def output_html_pvalue_color(pvalue: float, better: bool) -> None: 309 if not numpy.isfinite(pvalue): 310 hue = 200 311 else: 312 hue: int = round(min(-math.log(max(pvalue, 1e-10), 10), 2.5)/2.5*120) 313 saturation: int = 100 if better else 25 314 print(f'<td class="pvalue" style="background-color: hsl({hue}, {saturation}%, 50%);">{pvalue:.2e}</td>', end='') 315 316 317 def output_html(experiment: str, base_config: Dict[str, str], runs: Dict[Any, List[pathlib.Path]], run_results: Dict[Any, Dict[str, Any]], pvalues: Dict[str, Dict[Tuple[Any, Any], float]]) -> None: 318 run_order = sorted(list(run_results.keys())) 319 320 print('<section>') 321 print(f'<h1>{experiment}</h1>') 322 print('<details>') 323 print('<summary>Base config</summary>') 324 output_html_config(base_config, "base_config") 325 print('</details>') 326 print('<table>') 327 print(' <thead>') 328 print(' <tr><th rowspan="2">ID</th><th rowspan="2">Config</th><th rowspan="2">Macro-F1</th><th colspan="3">hero</th><th colspan="3">villain</th><th colspan="3">victim</th><th colspan="3">other</th></tr>') 329 print(' <tr><th>F1</th><th>P</th><th>R</th><th>F1</th><th>P</th><th>R</th><th>F1</th><th>P</th><th>R</th><th>F1</th><th>P</th><th>R</th></tr>') 330 print(' </thead>') 331 print(' <tbody>') 332 for irun, run in enumerate(run_order): 333 result = run_results[run] 334 print(' <tr>') 335 print(f' <td>{chr(ord("α")+irun)}</td>') 336 print(' <td>', end='') 337 print('<details><summary><code>', end='') 338 config = run_to_config(run) 339 output_html_config(config, "inline_config") 340 print('</code></summary>') 341 print(' <ul>') 342 for path in runs[run]: 343 print(f' <li>{path}</li>') 344 print(' </ul></details>') 345 print('</td>') 346 for key in ["macro-F1", "hero F1", "hero P", "hero R", "villain F1", "villain P", "villain R", "victim F1", "victim P", "victim R", "other F1", "other P", "other R"]: 347 data = result[key] 348 details = "\n".join(f"{key}: {value}" for key, value in data.items()) 349 print(' <td>', end='') 350 print(f'<abbr title="{details}" class="best_{data["best"]}">{data["mean"]:.2f}</abbr>', end='') 351 print('</td>') 352 print(' </tr>') 353 print(' </tbody>') 354 print('</table>') 355 356 for test, pvalue in pvalues.items(): 357 print('<section>') 358 print(f'<details><summary><h3>{test}</h3></summary>') 359 print('<table>') 360 print(' <tr>') 361 print(' <th></th>') 362 for irun in range(len(run_order)): 363 print(f' <th>{chr(ord("α")+irun)}</th>') 364 print(' </tr>') 365 for irun, lhs in enumerate(run_order): 366 print(' <tr>') 367 print(f' <th>{chr(ord("α")+irun)}</th>') 368 for rhs in run_order: 369 output_html_pvalue_color(pvalue[(lhs, rhs)], run_results[lhs]["macro-F1"]["mean"] > run_results[rhs]["macro-F1"]["mean"]) 370 print(' </tr>') 371 print('</table></details>') 372 print('</section>') 373 374 print('</section>') 375 376 377 def post_html() -> None: 378 print('</body>') 379 print('</html>') 380 381 382 def pre_latex() -> None: 383 print(r"""\documentclass[a4paper, landscape]{article} 384 \usepackage[a4paper,margin=1in,landscape]{geometry} 385 \usepackage{fontspec} 386 \usepackage{booktabs} 387 \usepackage{enumitem} 388 \usepackage{multirow} 389 \begin{document}""") 390 391 392 def output_latex_escape(x: str): 393 return x.replace('_', r'\_').replace('[', r'\([\)').replace(']', r'\(]\)').replace('{', r'\({\)').replace('}', r'\(}\)') 394 395 396 def output_latex(experiment: str, base_config: Dict[str, str], runs: Dict[Any, List[pathlib.Path]], run_results: Dict[Any, Dict[str, Any]], pvalues: Dict[str, Dict[Tuple[Any, Any], float]]) -> None: 397 print(f'\\section{{{output_latex_escape(experiment)}}}') 398 print('\paragraph{Base config:}') 399 print(r'\begin{description}[nosep]') 400 for key, value in base_config.items(): 401 key = output_latex_escape(key) 402 print(f'\\item[{key}=] \\verb|{value}|') 403 print(r'\end{description}') 404 print(r'\begin{tabular}{p{5cm} r r r r r r r r r r r r r}') 405 print(r' \toprule') 406 print(r' \multirow{2}{*}{Config} & \multirow{2}{*}{Macro-\(F_1\)} & \multicolumn{3}{c}{hero} & \multicolumn{3}{c}{villain} & \multicolumn{3}{c}{victim} & \multicolumn{3}{c}{other} \\') 407 print(r' & & \(F_1\) & \(P\) & \(R\) & \(F_1\) & \(P\) & \(R\) & \(F_1\) & \(P\) & \(R\) & \(F_1\) & \(P\) & \(R\) \\') 408 print(r' \midrule') 409 for run, result in run_results.items(): 410 config = run_to_config(run) 411 for key, value in config.items(): 412 key = output_latex_escape(key) 413 print(f' \\strong{{{key}=}} \\verb|{value}|', end='') 414 for key in ["macro-F1", "hero F1", "hero P", "hero R", "villain F1", "villain P", "villain R", "victim F1", "victim P", "victim R", "other F1", "other P", "other R"]: 415 data = result[key] 416 if data["best"]: 417 print(f' & \strong{{{data["mean"]:.2f}}}', end='') 418 else: 419 print(f' & {data["mean"]:.2f}', end='') 420 print(r'\\') 421 print(r' \bottomrule') 422 print(r'\end{tabular}') 423 424 425 def post_latex() -> None: 426 print(r"\end{document}") 427 428 429 def main(args: argparse.Namespace) -> None: 430 if args.html: 431 pre, output, post = pre_html, output_html, post_html 432 elif args.latex: 433 pre, output, post = pre_latex, output_latex, post_latex 434 pre() 435 experiments = get_experiments(args) 436 for experiment, paths in experiments.items(): 437 configs = {} 438 results = {} 439 for path in paths: 440 with path.open("r") as file: 441 data = list(map(lambda line: line.rstrip("\n"), file.readlines())) 442 result = get_result(args, data) 443 config = get_config(data) 444 if result and config: 445 results[path] = result 446 configs[path] = config 447 if not configs: 448 continue 449 base_config, delta_configs = merge_configs(configs) 450 runs = merge_runs(delta_configs) 451 run_results = merge_results(runs, results) 452 mark_best(run_results) 453 pvalues = compute_pvalues(args, runs, delta_configs, run_results) 454 output(experiment, base_config, runs, run_results, pvalues) 455 post() 456 457 458 if __name__=="__main__": 459 parser = argparse.ArgumentParser() 460 parser.add_argument("--all-test", action="store_true", help="Compute all significance test methods.") 461 parser.add_argument("--exclude", type=str, nargs='+', default=["old"], help="Ignore these directories") 462 parser.add_argument("--group-threshold", type=int, default=10, help="Maximum levenshtein (add=del=2, mod=1) between experiences grouped in the same experiment.") 463 parser.add_argument("--html", action="store_true", help="Output detailed table in HTML format.") 464 parser.add_argument("--latex", action="store_true", help="Output table in LaTeX format.") 465 parser.add_argument("--min-epoch", type=int, default=25, help="Ignore runs that did not reached this epoch number.") 466 parser.add_argument("--prefix", type=str, default="", help="Prefix of the stdout files to consider, can be used to select a specific experiment.") 467 parser.add_argument("--split", type=str, default="test", help="Output results on the given split (val or test).") 468 parser.add_argument("--suffix", type=str, default=".out", help="Suffix of the stdout files to consider.") 469 parser.add_argument("path", type=pathlib.Path, help="Path to the directory containing stdout files.") 470 args = parser.parse_args() 471 if args.html == args.latex: 472 raise RuntimeError("You must use exactly one of --html or --latex.") 473 main(args)