UCDP-AEC

Models and data for the UCDP Abstractive Event analysis Corpus paper
git clone https://esimon.eu/repos/UCDP-AEC.git
Log | Files | Refs | README | LICENSE

extraction_metrics.py (2831B)

      1 from typing import List
      2 
      3 from extraction.event_schema import EventSchema
      4 from extraction.predict_parser.predict_parser import Metric
      5 from extraction.predict_parser.tree_predict_parser import TreePredictParser, ListPredictParser
      6 from data_convert.ucdp_fields import UCDP_FIELDS
      7 
      8 decoding_format_dict = {
      9     'tree': TreePredictParser,
     10     'list': ListPredictParser,
     11     'treespan': TreePredictParser,
     12 }
     13 
     14 
     15 def get_predict_parser(format_name):
     16     return decoding_format_dict[format_name]
     17 
     18 
     19 def eval_pred(predict_parser, gold_list, pred_list, text_list=None, raw_list=None):
     20     well_formed_list, counter = predict_parser.decode(
     21         gold_list, pred_list, text_list, raw_list)
     22 
     23     event_metric = Metric()
     24     role_metric = Metric()
     25     ucdp_metric = {role: 0 for role, _, _ in UCDP_FIELDS}
     26 
     27     for instance in well_formed_list:
     28         event_metric.count_instance(instance['gold_event'],
     29                                     instance['pred_event'])
     30         role_metric.count_instance(instance['gold_role'],
     31                                    instance['pred_role'],
     32                                    verbose=False)
     33         gold_dict = {role: argument for etype, role, argument in instance["gold_role"]}
     34         pred_dict = {role: argument for etype, role, argument in instance["pred_role"]}
     35         for role, _, _ in UCDP_FIELDS:
     36             if role not in gold_dict:
     37                 continue
     38             if gold_dict[role] == pred_dict.get(role):
     39                 ucdp_metric[role] += 1
     40 
     41     for role, count in ucdp_metric.items():
     42         ucdp_metric[role] = count / len(well_formed_list)
     43     ucdp_agg = {category: 0 for _, _, category in UCDP_FIELDS}
     44     for role, _, category in UCDP_FIELDS:
     45         ucdp_agg[category] += ucdp_metric[role]
     46     for category, total in ucdp_agg.items():
     47         ucdp_agg[category] = total / [category for _, _, category in UCDP_FIELDS].count(category)
     48     aggregate = sum(ucdp_agg.values()) / len(ucdp_agg)
     49     ucdp_metric.update(ucdp_agg)
     50     ucdp_metric["aggregate"] = aggregate
     51 
     52 
     53     trigger_result = event_metric.compute_f1(prefix='trigger-')
     54     role_result = role_metric.compute_f1(prefix='role-')
     55 
     56     result = dict()
     57     result.update(trigger_result)
     58     result.update(role_result)
     59     result['AVG-F1'] = trigger_result.get('trigger-F1', 0.) + \
     60         role_result.get('role-F1', 0.)
     61     result.update(counter)
     62 
     63     for role, accuracy in ucdp_metric.items():
     64         result["UCDP-" + role + "-acc"] = accuracy
     65 
     66     return result
     67 
     68 
     69 def get_extract_metrics(pred_lns: List[str], tgt_lns: List[str], label_constraint: EventSchema, decoding_format='tree'):
     70     predict_parser = get_predict_parser(format_name=decoding_format)(
     71         label_constraint=label_constraint)
     72     return eval_pred(
     73         predict_parser=predict_parser,
     74         gold_list=tgt_lns,
     75         pred_list=pred_lns
     76     )