text-graph-generalization

Code for our paper Compositional Generalization with Grounded Language Models
git clone https://esimon.eu/repos/text-graph-generalization.git
Log | Files | Refs | README | LICENSE

results.json (3684B)


      1 {
      2     "disjoint_subst_2_hop": {
      3         "mean": 98.17,
      4         "std": 0.39
      5     },
      6     "disjoint_subst_3_hop": {
      7         "mean": 83.08,
      8         "std": 0.96
      9     },
     10     "disjoint_subst_4_hop": {
     11         "mean": 67.86,
     12         "std": 1.89
     13     },
     14     "disjoint_prod_T2a3_V4": {
     15         "mean": 59.42,
     16         "std": 2.07
     17     },
     18     "disjoint_prod_T2a4_V3": {
     19         "mean": 63.87,
     20         "std": 2.24
     21     },
     22     "disjoint_prod_T3a4_V2": {
     23         "mean": 60.34,
     24         "std": 3.25
     25     },
     26     "disjoint_system_2_hop": {
     27         "mean": 55.84,
     28         "std": 12.66
     29     },
     30     "disjoint_system_3_hop": {
     31         "mean": 67.44,
     32         "std": 4.84
     33     },
     34     "disjoint_system_4_hop": {
     35         "mean": 64.52,
     36         "std": 4.68
     37     },
     38     "grounded_subst_2_hop": {
     39         "mean": 92.46,
     40         "std": 0.49
     41     },
     42     "grounded_subst_3_hop": {
     43         "mean": 84.91,
     44         "std": 0.38
     45     },
     46     "grounded_subst_4_hop": {
     47         "mean": 69.32,
     48         "std": 1.96
     49     },
     50     "grounded_prod_T2a3_V4": {
     51         "mean": 61.79,
     52         "std": 0.52
     53     },
     54     "grounded_prod_T2a4_V3": {
     55         "mean": 66.1,
     56         "std": 2.05
     57     },
     58     "grounded_prod_T3a4_V2": {
     59         "mean": 62.32,
     60         "std": 1.98
     61     },
     62     "grounded_system_2_hop": {
     63         "mean": 55.23,
     64         "std": 21.76
     65     },
     66     "grounded_system_3_hop": {
     67         "mean": 72.55,
     68         "std": 4.35
     69     },
     70     "grounded_system_4_hop": {
     71         "mean": 64.29,
     72         "std": 5.45
     73     },
     74     "gnn_subst_2_hop": {
     75         "mean": 48.88,
     76         "std": 1.0
     77     },
     78     "gnn_subst_3_hop": {
     79         "mean": 49.8,
     80         "std": 0.64
     81     },
     82     "gnn_subst_4_hop": {
     83         "mean": 49.36,
     84         "std": 0.44
     85     },
     86     "gnn_prod_T2a3_V4": {
     87         "mean": 49.92,
     88         "std": 0.82
     89     },
     90     "gnn_prod_T2a4_V3": {
     91         "mean": 51.19,
     92         "std": 0.31
     93     },
     94     "gnn_prod_T3a4_V2": {
     95         "mean": 50.98,
     96         "std": 0.25
     97     },
     98     "gnn_system_2_hop": {
     99         "mean": 49.27,
    100         "std": 0.87
    101     },
    102     "gnn_system_3_hop": {
    103         "mean": 49.68,
    104         "std": 0.54
    105     },
    106     "gnn_system_4_hop": {
    107         "mean": 49.77,
    108         "std": 0.75
    109     },
    110     "unidirectional_subst_2_hop": {
    111         "mean": 91.78,
    112         "std": 0.72
    113     },
    114     "unidirectional_subst_3_hop": {
    115         "mean": 70.88,
    116         "std": 3.53
    117     },
    118     "unidirectional_subst_4_hop": {
    119         "mean": 62.0,
    120         "std": 4.97
    121     },
    122     "unidirectional_prod_T2a3_V4": {
    123         "mean": 62.4,
    124         "std": 1.02
    125     },
    126     "unidirectional_prod_T2a4_V3": {
    127         "mean": 66.65,
    128         "std": 2.24
    129     },
    130     "unidirectional_prod_T3a4_V2": {
    131         "mean": 64.04,
    132         "std": 4.47
    133     },
    134     "unidirectional_system_2_hop": {
    135         "mean": 55.42,
    136         "std": 22.79
    137     },
    138     "unidirectional_system_3_hop": {
    139         "mean": 66.52,
    140         "std": 4.9
    141     },
    142     "unidirectional_system_4_hop": {
    143         "mean": 61.96,
    144         "std": 5.74
    145     },
    146     "bidirectional_subst_2_hop": {
    147         "mean": 92.44,
    148         "std": 0.23
    149     },
    150     "bidirectional_subst_3_hop": {
    151         "mean": 73.66,
    152         "std": 1.04
    153     },
    154     "bidirectional_subst_4_hop": {
    155         "mean": 63.24,
    156         "std": 4.36
    157     },
    158     "bidirectional_prod_T2a3_V4": {
    159         "mean": 62.05,
    160         "std": 0.91
    161     },
    162     "bidirectional_prod_T2a4_V3": {
    163         "mean": 66.29,
    164         "std": 1.25
    165     },
    166     "bidirectional_prod_T3a4_V2": {
    167         "mean": 63.03,
    168         "std": 3.76
    169     },
    170     "bidirectional_system_2_hop": {
    171         "mean": 47.11,
    172         "std": 22.64
    173     },
    174     "bidirectional_system_3_hop": {
    175         "mean": 63.78,
    176         "std": 7.54
    177     },
    178     "bidirectional_system_4_hop": {
    179         "mean": 62.76,
    180         "std": 5.92
    181     }
    182 }