Upload holdout_metrics.json with huggingface_hub

holdout_metrics.json

@@ -0,0 +1,676 @@
+{
+  "real": {
+    "n_eval": 9540,
+    "overall": {
+      "tp": 6956,
+      "fp": 2047,
+      "fn": 2183,
+      "precision": 0.7726313451071865,
+      "recall": 0.7611336032388664,
+      "f05": 0.7703040907178135,
+      "f1": 0.7668393782383419
+    },
+    "per_key": {
+      "producer": {
+        "tp": 1474,
+        "fp": 568,
+        "fn": 752,
+        "precision": 0.7218413320274241,
+        "recall": 0.6621743036837376,
+        "f05": 0.7090629209159129,
+        "f1": 0.6907216494845361
+      },
+      "year": {
+        "tp": 1737,
+        "fp": 370,
+        "fn": 488,
+        "precision": 0.8243948742287612,
+        "recall": 0.7806741573033708,
+        "f05": 0.8152633061109545,
+        "f1": 0.8019390581717452
+      },
+      "geography": {
+        "tp": 2159,
+        "fp": 751,
+        "fn": 652,
+        "precision": 0.7419243986254296,
+        "recall": 0.768054073283529,
+        "f05": 0.7470071275344268,
+        "f1": 0.7547631532948785
+      },
+      "acronym": {
+        "tp": 1586,
+        "fp": 358,
+        "fn": 291,
+        "precision": 0.815843621399177,
+        "recall": 0.8449653702717101,
+        "f05": 0.821506267481612,
+        "f1": 0.8301491756084796
+      }
+    },
+    "verbatim": {
+      "emitted": 9003,
+      "verbatim": 8964,
+      "rate": 0.9956681106297901,
+      "per_key": {
+        "producer": {
+          "emitted": 2042,
+          "verbatim": 2024
+        },
+        "year": {
+          "emitted": 2107,
+          "verbatim": 2107
+        },
+        "geography": {
+          "emitted": 2910,
+          "verbatim": 2892
+        },
+        "acronym": {
+          "emitted": 1944,
+          "verbatim": 1941
+        }
+      }
+    },
+    "usage_impact": {
+      "data_type_macro_f1": 0.6985499627733744,
+      "usage_action_macro_f1": 0.6339285026503059,
+      "impact_label_macro_f1": 0.5977288525673021,
+      "per_label": {
+        "data_type": {
+          "administrative": {
+            "tp": 753,
+            "fp": 461,
+            "fn": 153,
+            "precision": 0.6202635914332785,
+            "recall": 0.8311258278145696,
+            "f1": 0.710377358490566
+          },
+          "database": {
+            "tp": 573,
+            "fp": 162,
+            "fn": 272,
+            "precision": 0.7795918367346939,
+            "recall": 0.6781065088757396,
+            "f1": 0.7253164556962025
+          },
+          "geospatial": {
+            "tp": 152,
+            "fp": 38,
+            "fn": 65,
+            "precision": 0.8,
+            "recall": 0.7004608294930875,
+            "f1": 0.7469287469287469
+          },
+          "indicator": {
+            "tp": 174,
+            "fp": 77,
+            "fn": 137,
+            "precision": 0.6932270916334662,
+            "recall": 0.5594855305466238,
+            "f1": 0.6192170818505338
+          },
+          "other": {
+            "tp": 237,
+            "fp": 142,
+            "fn": 193,
+            "precision": 0.6253298153034301,
+            "recall": 0.5511627906976744,
+            "f1": 0.5859085290482077
+          },
+          "report": {
+            "tp": 211,
+            "fp": 111,
+            "fn": 168,
+            "precision": 0.65527950310559,
+            "recall": 0.5567282321899736,
+            "f1": 0.6019971469329529
+          },
+          "survey": {
+            "tp": 1293,
+            "fp": 142,
+            "fn": 145,
+            "precision": 0.9010452961672474,
+            "recall": 0.8991655076495132,
+            "f1": 0.9001044204664115
+          }
+        },
+        "usage_action": {
+          "analyze": {
+            "tp": 979,
+            "fp": 349,
+            "fn": 629,
+            "precision": 0.7371987951807228,
+            "recall": 0.6088308457711443,
+            "f1": 0.6668937329700272
+          },
+          "curate": {
+            "tp": 553,
+            "fp": 234,
+            "fn": 290,
+            "precision": 0.7026683608640406,
+            "recall": 0.6559905100830368,
+            "f1": 0.6785276073619633
+          },
+          "inform": {
+            "tp": 546,
+            "fp": 473,
+            "fn": 229,
+            "precision": 0.535819430814524,
+            "recall": 0.704516129032258,
+            "f1": 0.608695652173913
+          },
+          "source": {
+            "tp": 730,
+            "fp": 377,
+            "fn": 224,
+            "precision": 0.6594399277326106,
+            "recall": 0.7651991614255765,
+            "f1": 0.7083939835031537
+          },
+          "validate": {
+            "tp": 160,
+            "fp": 125,
+            "fn": 186,
+            "precision": 0.5614035087719298,
+            "recall": 0.4624277456647399,
+            "f1": 0.5071315372424722
+          }
+        },
+        "impact_label": {
+          "evidence-base": {
+            "tp": 2050,
+            "fp": 350,
+            "fn": 647,
+            "precision": 0.8541666666666666,
+            "recall": 0.7601038190582128,
+            "f1": 0.8043947420051011
+          },
+          "none": {
+            "tp": 730,
+            "fp": 426,
+            "fn": 270,
+            "precision": 0.6314878892733564,
+            "recall": 0.73,
+            "f1": 0.6771799628942485
+          },
+          "policy": {
+            "tp": 586,
+            "fp": 366,
+            "fn": 192,
+            "precision": 0.615546218487395,
+            "recall": 0.7532133676092545,
+            "f1": 0.677456647398844
+          },
+          "uptake": {
+            "tp": 8,
+            "fp": 10,
+            "fn": 43,
+            "precision": 0.4444444444444444,
+            "recall": 0.1568627450980392,
+            "f1": 0.23188405797101447
+          }
+        }
+      },
+      "usage_summary": {
+        "n": 4526,
+        "mean_sim": 0.5934299230575562,
+        "grounded_rate": 0.7551922227132125
+      }
+    },
+    "examples": [
+      {
+        "mention": "SARA",
+        "gold": "{\"acronym\": \"SARA\", \"producer\": \"World Health Organization\"}",
+        "pred": "{\"acronym\": \"SARA\", \"producer\": \"USAID\"}",
+        "correct": "acronym",
+        "verbatim": "producer,acronym"
+      },
+      {
+        "mention": "Encuesta Nacional Industrial Annual",
+        "gold": "{\"acronym\": \"ENIA\", \"geography\": \"Chilean\", \"producer\": \"National Statistical Institute\"}",
+        "pred": "{\"acronym\": \"ENIA\", \"geography\": \"Chilean\"}",
+        "correct": "geography,acronym",
+        "verbatim": "geography,acronym"
+      },
+      {
+        "mention": "child protection services data",
+        "gold": "{\"year\": \"2018\", \"geography\": \"Jordan\"}",
+        "pred": "{\"year\": \"2018\", \"geography\": \"Jordan\"}",
+        "correct": "year,geography",
+        "verbatim": "year,geography"
+      },
+      {
+        "mention": "GBD data",
+        "gold": "{\"geography\": \"Georgia\", \"year\": \"2017\"}",
+        "pred": "{\"acronym\": \"GBD\", \"geography\": \"Georgia\", \"year\": \"2017\"}",
+        "correct": "year,geography",
+        "verbatim": "year,geography,acronym"
+      },
+      {
+        "mention": "Human Capital Index",
+        "gold": "{\"producer\": \"World Bank\", \"geography\": \"Djibouti\"}",
+        "pred": "{\"producer\": \"World Bank\", \"year\": \"2019\", \"geography\": \"Djibouti\", \"acronym\": \"HDI\"}",
+        "correct": "producer,geography",
+        "verbatim": "producer,year,geography,acronym"
+      },
+      {
+        "mention": "night-time lights data",
+        "gold": "{\"geography\": \"district level\"}",
+        "pred": "{\"geography\": \"rural India\"}",
+        "correct": "-",
+        "verbatim": "geography"
+      },
+      {
+        "mention": "annual production of four major crops",
+        "gold": "{\"geography\": \"global\", \"producer\": \"Iizumi and Sakai ( 2020 )\", \"year\": \"2020\"}",
+        "pred": "{\"geography\": \"global\", \"year\": \"2020\"}",
+        "correct": "year,geography",
+        "verbatim": "year,geography"
+      },
+      {
+        "mention": "registry data",
+        "gold": "{\"producer\": \"UNHCR\"}",
+        "pred": "{\"producer\": \"UNHCR\"}",
+        "correct": "producer",
+        "verbatim": "producer"
+      },
+      {
+        "mention": "DTM Mali – Septembre 2019",
+        "gold": "{\"year\": \"2019\", \"geography\": \"Mali\", \"acronym\": \"DTM\"}",
+        "pred": "{\"year\": \"2019\", \"geography\": \"Mali\", \"acronym\": \"DTM\"}",
+        "correct": "year,geography,acronym",
+        "verbatim": "year,geography,acronym"
+      },
+      {
+        "mention": "natural disaster information",
+        "gold": "{\"geography\": \"LAC\"}",
+        "pred": "{\"geography\": \"LAC\"}",
+        "correct": "geography",
+        "verbatim": "geography"
+      },
+      {
+        "mention": "Enquˆete Nationale sur les Conditions de Vie des",
+        "gold": "{\"acronym\": \"ECVMA\", \"geography\": \"Niger\"}",
+        "pred": "{\"acronym\": \"ECVMA\", \"geography\": \"Niger\"}",
+        "correct": "geography,acronym",
+        "verbatim": "geography,acronym"
+      },
+      {
+        "mention": "Covid-19 Trade Watch",
+        "gold": "{\"producer\": \"World Bank\", \"year\": \"2020\"}",
+        "pred": "{\"producer\": \"World Bank\", \"year\": \"2020\"}",
+        "correct": "producer,year",
+        "verbatim": "producer,year"
+      }
+    ]
+  },
+  "synthetic": {
+    "n_eval": 2305,
+    "overall": {
+      "tp": 1005,
+      "fp": 155,
+      "fn": 165,
+      "precision": 0.8663793103448276,
+      "recall": 0.8589743589743589,
+      "f05": 0.8648881239242684,
+      "f1": 0.8626609442060087
+    },
+    "per_key": {
+      "producer": {
+        "tp": 252,
+        "fp": 23,
+        "fn": 35,
+        "precision": 0.9163636363636364,
+        "recall": 0.8780487804878049,
+        "f05": 0.9084354722422495,
+        "f1": 0.8967971530249111
+      },
+      "year": {
+        "tp": 259,
+        "fp": 30,
+        "fn": 46,
+        "precision": 0.8961937716262975,
+        "recall": 0.8491803278688524,
+        "f05": 0.8863791923340176,
+        "f1": 0.8720538720538721
+      },
+      "geography": {
+        "tp": 355,
+        "fp": 62,
+        "fn": 42,
+        "precision": 0.8513189448441247,
+        "recall": 0.8942065491183879,
+        "f05": 0.8595641646489104,
+        "f1": 0.8722358722358723
+      },
+      "acronym": {
+        "tp": 139,
+        "fp": 40,
+        "fn": 42,
+        "precision": 0.776536312849162,
+        "recall": 0.7679558011049724,
+        "f05": 0.7748049052396878,
+        "f1": 0.7722222222222223
+      }
+    },
+    "verbatim": {
+      "emitted": 1160,
+      "verbatim": 1156,
+      "rate": 0.996551724137931,
+      "per_key": {
+        "producer": {
+          "emitted": 275,
+          "verbatim": 275
+        },
+        "year": {
+          "emitted": 289,
+          "verbatim": 289
+        },
+        "geography": {
+          "emitted": 417,
+          "verbatim": 413
+        },
+        "acronym": {
+          "emitted": 179,
+          "verbatim": 179
+        }
+      }
+    },
+    "usage_impact": {
+      "data_type_macro_f1": 0.7857483174178359,
+      "usage_action_macro_f1": 0.6624851717052487,
+      "impact_label_macro_f1": 0.5738134926381705,
+      "per_label": {
+        "data_type": {
+          "administrative": {
+            "tp": 247,
+            "fp": 82,
+            "fn": 31,
+            "precision": 0.7507598784194529,
+            "recall": 0.8884892086330936,
+            "f1": 0.8138385502471169
+          },
+          "database": {
+            "tp": 190,
+            "fp": 50,
+            "fn": 69,
+            "precision": 0.7916666666666666,
+            "recall": 0.7335907335907336,
+            "f1": 0.7615230460921844
+          },
+          "geospatial": {
+            "tp": 93,
+            "fp": 18,
+            "fn": 24,
+            "precision": 0.8378378378378378,
+            "recall": 0.7948717948717948,
+            "f1": 0.8157894736842105
+          },
+          "indicator": {
+            "tp": 117,
+            "fp": 18,
+            "fn": 37,
+            "precision": 0.8666666666666667,
+            "recall": 0.7597402597402597,
+            "f1": 0.8096885813148789
+          },
+          "other": {
+            "tp": 114,
+            "fp": 60,
+            "fn": 56,
+            "precision": 0.6551724137931034,
+            "recall": 0.6705882352941176,
+            "f1": 0.6627906976744184
+          },
+          "report": {
+            "tp": 30,
+            "fp": 9,
+            "fn": 15,
+            "precision": 0.7692307692307693,
+            "recall": 0.6666666666666666,
+            "f1": 0.7142857142857142
+          },
+          "survey": {
+            "tp": 564,
+            "fp": 45,
+            "fn": 50,
+            "precision": 0.9261083743842364,
+            "recall": 0.9185667752442996,
+            "f1": 0.9223221586263287
+          }
+        },
+        "usage_action": {
+          "analyze": {
+            "tp": 76,
+            "fp": 4,
+            "fn": 8,
+            "precision": 0.95,
+            "recall": 0.9047619047619048,
+            "f1": 0.9268292682926829
+          },
+          "causal": {
+            "tp": 56,
+            "fp": 19,
+            "fn": 52,
+            "precision": 0.7466666666666667,
+            "recall": 0.5185185185185185,
+            "f1": 0.6120218579234973
+          },
+          "curate": {
+            "tp": 32,
+            "fp": 2,
+            "fn": 8,
+            "precision": 0.9411764705882353,
+            "recall": 0.8,
+            "f1": 0.8648648648648648
+          },
+          "inform": {
+            "tp": 138,
+            "fp": 48,
+            "fn": 51,
+            "precision": 0.7419354838709677,
+            "recall": 0.7301587301587301,
+            "f1": 0.736
+          },
+          "measure": {
+            "tp": 188,
+            "fp": 146,
+            "fn": 82,
+            "precision": 0.562874251497006,
+            "recall": 0.6962962962962963,
+            "f1": 0.6225165562913907
+          },
+          "model": {
+            "tp": 90,
+            "fp": 49,
+            "fn": 68,
+            "precision": 0.6474820143884892,
+            "recall": 0.569620253164557,
+            "f1": 0.6060606060606062
+          },
+          "other": {
+            "tp": 25,
+            "fp": 21,
+            "fn": 36,
+            "precision": 0.5434782608695652,
+            "recall": 0.4098360655737705,
+            "f1": 0.4672897196261682
+          },
+          "produce": {
+            "tp": 57,
+            "fp": 29,
+            "fn": 61,
+            "precision": 0.6627906976744186,
+            "recall": 0.4830508474576271,
+            "f1": 0.5588235294117647
+          },
+          "source": {
+            "tp": 276,
+            "fp": 165,
+            "fn": 121,
+            "precision": 0.6258503401360545,
+            "recall": 0.6952141057934509,
+            "f1": 0.6587112171837709
+          },
+          "validate": {
+            "tp": 114,
+            "fp": 70,
+            "fn": 46,
+            "precision": 0.6195652173913043,
+            "recall": 0.7125,
+            "f1": 0.6627906976744186
+          },
+          "wrangle": {
+            "tp": 24,
+            "fp": 8,
+            "fn": 28,
+            "precision": 0.75,
+            "recall": 0.46153846153846156,
+            "f1": 0.5714285714285714
+          }
+        },
+        "impact_label": {
+          "evidence-base": {
+            "tp": 942,
+            "fp": 197,
+            "fn": 86,
+            "precision": 0.8270412642669008,
+            "recall": 0.9163424124513618,
+            "f1": 0.8694047069681587
+          },
+          "market-use": {
+            "tp": 0,
+            "fp": 1,
+            "fn": 8,
+            "precision": 0.0,
+            "recall": 0.0,
+            "f1": 0.0
+          },
+          "none": {
+            "tp": 54,
+            "fp": 19,
+            "fn": 42,
+            "precision": 0.7397260273972602,
+            "recall": 0.5625,
+            "f1": 0.6390532544378699
+          },
+          "operational-use": {
+            "tp": 9,
+            "fp": 6,
+            "fn": 57,
+            "precision": 0.6,
+            "recall": 0.13636363636363635,
+            "f1": 0.22222222222222218
+          },
+          "policy": {
+            "tp": 254,
+            "fp": 82,
+            "fn": 110,
+            "precision": 0.7559523809523809,
+            "recall": 0.6978021978021978,
+            "f1": 0.7257142857142858
+          },
+          "uptake": {
+            "tp": 73,
+            "fp": 0,
+            "fn": 2,
+            "precision": 1.0,
+            "recall": 0.9733333333333334,
+            "f1": 0.9864864864864865
+          }
+        }
+      },
+      "usage_summary": {
+        "n": 1637,
+        "mean_sim": 0.5707868337631226,
+        "grounded_rate": 0.6823457544288333
+      }
+    },
+    "examples": [
+      {
+        "mention": "Women Business and Law",
+        "gold": "{\"acronym\": \"WBL\", \"producer\": \"World Bank\"}",
+        "pred": "{\"acronym\": \"WBL\", \"producer\": \"World Bank\"}",
+        "correct": "producer,acronym",
+        "verbatim": "producer,acronym"
+      },
+      {
+        "mention": "Quarterly Public Finance Database",
+        "gold": "{\"producer\": \"national sources\"}",
+        "pred": "{\"producer\": \"national sources\"}",
+        "correct": "producer",
+        "verbatim": "producer"
+      },
+      {
+        "mention": "IMF data",
+        "gold": "{\"geography\": \"40 low-income countries\", \"producer\": \"IMF\"}",
+        "pred": "{\"geography\": \"40 low-income countries\", \"producer\": \"IMF\"}",
+        "correct": "producer,geography",
+        "verbatim": "producer,geography"
+      },
+      {
+        "mention": "yearly data on firms ’ export transactions",
+        "gold": "{\"acronym\": \"ECI\", \"producer\": \"INE\"}",
+        "pred": "{\"acronym\": \"ECI\", \"producer\": \"INE\"}",
+        "correct": "producer,acronym",
+        "verbatim": "producer,acronym"
+      },
+      {
+        "mention": "Inspectorate 2011 data",
+        "gold": "{\"geography\": \"Tanzania\", \"year\": \"2011\"}",
+        "pred": "{\"geography\": \"Tanzania\", \"year\": \"2011\"}",
+        "correct": "year,geography",
+        "verbatim": "year,geography"
+      },
+      {
+        "mention": "2018 / 19 NLSS data",
+        "gold": "{\"acronym\": \"NLSS\", \"geography\": \"Nigeria\", \"producer\": \"National Bureau of Statistics\"}",
+        "pred": "{\"acronym\": \"NLSS\", \"geography\": \"Nigeria\", \"producer\": \"Nigeria ’ s National Bureau of Statistics ( NBS )\"}",
+        "correct": "geography,acronym",
+        "verbatim": "producer,geography,acronym"
+      },
+      {
+        "mention": "administrative panel data",
+        "gold": "{\"geography\": \"Colombia\"}",
+        "pred": "{\"geography\": \"Colombia\"}",
+        "correct": "geography",
+        "verbatim": "geography"
+      },
+      {
+        "mention": "India Human Development Survey ( IHDS )",
+        "gold": "{\"acronym\": \"( IHDS )\", \"geography\": \"India\"}",
+        "pred": "{\"acronym\": \"IHDS\", \"geography\": \"India\"}",
+        "correct": "geography",
+        "verbatim": "geography,acronym"
+      },
+      {
+        "mention": "OECD-PISA 2012 database",
+        "gold": "{\"producer\": \"OECD\", \"year\": \"2012\"}",
+        "pred": "{\"geography\": \"Vietnam\", \"producer\": \"OECD\", \"year\": \"2012\"}",
+        "correct": "producer,year",
+        "verbatim": "producer,year,geography"
+      },
+      {
+        "mention": "DHS figures on access",
+        "gold": "{\"year\": \"1997\"}",
+        "pred": "{\"year\": \"1997\"}",
+        "correct": "year",
+        "verbatim": "year"
+      },
+      {
+        "mention": "DHS surveys",
+        "gold": "{\"acronym\": \"DHS\", \"geography\": \"Nigeria\"}",
+        "pred": "{\"geography\": \"Nigeria, Cameroon, and Chad\"}",
+        "correct": "-",
+        "verbatim": "geography"
+      },
+      {
+        "mention": "International Country Risk Guide",
+        "gold": "{\"acronym\": \"( ICRG )\"}",
+        "pred": "{\"acronym\": \"ICRG\"}",
+        "correct": "-",
+        "verbatim": "acronym"
+      }
+    ]
+  }
+}