{
 "protocol": {
  "original": "Raha/Baran convention: value_normalizer (html-unescape + whitespace collapse) then raw string equality; P = exact-gold repairs / changed cells, R = exact-gold repairs / (dirty->clean diff); no churn-neutrality, no damage",
  "churn_neutral": "eval.run_real_multi.score \u2014 the scoring contract",
  "baran_rows": "oracle error positions + n gold labels, 3 seeds, seed-mean; reconstructed from repaired CSVs (no-op corrections vanish: |changed| lower-bounds Baran's output_size)",
  "movies_1": "first 2000 rows (_raha_pair), as everywhere in the suite"
 },
 "systems": {
  "grounded": {
   "per_dataset": [
    {
     "dataset": "hospital",
     "original": {
      "f1": 0.13683688646504708,
      "precision": 0.07829787234042553,
      "recall": 0.5422396856581533,
      "_errors": 509,
      "_changed": 3525,
      "_tp": 276
     },
     "churn_neutral": {
      "f1": 0.25794392523364484,
      "precision": 0.16922133660331085,
      "recall": 0.5422396856581533,
      "damage": 0.06623569852752552,
      "_errors": 509,
      "_changed": 1631,
      "_fixed": 276
     }
    },
    {
     "dataset": "beers",
     "original": {
      "f1": 0.010414997596539017,
      "precision": 0.007330551483026954,
      "recall": 0.017980636237897647,
      "_errors": 3615,
      "_changed": 8867,
      "_tp": 65
     },
     "churn_neutral": {
      "f1": 0.024956805528892304,
      "precision": 0.04077791718946048,
      "recall": 0.017980636237897647,
      "damage": 0.004568942072341583,
      "_errors": 3615,
      "_changed": 1594,
      "_fixed": 65
     }
    },
    {
     "dataset": "flights",
     "original": {
      "f1": 0.09555972482801753,
      "precision": 0.10213903743315508,
      "recall": 0.08977673325499412,
      "_errors": 4255,
      "_changed": 3740,
      "_tp": 382
     },
     "churn_neutral": {
      "f1": 0.12701579384871156,
      "precision": 0.21704545454545454,
      "recall": 0.08977673325499412,
      "damage": 0.08229861467419189,
      "_errors": 4255,
      "_changed": 1760,
      "_fixed": 382
     }
    },
    {
     "dataset": "rayyan",
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 948,
      "_changed": 4124,
      "_tp": 0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 0.1177875049741345,
      "_errors": 948,
      "_changed": 1274,
      "_fixed": 0
     }
    },
    {
     "dataset": "movies_1",
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 2224,
      "_changed": 4798,
      "_tp": 0
     },
     "churn_neutral": {
      "f1": 0.714254624591948,
      "precision": 0.6447937131630649,
      "recall": 0.8004878048780488,
      "damage": 0.024788732394366197,
      "_errors": 2050,
      "_changed": 2545,
      "_fixed": 1641
     }
    }
   ],
   "macro_f1_original": 0.048562321777920726,
   "macro_f1_churn_neutral": 0.22483422984063933
  },
  "openrefine_fingerprint": {
   "per_dataset": [
    {
     "dataset": "hospital",
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 509,
      "_changed": 0,
      "_tp": 0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 1.0,
      "recall": 0.0,
      "damage": 0.0,
      "_errors": 509,
      "_changed": 0,
      "_fixed": 0
     }
    },
    {
     "dataset": "beers",
     "original": {
      "f1": 0.18844596034960565,
      "precision": 0.4107806691449814,
      "recall": 0.122268326417704,
      "_errors": 3615,
      "_changed": 1076,
      "_tp": 442
     },
     "churn_neutral": {
      "f1": 0.1936897458369851,
      "precision": 0.4657534246575342,
      "recall": 0.122268326417704,
      "damage": 0.0,
      "_errors": 3615,
      "_changed": 949,
      "_fixed": 442
     }
    },
    {
     "dataset": "flights",
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 4255,
      "_changed": 8,
      "_tp": 0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 0.0,
      "_errors": 4255,
      "_changed": 6,
      "_fixed": 0
     }
    },
    {
     "dataset": "rayyan",
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 948,
      "_changed": 63,
      "_tp": 0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 0.0005968961400716276,
      "_errors": 948,
      "_changed": 17,
      "_fixed": 0
     }
    },
    {
     "dataset": "movies_1",
     "original": {
      "f1": 0.0021306818181818185,
      "precision": 0.005067567567567568,
      "recall": 0.0013489208633093526,
      "_errors": 2224,
      "_changed": 592,
      "_tp": 3
     },
     "churn_neutral": {
      "f1": 0.002271006813020439,
      "precision": 0.005067567567567568,
      "recall": 0.0014634146341463415,
      "damage": 0.01812206572769953,
      "_errors": 2050,
      "_changed": 592,
      "_fixed": 3
     }
    }
   ],
   "macro_f1_original": 0.03811532843355749,
   "macro_f1_churn_neutral": 0.03919215053000111
  },
  "openrefine_knn": {
   "per_dataset": [
    {
     "dataset": "hospital",
     "original": {
      "f1": 0.18923465096719932,
      "precision": 0.12038523274478331,
      "recall": 0.44204322200392926,
      "_errors": 509,
      "_changed": 1869,
      "_tp": 225
     },
     "churn_neutral": {
      "f1": 0.18923465096719932,
      "precision": 0.12038523274478331,
      "recall": 0.44204322200392926,
      "damage": 0.0829100610538197,
      "_errors": 509,
      "_changed": 1869,
      "_fixed": 225
     }
    },
    {
     "dataset": "beers",
     "original": {
      "f1": 0.0843618184658647,
      "precision": 0.09691313711414214,
      "recall": 0.07468879668049792,
      "_errors": 3615,
      "_changed": 2786,
      "_tp": 270
     },
     "churn_neutral": {
      "f1": 0.08606949314631814,
      "precision": 0.10154193305754043,
      "recall": 0.07468879668049792,
      "damage": 0.07391895567038347,
      "_errors": 3615,
      "_changed": 2659,
      "_fixed": 270
     }
    },
    {
     "dataset": "flights",
     "original": {
      "f1": 0.013764880952380954,
      "precision": 0.03300624442462088,
      "recall": 0.008695652173913044,
      "_errors": 4255,
      "_changed": 1121,
      "_tp": 37
     },
     "churn_neutral": {
      "f1": 0.013770003721622627,
      "precision": 0.03306523681858802,
      "recall": 0.008695652173913044,
      "damage": 0.06536685479733197,
      "_errors": 4255,
      "_changed": 1119,
      "_fixed": 37
     }
    },
    {
     "dataset": "rayyan",
     "original": {
      "f1": 0.001748251748251748,
      "precision": 0.00510204081632653,
      "recall": 0.0010548523206751054,
      "_errors": 948,
      "_changed": 196,
      "_tp": 1
     },
     "churn_neutral": {
      "f1": 0.0018001800180017999,
      "precision": 0.006134969325153374,
      "recall": 0.0010548523206751054,
      "damage": 0.007958615200955034,
      "_errors": 948,
      "_changed": 163,
      "_fixed": 1
     }
    },
    {
     "dataset": "movies_1",
     "original": {
      "f1": 0.0008737439930100482,
      "precision": 0.0008496176720475786,
      "recall": 0.0008992805755395684,
      "_errors": 2224,
      "_changed": 2354,
      "_tp": 2
     },
     "churn_neutral": {
      "f1": 0.0009082652134423252,
      "precision": 0.0008496176720475786,
      "recall": 0.000975609756097561,
      "damage": 0.07176838810641628,
      "_errors": 2050,
      "_changed": 2354,
      "_fixed": 2
     }
    }
   ],
   "macro_f1_original": 0.05799666922534136,
   "macro_f1_churn_neutral": 0.058356518613316845
  },
  "verified_union_v6_tau0.5": {
   "per_dataset": [
    {
     "dataset": "hospital",
     "original": {
      "f1": 0.5668016194331984,
      "precision": 0.9051724137931034,
      "recall": 0.412573673870334,
      "_errors": 509,
      "_changed": 232,
      "_tp": 210
     },
     "churn_neutral": {
      "f1": 0.5668016194331984,
      "precision": 0.9051724137931034,
      "recall": 0.412573673870334,
      "damage": 0.0010261146170027192,
      "_errors": 509,
      "_changed": 232,
      "_fixed": 210
     }
    },
    {
     "dataset": "beers",
     "original": {
      "f1": 0.03367875647668394,
      "precision": 0.2653061224489796,
      "recall": 0.017980636237897647,
      "_errors": 3615,
      "_changed": 245,
      "_tp": 65
     },
     "churn_neutral": {
      "f1": 0.034815211569362604,
      "precision": 0.5462184873949579,
      "recall": 0.017980636237897647,
      "damage": 0.0009790590155017678,
      "_errors": 3615,
      "_changed": 119,
      "_fixed": 65
     }
    },
    {
     "dataset": "flights",
     "original": {
      "f1": 0.03499047289104452,
      "precision": 0.06653491436100131,
      "recall": 0.02373678025851939,
      "_errors": 4255,
      "_changed": 1518,
      "_tp": 101
     },
     "churn_neutral": {
      "f1": 0.03500259920291111,
      "precision": 0.06662269129287599,
      "recall": 0.02373678025851939,
      "damage": 0.08045151359671626,
      "_errors": 4255,
      "_changed": 1516,
      "_fixed": 101
     }
    },
    {
     "dataset": "rayyan",
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 948,
      "_changed": 19,
      "_tp": 0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 0.001193792280143255,
      "_errors": 948,
      "_changed": 12,
      "_fixed": 0
     }
    },
    {
     "dataset": "movies_1",
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 2224,
      "_changed": 808,
      "_tp": 0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 0.02485133020344288,
      "_errors": 2050,
      "_changed": 808,
      "_fixed": 0
     }
    }
   ],
   "macro_f1_original": 0.12709416976018537,
   "macro_f1_churn_neutral": 0.1273238860410944
  },
  "baran_oracle0": {
   "per_dataset": [
    {
     "dataset": "hospital",
     "n_seeds": 3,
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 509.0,
      "_changed": 0.0,
      "_tp": 0.0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 0.004001847006310605,
      "_errors": 509.0,
      "_changed": 78.0,
      "_fixed": 0.0
     }
    },
    {
     "dataset": "beers",
     "n_seeds": 3,
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 3615.0,
      "_changed": 0.0,
      "_tp": 0.0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 5.439216752787599e-05,
      "_errors": 3615.0,
      "_changed": 1.0,
      "_fixed": 0.0
     }
    },
    {
     "dataset": "flights",
     "n_seeds": 3,
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 4255.0,
      "_changed": 0.0,
      "_tp": 0.0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 0.0,
      "_errors": 4255.0,
      "_changed": 31.0,
      "_fixed": 0.0
     }
    },
    {
     "dataset": "rayyan",
     "n_seeds": 3,
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 948.0,
      "_changed": 0.0,
      "_tp": 0.0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "damage": 0.009649820931157979,
      "_errors": 948.0,
      "_changed": 101.0,
      "_fixed": 0.0
     }
    },
    {
     "dataset": "movies_1",
     "n_seeds": 3,
     "original": {
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "_errors": 2224.0,
      "_changed": 0.0,
      "_tp": 0.0
     },
     "churn_neutral": {
      "f1": 0.0,
      "precision": 1.0,
      "recall": 0.0,
      "damage": 0.0,
      "_errors": 2050.0,
      "_changed": 0.0,
      "_fixed": 0.0
     }
    }
   ],
   "macro_f1_original": 0.0,
   "macro_f1_churn_neutral": 0.0
  },
  "baran_oracle5": {
   "per_dataset": [
    {
     "dataset": "hospital",
     "n_seeds": 3,
     "original": {
      "f1": 0.6551007568434103,
      "precision": 0.8789393797446383,
      "recall": 0.5225933202357563,
      "_errors": 509.0,
      "_changed": 302.3333333333333,
      "_tp": 266.0
     },
     "churn_neutral": {
      "f1": 0.5976770747168747,
      "precision": 0.6985601097769835,
      "recall": 0.5225933202357563,
      "damage": 0.004001847006310605,
      "_errors": 509.0,
      "_changed": 380.3333333333333,
      "_fixed": 266.0
     }
    },
    {
     "dataset": "beers",
     "n_seeds": 3,
     "original": {
      "f1": 0.7528420602908671,
      "precision": 0.7693038449137194,
      "recall": 0.7372982941447672,
      "_errors": 3615.0,
      "_changed": 3471.0,
      "_tp": 2665.3333333333335
     },
     "churn_neutral": {
      "f1": 0.7527357363043982,
      "precision": 0.7690817230910615,
      "recall": 0.7372982941447672,
      "damage": 5.439216752787599e-05,
      "_errors": 3615.0,
      "_changed": 3472.0,
      "_fixed": 2665.3333333333335
     }
    },
    {
     "dataset": "flights",
     "n_seeds": 3,
     "original": {
      "f1": 0.9716621879411721,
      "precision": 0.9907413332218985,
      "recall": 0.9533098315707011,
      "_errors": 4255.0,
      "_changed": 4094.3333333333335,
      "_tp": 4056.3333333333335
     },
     "churn_neutral": {
      "f1": 0.9715071390508543,
      "precision": 0.990419230744583,
      "recall": 0.9533098315707011,
      "damage": 0.0,
      "_errors": 4255.0,
      "_changed": 4095.6666666666665,
      "_fixed": 4056.3333333333335
     }
    },
    {
     "dataset": "rayyan",
     "n_seeds": 3,
     "original": {
      "f1": 0.13519068028219686,
      "precision": 0.20218769308487497,
      "recall": 0.10654008438818564,
      "_errors": 948.0,
      "_changed": 497.0,
      "_tp": 101.0
     },
     "churn_neutral": {
      "f1": 0.126498861945361,
      "precision": 0.16496175881858396,
      "recall": 0.10654008438818564,
      "damage": 0.009649820931157979,
      "_errors": 948.0,
      "_changed": 598.0,
      "_fixed": 101.0
     }
    },
    {
     "dataset": "movies_1",
     "n_seeds": 3,
     "original": {
      "f1": 0.06651272736623408,
      "precision": 0.38518756403094745,
      "recall": 0.036420863309352514,
      "_errors": 2224.0,
      "_changed": 207.66666666666666,
      "_tp": 81.0
     },
     "churn_neutral": {
      "f1": 0.07163022789809685,
      "precision": 0.38518756403094745,
      "recall": 0.03951219512195122,
      "damage": 0.0004694835680751174,
      "_errors": 2050.0,
      "_changed": 207.66666666666666,
      "_fixed": 81.0
     }
    }
   ],
   "macro_f1_original": 0.5162616825447761,
   "macro_f1_churn_neutral": 0.504009807983117
  },
  "baran_oracle20": {
   "per_dataset": [
    {
     "dataset": "hospital",
     "n_seeds": 3,
     "original": {
      "f1": 0.891377318613051,
      "precision": 0.9080916290693971,
      "recall": 0.8755730189914867,
      "_errors": 509.0,
      "_changed": 491.0,
      "_tp": 445.6666666666667
     },
     "churn_neutral": {
      "f1": 0.8268680969233834,
      "precision": 0.7834926241010877,
      "recall": 0.8755730189914867,
      "damage": 0.004001847006310605,
      "_errors": 509.0,
      "_changed": 569.0,
      "_fixed": 445.6666666666667
     }
    },
    {
     "dataset": "beers",
     "n_seeds": 3,
     "original": {
      "f1": 0.9182707659114943,
      "precision": 0.9265731396812268,
      "recall": 0.9101890272014753,
      "_errors": 3615.0,
      "_changed": 3552.0,
      "_tp": 3290.3333333333335
     },
     "churn_neutral": {
      "f1": 0.9181426425323927,
      "precision": 0.9263122436892998,
      "recall": 0.9101890272014753,
      "damage": 5.439216752787599e-05,
      "_errors": 3615.0,
      "_changed": 3553.0,
      "_fixed": 3290.3333333333335
     }
    },
    {
     "dataset": "flights",
     "n_seeds": 3,
     "original": {
      "f1": 1.0,
      "precision": 1.0,
      "recall": 1.0,
      "_errors": 4255.0,
      "_changed": 4255.0,
      "_tp": 4255.0
     },
     "churn_neutral": {
      "f1": 1.0,
      "precision": 1.0,
      "recall": 1.0,
      "damage": 0.0,
      "_errors": 4255.0,
      "_changed": 4255.0,
      "_fixed": 4255.0
     }
    },
    {
     "dataset": "rayyan",
     "n_seeds": 3,
     "original": {
      "f1": 0.4301217203128622,
      "precision": 0.5555661159663857,
      "recall": 0.3537271448663854,
      "_errors": 948.0,
      "_changed": 641.0,
      "_tp": 335.3333333333333
     },
     "churn_neutral": {
      "f1": 0.4023919283154553,
      "precision": 0.4732993292645808,
      "recall": 0.35232067510548526,
      "damage": 0.009649820931157979,
      "_errors": 948.0,
      "_changed": 741.6666666666666,
      "_fixed": 334.0
     }
    },
    {
     "dataset": "movies_1",
     "n_seeds": 3,
     "original": {
      "f1": 0.8708333112632177,
      "precision": 0.9376290775283244,
      "recall": 0.8129496402877697,
      "_errors": 2224.0,
      "_changed": 1928.3333333333333,
      "_tp": 1808.0
     },
     "churn_neutral": {
      "f1": 0.9089214230345909,
      "precision": 0.9376290775283244,
      "recall": 0.8819512195121951,
      "damage": 0.0005007824726134585,
      "_errors": 2050.0,
      "_changed": 1928.3333333333333,
      "_fixed": 1808.0
     }
    }
   ],
   "macro_f1_original": 0.822120623220125,
   "macro_f1_churn_neutral": 0.8112648181611645
  }
 },
 "acceptance": {
  "vs": "raha_per_dataset.json",
  "n_cells": 100,
  "pass": true
 },
 "kendall_tau_b": {
  "primary": {
   "systems": [
    "grounded",
    "verified_union_v6_tau0.5",
    "openrefine_fingerprint",
    "openrefine_knn",
    "baran_oracle20"
   ],
   "per_dataset": {
    "hospital": 0.8,
    "beers": 1.0,
    "flights": 1.0,
    "rayyan": 1.0,
    "movies_1": 0.5270462766947299
   },
   "macro": 0.6
  },
  "extended": {
   "systems": [
    "grounded",
    "verified_union_v6_tau0.5",
    "openrefine_fingerprint",
    "openrefine_knn",
    "baran_oracle20",
    "baran_oracle0",
    "baran_oracle5"
   ],
   "per_dataset": {
    "hospital": 0.9,
    "beers": 1.0,
    "flights": 1.0,
    "rayyan": 1.0,
    "movies_1": 0.6324555320336759
   },
   "macro": 0.8095238095238095
  }
 },
 "calibration": {
  "source": "Mahdavi & Abedjan, PVLDB 13(12) p1948, Table 3 row 'Baran' (no TL): complete error set given (oracle detection), budget 20, mean of 10 runs; PDF read 2026-06-12",
  "notes": "their runs: full datasets, 10 label seeds, Wikipedia value models available in package but Table-3 row is without TL; ours: 3 label seeds, no pretraining, movies_1 not in their paper; our churn-neutral macro for this row is the paper's 0.811",
  "rows": [
   {
    "dataset": "hospital",
    "published_f1": 0.87,
    "published_precision": 0.88,
    "published_recall": 0.86,
    "repro_f1": 0.891377318613051,
    "repro_precision": 0.9080916290693971,
    "repro_recall": 0.8755730189914867,
    "delta_f1": 0.021377318613051033
   },
   {
    "dataset": "flights",
    "published_f1": 1.0,
    "published_precision": 1.0,
    "published_recall": 1.0,
    "repro_f1": 1.0,
    "repro_precision": 1.0,
    "repro_recall": 1.0,
    "delta_f1": 0.0
   },
   {
    "dataset": "beers",
    "published_f1": 0.9,
    "published_precision": 0.91,
    "published_recall": 0.89,
    "repro_f1": 0.9182707659114943,
    "repro_precision": 0.9265731396812268,
    "repro_recall": 0.9101890272014753,
    "delta_f1": 0.018270765911494236
   },
   {
    "dataset": "rayyan",
    "published_f1": 0.52,
    "published_precision": 0.76,
    "published_recall": 0.4,
    "repro_f1": 0.4301217203128622,
    "repro_precision": 0.5555661159663857,
    "repro_recall": 0.3537271448663854,
    "delta_f1": -0.0898782796871378
   }
  ]
 }
}