{ "benchmark_name": "DataForge-Bench-light-verified", "benchmark_seeds": [ 0, 1, 2 ], "failure_samples": [ { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "172 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35957", "_row": "55", "index": "56" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boxz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "120 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-xb", "ZipCode": "35957", "_row": "56", "index": "57" }, { "Address1": "2505xuxsxhighwayx431xnorth", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2x6x938310", "ProviderNumber": "10005", "Sample": "98 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-4", "ZipCode": "35957", "_row": "57", "index": "58" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-5c", "MeasureName": "pneumonia patients given initial antibiotic(s) within 6 hours after arrival", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "211 patients", "Score": "96%", "State": "al", "Stateavg": "al_pn-5c", "ZipCode": "35957", "_row": "58", "index": "59" } ], "task_id": "DataForge-Bench-light-verified:hospital:10000:0000", "task_index": 1, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "ground_truth_count": 12, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1651", "act_arr_time": "", "act_dep_time": "", "flight": "AA-1544-SAN-ORD", "sched_arr_time": "11/30 5:25 p.m.", "sched_dep_time": "11/30 11:25 a.m.", "src": "ord", "tuple_id": "1652" }, { "_row": "1652", "act_arr_time": "", "act_dep_time": "", "flight": "AA-4344-ORD-DTW", "sched_arr_time": "2:00 p.m.", "sched_dep_time": "11:45 a.m.", "src": "ord", "tuple_id": "1653" }, { "_row": "1653", "act_arr_time": "", "act_dep_time": "", "flight": "AA-1733-ORD-PHX", "sched_arr_time": "11/30 10:30 p.m.", "sched_dep_time": "11/30 7:45 p.m.", "src": "ord", "tuple_id": "1654" }, { "_row": "1654", "act_arr_time": "", "act_dep_time": "", "flight": "AA-4198-ORD-CLE", "sched_arr_time": "12:55 p.m.", "sched_dep_time": "10:40 a.m.", "src": "ord", "tuple_id": "1655" } ], "task_id": "DataForge-Bench-light-verified:flights:10004:0001", "task_index": 2, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "ground_truth_count": 5, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "657", "abv": "0.052", "beer-name": "Lava Lake Wit", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "15", "id": "685", "index": "658", "ounces": "12.0 oz.", "state": "CO", "style": "Witbier" }, { "_row": "658", "abv": "0.06", "beer-name": "Mountain Livin' Pale Ale", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "N/A", "id": "613", "index": "659", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Ale (APA)" }, { "_row": "659", "abv": "0.052", "beer-name": "Crazy Mountain Amber Ale", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "25", "id": "356", "index": "660", "ounces": "12.0 ounce", "state": "CO", "style": "American Amber / Red Ale" }, { "_row": "660", "abv": "0.065", "beer-name": "Tropicalia", "brewery-name": "Creature Comforts", "brewery_id": "247", "city": "Athens", "ibu": "65", "id": "2029", "index": "661", "ounces": "12.0 oz.", "state": "GA", "style": "American IPA" } ], "task_id": "DataForge-Bench-light-verified:beers:10008:0002", "task_index": 3, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "201 pine sxreex norxhwesx", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-card-2", "MeasureName": "surgery patients who were taking heart drugs called beta blockers before coming to the hospital who were kept on the beta blockers during the period just before and after their surgery", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "3 patients", "Score": "67%", "State": "al", "Stateavg": "al_scip-card-2", "ZipCode": "35640", "_row": "161", "index": "162" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "xroprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-1", "MeasureName": "surgery patients who were given an antibiotic at the right time (within one hour before surgery) to help prevent infection", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "6 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-inf-1", "ZipCode": "35640", "_row": "162", "index": "163" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-2", "MeasureName": "surgery patients who were given the right kind of antibiotic to help prevent infection", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "7 patients", "Score": "x7%", "State": "al", "Stateavg": "al_scip-inf-2", "ZipCode": "35640", "_row": "163", "index": "164" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "pxopxietaxy", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-3", "MeasureName": "surgery patients whose preventive antibiotics were stopped at the right time (within 24 hours after surgery)", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "5 patients", "Score": "80%", "State": "al", "Stateavg": "al_scip-inf-3", "ZipCode": "35640", "_row": "164", "index": "165" } ], "task_id": "DataForge-Bench-light-verified:hospital:10009:0003", "task_index": 4, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1589", "abv": "0.08", "beer-name": "Old Chub (2003)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "35", "id": "785", "index": "1590", "ounces": "12.0 oz", "state": "CO", "style": "Scottish Ale" }, { "_row": "1590", "abv": "0.065", "beer-name": "Dale's Pale Ale (2008)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "65", "id": "745", "index": "1591", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Ale (APA)" }, { "_row": "1591", "abv": "0.065%", "beer-name": "Dale's Pale Ale (2006)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons CO", "ibu": "65", "id": "744", "index": "1592", "ounces": "12.0 oz.", "state": "", "style": "American Pale Ale (APA)" }, { "_row": "1592", "abv": "0.065", "beer-name": "Dale's Pale Ale (2004)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "65", "id": "743", "index": "1593", "ounces": "12.0 oz", "state": "CO", "style": "American Pale Ale (APA)" } ], "task_id": "DataForge-Bench-light-verified:beers:10017:0005", "task_index": 6, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "ground_truth_count": 2, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-4", "MeasureName": "all heart surgery patients whose blood sugar (blood glucose) is kept under good control in the days right after surgery", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "0 patients", "Score": "empty", "State": "al", "Stateavg": "al_scip-inf-4", "ZipCode": "35960", "_row": "512", "index": "513" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "chxrokxx", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-6", "MeasureName": "surgery patients needing hair removed from the surgical area before surgery who had hair removed using a safer method (electric clippers or hair removal cream c not a razor)", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "6 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-inf-6", "ZipCode": "35960", "_row": "513", "index": "514" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-vte-1", "MeasureName": "surgery patients whose doctors ordered treatments to prevent blood clots after certain types of surgeries", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "5 patients", "Score": "1xx%", "State": "al", "Stateavg": "al_scip-vte-1", "ZipCode": "35960", "_row": "514", "index": "515" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-vte-2", "MeasureName": "patients who got treatment at the right time (within 24 hours before or after their surgery) to help prevent blood clots after certain types of surgery", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "5 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-vte-2", "ZipCode": "35960", "_row": "515", "index": "516" } ], "task_id": "DataForge-Bench-light-verified:hospital:10018:0006", "task_index": 7, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "ground_truth_count": 8, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "324", "act_arr_time": "2:49 p.m.", "act_dep_time": "1:20 p.m.", "flight": "AA-789-ORD-DEN", "sched_arr_time": "2:35 p.m.", "sched_dep_time": "1:05 p.m.", "src": "weather", "tuple_id": "325" }, { "_row": "325", "act_arr_time": "1:23 p.m.", "act_dep_time": "8:07 a.m.", "flight": "UA-854-SFO-IAH", "sched_arr_time": "1:39 p.m.", "sched_dep_time": "7:53 a.m.", "src": "weather", "tuple_id": "326" }, { "_row": "326", "act_arr_time": "", "act_dep_time": "", "flight": "AA-415-BOS-ORD", "sched_arr_time": "10:40 a.m.", "sched_dep_time": "8:35 a.m.", "src": "weather", "tuple_id": "327" }, { "_row": "327", "act_arr_time": "", "act_dep_time": "", "flight": "AA-85-JFK-SFO", "sched_arr_time": "6:30 p.m.", "sched_dep_time": "3:05 p.m.", "src": "weather", "tuple_id": "328" } ], "task_id": "DataForge-Bench-light-verified:flights:10022:0007", "task_index": 8, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "ground_truth_count": 8, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "379", "abv": "0.045", "beer-name": "SummerBright Ale", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver", "ibu": "15", "id": "397", "index": "380", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Wheat Ale" }, { "_row": "380", "abv": "0.062%", "beer-name": "Lucky U IPA", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver", "ibu": "68", "id": "193", "index": "381", "ounces": "12.0 ounce", "state": "CO", "style": "American IPA" }, { "_row": "381", "abv": "0.054", "beer-name": "Avalanche Ale", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver CO", "ibu": "19", "id": "83", "index": "382", "ounces": "12.0 oz", "state": "", "style": "American Amber / Red Ale" }, { "_row": "382", "abv": "0.05", "beer-name": "You're My Boy, Blue", "brewery-name": "Brew Bus Brewing", "brewery_id": "308", "city": "Tampa", "ibu": "N/A", "id": "1802", "index": "383", "ounces": "12.0 oz.", "state": "FL", "style": "Fruit / Vegetable Beer" } ], "task_id": "DataForge-Bench-light-verified:beers:10026:0008", "task_index": 9, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"_row\":655,\"column\":\"act_arr_time\",\"new_value\":\"9:05 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":656,\"column\":\"act_arr_time\",\"new_value\":\"5:03 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":657,\"column\":\"act_arr_time\",\"new_value\":\"10:27 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":658,\"column\":\"act_arr_time\",\"new_value\":\"10:56 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "ground_truth_count": 9, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "655", "act_arr_time": "9:05 a.m.", "act_dep_time": "6:09 a.m.", "flight": "AA-466-IAH-MIA", "sched_arr_time": "9:20 a.m.", "sched_dep_time": "6:00 a.m.", "src": "panynj", "tuple_id": "656" }, { "_row": "656", "act_arr_time": "5:03 p.m.", "act_dep_time": "11:45 a.m.", "flight": "UA-3050-PHX-CLT", "sched_arr_time": "4:57 p.m.", "sched_dep_time": "11:05 a.m.", "src": "panynj", "tuple_id": "657" }, { "_row": "657", "act_arr_time": "10:27 a.m.", "act_dep_time": "6:17 a.m.", "flight": "CO-89-IAH-EWR", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "658" }, { "_row": "658", "act_arr_time": "10:56 a.m.", "act_dep_time": "9:01 a.m.", "flight": "CO-1088-CLE-IAH", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "659" } ], "task_id": "DataForge-Bench-light-verified:flights:10031:0010", "task_index": 11, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "ground_truth_count": 8, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "2211", "abv": "0.05", "beer-name": "Triangle White Ale", "brewery-name": "Triangle Brewing Company", "brewery_id": "524", "city": "Durham", "ibu": "N/A", "id": "114", "index": "2212", "ounces": "12.0 ounce", "state": "NC", "style": "Witbier" }, { "_row": "2212", "abv": "0.08%", "beer-name": "Triangle Belgian Golden Ale", "brewery-name": "Triangle Brewing Company", "brewery_id": "524", "city": "Durham", "ibu": "N/A", "id": "113", "index": "2213", "ounces": "12.0 ounce", "state": "NC", "style": "Belgian Strong Pale Ale" }, { "_row": "2213", "abv": "0.082", "beer-name": "Troegenator", "brewery-name": "Tr\u00f6egs Brewing Company", "brewery_id": "97", "city": "Hershey", "ibu": "N/A", "id": "2445", "index": "2214", "ounces": "16.0 ounce", "state": "PA", "style": "Doppelbock" }, { "_row": "2214", "abv": "0.075", "beer-name": "Nugget Nectar", "brewery-name": "Tr\u00f6egs Brewing Company", "brewery_id": "97", "city": "Hershey", "ibu": "93", "id": "2444", "index": "2215", "ounces": "16.0 ounce", "state": "PA", "style": "American Amber / Red Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10035:0011", "task_index": 12, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "ground_truth_count": 3, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "chxrokxx", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "44 paxienxs", "Score": "93%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35960", "_row": "502", "index": "503" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "33 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-3b", "ZipCode": "35960", "_row": "503", "index": "504" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "20 patients", "Score": "100%", "State": "ax", "Stateavg": "al_pn-4", "ZipCode": "35960", "_row": "504", "index": "505" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-5c", "MeasureName": "pneumonia patients given initial antibiotic(s) within 6 hours after arrival", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "56 patients", "Score": "89%", "State": "al", "Stateavg": "al_pn-5c", "ZipCode": "35960", "_row": "505", "index": "506" } ], "task_id": "DataForge-Bench-light-verified:hospital:10036:0012", "task_index": 13, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 10 }, "fn": 10, "fp": 0, "ground_truth_count": 10, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "679", "act_arr_time": "7:53 p.m.", "act_dep_time": "5:09 p.m.", "flight": "AA-1007-MIA-PHX", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "680" }, { "_row": "680", "act_arr_time": "6:35 p.m.", "act_dep_time": "4:46 p.m.", "flight": "UA-2906-PHL-MCO", "sched_arr_time": "6:23 p.m.", "sched_dep_time": "3:50 p.m.", "src": "panynj", "tuple_id": "681" }, { "_row": "681", "act_arr_time": "7:36 p.m.", "act_dep_time": "4:37 p.m.", "flight": "UA-938-DEN-ORD", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "682" }, { "_row": "682", "act_arr_time": "9:49 p.m.", "act_dep_time": "8:23 p.m.", "flight": "AA-1221-MCO-ORD", "sched_arr_time": "9:45 p.m.", "sched_dep_time": "8:00 p.m.", "src": "panynj", "tuple_id": "683" } ], "task_id": "DataForge-Bench-light-verified:flights:10040:0013", "task_index": 14, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "739", "abv": "0.043", "beer-name": "Dry Dock Hefeweizen", "brewery-name": "Dry Dock Brewing Company", "brewery_id": "496", "city": "Aurora", "ibu": "12", "id": "872", "index": "740", "ounces": "12.0 oz", "state": "CO", "style": "Hefeweizen" }, { "_row": "740", "abv": "0.057999999999999996%", "beer-name": "Dry Dock Amber Ale", "brewery-name": "Dry Dock Brewing Company", "brewery_id": "496", "city": "Aurora", "ibu": "49", "id": "871", "index": "741", "ounces": "12.0 ounce", "state": "CO", "style": "American Amber / Red Ale" }, { "_row": "741", "abv": "0.061%", "beer-name": "Category 3 IPA", "brewery-name": "Due South Brewing Company", "brewery_id": "340", "city": "Boynton Beach", "ibu": "64", "id": "1685", "index": "742", "ounces": "12.0 ounce", "state": "FL", "style": "American IPA" }, { "_row": "742", "abv": "0.045%", "beer-name": "Dundee Summer Wheat Beer", "brewery-name": "Dundee Brewing Company", "brewery_id": "538", "city": "Rochester", "ibu": "18", "id": "457", "index": "743", "ounces": "12.0 oz.", "state": "NY", "style": "American Pale Wheat Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10044:0014", "task_index": 15, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart attack", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ ox south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "ami-7a", "MeasureName": "heart attack patients given fibrinolytic medication within 30 minutes of arrival", "PhoneNumber": "25x47x7xx0", "ProviderNumber": "10087", "Sample": "0 patients", "Score": "empty", "State": "al", "Stateavg": "al_ami-7a", "ZipCode": "36617", "_row": "384", "index": "385" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart attack", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "xniv of soxth alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "ami-8a", "MeasureName": "heart attack patients given pci within 90 minutes of arrival", "PhoneNumber": "2514717110", "ProviderNumber": "10087", "Sample": "16 patients", "Score": "50%", "State": "al", "Stateavg": "al_ami-8a", "ZipCode": "36617", "_row": "385", "index": "386" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "hxart failurx", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ of south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "hf-1", "MeasureName": "heart failure patients given discharge instructions", "PhoneNumber": "25x47x7xx0", "ProviderNumber": "10087", "Sample": "146 patients", "Score": "83%", "State": "xl", "Stateavg": "al_hf-1", "ZipCode": "36617", "_row": "386", "index": "387" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart failure", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ of south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "hf-2", "MeasureName": "heart failure patients given an evaluation of left ventricular systolic (lvs) function", "PhoneNumber": "2514717110", "ProviderNumber": "10087", "Sample": "150 patients", "Score": "98%", "State": "al", "Stateavg": "al_hx-2", "ZipCode": "36617", "_row": "387", "index": "388" } ], "task_id": "DataForge-Bench-light-verified:hospital:10045:0015", "task_index": 16, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1215", "act_arr_time": "3:13 p.m.", "act_dep_time": "1:19 p.m.", "flight": "AA-789-ORD-DEN", "sched_arr_time": "2:35 p.m.", "sched_dep_time": "1:05 p.m.", "src": "usatoday", "tuple_id": "1216" }, { "_row": "1216", "act_arr_time": "1:30 p.m.", "act_dep_time": "7:49 a.m.", "flight": "UA-854-SFO-IAH", "sched_arr_time": "1:39 p.m.", "sched_dep_time": "7:53 a.m.", "src": "usatoday", "tuple_id": "1217" }, { "_row": "1217", "act_arr_time": "6:36 p.m.", "act_dep_time": "3:38 p.m.", "flight": "AA-85-JFK-SFO", "sched_arr_time": "6:30 p.m.", "sched_dep_time": "3:05 p.m.", "src": "usatoday", "tuple_id": "1218" }, { "_row": "1218", "act_arr_time": "7:35 p.m.", "act_dep_time": "6:56 p.m.", "flight": "AA-1640-MIA-MCO", "sched_arr_time": "7:25 p.m.", "sched_dep_time": "6:30 p.m.", "src": "usatoday", "tuple_id": "1219" } ], "task_id": "DataForge-Bench-light-verified:flights:10049:0016", "task_index": 17, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "ground_truth_count": 9, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "123", "abv": "0.05%", "beer-name": "Angry Orchard Crisp Apple", "brewery-name": "Angry Orchard Cider Company", "brewery_id": "434", "city": "Cincinnati", "ibu": "N/A", "id": "1293", "index": "124", "ounces": "16.0 oz.", "state": "OH", "style": "Cider" }, { "_row": "124", "abv": "0.05", "beer-name": "Angry Orchard Crisp Apple", "brewery-name": "Angry Orchard Cider Company", "brewery_id": "434", "city": "Cincinnati", "ibu": "N/A", "id": "1292", "index": "125", "ounces": "12.0 OZ.", "state": "OH", "style": "Cider" }, { "_row": "125", "abv": "0.068", "beer-name": "Golden One", "brewery-name": "Anthem Brewing Company", "brewery_id": "193", "city": "Oklahoma City", "ibu": "N/A", "id": "2207", "index": "126", "ounces": "12.0 oz.", "state": "OK", "style": "Belgian Pale Ale" }, { "_row": "126", "abv": "0.06", "beer-name": "Arjuna", "brewery-name": "Anthem Brewing Company", "brewery_id": "193", "city": "Oklahoma City", "ibu": "N/A", "id": "2040", "index": "127", "ounces": "12.0 ounce", "state": "OK", "style": "Witbier" } ], "task_id": "DataForge-Bench-light-verified:beers:10053:0017", "task_index": 18, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "ground_truth_count": 2, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "fxorence", "Condition": "heart failure", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "hf-4", "MeasureName": "heart failure patients given smoking cessation advice/counseling", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "53 patients", "Score": "100%", "State": "al", "Stateavg": "al_hf-4", "ZipCode": "35631", "_row": "79", "index": "80" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "174 patients", "Score": "90%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35631", "_row": "80", "index": "81" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2567688400", "ProviderNumber": "1000x", "Sample": "137 patients", "Score": "95%", "State": "al", "Stateavg": "al_pn-3b", "ZipCode": "35631", "_row": "81", "index": "82" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "65 patients", "Score": "100%", "State": "al", "Stateavg": "al_pn-4", "ZipCode": "35631", "_row": "82", "index": "83" } ], "task_id": "DataForge-Bench-light-verified:hospital:10054:0018", "task_index": 19, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "ground_truth_count": 5, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "909", "abv": "0.093", "beer-name": "Snake Handler Double IPA", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "103", "id": "312", "index": "910", "ounces": "12.0 oz", "state": "AL", "style": "American Double / Imperial IPA" }, { "_row": "910", "abv": "0.06", "beer-name": "Coffee Oatmeal Stout", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "54", "id": "311", "index": "911", "ounces": "12.0 oz.", "state": "AL", "style": "Oatmeal Stout" }, { "_row": "911", "abv": "0.06", "beer-name": "Good People IPA", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "64", "id": "309", "index": "912", "ounces": "12.0 ounce", "state": "AL", "style": "American IPA" }, { "_row": "912", "abv": "0.057999999999999996%", "beer-name": "Good People American Brown Ale", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "36", "id": "308", "index": "913", "ounces": "12.0 ounce", "state": "AL", "style": "American Brown Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10062:0020", "task_index": 21, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1 }, "fn": 1, "fp": 0, "ground_truth_count": 1, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "315 w hickory st", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "pneumonia", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-6", "MeasureName": "pneumonia patients given the most appropriate initial antibiotic(s)", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "57 patients", "Score": "96%", "State": "al", "Stateavg": "al_pn-6", "ZipCode": "35150", "_row": "429", "index": "430" }, { "Address1": "315 w hickory st", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "pneumonia", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-7", "MeasureName": "pneumonia patients assessed and given influenza vaccination", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "51 patients", "Score": "88%", "State": "al", "Stateavg": "al_pn-7", "ZipCode": "35150", "_row": "430", "index": "431" }, { "Address1": "315xwxhickoryxst", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "surgical infection prevention", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "scip-card-2", "MeasureName": "surgery patients who were taking heart drugs called beta blockers before coming to the hospital who were kept on the beta blockers during the period just before and after their surgery", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "22 patients", "Score": "82%", "State": "al", "Stateavg": "al_scip-card-2", "ZipCode": "35150", "_row": "431", "index": "432" }, { "Address1": "2801 debarr road", "Address2": "empty", "Address3": "empty", "City": "anchorage", "Condition": "heart failure", "CountyName": "anchorage", "EmergencyService": "yes", "HospitalName": "alaska regional hospital", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "hf-1", "MeasureName": "heart failure patients given discharge instructions", "PhoneNumber": "9072761131", "ProviderNumber": "20017", "Sample": "57 patients", "Score": "91%", "State": "ak", "Stateavg": "ak_hf-1", "ZipCode": "99508", "_row": "432", "index": "433" } ], "task_id": "DataForge-Bench-light-verified:hospital:10063:0021", "task_index": 22, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 16 }, "fn": 16, "fp": 0, "ground_truth_count": 16, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "58", "act_arr_time": "4:54 p.m.", "act_dep_time": "11:41 a.m.", "flight": "AA-1544-SAN-ORD", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "59" }, { "_row": "59", "act_arr_time": "11:11 a.m.", "act_dep_time": "10:53 a.m.", "flight": "UA-5487-SFO-MRY", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "60" }, { "_row": "60", "act_arr_time": "12/2/11 6:35 a.m.", "act_dep_time": "11:29 p.m.", "flight": "AA-204-LAX-MCO", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "61" }, { "_row": "61", "act_arr_time": "7:04 a.m.", "act_dep_time": "6:10 a.m.", "flight": "UA-414-LAX-SFO", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "62" } ], "task_id": "DataForge-Bench-light-verified:flights:10067:0022", "task_index": 23, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "ground_truth_count": 5, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1325", "abv": "0.053%", "beer-name": "Shark Bait", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "11", "id": "1508", "index": "1326", "ounces": "12.0 oz", "state": "FL", "style": "Fruit / Vegetable Beer" }, { "_row": "1326", "abv": "0.053", "beer-name": "Gator Tail Brown Ale", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "30", "id": "1507", "index": "1327", "ounces": "12.0 ounce", "state": "FL", "style": "American Brown Ale" }, { "_row": "1327", "abv": "0.071", "beer-name": "Miami Vice IPA", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "62", "id": "1506", "index": "1328", "ounces": "12.0 oz.", "state": "FL", "style": "American IPA" }, { "_row": "1328", "abv": "0.053", "beer-name": "Big Rod Coconut Ale", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "16", "id": "1325", "index": "1329", "ounces": "12.0 oz", "state": "FL", "style": "American Blonde Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10071:0023", "task_index": 24, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "ground_truth_count": 12, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1523", "act_arr_time": "1:11 p.m.", "act_dep_time": "12:07 p.m.", "flight": "UA-2945-PHL-CLT", "sched_arr_time": "", "sched_dep_time": "", "src": "flytecomm", "tuple_id": "1524" }, { "_row": "1524", "act_arr_time": "", "act_dep_time": "", "flight": "CO-63-EWR-IAH", "sched_arr_time": "8:22 p.m.", "sched_dep_time": "5:25 p.m.", "src": "flytecomm", "tuple_id": "1525" }, { "_row": "1525", "act_arr_time": "", "act_dep_time": "", "flight": "CO-47-IAH-LAX", "sched_arr_time": "8:56 p.m.", "sched_dep_time": "7:10 p.m.", "src": "flytecomm", "tuple_id": "1526" }, { "_row": "1526", "act_arr_time": "2:39 p.m.", "act_dep_time": "11:06 a.m.", "flight": "AA-2050-ORD-MIA", "sched_arr_time": "", "sched_dep_time": "", "src": "flytecomm", "tuple_id": "1527" } ], "task_id": "DataForge-Bench-light-verified:flights:10076:0025", "task_index": 26, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "ground_truth_count": 8, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "299", "abv": "", "beer-name": "N Street Drive-In 50th Anniversary IPA", "brewery-name": "Blue Blood Brewing Company", "brewery_id": "380", "city": "Lincoln", "ibu": "N/A", "id": "1025", "index": "300", "ounces": "12.0 ounce", "state": "NE", "style": "American Double / Imperial IPA" }, { "_row": "300", "abv": "0.05", "beer-name": "467 Ethan's Stout", "brewery-name": "Blue Blood Brewing Company", "brewery_id": "380", "city": "Lincoln", "ibu": "N/A", "id": "1012", "index": "301", "ounces": "12.0 ounce", "state": "NE", "style": "American Stout" }, { "_row": "301", "abv": "0.064", "beer-name": "1335 Wicked Snout", "brewery-name": "Blue Blood Brewing Company", "brewery_id": "380", "city": "Lincoln", "ibu": "N/A", "id": "942", "index": "302", "ounces": "12.0 ounce", "state": "NE", "style": "Saison / Farmhouse Ale" }, { "_row": "302", "abv": "0.045", "beer-name": "543 Skull Creek Fresh Hopped Pale Ale", "brewery-name": "Blue Blood Brewing Company", "brewery_id": "380", "city": "Lincoln", "ibu": "N/A", "id": "937", "index": "303", "ounces": "12.0 oz.", "state": "NE", "style": "American Pale Ale (APA)" } ], "task_id": "DataForge-Bench-light-verified:beers:10080:0026", "task_index": 27, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "ground_truth_count": 3, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "heart attack", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "ami-2", "MeasureName": "heart attack patients given aspirin at discharge", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "247 patients", "Score": "96%", "State": "al", "Stateavg": "al_ami-2", "ZipCode": "3563x", "_row": "70", "index": "71" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "heart attack", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "ami-3", "MeasureName": "heart attack patients given ace inhibitor or arb for left ventricular systolic dysfunction (lvsd)", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "43 patients", "Score": "93%", "State": "al", "Stateavg": "al_ami-3", "ZipCode": "35631", "_row": "71", "index": "72" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "heart attack", "CountyName": "laudxrdalx", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "ami-4", "MeasureName": "heart attack patients given smoking cessation advice/counseling", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "103 patients", "Score": "100%", "State": "al", "Stateavg": "al_ami-4", "ZipCode": "35631", "_row": "72", "index": "73" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "hearx axxack", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "ami-5", "MeasureName": "heart attack patients given beta blocker at discharge", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "227 patients", "Score": "96%", "State": "al", "Stateavg": "al_ami-5", "ZipCode": "35631", "_row": "73", "index": "74" } ], "task_id": "DataForge-Bench-light-verified:hospital:10081:0027", "task_index": 28, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"row\":521,\"column\":\"act_arr_time\",\"new_value\":\"3:01 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":522,\"column\":\"act_arr_time\",\"new_value\":\"10:10 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":523,\"column\":\"act_arr_time\",\"new_value\":\"6:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":524,\"column\":\"act_arr_time\",\"new_value\":\"7:57 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "521", "act_arr_time": "3:01 p.m.", "act_dep_time": "1:20 p.m.", "flight": "AA-789-ORD-DEN", "sched_arr_time": "2:35 p.m.", "sched_dep_time": "1:05 p.m.", "src": "flightview", "tuple_id": "522" }, { "_row": "522", "act_arr_time": "10:10 a.m.", "act_dep_time": "8:56 a.m.", "flight": "AA-415-BOS-ORD", "sched_arr_time": "", "sched_dep_time": "", "src": "flightview", "tuple_id": "523" }, { "_row": "523", "act_arr_time": "6:43 p.m.", "act_dep_time": "3:38 p.m.", "flight": "AA-85-JFK-SFO", "sched_arr_time": "6:30 p.m.", "sched_dep_time": "3:05 p.m.", "src": "flightview", "tuple_id": "524" }, { "_row": "524", "act_arr_time": "7:57 a.m.", "act_dep_time": "8:04 a.m.", "flight": "AA-3979-CVG-ORD", "sched_arr_time": "8:00 a.m.", "sched_dep_time": "7:30 a.m.", "src": "flightview", "tuple_id": "525" } ], "task_id": "DataForge-Bench-light-verified:flights:10085:0028", "task_index": 29, "tp": 0 } ], "gate_failures": [], "grpo": { "failure_samples": [ { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "172 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35957", "_row": "55", "index": "56" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boxz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "120 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-xb", "ZipCode": "35957", "_row": "56", "index": "57" }, { "Address1": "2505xuxsxhighwayx431xnorth", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2x6x938310", "ProviderNumber": "10005", "Sample": "98 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-4", "ZipCode": "35957", "_row": "57", "index": "58" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-5c", "MeasureName": "pneumonia patients given initial antibiotic(s) within 6 hours after arrival", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "211 patients", "Score": "96%", "State": "al", "Stateavg": "al_pn-5c", "ZipCode": "35957", "_row": "58", "index": "59" } ], "task_id": "DataForge-Bench-light-verified:hospital:10000:0000", "task_index": 1, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "ground_truth_count": 12, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1651", "act_arr_time": "", "act_dep_time": "", "flight": "AA-1544-SAN-ORD", "sched_arr_time": "11/30 5:25 p.m.", "sched_dep_time": "11/30 11:25 a.m.", "src": "ord", "tuple_id": "1652" }, { "_row": "1652", "act_arr_time": "", "act_dep_time": "", "flight": "AA-4344-ORD-DTW", "sched_arr_time": "2:00 p.m.", "sched_dep_time": "11:45 a.m.", "src": "ord", "tuple_id": "1653" }, { "_row": "1653", "act_arr_time": "", "act_dep_time": "", "flight": "AA-1733-ORD-PHX", "sched_arr_time": "11/30 10:30 p.m.", "sched_dep_time": "11/30 7:45 p.m.", "src": "ord", "tuple_id": "1654" }, { "_row": "1654", "act_arr_time": "", "act_dep_time": "", "flight": "AA-4198-ORD-CLE", "sched_arr_time": "12:55 p.m.", "sched_dep_time": "10:40 a.m.", "src": "ord", "tuple_id": "1655" } ], "task_id": "DataForge-Bench-light-verified:flights:10004:0001", "task_index": 2, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "ground_truth_count": 5, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "657", "abv": "0.052", "beer-name": "Lava Lake Wit", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "15", "id": "685", "index": "658", "ounces": "12.0 oz.", "state": "CO", "style": "Witbier" }, { "_row": "658", "abv": "0.06", "beer-name": "Mountain Livin' Pale Ale", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "N/A", "id": "613", "index": "659", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Ale (APA)" }, { "_row": "659", "abv": "0.052", "beer-name": "Crazy Mountain Amber Ale", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "25", "id": "356", "index": "660", "ounces": "12.0 ounce", "state": "CO", "style": "American Amber / Red Ale" }, { "_row": "660", "abv": "0.065", "beer-name": "Tropicalia", "brewery-name": "Creature Comforts", "brewery_id": "247", "city": "Athens", "ibu": "65", "id": "2029", "index": "661", "ounces": "12.0 oz.", "state": "GA", "style": "American IPA" } ], "task_id": "DataForge-Bench-light-verified:beers:10008:0002", "task_index": 3, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "201 pine sxreex norxhwesx", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-card-2", "MeasureName": "surgery patients who were taking heart drugs called beta blockers before coming to the hospital who were kept on the beta blockers during the period just before and after their surgery", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "3 patients", "Score": "67%", "State": "al", "Stateavg": "al_scip-card-2", "ZipCode": "35640", "_row": "161", "index": "162" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "xroprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-1", "MeasureName": "surgery patients who were given an antibiotic at the right time (within one hour before surgery) to help prevent infection", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "6 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-inf-1", "ZipCode": "35640", "_row": "162", "index": "163" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-2", "MeasureName": "surgery patients who were given the right kind of antibiotic to help prevent infection", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "7 patients", "Score": "x7%", "State": "al", "Stateavg": "al_scip-inf-2", "ZipCode": "35640", "_row": "163", "index": "164" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "pxopxietaxy", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-3", "MeasureName": "surgery patients whose preventive antibiotics were stopped at the right time (within 24 hours after surgery)", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "5 patients", "Score": "80%", "State": "al", "Stateavg": "al_scip-inf-3", "ZipCode": "35640", "_row": "164", "index": "165" } ], "task_id": "DataForge-Bench-light-verified:hospital:10009:0003", "task_index": 4, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1589", "abv": "0.08", "beer-name": "Old Chub (2003)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "35", "id": "785", "index": "1590", "ounces": "12.0 oz", "state": "CO", "style": "Scottish Ale" }, { "_row": "1590", "abv": "0.065", "beer-name": "Dale's Pale Ale (2008)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "65", "id": "745", "index": "1591", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Ale (APA)" }, { "_row": "1591", "abv": "0.065%", "beer-name": "Dale's Pale Ale (2006)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons CO", "ibu": "65", "id": "744", "index": "1592", "ounces": "12.0 oz.", "state": "", "style": "American Pale Ale (APA)" }, { "_row": "1592", "abv": "0.065", "beer-name": "Dale's Pale Ale (2004)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "65", "id": "743", "index": "1593", "ounces": "12.0 oz", "state": "CO", "style": "American Pale Ale (APA)" } ], "task_id": "DataForge-Bench-light-verified:beers:10017:0005", "task_index": 6, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "ground_truth_count": 2, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-4", "MeasureName": "all heart surgery patients whose blood sugar (blood glucose) is kept under good control in the days right after surgery", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "0 patients", "Score": "empty", "State": "al", "Stateavg": "al_scip-inf-4", "ZipCode": "35960", "_row": "512", "index": "513" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "chxrokxx", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-6", "MeasureName": "surgery patients needing hair removed from the surgical area before surgery who had hair removed using a safer method (electric clippers or hair removal cream c not a razor)", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "6 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-inf-6", "ZipCode": "35960", "_row": "513", "index": "514" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-vte-1", "MeasureName": "surgery patients whose doctors ordered treatments to prevent blood clots after certain types of surgeries", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "5 patients", "Score": "1xx%", "State": "al", "Stateavg": "al_scip-vte-1", "ZipCode": "35960", "_row": "514", "index": "515" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-vte-2", "MeasureName": "patients who got treatment at the right time (within 24 hours before or after their surgery) to help prevent blood clots after certain types of surgery", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "5 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-vte-2", "ZipCode": "35960", "_row": "515", "index": "516" } ], "task_id": "DataForge-Bench-light-verified:hospital:10018:0006", "task_index": 7, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "ground_truth_count": 8, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "324", "act_arr_time": "2:49 p.m.", "act_dep_time": "1:20 p.m.", "flight": "AA-789-ORD-DEN", "sched_arr_time": "2:35 p.m.", "sched_dep_time": "1:05 p.m.", "src": "weather", "tuple_id": "325" }, { "_row": "325", "act_arr_time": "1:23 p.m.", "act_dep_time": "8:07 a.m.", "flight": "UA-854-SFO-IAH", "sched_arr_time": "1:39 p.m.", "sched_dep_time": "7:53 a.m.", "src": "weather", "tuple_id": "326" }, { "_row": "326", "act_arr_time": "", "act_dep_time": "", "flight": "AA-415-BOS-ORD", "sched_arr_time": "10:40 a.m.", "sched_dep_time": "8:35 a.m.", "src": "weather", "tuple_id": "327" }, { "_row": "327", "act_arr_time": "", "act_dep_time": "", "flight": "AA-85-JFK-SFO", "sched_arr_time": "6:30 p.m.", "sched_dep_time": "3:05 p.m.", "src": "weather", "tuple_id": "328" } ], "task_id": "DataForge-Bench-light-verified:flights:10022:0007", "task_index": 8, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "ground_truth_count": 8, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "379", "abv": "0.045", "beer-name": "SummerBright Ale", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver", "ibu": "15", "id": "397", "index": "380", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Wheat Ale" }, { "_row": "380", "abv": "0.062%", "beer-name": "Lucky U IPA", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver", "ibu": "68", "id": "193", "index": "381", "ounces": "12.0 ounce", "state": "CO", "style": "American IPA" }, { "_row": "381", "abv": "0.054", "beer-name": "Avalanche Ale", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver CO", "ibu": "19", "id": "83", "index": "382", "ounces": "12.0 oz", "state": "", "style": "American Amber / Red Ale" }, { "_row": "382", "abv": "0.05", "beer-name": "You're My Boy, Blue", "brewery-name": "Brew Bus Brewing", "brewery_id": "308", "city": "Tampa", "ibu": "N/A", "id": "1802", "index": "383", "ounces": "12.0 oz.", "state": "FL", "style": "Fruit / Vegetable Beer" } ], "task_id": "DataForge-Bench-light-verified:beers:10026:0008", "task_index": 9, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"_row\":655,\"column\":\"act_arr_time\",\"new_value\":\"9:05 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":656,\"column\":\"act_arr_time\",\"new_value\":\"5:03 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":657,\"column\":\"act_arr_time\",\"new_value\":\"10:27 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":658,\"column\":\"act_arr_time\",\"new_value\":\"10:56 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "ground_truth_count": 9, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "655", "act_arr_time": "9:05 a.m.", "act_dep_time": "6:09 a.m.", "flight": "AA-466-IAH-MIA", "sched_arr_time": "9:20 a.m.", "sched_dep_time": "6:00 a.m.", "src": "panynj", "tuple_id": "656" }, { "_row": "656", "act_arr_time": "5:03 p.m.", "act_dep_time": "11:45 a.m.", "flight": "UA-3050-PHX-CLT", "sched_arr_time": "4:57 p.m.", "sched_dep_time": "11:05 a.m.", "src": "panynj", "tuple_id": "657" }, { "_row": "657", "act_arr_time": "10:27 a.m.", "act_dep_time": "6:17 a.m.", "flight": "CO-89-IAH-EWR", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "658" }, { "_row": "658", "act_arr_time": "10:56 a.m.", "act_dep_time": "9:01 a.m.", "flight": "CO-1088-CLE-IAH", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "659" } ], "task_id": "DataForge-Bench-light-verified:flights:10031:0010", "task_index": 11, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "ground_truth_count": 8, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "2211", "abv": "0.05", "beer-name": "Triangle White Ale", "brewery-name": "Triangle Brewing Company", "brewery_id": "524", "city": "Durham", "ibu": "N/A", "id": "114", "index": "2212", "ounces": "12.0 ounce", "state": "NC", "style": "Witbier" }, { "_row": "2212", "abv": "0.08%", "beer-name": "Triangle Belgian Golden Ale", "brewery-name": "Triangle Brewing Company", "brewery_id": "524", "city": "Durham", "ibu": "N/A", "id": "113", "index": "2213", "ounces": "12.0 ounce", "state": "NC", "style": "Belgian Strong Pale Ale" }, { "_row": "2213", "abv": "0.082", "beer-name": "Troegenator", "brewery-name": "Tr\u00f6egs Brewing Company", "brewery_id": "97", "city": "Hershey", "ibu": "N/A", "id": "2445", "index": "2214", "ounces": "16.0 ounce", "state": "PA", "style": "Doppelbock" }, { "_row": "2214", "abv": "0.075", "beer-name": "Nugget Nectar", "brewery-name": "Tr\u00f6egs Brewing Company", "brewery_id": "97", "city": "Hershey", "ibu": "93", "id": "2444", "index": "2215", "ounces": "16.0 ounce", "state": "PA", "style": "American Amber / Red Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10035:0011", "task_index": 12, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "ground_truth_count": 3, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "chxrokxx", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "44 paxienxs", "Score": "93%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35960", "_row": "502", "index": "503" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "33 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-3b", "ZipCode": "35960", "_row": "503", "index": "504" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "20 patients", "Score": "100%", "State": "ax", "Stateavg": "al_pn-4", "ZipCode": "35960", "_row": "504", "index": "505" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-5c", "MeasureName": "pneumonia patients given initial antibiotic(s) within 6 hours after arrival", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "56 patients", "Score": "89%", "State": "al", "Stateavg": "al_pn-5c", "ZipCode": "35960", "_row": "505", "index": "506" } ], "task_id": "DataForge-Bench-light-verified:hospital:10036:0012", "task_index": 13, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 10 }, "fn": 10, "fp": 0, "ground_truth_count": 10, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "679", "act_arr_time": "7:53 p.m.", "act_dep_time": "5:09 p.m.", "flight": "AA-1007-MIA-PHX", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "680" }, { "_row": "680", "act_arr_time": "6:35 p.m.", "act_dep_time": "4:46 p.m.", "flight": "UA-2906-PHL-MCO", "sched_arr_time": "6:23 p.m.", "sched_dep_time": "3:50 p.m.", "src": "panynj", "tuple_id": "681" }, { "_row": "681", "act_arr_time": "7:36 p.m.", "act_dep_time": "4:37 p.m.", "flight": "UA-938-DEN-ORD", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "682" }, { "_row": "682", "act_arr_time": "9:49 p.m.", "act_dep_time": "8:23 p.m.", "flight": "AA-1221-MCO-ORD", "sched_arr_time": "9:45 p.m.", "sched_dep_time": "8:00 p.m.", "src": "panynj", "tuple_id": "683" } ], "task_id": "DataForge-Bench-light-verified:flights:10040:0013", "task_index": 14, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "739", "abv": "0.043", "beer-name": "Dry Dock Hefeweizen", "brewery-name": "Dry Dock Brewing Company", "brewery_id": "496", "city": "Aurora", "ibu": "12", "id": "872", "index": "740", "ounces": "12.0 oz", "state": "CO", "style": "Hefeweizen" }, { "_row": "740", "abv": "0.057999999999999996%", "beer-name": "Dry Dock Amber Ale", "brewery-name": "Dry Dock Brewing Company", "brewery_id": "496", "city": "Aurora", "ibu": "49", "id": "871", "index": "741", "ounces": "12.0 ounce", "state": "CO", "style": "American Amber / Red Ale" }, { "_row": "741", "abv": "0.061%", "beer-name": "Category 3 IPA", "brewery-name": "Due South Brewing Company", "brewery_id": "340", "city": "Boynton Beach", "ibu": "64", "id": "1685", "index": "742", "ounces": "12.0 ounce", "state": "FL", "style": "American IPA" }, { "_row": "742", "abv": "0.045%", "beer-name": "Dundee Summer Wheat Beer", "brewery-name": "Dundee Brewing Company", "brewery_id": "538", "city": "Rochester", "ibu": "18", "id": "457", "index": "743", "ounces": "12.0 oz.", "state": "NY", "style": "American Pale Wheat Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10044:0014", "task_index": 15, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart attack", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ ox south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "ami-7a", "MeasureName": "heart attack patients given fibrinolytic medication within 30 minutes of arrival", "PhoneNumber": "25x47x7xx0", "ProviderNumber": "10087", "Sample": "0 patients", "Score": "empty", "State": "al", "Stateavg": "al_ami-7a", "ZipCode": "36617", "_row": "384", "index": "385" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart attack", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "xniv of soxth alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "ami-8a", "MeasureName": "heart attack patients given pci within 90 minutes of arrival", "PhoneNumber": "2514717110", "ProviderNumber": "10087", "Sample": "16 patients", "Score": "50%", "State": "al", "Stateavg": "al_ami-8a", "ZipCode": "36617", "_row": "385", "index": "386" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "hxart failurx", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ of south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "hf-1", "MeasureName": "heart failure patients given discharge instructions", "PhoneNumber": "25x47x7xx0", "ProviderNumber": "10087", "Sample": "146 patients", "Score": "83%", "State": "xl", "Stateavg": "al_hf-1", "ZipCode": "36617", "_row": "386", "index": "387" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart failure", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ of south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "hf-2", "MeasureName": "heart failure patients given an evaluation of left ventricular systolic (lvs) function", "PhoneNumber": "2514717110", "ProviderNumber": "10087", "Sample": "150 patients", "Score": "98%", "State": "al", "Stateavg": "al_hx-2", "ZipCode": "36617", "_row": "387", "index": "388" } ], "task_id": "DataForge-Bench-light-verified:hospital:10045:0015", "task_index": 16, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1215", "act_arr_time": "3:13 p.m.", "act_dep_time": "1:19 p.m.", "flight": "AA-789-ORD-DEN", "sched_arr_time": "2:35 p.m.", "sched_dep_time": "1:05 p.m.", "src": "usatoday", "tuple_id": "1216" }, { "_row": "1216", "act_arr_time": "1:30 p.m.", "act_dep_time": "7:49 a.m.", "flight": "UA-854-SFO-IAH", "sched_arr_time": "1:39 p.m.", "sched_dep_time": "7:53 a.m.", "src": "usatoday", "tuple_id": "1217" }, { "_row": "1217", "act_arr_time": "6:36 p.m.", "act_dep_time": "3:38 p.m.", "flight": "AA-85-JFK-SFO", "sched_arr_time": "6:30 p.m.", "sched_dep_time": "3:05 p.m.", "src": "usatoday", "tuple_id": "1218" }, { "_row": "1218", "act_arr_time": "7:35 p.m.", "act_dep_time": "6:56 p.m.", "flight": "AA-1640-MIA-MCO", "sched_arr_time": "7:25 p.m.", "sched_dep_time": "6:30 p.m.", "src": "usatoday", "tuple_id": "1219" } ], "task_id": "DataForge-Bench-light-verified:flights:10049:0016", "task_index": 17, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "ground_truth_count": 9, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "123", "abv": "0.05%", "beer-name": "Angry Orchard Crisp Apple", "brewery-name": "Angry Orchard Cider Company", "brewery_id": "434", "city": "Cincinnati", "ibu": "N/A", "id": "1293", "index": "124", "ounces": "16.0 oz.", "state": "OH", "style": "Cider" }, { "_row": "124", "abv": "0.05", "beer-name": "Angry Orchard Crisp Apple", "brewery-name": "Angry Orchard Cider Company", "brewery_id": "434", "city": "Cincinnati", "ibu": "N/A", "id": "1292", "index": "125", "ounces": "12.0 OZ.", "state": "OH", "style": "Cider" }, { "_row": "125", "abv": "0.068", "beer-name": "Golden One", "brewery-name": "Anthem Brewing Company", "brewery_id": "193", "city": "Oklahoma City", "ibu": "N/A", "id": "2207", "index": "126", "ounces": "12.0 oz.", "state": "OK", "style": "Belgian Pale Ale" }, { "_row": "126", "abv": "0.06", "beer-name": "Arjuna", "brewery-name": "Anthem Brewing Company", "brewery_id": "193", "city": "Oklahoma City", "ibu": "N/A", "id": "2040", "index": "127", "ounces": "12.0 ounce", "state": "OK", "style": "Witbier" } ], "task_id": "DataForge-Bench-light-verified:beers:10053:0017", "task_index": 18, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "ground_truth_count": 2, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "fxorence", "Condition": "heart failure", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "hf-4", "MeasureName": "heart failure patients given smoking cessation advice/counseling", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "53 patients", "Score": "100%", "State": "al", "Stateavg": "al_hf-4", "ZipCode": "35631", "_row": "79", "index": "80" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "174 patients", "Score": "90%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35631", "_row": "80", "index": "81" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2567688400", "ProviderNumber": "1000x", "Sample": "137 patients", "Score": "95%", "State": "al", "Stateavg": "al_pn-3b", "ZipCode": "35631", "_row": "81", "index": "82" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "65 patients", "Score": "100%", "State": "al", "Stateavg": "al_pn-4", "ZipCode": "35631", "_row": "82", "index": "83" } ], "task_id": "DataForge-Bench-light-verified:hospital:10054:0018", "task_index": 19, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "ground_truth_count": 5, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "909", "abv": "0.093", "beer-name": "Snake Handler Double IPA", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "103", "id": "312", "index": "910", "ounces": "12.0 oz", "state": "AL", "style": "American Double / Imperial IPA" }, { "_row": "910", "abv": "0.06", "beer-name": "Coffee Oatmeal Stout", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "54", "id": "311", "index": "911", "ounces": "12.0 oz.", "state": "AL", "style": "Oatmeal Stout" }, { "_row": "911", "abv": "0.06", "beer-name": "Good People IPA", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "64", "id": "309", "index": "912", "ounces": "12.0 ounce", "state": "AL", "style": "American IPA" }, { "_row": "912", "abv": "0.057999999999999996%", "beer-name": "Good People American Brown Ale", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "36", "id": "308", "index": "913", "ounces": "12.0 ounce", "state": "AL", "style": "American Brown Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10062:0020", "task_index": 21, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1 }, "fn": 1, "fp": 0, "ground_truth_count": 1, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "315 w hickory st", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "pneumonia", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-6", "MeasureName": "pneumonia patients given the most appropriate initial antibiotic(s)", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "57 patients", "Score": "96%", "State": "al", "Stateavg": "al_pn-6", "ZipCode": "35150", "_row": "429", "index": "430" }, { "Address1": "315 w hickory st", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "pneumonia", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-7", "MeasureName": "pneumonia patients assessed and given influenza vaccination", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "51 patients", "Score": "88%", "State": "al", "Stateavg": "al_pn-7", "ZipCode": "35150", "_row": "430", "index": "431" }, { "Address1": "315xwxhickoryxst", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "surgical infection prevention", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "scip-card-2", "MeasureName": "surgery patients who were taking heart drugs called beta blockers before coming to the hospital who were kept on the beta blockers during the period just before and after their surgery", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "22 patients", "Score": "82%", "State": "al", "Stateavg": "al_scip-card-2", "ZipCode": "35150", "_row": "431", "index": "432" }, { "Address1": "2801 debarr road", "Address2": "empty", "Address3": "empty", "City": "anchorage", "Condition": "heart failure", "CountyName": "anchorage", "EmergencyService": "yes", "HospitalName": "alaska regional hospital", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "hf-1", "MeasureName": "heart failure patients given discharge instructions", "PhoneNumber": "9072761131", "ProviderNumber": "20017", "Sample": "57 patients", "Score": "91%", "State": "ak", "Stateavg": "ak_hf-1", "ZipCode": "99508", "_row": "432", "index": "433" } ], "task_id": "DataForge-Bench-light-verified:hospital:10063:0021", "task_index": 22, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 16 }, "fn": 16, "fp": 0, "ground_truth_count": 16, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "58", "act_arr_time": "4:54 p.m.", "act_dep_time": "11:41 a.m.", "flight": "AA-1544-SAN-ORD", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "59" }, { "_row": "59", "act_arr_time": "11:11 a.m.", "act_dep_time": "10:53 a.m.", "flight": "UA-5487-SFO-MRY", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "60" }, { "_row": "60", "act_arr_time": "12/2/11 6:35 a.m.", "act_dep_time": "11:29 p.m.", "flight": "AA-204-LAX-MCO", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "61" }, { "_row": "61", "act_arr_time": "7:04 a.m.", "act_dep_time": "6:10 a.m.", "flight": "UA-414-LAX-SFO", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "62" } ], "task_id": "DataForge-Bench-light-verified:flights:10067:0022", "task_index": 23, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "ground_truth_count": 5, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1325", "abv": "0.053%", "beer-name": "Shark Bait", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "11", "id": "1508", "index": "1326", "ounces": "12.0 oz", "state": "FL", "style": "Fruit / Vegetable Beer" }, { "_row": "1326", "abv": "0.053", "beer-name": "Gator Tail Brown Ale", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "30", "id": "1507", "index": "1327", "ounces": "12.0 ounce", "state": "FL", "style": "American Brown Ale" }, { "_row": "1327", "abv": "0.071", "beer-name": "Miami Vice IPA", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "62", "id": "1506", "index": "1328", "ounces": "12.0 oz.", "state": "FL", "style": "American IPA" }, { "_row": "1328", "abv": "0.053", "beer-name": "Big Rod Coconut Ale", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "16", "id": "1325", "index": "1329", "ounces": "12.0 oz", "state": "FL", "style": "American Blonde Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10071:0023", "task_index": 24, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "ground_truth_count": 12, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1523", "act_arr_time": "1:11 p.m.", "act_dep_time": "12:07 p.m.", "flight": "UA-2945-PHL-CLT", "sched_arr_time": "", "sched_dep_time": "", "src": "flytecomm", "tuple_id": "1524" }, { "_row": "1524", "act_arr_time": "", "act_dep_time": "", "flight": "CO-63-EWR-IAH", "sched_arr_time": "8:22 p.m.", "sched_dep_time": "5:25 p.m.", "src": "flytecomm", "tuple_id": "1525" }, { "_row": "1525", "act_arr_time": "", "act_dep_time": "", "flight": "CO-47-IAH-LAX", "sched_arr_time": "8:56 p.m.", "sched_dep_time": "7:10 p.m.", "src": "flytecomm", "tuple_id": "1526" }, { "_row": "1526", "act_arr_time": "2:39 p.m.", "act_dep_time": "11:06 a.m.", "flight": "AA-2050-ORD-MIA", "sched_arr_time": "", "sched_dep_time": "", "src": "flytecomm", "tuple_id": "1527" } ], "task_id": "DataForge-Bench-light-verified:flights:10076:0025", "task_index": 26, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "ground_truth_count": 8, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "299", "abv": "", "beer-name": "N Street Drive-In 50th Anniversary IPA", "brewery-name": "Blue Blood Brewing Company", "brewery_id": "380", "city": "Lincoln", "ibu": "N/A", "id": "1025", "index": "300", "ounces": "12.0 ounce", "state": "NE", "style": "American Double / Imperial IPA" }, { "_row": "300", "abv": "0.05", "beer-name": "467 Ethan's Stout", "brewery-name": "Blue Blood Brewing Company", "brewery_id": "380", "city": "Lincoln", "ibu": "N/A", "id": "1012", "index": "301", "ounces": "12.0 ounce", "state": "NE", "style": "American Stout" }, { "_row": "301", "abv": "0.064", "beer-name": "1335 Wicked Snout", "brewery-name": "Blue Blood Brewing Company", "brewery_id": "380", "city": "Lincoln", "ibu": "N/A", "id": "942", "index": "302", "ounces": "12.0 ounce", "state": "NE", "style": "Saison / Farmhouse Ale" }, { "_row": "302", "abv": "0.045", "beer-name": "543 Skull Creek Fresh Hopped Pale Ale", "brewery-name": "Blue Blood Brewing Company", "brewery_id": "380", "city": "Lincoln", "ibu": "N/A", "id": "937", "index": "303", "ounces": "12.0 oz.", "state": "NE", "style": "American Pale Ale (APA)" } ], "task_id": "DataForge-Bench-light-verified:beers:10080:0026", "task_index": 27, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "ground_truth_count": 3, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "heart attack", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "ami-2", "MeasureName": "heart attack patients given aspirin at discharge", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "247 patients", "Score": "96%", "State": "al", "Stateavg": "al_ami-2", "ZipCode": "3563x", "_row": "70", "index": "71" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "heart attack", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "ami-3", "MeasureName": "heart attack patients given ace inhibitor or arb for left ventricular systolic dysfunction (lvsd)", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "43 patients", "Score": "93%", "State": "al", "Stateavg": "al_ami-3", "ZipCode": "35631", "_row": "71", "index": "72" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "heart attack", "CountyName": "laudxrdalx", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "ami-4", "MeasureName": "heart attack patients given smoking cessation advice/counseling", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "103 patients", "Score": "100%", "State": "al", "Stateavg": "al_ami-4", "ZipCode": "35631", "_row": "72", "index": "73" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "hearx axxack", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "ami-5", "MeasureName": "heart attack patients given beta blocker at discharge", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "227 patients", "Score": "96%", "State": "al", "Stateavg": "al_ami-5", "ZipCode": "35631", "_row": "73", "index": "74" } ], "task_id": "DataForge-Bench-light-verified:hospital:10081:0027", "task_index": 28, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"row\":521,\"column\":\"act_arr_time\",\"new_value\":\"3:01 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":522,\"column\":\"act_arr_time\",\"new_value\":\"10:10 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":523,\"column\":\"act_arr_time\",\"new_value\":\"6:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":524,\"column\":\"act_arr_time\",\"new_value\":\"7:57 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "521", "act_arr_time": "3:01 p.m.", "act_dep_time": "1:20 p.m.", "flight": "AA-789-ORD-DEN", "sched_arr_time": "2:35 p.m.", "sched_dep_time": "1:05 p.m.", "src": "flightview", "tuple_id": "522" }, { "_row": "522", "act_arr_time": "10:10 a.m.", "act_dep_time": "8:56 a.m.", "flight": "AA-415-BOS-ORD", "sched_arr_time": "", "sched_dep_time": "", "src": "flightview", "tuple_id": "523" }, { "_row": "523", "act_arr_time": "6:43 p.m.", "act_dep_time": "3:38 p.m.", "flight": "AA-85-JFK-SFO", "sched_arr_time": "6:30 p.m.", "sched_dep_time": "3:05 p.m.", "src": "flightview", "tuple_id": "524" }, { "_row": "524", "act_arr_time": "7:57 a.m.", "act_dep_time": "8:04 a.m.", "flight": "AA-3979-CVG-ORD", "sched_arr_time": "8:00 a.m.", "sched_dep_time": "7:30 a.m.", "src": "flightview", "tuple_id": "525" } ], "task_id": "DataForge-Bench-light-verified:flights:10085:0028", "task_index": 29, "tp": 0 } ], "model_label": "grpo", "runtime_seconds": 110.139, "task_scores": [ { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10000:0000", "task_index": 1, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10004:0001", "task_index": 2, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10008:0002", "task_index": 3, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10009:0003", "task_index": 4, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10013:0004", "task_index": 5, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10017:0005", "task_index": 6, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10018:0006", "task_index": 7, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10022:0007", "task_index": 8, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10026:0008", "task_index": 9, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10027:0009", "task_index": 10, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"_row\":655,\"column\":\"act_arr_time\",\"new_value\":\"9:05 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":656,\"column\":\"act_arr_time\",\"new_value\":\"5:03 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":657,\"column\":\"act_arr_time\",\"new_value\":\"10:27 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":658,\"column\":\"act_arr_time\",\"new_value\":\"10:56 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10031:0010", "task_index": 11, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10035:0011", "task_index": 12, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10036:0012", "task_index": 13, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 10 }, "fn": 10, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10040:0013", "task_index": 14, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10044:0014", "task_index": 15, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10045:0015", "task_index": 16, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10049:0016", "task_index": 17, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10053:0017", "task_index": 18, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10054:0018", "task_index": 19, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10058:0019", "task_index": 20, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10062:0020", "task_index": 21, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1 }, "fn": 1, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10063:0021", "task_index": 22, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 16 }, "fn": 16, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10067:0022", "task_index": 23, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10071:0023", "task_index": 24, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10072:0024", "task_index": 25, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10076:0025", "task_index": 26, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10080:0026", "task_index": 27, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10081:0027", "task_index": 28, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"row\":521,\"column\":\"act_arr_time\",\"new_value\":\"3:01 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":522,\"column\":\"act_arr_time\",\"new_value\":\"10:10 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":523,\"column\":\"act_arr_time\",\"new_value\":\"6:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":524,\"column\":\"act_arr_time\",\"new_value\":\"7:57 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10085:0028", "task_index": 29, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 10 }, "fn": 10, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10089:0029", "task_index": 30, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10090:0030", "task_index": 31, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10094:0031", "task_index": 32, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10098:0032", "task_index": 33, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10099:0033", "task_index": 34, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"row\":612,\"column\":\"act_arr_time\",\"new_value\":\"7:10 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":613,\"column\":\"act_arr_time\",\"new_value\":\"3:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":614,\"column\":\"act_arr_time\",\"new_value\":\"5:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":615,\"column\":\"act_arr_time\",\"new_value\":\"11:06 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10103:0034", "task_index": 35, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10107:0035", "task_index": 36, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10108:0036", "task_index": 37, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10112:0037", "task_index": 38, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10116:0038", "task_index": 39, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10117:0039", "task_index": 40, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 15 }, "fn": 15, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10121:0040", "task_index": 41, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10125:0041", "task_index": 42, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10126:0042", "task_index": 43, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10130:0043", "task_index": 44, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 11 }, "fn": 11, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10134:0044", "task_index": 45, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10135:0045", "task_index": 46, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 14 }, "fn": 14, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10139:0046", "task_index": 47, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10143:0047", "task_index": 48, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10144:0048", "task_index": 49, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10148:0049", "task_index": 50, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10152:0050", "task_index": 51, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1 }, "fn": 1, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10153:0051", "task_index": 52, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10157:0052", "task_index": 53, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10161:0053", "task_index": 54, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10162:0054", "task_index": 55, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 13 }, "fn": 13, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10166:0055", "task_index": 56, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10170:0056", "task_index": 57, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10171:0057", "task_index": 58, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"_row\":2174,\"column\":\"act_arr_time\",\"new_value\":\"7:39 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":2175,\"column\":\"act_arr_time\",\"new_value\":\"1:40 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":2176,\"column\":\"act_arr_time\",\"new_value\":\"7:10 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":2177,\"column\":\"act_arr_time\",\"new_value\":\"11:32 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10175:0058", "task_index": 59, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4 }, "fn": 4, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10179:0059", "task_index": 60, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10180:0060", "task_index": 61, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"_row\":2241,\"column\":\"act_arr_time\",\"new_value\":\"6:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":2242,\"column\":\"act_arr_time\",\"new_value\":\"7:57 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":2243,\"column\":\"act_arr_time\",\"new_value\":\"4:56 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":2244,\"column\":\"act_arr_time\",\"new_value\":\"6:09 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10184:0061", "task_index": 62, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10188:0062", "task_index": 63, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10189:0063", "task_index": 64, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10193:0064", "task_index": 65, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10197:0065", "task_index": 66, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1 }, "fn": 1, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10198:0066", "task_index": 67, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"row\":1048,\"column\":\"act_arr_time\",\"new_value\":\"9:19 p.m.*\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1049,\"column\":\"act_arr_time\",\"new_value\":\"Contact Airline\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1050,\"column\":\"act_arr_time\",\"new_value\":\"Contact Airline\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1051,\"column\":\"act_arr_time\",\"new_value\":\"3:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10202:0067", "task_index": 68, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10206:0068", "task_index": 69, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10207:0069", "task_index": 70, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12 }, "fn": 12, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10211:0070", "task_index": 71, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10215:0071", "task_index": 72, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10216:0072", "task_index": 73, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"_row\":1001,\"column\":\"act_arr_time\",\"new_value\":\"Not Available\",\"reason\":\"The requested departure time is not available.\"},{\"_row\":1002,\"column\":\"act_arr_time\",\"new_value\":\"9:56 a.m.\",\"reason\":\"The requested departure time is 9:56 AM.\"},{\"_row\":1003,\"column\":\"act_arr_time\",\"new_value\":\"7:39 p.m.\",\"reason\":\"The requested departure time is 7:39 PM.\"},{\"_row\":1004,\"column\":\"act_arr_time\",\"new_value\":\"Not Available\",\"reason\":\"The requested departure time is not available.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10220:0073", "task_index": 74, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10224:0074", "task_index": 75, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10225:0075", "task_index": 76, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 11 }, "fn": 11, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10229:0076", "task_index": 77, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5 }, "fn": 5, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10233:0077", "task_index": 78, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10234:0078", "task_index": 79, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"row\":24,\"column\":\"act_arr_time\",\"new_value\":\"3:16 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":25,\"column\":\"act_arr_time\",\"new_value\":\"7:51 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":26,\"column\":\"act_arr_time\",\"new_value\":\"9:31 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":27,\"column\":\"act_arr_time\",\"new_value\":\"7:55 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10238:0079", "task_index": 80, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10242:0080", "task_index": 81, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1 }, "fn": 1, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10243:0081", "task_index": 82, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"_row\":1980,\"column\":\"act_arr_time\",\"new_value\":\"11:50 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":1981,\"column\":\"act_arr_time\",\"new_value\":\"3:19 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":1982,\"column\":\"act_arr_time\",\"new_value\":\"1:30 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":1983,\"column\":\"act_arr_time\",\"new_value\":\"3:21 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10247:0082", "task_index": 83, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10251:0083", "task_index": 84, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10252:0084", "task_index": 85, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 11 }, "fn": 11, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10256:0085", "task_index": 86, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10260:0086", "task_index": 87, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10261:0087", "task_index": 88, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10265:0088", "task_index": 89, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6 }, "fn": 6, "fp": 0, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10269:0089", "task_index": 90, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10270:0090", "task_index": 91, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 16 }, "fn": 16, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10274:0091", "task_index": 92, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8 }, "fn": 8, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10278:0092", "task_index": 93, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10279:0093", "task_index": 94, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"message\":\"No changes needed.\"}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10283:0094", "task_index": 95, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10287:0095", "task_index": 96, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10288:0096", "task_index": 97, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[],\"rows\":[{\"_row\":1033,\"column\":\"act_arr_time\",\"new_value\":\"3:04 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":1034,\"column\":\"act_arr_time\",\"new_value\":\"4:23 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":1035,\"column\":\"act_arr_time\",\"new_value\":\"Dec 02 Not Available\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"_row\":1036,\"column\":\"act_arr_time\",\"new_value\":\"Not Available\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10292:0097", "task_index": 98, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9 }, "fn": 9, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10296:0098", "task_index": 99, "tp": 0 }, { "canonicalized_f1": 1.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"finish\",\"repairs\":[]}", "f1": 1.0, "failure_taxonomy": {}, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 1.0, "predicted_repairs": [], "recall": 1.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10297:0099", "task_index": 100, "tp": 0 } ] }, "grpo_eval": { "canonicalized_macro_f1": 0.1393, "dataset_f1": { "beers": 0.0, "flights": 0.2121, "hospital": 0.2059 }, "failure_taxonomy": { "missed_repair": 576 }, "macro_f1": 0.1393, "mean_f1": 0.14, "parse_success_rate": 1.0, "schema_case_error_count": 0, "slice_scores": { "deterministic_normalization": { "macro_f1": 0.0, "parse_success_rate": 1.0, "schema_case_error_count": 0, "tasks": 20 }, "external_reference_required": { "macro_f1": 0.0, "parse_success_rate": 1.0, "schema_case_error_count": 0, "tasks": 66 }, "not_inferable_from_prompt": { "macro_f1": 1.0, "parse_success_rate": 1.0, "schema_case_error_count": 0, "tasks": 14 } }, "tasks": 100 }, "limitations": [ "Strict held-out eval verifies this model only for DataForge repair research tasks.", "This is not production autonomous repair evidence." ], "schema_version": "dataforge_grpo_eval_diagnostics_v1", "sft": { "failure_samples": [ { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":55,\"column\":\"ProviderNumber\",\"new_value\":\"10005\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to any hospital records.\"},{\"row\":56,\"column\":\"ProviderNumber\",\"new_value\":\"10005\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to any hospital records.\"},{\"row\":57,\"column\":\"ProviderNumber\",\"new_value\":\"10005\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to any hospital records.\"},{\"row\":58,\"column\":\"ProviderNumber\",\"new_value\":\"10005\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to any hospital records.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 4 }, "fn": 4, "fp": 4, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "10005", "reason": "The patient's address does not match any hospital records.", "row": 55 }, { "column": "ProviderNumber", "new_value": "10005", "reason": "The patient's address does not match any hospital records.", "row": 56 }, { "column": "ProviderNumber", "new_value": "10005", "reason": "The patient's address does not match any hospital records.", "row": 57 }, { "column": "ProviderNumber", "new_value": "10005", "reason": "The patient's address does not match any hospital records.", "row": 58 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "172 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35957", "_row": "55", "index": "56" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boxz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "120 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-xb", "ZipCode": "35957", "_row": "56", "index": "57" }, { "Address1": "2505xuxsxhighwayx431xnorth", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2x6x938310", "ProviderNumber": "10005", "Sample": "98 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-4", "ZipCode": "35957", "_row": "57", "index": "58" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-5c", "MeasureName": "pneumonia patients given initial antibiotic(s) within 6 hours after arrival", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "211 patients", "Score": "96%", "State": "al", "Stateavg": "al_pn-5c", "ZipCode": "35957", "_row": "58", "index": "59" } ], "task_id": "DataForge-Bench-light-verified:hospital:10000:0000", "task_index": 1, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1651,\"column\":\"act_arr_time\",\"new_value\":\"11:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1652,\"column\":\"act_arr_time\",\"new_value\":\"11:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1653,\"column\":\"act_arr_time\",\"new_value\":\"11:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1654,\"column\":\"act_arr_time\",\"new_value\":\"11:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "wrong_value": 4 }, "fn": 12, "fp": 4, "ground_truth_count": 12, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "11:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1651 }, { "column": "act_arr_time", "new_value": "11:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1652 }, { "column": "act_arr_time", "new_value": "11:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1653 }, { "column": "act_arr_time", "new_value": "11:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1654 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1651", "act_arr_time": "", "act_dep_time": "", "flight": "AA-1544-SAN-ORD", "sched_arr_time": "11/30 5:25 p.m.", "sched_dep_time": "11/30 11:25 a.m.", "src": "ord", "tuple_id": "1652" }, { "_row": "1652", "act_arr_time": "", "act_dep_time": "", "flight": "AA-4344-ORD-DTW", "sched_arr_time": "2:00 p.m.", "sched_dep_time": "11:45 a.m.", "src": "ord", "tuple_id": "1653" }, { "_row": "1653", "act_arr_time": "", "act_dep_time": "", "flight": "AA-1733-ORD-PHX", "sched_arr_time": "11/30 10:30 p.m.", "sched_dep_time": "11/30 7:45 p.m.", "src": "ord", "tuple_id": "1654" }, { "_row": "1654", "act_arr_time": "", "act_dep_time": "", "flight": "AA-4198-ORD-CLE", "sched_arr_time": "12:55 p.m.", "sched_dep_time": "10:40 a.m.", "src": "ord", "tuple_id": "1655" } ], "task_id": "DataForge-Bench-light-verified:flights:10004:0001", "task_index": 2, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":657,\"column\":\"abv\",\"new_value\":\"0.052\",\"reason\":\"The new value for abv is 0.052, which does not exist in the beer records.\"},{\"row\":658,\"column\":\"abv\",\"new_value\":\"0.06\",\"reason\":\"The new value for abv is 0.06, which does not exist in the beer records.\"},{\"row\":659,\"column\":\"abv\",\"new_value\":\"0.052\",\"reason\":\"The new value for abv is 0.052, which exists but has a different meaning than the original value.\"},{\"row\":660,\"column\":\"abv\",\"new_value\":\"0.065\",\"reason\":\"The new value for abv is 0.065, which does not exist in the beer records.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 4 }, "fn": 5, "fp": 4, "ground_truth_count": 5, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.052", "reason": "The new value for abv is 0.052, which does not exist in the beer records.", "row": 657 }, { "column": "abv", "new_value": "0.06", "reason": "The new value for abv is 0.06, which does not exist in the beer records.", "row": 658 }, { "column": "abv", "new_value": "0.052", "reason": "The new value for abv is 0.052, which exists but has a different meaning than the original value.", "row": 659 }, { "column": "abv", "new_value": "0.065", "reason": "The new value for abv is 0.065, which does not exist in the beer records.", "row": 660 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "657", "abv": "0.052", "beer-name": "Lava Lake Wit", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "15", "id": "685", "index": "658", "ounces": "12.0 oz.", "state": "CO", "style": "Witbier" }, { "_row": "658", "abv": "0.06", "beer-name": "Mountain Livin' Pale Ale", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "N/A", "id": "613", "index": "659", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Ale (APA)" }, { "_row": "659", "abv": "0.052", "beer-name": "Crazy Mountain Amber Ale", "brewery-name": "Crazy Mountain Brewing Company", "brewery_id": "63", "city": "Edwards", "ibu": "25", "id": "356", "index": "660", "ounces": "12.0 ounce", "state": "CO", "style": "American Amber / Red Ale" }, { "_row": "660", "abv": "0.065", "beer-name": "Tropicalia", "brewery-name": "Creature Comforts", "brewery_id": "247", "city": "Athens", "ibu": "65", "id": "2029", "index": "661", "ounces": "12.0 oz.", "state": "GA", "style": "American IPA" } ], "task_id": "DataForge-Bench-light-verified:beers:10008:0002", "task_index": 3, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":161,\"column\":\"Index\",\"new_value\":\"10009\",\"reason\":\"The patient's name matches the expected value.\",\"_type\":\"patient\"},{\"row\":162,\"column\":\"Index\",\"new_value\":\"10009\",\"reason\":\"The patient's name matches the expected value.\",\"_type\":\"patient\"},{\"row\":163,\"column\":\"Index\",\"new_value\":\"10009\",\"reason\":\"The patient's name matches the expected value.\",\"_type\":\"patient\"},{\"row\":164,\"column\":\"Index\",\"new_value\":\"10009\",\"reason\":\"The patient's name matches the expected value.\",\"_type\":\"patient\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "schema_case_error": 4 }, "fn": 4, "fp": 0, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "10009", "reason": "The patient's name matches the expected value.", "row": 161 }, { "column": "Index", "new_value": "10009", "reason": "The patient's name matches the expected value.", "row": 162 }, { "column": "Index", "new_value": "10009", "reason": "The patient's name matches the expected value.", "row": 163 }, { "column": "Index", "new_value": "10009", "reason": "The patient's name matches the expected value.", "row": 164 } ], "recall": 0.0, "schema_case_errors": 4, "target_rows": [ { "Address1": "201 pine sxreex norxhwesx", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-card-2", "MeasureName": "surgery patients who were taking heart drugs called beta blockers before coming to the hospital who were kept on the beta blockers during the period just before and after their surgery", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "3 patients", "Score": "67%", "State": "al", "Stateavg": "al_scip-card-2", "ZipCode": "35640", "_row": "161", "index": "162" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "xroprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-1", "MeasureName": "surgery patients who were given an antibiotic at the right time (within one hour before surgery) to help prevent infection", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "6 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-inf-1", "ZipCode": "35640", "_row": "162", "index": "163" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-2", "MeasureName": "surgery patients who were given the right kind of antibiotic to help prevent infection", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "7 patients", "Score": "x7%", "State": "al", "Stateavg": "al_scip-inf-2", "ZipCode": "35640", "_row": "163", "index": "164" }, { "Address1": "201 pine street northwest", "Address2": "empty", "Address3": "empty", "City": "hartselle", "Condition": "surgical infection prevention", "CountyName": "morgan", "EmergencyService": "yes", "HospitalName": "hartselle medical center", "HospitalOwner": "pxopxietaxy", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-3", "MeasureName": "surgery patients whose preventive antibiotics were stopped at the right time (within 24 hours after surgery)", "PhoneNumber": "2567736511", "ProviderNumber": "10009", "Sample": "5 patients", "Score": "80%", "State": "al", "Stateavg": "al_scip-inf-3", "ZipCode": "35640", "_row": "164", "index": "165" } ], "task_id": "DataForge-Bench-light-verified:hospital:10009:0003", "task_index": 4, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1433,\"column\":\"act_arr_time\",\"new_value\":\"3:31 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1434,\"column\":\"act_arr_time\",\"new_value\":\"5:26 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1435,\"column\":\"act_arr_time\",\"new_value\":\"7:10 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1436,\"column\":\"act_arr_time\",\"new_value\":\"12:09 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "ground_truth_count": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "3:31 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1433 }, { "column": "act_arr_time", "new_value": "5:26 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1434 }, { "column": "act_arr_time", "new_value": "7:10 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1435 }, { "column": "act_arr_time", "new_value": "12:09 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1436 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1433", "act_arr_time": "3:31 p.m.", "act_dep_time": "11:32 a.m.", "flight": "CO-58-DEN-IAH", "sched_arr_time": "2:50 p.m.", "sched_dep_time": "11:25 a.m.", "src": "CO", "tuple_id": "1434" }, { "_row": "1434", "act_arr_time": "5:26 p.m.", "act_dep_time": "3:31 p.m.", "flight": "CO-50-CLE-EWR", "sched_arr_time": "4:49 p.m.", "sched_dep_time": "2:55 p.m.", "src": "CO", "tuple_id": "1435" }, { "_row": "1435", "act_arr_time": "7:10 p.m.", "act_dep_time": "2:48 p.m.", "flight": "CO-62-IAH-EWR", "sched_arr_time": "7:03 p.m.", "sched_dep_time": "2:30 p.m.", "src": "CO", "tuple_id": "1436" }, { "_row": "1436", "act_arr_time": "12:09 p.m.", "act_dep_time": "9:14 a.m.", "flight": "CO-1193-EWR-MCO", "sched_arr_time": "12:18 p.m.", "sched_dep_time": "9:15 a.m.", "src": "CO", "tuple_id": "1437" } ], "task_id": "DataForge-Bench-light-verified:flights:10013:0004", "task_index": 5, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1592,\"column\":\"style\",\"new_value\":\"Scottish Ale\",\"reason\":\"The beer name has been changed to 'Scottish Ale' which is a different style than the previous one.\"},{\"row\":1591,\"column\":\"style\",\"new_value\":\"American Pale Ale (APA)\",\"reason\":\"The beer name has been changed to 'American Pale Ale (APA)' which is a different style than the previous one.\"},{\"row\":1590,\"column\":\"style\",\"new_value\":\"Scottish Ale\",\"reason\":\"The beer name has been changed to 'Scottish Ale' which is a different style than the previous one.\"},{\"row\":1589,\"column\":\"style\",\"new_value\":\"American Pale Ale (APA)\",\"reason\":\"The beer name has been changed to 'American Pale Ale (APA)' which is a different style than the previous one.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 4 }, "fn": 7, "fp": 4, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "style", "new_value": "Scottish Ale", "reason": "The beer name has been changed to 'Scottish Ale' which is a different style than the previous one.", "row": 1592 }, { "column": "style", "new_value": "American Pale Ale (APA)", "reason": "The beer name has been changed to 'American Pale Ale (APA)' which is a different style than the previous one.", "row": 1591 }, { "column": "style", "new_value": "Scottish Ale", "reason": "The beer name has been changed to 'Scottish Ale' which is a different style than the previous one.", "row": 1590 }, { "column": "style", "new_value": "American Pale Ale (APA)", "reason": "The beer name has been changed to 'American Pale Ale (APA)' which is a different style than the previous one.", "row": 1589 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1589", "abv": "0.08", "beer-name": "Old Chub (2003)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "35", "id": "785", "index": "1590", "ounces": "12.0 oz", "state": "CO", "style": "Scottish Ale" }, { "_row": "1590", "abv": "0.065", "beer-name": "Dale's Pale Ale (2008)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "65", "id": "745", "index": "1591", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Ale (APA)" }, { "_row": "1591", "abv": "0.065%", "beer-name": "Dale's Pale Ale (2006)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons CO", "ibu": "65", "id": "744", "index": "1592", "ounces": "12.0 oz.", "state": "", "style": "American Pale Ale (APA)" }, { "_row": "1592", "abv": "0.065", "beer-name": "Dale's Pale Ale (2004)", "brewery-name": "Oskar Blues Brewery", "brewery_id": "503", "city": "Lyons", "ibu": "65", "id": "743", "index": "1593", "ounces": "12.0 oz", "state": "CO", "style": "American Pale Ale (APA)" } ], "task_id": "DataForge-Bench-light-verified:beers:10017:0005", "task_index": 6, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":512,\"column\":\"ProviderNumber\",\"new_value\":\"10022\",\"reason\":\"The patient's condition requires immediate attention and may require urgent care.\",\"why\":\"The proposed new value '10022' aligns with the emergency service requirement for this specific patient.\"},{\"row\":513,\"column\":\"ProviderNumber\",\"new_value\":\"10022\",\"reason\":\"The patient's condition requires immediate attention and may require urgent care.\",\"why\":\"The proposed new value '10022' aligns with the emergency service requirement for this specific patient.\"},{\"row\":514,\"column\":\"ProviderNumber\",\"new_value\":\"10022\",\"reason\":\"The patient's condition requires immediate attention and may require urgent care.\",\"why\":\"The proposed new value '10022' aligns with the emergency service requirement for this specific patient.\"},{\"row\":515,\"column\":\"ProviderNumber\",\"new_value\":\"10022\",\"reason\":\"The patient's condition requires immediate attention and may require urgent care.\",\"why\":\"The proposed new value '10022' aligns with the emergency service requirement for this specific patient.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "overrepair": 4 }, "fn": 2, "fp": 4, "ground_truth_count": 2, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "10022", "reason": "The patient's condition requires immediate attention and may require urgent care.", "row": 512 }, { "column": "ProviderNumber", "new_value": "10022", "reason": "The patient's condition requires immediate attention and may require urgent care.", "row": 513 }, { "column": "ProviderNumber", "new_value": "10022", "reason": "The patient's condition requires immediate attention and may require urgent care.", "row": 514 }, { "column": "ProviderNumber", "new_value": "10022", "reason": "The patient's condition requires immediate attention and may require urgent care.", "row": 515 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-4", "MeasureName": "all heart surgery patients whose blood sugar (blood glucose) is kept under good control in the days right after surgery", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "0 patients", "Score": "empty", "State": "al", "Stateavg": "al_scip-inf-4", "ZipCode": "35960", "_row": "512", "index": "513" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "chxrokxx", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-6", "MeasureName": "surgery patients needing hair removed from the surgical area before surgery who had hair removed using a safer method (electric clippers or hair removal cream c not a razor)", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "6 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-inf-6", "ZipCode": "35960", "_row": "513", "index": "514" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-vte-1", "MeasureName": "surgery patients whose doctors ordered treatments to prevent blood clots after certain types of surgeries", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "5 patients", "Score": "1xx%", "State": "al", "Stateavg": "al_scip-vte-1", "ZipCode": "35960", "_row": "514", "index": "515" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "surgical infection prevention", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "scip-vte-2", "MeasureName": "patients who got treatment at the right time (within 24 hours before or after their surgery) to help prevent blood clots after certain types of surgery", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "5 patients", "Score": "100%", "State": "al", "Stateavg": "al_scip-vte-2", "ZipCode": "35960", "_row": "515", "index": "516" } ], "task_id": "DataForge-Bench-light-verified:hospital:10018:0006", "task_index": 7, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":324,\"column\":\"act_arr_time\",\"new_value\":\"2:49 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":325,\"column\":\"act_arr_time\",\"new_value\":\"1:23 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":326,\"column\":\"act_arr_time\",\"new_value\":\"10:40 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":327,\"column\":\"act_arr_time\",\"new_value\":\"\",\"reason\":\"The new value for act_arr_time is empty.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "wrong_value": 4 }, "fn": 8, "fp": 4, "ground_truth_count": 8, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "2:49 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 324 }, { "column": "act_arr_time", "new_value": "1:23 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 325 }, { "column": "act_arr_time", "new_value": "10:40 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 326 }, { "column": "act_arr_time", "new_value": "", "reason": "The new value for act_arr_time is empty.", "row": 327 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "324", "act_arr_time": "2:49 p.m.", "act_dep_time": "1:20 p.m.", "flight": "AA-789-ORD-DEN", "sched_arr_time": "2:35 p.m.", "sched_dep_time": "1:05 p.m.", "src": "weather", "tuple_id": "325" }, { "_row": "325", "act_arr_time": "1:23 p.m.", "act_dep_time": "8:07 a.m.", "flight": "UA-854-SFO-IAH", "sched_arr_time": "1:39 p.m.", "sched_dep_time": "7:53 a.m.", "src": "weather", "tuple_id": "326" }, { "_row": "326", "act_arr_time": "", "act_dep_time": "", "flight": "AA-415-BOS-ORD", "sched_arr_time": "10:40 a.m.", "sched_dep_time": "8:35 a.m.", "src": "weather", "tuple_id": "327" }, { "_row": "327", "act_arr_time": "", "act_dep_time": "", "flight": "AA-85-JFK-SFO", "sched_arr_time": "6:30 p.m.", "sched_dep_time": "3:05 p.m.", "src": "weather", "tuple_id": "328" } ], "task_id": "DataForge-Bench-light-verified:flights:10022:0007", "task_index": 8, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":379,\"column\":\"abv\",\"new_value\":\"0.045\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":380,\"column\":\"abv\",\"new_value\":\"0.062%\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":381,\"column\":\"abv\",\"new_value\":\"0.054\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":382,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The new value for ABV is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 3, "wrong_value": 1 }, "fn": 8, "fp": 4, "ground_truth_count": 8, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.045", "reason": "The new value for ABV is incorrect.", "row": 379 }, { "column": "abv", "new_value": "0.062%", "reason": "The new value for ABV is incorrect.", "row": 380 }, { "column": "abv", "new_value": "0.054", "reason": "The new value for ABV is incorrect.", "row": 381 }, { "column": "abv", "new_value": "0.05", "reason": "The new value for ABV is incorrect.", "row": 382 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "379", "abv": "0.045", "beer-name": "SummerBright Ale", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver", "ibu": "15", "id": "397", "index": "380", "ounces": "12.0 ounce", "state": "CO", "style": "American Pale Wheat Ale" }, { "_row": "380", "abv": "0.062%", "beer-name": "Lucky U IPA", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver", "ibu": "68", "id": "193", "index": "381", "ounces": "12.0 ounce", "state": "CO", "style": "American IPA" }, { "_row": "381", "abv": "0.054", "beer-name": "Avalanche Ale", "brewery-name": "Breckenridge Brewery", "brewery_id": "391", "city": "Denver CO", "ibu": "19", "id": "83", "index": "382", "ounces": "12.0 oz", "state": "", "style": "American Amber / Red Ale" }, { "_row": "382", "abv": "0.05", "beer-name": "You're My Boy, Blue", "brewery-name": "Brew Bus Brewing", "brewery_id": "308", "city": "Tampa", "ibu": "N/A", "id": "1802", "index": "383", "ounces": "12.0 oz.", "state": "FL", "style": "Fruit / Vegetable Beer" } ], "task_id": "DataForge-Bench-light-verified:beers:10026:0008", "task_index": 9, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":52,\"column\":\"Address1\",\"new_value\":\"2505 u s highway 431 north\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be 2505 U.S. Highway 431 North.\"},{\"row\":53,\"column\":\"Address1\",\"new_value\":\"2505 u s highway 431 north\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be 2505 U.S. Highway 431 North.\"},{\"row\":54,\"column\":\"Address1\",\"new_value\":\"2505 u s highway 431 north\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be 2505 U.S. Highway 431 North.\"},{\"row\":55,\"column\":\"Address1\",\"new_value\":\"2505 u s highway 431 north\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be 2505 U.S. Highway 431 North.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "ground_truth_count": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "2505 u s highway 431 north", "reason": "The address is incorrect.", "row": 52 }, { "column": "Address1", "new_value": "2505 u s highway 431 north", "reason": "The address is incorrect.", "row": 53 }, { "column": "Address1", "new_value": "2505 u s highway 431 north", "reason": "The address is incorrect.", "row": 54 }, { "column": "Address1", "new_value": "2505 u s highway 431 north", "reason": "The address is incorrect.", "row": 55 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "heart failure", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "hf-2", "MeasureName": "heart failure patients given an evaluation of left ventricular systolic (lvs) function", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "160 patients", "Score": "99%", "State": "al", "Stateavg": "al_hf-2", "ZipCode": "35957", "_row": "52", "index": "53" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "heart failure", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "hf-3", "MeasureName": "heart failure patients given ace inhibitor or arb for left ventricular systolic dysfunction (lvsd)", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "69 patients", "Score": "90%", "State": "al", "Stateavg": "al_hf-3", "ZipCode": "35957", "_row": "53", "index": "54" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "heart failure", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "hf-4", "MeasureName": "heart failure patients given smoking cessation advice/counseling", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "34 patients", "Score": "94%", "State": "al", "Stateavg": "al_hf-4", "ZipCode": "35957", "_row": "54", "index": "55" }, { "Address1": "2505 u s highway 431 north", "Address2": "empty", "Address3": "empty", "City": "boaz", "Condition": "pneumonia", "CountyName": "marshall", "EmergencyService": "yes", "HospitalName": "marshall medical center south", "HospitalOwner": "government - hospital district or authority", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2565938310", "ProviderNumber": "10005", "Sample": "172 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35957", "_row": "55", "index": "56" } ], "task_id": "DataForge-Bench-light-verified:hospital:10027:0009", "task_index": 10, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":655,\"column\":\"act_arr_time\",\"new_value\":\"9:05 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":656,\"column\":\"act_arr_time\",\"new_value\":\"5:03 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":657,\"column\":\"act_arr_time\",\"new_value\":\"10:27 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":658,\"column\":\"act_arr_time\",\"new_value\":\"10:56 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 3, "wrong_value": 1 }, "fn": 9, "fp": 4, "ground_truth_count": 9, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "9:05 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 655 }, { "column": "act_arr_time", "new_value": "5:03 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 656 }, { "column": "act_arr_time", "new_value": "10:27 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 657 }, { "column": "act_arr_time", "new_value": "10:56 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 658 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "655", "act_arr_time": "9:05 a.m.", "act_dep_time": "6:09 a.m.", "flight": "AA-466-IAH-MIA", "sched_arr_time": "9:20 a.m.", "sched_dep_time": "6:00 a.m.", "src": "panynj", "tuple_id": "656" }, { "_row": "656", "act_arr_time": "5:03 p.m.", "act_dep_time": "11:45 a.m.", "flight": "UA-3050-PHX-CLT", "sched_arr_time": "4:57 p.m.", "sched_dep_time": "11:05 a.m.", "src": "panynj", "tuple_id": "657" }, { "_row": "657", "act_arr_time": "10:27 a.m.", "act_dep_time": "6:17 a.m.", "flight": "CO-89-IAH-EWR", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "658" }, { "_row": "658", "act_arr_time": "10:56 a.m.", "act_dep_time": "9:01 a.m.", "flight": "CO-1088-CLE-IAH", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "659" } ], "task_id": "DataForge-Bench-light-verified:flights:10031:0010", "task_index": 11, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2211,\"column\":\"beer-name\",\"new_value\":\"Triangle Belgian Golden Ale\",\"reason\":\"The beer name has been changed to 'Triangle Belgian Golden Ale'.\"},{\"row\":2212,\"column\":\"beer-name\",\"new_value\":\"Troegenator\",\"reason\":\"The beer name has been changed to 'Troegenator'.\"},{\"row\":2213,\"column\":\"beer-name\",\"new_value\":\"Nugget Nectar\",\"reason\":\"The beer name has been changed to 'Nugget Nectar'.\"},{\"row\":2214,\"column\":\"beer-name\",\"new_value\":\"Triangle Belgian Golden Ale\",\"reason\":\"The beer name has been changed to 'Triangle Belgian Golden Ale'.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 4 }, "fn": 8, "fp": 4, "ground_truth_count": 8, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "beer-name", "new_value": "Triangle Belgian Golden Ale", "reason": "The beer name has been changed to 'Triangle Belgian Golden Ale'.", "row": 2211 }, { "column": "beer-name", "new_value": "Troegenator", "reason": "The beer name has been changed to 'Troegenator'.", "row": 2212 }, { "column": "beer-name", "new_value": "Nugget Nectar", "reason": "The beer name has been changed to 'Nugget Nectar'.", "row": 2213 }, { "column": "beer-name", "new_value": "Triangle Belgian Golden Ale", "reason": "The beer name has been changed to 'Triangle Belgian Golden Ale'.", "row": 2214 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "2211", "abv": "0.05", "beer-name": "Triangle White Ale", "brewery-name": "Triangle Brewing Company", "brewery_id": "524", "city": "Durham", "ibu": "N/A", "id": "114", "index": "2212", "ounces": "12.0 ounce", "state": "NC", "style": "Witbier" }, { "_row": "2212", "abv": "0.08%", "beer-name": "Triangle Belgian Golden Ale", "brewery-name": "Triangle Brewing Company", "brewery_id": "524", "city": "Durham", "ibu": "N/A", "id": "113", "index": "2213", "ounces": "12.0 ounce", "state": "NC", "style": "Belgian Strong Pale Ale" }, { "_row": "2213", "abv": "0.082", "beer-name": "Troegenator", "brewery-name": "Tr\u00f6egs Brewing Company", "brewery_id": "97", "city": "Hershey", "ibu": "N/A", "id": "2445", "index": "2214", "ounces": "16.0 ounce", "state": "PA", "style": "Doppelbock" }, { "_row": "2214", "abv": "0.075", "beer-name": "Nugget Nectar", "brewery-name": "Tr\u00f6egs Brewing Company", "brewery_id": "97", "city": "Hershey", "ibu": "93", "id": "2444", "index": "2215", "ounces": "16.0 ounce", "state": "PA", "style": "American Amber / Red Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10035:0011", "task_index": 12, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":502,\"column\":\"ProviderNumber\",\"new_value\":\"Voluntary Non-Profit - Private\",\"reason\":\"The Provider number is already assigned.\",\"}}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "ground_truth_count": 3, "inferability": "external_reference_required", "parse_error_kind": "truncated_json", "parse_ok": false, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "chxrokxx", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "44 paxienxs", "Score": "93%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35960", "_row": "502", "index": "503" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "33 patients", "Score": "97%", "State": "al", "Stateavg": "al_pn-3b", "ZipCode": "35960", "_row": "503", "index": "504" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "20 patients", "Score": "100%", "State": "ax", "Stateavg": "al_pn-4", "ZipCode": "35960", "_row": "504", "index": "505" }, { "Address1": "400 northwood dr", "Address2": "empty", "Address3": "empty", "City": "centre", "Condition": "pneumonia", "CountyName": "cherokee", "EmergencyService": "yes", "HospitalName": "cherokee medical center", "HospitalOwner": "voluntary non-profit - private", "HospitalType": "acute care hospitals", "MeasureCode": "pn-5c", "MeasureName": "pneumonia patients given initial antibiotic(s) within 6 hours after arrival", "PhoneNumber": "2569275531", "ProviderNumber": "10022", "Sample": "56 patients", "Score": "89%", "State": "al", "Stateavg": "al_pn-5c", "ZipCode": "35960", "_row": "505", "index": "506" } ], "task_id": "DataForge-Bench-light-verified:hospital:10036:0012", "task_index": 13, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":679,\"column\":\"act_arr_time\",\"new_value\":\"7:53 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":680,\"column\":\"act_arr_time\",\"new_value\":\"6:35 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":681,\"column\":\"act_arr_time\",\"new_value\":\"7:36 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":682,\"column\":\"act_arr_time\",\"new_value\":\"9:49 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 1, "wrong_value": 3 }, "fn": 10, "fp": 4, "ground_truth_count": 10, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "7:53 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 679 }, { "column": "act_arr_time", "new_value": "6:35 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 680 }, { "column": "act_arr_time", "new_value": "7:36 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 681 }, { "column": "act_arr_time", "new_value": "9:49 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 682 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "679", "act_arr_time": "7:53 p.m.", "act_dep_time": "5:09 p.m.", "flight": "AA-1007-MIA-PHX", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "680" }, { "_row": "680", "act_arr_time": "6:35 p.m.", "act_dep_time": "4:46 p.m.", "flight": "UA-2906-PHL-MCO", "sched_arr_time": "6:23 p.m.", "sched_dep_time": "3:50 p.m.", "src": "panynj", "tuple_id": "681" }, { "_row": "681", "act_arr_time": "7:36 p.m.", "act_dep_time": "4:37 p.m.", "flight": "UA-938-DEN-ORD", "sched_arr_time": "", "sched_dep_time": "", "src": "panynj", "tuple_id": "682" }, { "_row": "682", "act_arr_time": "9:49 p.m.", "act_dep_time": "8:23 p.m.", "flight": "AA-1221-MCO-ORD", "sched_arr_time": "9:45 p.m.", "sched_dep_time": "8:00 p.m.", "src": "panynj", "tuple_id": "683" } ], "task_id": "DataForge-Bench-light-verified:flights:10040:0013", "task_index": 14, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":739,\"column\":\"abv\",\"new_value\":\"0.043\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":740,\"column\":\"ibu\",\"new_value\":\"12\",\"reason\":\"The new value for ibu is incorrect.\"},{\"row\":741,\"column\":\"ounces\",\"new_value\":\"12.0 ounce\",\"reason\":\"The new value for ounces is incorrect.\"},{\"row\":742,\"column\":\"beer-name\",\"new_value\":\"Dundee Summer Wheat Beer\",\"reason\":\"The new value for beer-name is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 3, "wrong_value": 1 }, "fn": 7, "fp": 4, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.043", "reason": "The new value for abv is incorrect.", "row": 739 }, { "column": "ibu", "new_value": "12", "reason": "The new value for ibu is incorrect.", "row": 740 }, { "column": "ounces", "new_value": "12.0 ounce", "reason": "The new value for ounces is incorrect.", "row": 741 }, { "column": "beer-name", "new_value": "Dundee Summer Wheat Beer", "reason": "The new value for beer-name is incorrect.", "row": 742 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "739", "abv": "0.043", "beer-name": "Dry Dock Hefeweizen", "brewery-name": "Dry Dock Brewing Company", "brewery_id": "496", "city": "Aurora", "ibu": "12", "id": "872", "index": "740", "ounces": "12.0 oz", "state": "CO", "style": "Hefeweizen" }, { "_row": "740", "abv": "0.057999999999999996%", "beer-name": "Dry Dock Amber Ale", "brewery-name": "Dry Dock Brewing Company", "brewery_id": "496", "city": "Aurora", "ibu": "49", "id": "871", "index": "741", "ounces": "12.0 ounce", "state": "CO", "style": "American Amber / Red Ale" }, { "_row": "741", "abv": "0.061%", "beer-name": "Category 3 IPA", "brewery-name": "Due South Brewing Company", "brewery_id": "340", "city": "Boynton Beach", "ibu": "64", "id": "1685", "index": "742", "ounces": "12.0 ounce", "state": "FL", "style": "American IPA" }, { "_row": "742", "abv": "0.045%", "beer-name": "Dundee Summer Wheat Beer", "brewery-name": "Dundee Brewing Company", "brewery_id": "538", "city": "Rochester", "ibu": "18", "id": "457", "index": "743", "ounces": "12.0 oz.", "state": "NY", "style": "American Pale Wheat Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10044:0014", "task_index": 15, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":384,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's provider number is missing.\",\"why\":\"This will prevent us from accurately updating their health information.\"},{\"row\":385,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's provider number is missing.\",\"why\":\"This will prevent us from accurately updating their health information.\"},{\"row\":386,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's provider number is missing.\",\"why\":\"This will prevent us from accurately updating their health information.\"},{\"row\":387,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's provider number is missing.\",\"why\":\"This will prevent us from accurately updating their health information.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 4 }, "fn": 7, "fp": 4, "ground_truth_count": 7, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's provider number is missing.", "row": 384 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's provider number is missing.", "row": 385 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's provider number is missing.", "row": 386 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's provider number is missing.", "row": 387 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart attack", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ ox south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "ami-7a", "MeasureName": "heart attack patients given fibrinolytic medication within 30 minutes of arrival", "PhoneNumber": "25x47x7xx0", "ProviderNumber": "10087", "Sample": "0 patients", "Score": "empty", "State": "al", "Stateavg": "al_ami-7a", "ZipCode": "36617", "_row": "384", "index": "385" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart attack", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "xniv of soxth alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "ami-8a", "MeasureName": "heart attack patients given pci within 90 minutes of arrival", "PhoneNumber": "2514717110", "ProviderNumber": "10087", "Sample": "16 patients", "Score": "50%", "State": "al", "Stateavg": "al_ami-8a", "ZipCode": "36617", "_row": "385", "index": "386" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "hxart failurx", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ of south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "hf-1", "MeasureName": "heart failure patients given discharge instructions", "PhoneNumber": "25x47x7xx0", "ProviderNumber": "10087", "Sample": "146 patients", "Score": "83%", "State": "xl", "Stateavg": "al_hf-1", "ZipCode": "36617", "_row": "386", "index": "387" }, { "Address1": "2451 fillingim street", "Address2": "empty", "Address3": "empty", "City": "mobile", "Condition": "heart failure", "CountyName": "mobile", "EmergencyService": "yes", "HospitalName": "univ of south alabama medical center", "HospitalOwner": "government - state", "HospitalType": "acute care hospitals", "MeasureCode": "hf-2", "MeasureName": "heart failure patients given an evaluation of left ventricular systolic (lvs) function", "PhoneNumber": "2514717110", "ProviderNumber": "10087", "Sample": "150 patients", "Score": "98%", "State": "al", "Stateavg": "al_hx-2", "ZipCode": "36617", "_row": "387", "index": "388" } ], "task_id": "DataForge-Bench-light-verified:hospital:10045:0015", "task_index": 16, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1215,\"column\":\"act_arr_time\",\"new_value\":\"3:13 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1216,\"column\":\"act_arr_time\",\"new_value\":\"1:30 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1217,\"column\":\"act_arr_time\",\"new_value\":\"6:36 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1218,\"column\":\"act_arr_time\",\"new_value\":\"7:35 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "overrepair": 2, "wrong_value": 2 }, "fn": 4, "fp": 4, "ground_truth_count": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "3:13 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1215 }, { "column": "act_arr_time", "new_value": "1:30 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1216 }, { "column": "act_arr_time", "new_value": "6:36 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1217 }, { "column": "act_arr_time", "new_value": "7:35 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1218 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1215", "act_arr_time": "3:13 p.m.", "act_dep_time": "1:19 p.m.", "flight": "AA-789-ORD-DEN", "sched_arr_time": "2:35 p.m.", "sched_dep_time": "1:05 p.m.", "src": "usatoday", "tuple_id": "1216" }, { "_row": "1216", "act_arr_time": "1:30 p.m.", "act_dep_time": "7:49 a.m.", "flight": "UA-854-SFO-IAH", "sched_arr_time": "1:39 p.m.", "sched_dep_time": "7:53 a.m.", "src": "usatoday", "tuple_id": "1217" }, { "_row": "1217", "act_arr_time": "6:36 p.m.", "act_dep_time": "3:38 p.m.", "flight": "AA-85-JFK-SFO", "sched_arr_time": "6:30 p.m.", "sched_dep_time": "3:05 p.m.", "src": "usatoday", "tuple_id": "1218" }, { "_row": "1218", "act_arr_time": "7:35 p.m.", "act_dep_time": "6:56 p.m.", "flight": "AA-1640-MIA-MCO", "sched_arr_time": "7:25 p.m.", "sched_dep_time": "6:30 p.m.", "src": "usatoday", "tuple_id": "1219" } ], "task_id": "DataForge-Bench-light-verified:flights:10049:0016", "task_index": 17, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":123,\"column\":\"abv\",\"new_value\":\"0.05%\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":124,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":125,\"column\":\"abv\",\"new_value\":\"0.068\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":126,\"column\":\"abv\",\"new_value\":\"0.06\",\"reason\":\"The new value for ABV is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 3, "wrong_value": 1 }, "fn": 9, "fp": 4, "ground_truth_count": 9, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.05%", "reason": "The new value for ABV is incorrect.", "row": 123 }, { "column": "abv", "new_value": "0.05", "reason": "The new value for ABV is incorrect.", "row": 124 }, { "column": "abv", "new_value": "0.068", "reason": "The new value for ABV is incorrect.", "row": 125 }, { "column": "abv", "new_value": "0.06", "reason": "The new value for ABV is incorrect.", "row": 126 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "123", "abv": "0.05%", "beer-name": "Angry Orchard Crisp Apple", "brewery-name": "Angry Orchard Cider Company", "brewery_id": "434", "city": "Cincinnati", "ibu": "N/A", "id": "1293", "index": "124", "ounces": "16.0 oz.", "state": "OH", "style": "Cider" }, { "_row": "124", "abv": "0.05", "beer-name": "Angry Orchard Crisp Apple", "brewery-name": "Angry Orchard Cider Company", "brewery_id": "434", "city": "Cincinnati", "ibu": "N/A", "id": "1292", "index": "125", "ounces": "12.0 OZ.", "state": "OH", "style": "Cider" }, { "_row": "125", "abv": "0.068", "beer-name": "Golden One", "brewery-name": "Anthem Brewing Company", "brewery_id": "193", "city": "Oklahoma City", "ibu": "N/A", "id": "2207", "index": "126", "ounces": "12.0 oz.", "state": "OK", "style": "Belgian Pale Ale" }, { "_row": "126", "abv": "0.06", "beer-name": "Arjuna", "brewery-name": "Anthem Brewing Company", "brewery_id": "193", "city": "Oklahoma City", "ibu": "N/A", "id": "2040", "index": "127", "ounces": "12.0 ounce", "state": "OK", "style": "Witbier" } ], "task_id": "DataForge-Bench-light-verified:beers:10053:0017", "task_index": 18, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":82,\"column\":\"Index\",\"new_value\":\"83\",\"reason\":\"The patient's condition has changed from heart failure to pneumonia.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "schema_case_error": 1 }, "fn": 2, "fp": 0, "ground_truth_count": 2, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "83", "reason": "The patient's condition has changed from heart failure to pneumonia.", "row": 82 } ], "recall": 0.0, "schema_case_errors": 1, "target_rows": [ { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "fxorence", "Condition": "heart failure", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "hf-4", "MeasureName": "heart failure patients given smoking cessation advice/counseling", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "53 patients", "Score": "100%", "State": "al", "Stateavg": "al_hf-4", "ZipCode": "35631", "_row": "79", "index": "80" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-2", "MeasureName": "pneumonia patients assessed and given pneumococcal vaccination", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "174 patients", "Score": "90%", "State": "al", "Stateavg": "al_pn-2", "ZipCode": "35631", "_row": "80", "index": "81" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-3b", "MeasureName": "pneumonia patients whose initial emergency room blood culture was performed prior to the administration of the first hospital dose of antibiotics", "PhoneNumber": "2567688400", "ProviderNumber": "1000x", "Sample": "137 patients", "Score": "95%", "State": "al", "Stateavg": "al_pn-3b", "ZipCode": "35631", "_row": "81", "index": "82" }, { "Address1": "205 marengo street", "Address2": "empty", "Address3": "empty", "City": "florence", "Condition": "pneumonia", "CountyName": "lauderdale", "EmergencyService": "yes", "HospitalName": "eliza coffee memorial hospital", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-4", "MeasureName": "pneumonia patients given smoking cessation advice/counseling", "PhoneNumber": "2567688400", "ProviderNumber": "10006", "Sample": "65 patients", "Score": "100%", "State": "al", "Stateavg": "al_pn-4", "ZipCode": "35631", "_row": "82", "index": "83" } ], "task_id": "DataForge-Bench-light-verified:hospital:10054:0018", "task_index": 19, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1732,\"column\":\"flight\",\"new_value\":\"UA-3050-PHX-CLT\",\"reason\":\"The flight is currently on a different schedule than the other flights listed.\"},{\"row\":1731,\"column\":\"flight\",\"new_value\":\"UA-5487-SFO-MRY\",\"reason\":\"The flight is currently on a different schedule than the other flights listed.\"},{\"row\":1730,\"column\":\"flight\",\"new_value\":\"UA-2830-MCO-CLT\",\"reason\":\"The flight is currently on a different schedule than the other flights listed.\"},{\"row\":1729,\"column\":\"flight\",\"new_value\":\"UA-2726-FLL-PHL\",\"reason\":\"The flight is currently on a different schedule than the other flights listed.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "ground_truth_count": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "UA-3050-PHX-CLT", "reason": "The flight is currently on a different schedule than the other flights listed.", "row": 1732 }, { "column": "flight", "new_value": "UA-5487-SFO-MRY", "reason": "The flight is currently on a different schedule than the other flights listed.", "row": 1731 }, { "column": "flight", "new_value": "UA-2830-MCO-CLT", "reason": "The flight is currently on a different schedule than the other flights listed.", "row": 1730 }, { "column": "flight", "new_value": "UA-2726-FLL-PHL", "reason": "The flight is currently on a different schedule than the other flights listed.", "row": 1729 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1729", "act_arr_time": "5:17 p.m.", "act_dep_time": "3:27 p.m.", "flight": "UA-2830-MCO-CLT", "sched_arr_time": "5:05 p.m.", "sched_dep_time": "3:27 p.m.", "src": "ua", "tuple_id": "1730" }, { "_row": "1730", "act_arr_time": "11:21 a.m.", "act_dep_time": "10:33 a.m.", "flight": "UA-5487-SFO-MRY", "sched_arr_time": "11:19 a.m.", "sched_dep_time": "10:39 a.m.", "src": "ua", "tuple_id": "1731" }, { "_row": "1731", "act_arr_time": "6:28 p.m.", "act_dep_time": "4:00 p.m.", "flight": "UA-2726-FLL-PHL", "sched_arr_time": "6:46 p.m.", "sched_dep_time": "4:00 p.m.", "src": "ua", "tuple_id": "1732" }, { "_row": "1732", "act_arr_time": "5:05 p.m.", "act_dep_time": "11:30 a.m.", "flight": "UA-3050-PHX-CLT", "sched_arr_time": "4:57 p.m.", "sched_dep_time": "11:05 a.m.", "src": "ua", "tuple_id": "1733" } ], "task_id": "DataForge-Bench-light-verified:flights:10058:0019", "task_index": 20, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":909,\"column\":\"abv\",\"new_value\":\"0.093\",\"reason\":\"The new value for 'abv' is incorrect.\"},{\"row\":910,\"column\":\"beer-name\",\"new_value\":\"Coffee Oatmeal Stout\",\"reason\":\"The new value for 'beer-name' is incorrect.\"},{\"row\":911,\"column\":\"abv\",\"new_value\":\"0.06\",\"reason\":\"The new value for 'abv' is incorrect.\"},{\"row\":912,\"column\":\"beer-name\",\"new_value\":\"Good People American Brown Ale\",\"reason\":\"The new value for 'beer-name' is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 4 }, "fn": 5, "fp": 4, "ground_truth_count": 5, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.093", "reason": "The new value for 'abv' is incorrect.", "row": 909 }, { "column": "beer-name", "new_value": "Coffee Oatmeal Stout", "reason": "The new value for 'beer-name' is incorrect.", "row": 910 }, { "column": "abv", "new_value": "0.06", "reason": "The new value for 'abv' is incorrect.", "row": 911 }, { "column": "beer-name", "new_value": "Good People American Brown Ale", "reason": "The new value for 'beer-name' is incorrect.", "row": 912 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "909", "abv": "0.093", "beer-name": "Snake Handler Double IPA", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "103", "id": "312", "index": "910", "ounces": "12.0 oz", "state": "AL", "style": "American Double / Imperial IPA" }, { "_row": "910", "abv": "0.06", "beer-name": "Coffee Oatmeal Stout", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "54", "id": "311", "index": "911", "ounces": "12.0 oz.", "state": "AL", "style": "Oatmeal Stout" }, { "_row": "911", "abv": "0.06", "beer-name": "Good People IPA", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "64", "id": "309", "index": "912", "ounces": "12.0 ounce", "state": "AL", "style": "American IPA" }, { "_row": "912", "abv": "0.057999999999999996%", "beer-name": "Good People American Brown Ale", "brewery-name": "Good People Brewing Company", "brewery_id": "478", "city": "Birmingham", "ibu": "36", "id": "308", "index": "913", "ounces": "12.0 ounce", "state": "AL", "style": "American Brown Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10062:0020", "task_index": 21, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":429,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":430,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":431,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":432,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1, "overrepair": 4 }, "fn": 1, "fp": 4, "ground_truth_count": 1, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 429 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 430 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 431 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 432 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "Address1": "315 w hickory st", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "pneumonia", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-6", "MeasureName": "pneumonia patients given the most appropriate initial antibiotic(s)", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "57 patients", "Score": "96%", "State": "al", "Stateavg": "al_pn-6", "ZipCode": "35150", "_row": "429", "index": "430" }, { "Address1": "315 w hickory st", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "pneumonia", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "pn-7", "MeasureName": "pneumonia patients assessed and given influenza vaccination", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "51 patients", "Score": "88%", "State": "al", "Stateavg": "al_pn-7", "ZipCode": "35150", "_row": "430", "index": "431" }, { "Address1": "315xwxhickoryxst", "Address2": "empty", "Address3": "empty", "City": "sylacauga", "Condition": "surgical infection prevention", "CountyName": "talladega", "EmergencyService": "no", "HospitalName": "coosa valley medical center", "HospitalOwner": "voluntary non-profit - other", "HospitalType": "acute care hospitals", "MeasureCode": "scip-card-2", "MeasureName": "surgery patients who were taking heart drugs called beta blockers before coming to the hospital who were kept on the beta blockers during the period just before and after their surgery", "PhoneNumber": "2562495000", "ProviderNumber": "10164", "Sample": "22 patients", "Score": "82%", "State": "al", "Stateavg": "al_scip-card-2", "ZipCode": "35150", "_row": "431", "index": "432" }, { "Address1": "2801 debarr road", "Address2": "empty", "Address3": "empty", "City": "anchorage", "Condition": "heart failure", "CountyName": "anchorage", "EmergencyService": "yes", "HospitalName": "alaska regional hospital", "HospitalOwner": "proprietary", "HospitalType": "acute care hospitals", "MeasureCode": "hf-1", "MeasureName": "heart failure patients given discharge instructions", "PhoneNumber": "9072761131", "ProviderNumber": "20017", "Sample": "57 patients", "Score": "91%", "State": "ak", "Stateavg": "ak_hf-1", "ZipCode": "99508", "_row": "432", "index": "433" } ], "task_id": "DataForge-Bench-light-verified:hospital:10063:0021", "task_index": 22, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":61,\"column\":\"flight\",\"new_value\":\"UA-414-LAX-SFO\",\"reason\":\"Flight UA-414 is scheduled to arrive at SFO instead of LAX.\"},{\"row\":62,\"column\":\"flight\",\"new_value\":\"UA-5487-SFO-MRY\",\"reason\":\"Flight UA-5487 is scheduled to depart from SFO instead of MRY.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 16, "overrepair": 1, "wrong_cell": 1 }, "fn": 16, "fp": 0, "ground_truth_count": 16, "inferability": "external_reference_required", "parse_error_kind": "invalid_row", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "UA-414-LAX-SFO", "reason": "Flight UA-414 is scheduled to arrive at SFO instead of LAX.", "row": 61 }, { "column": "flight", "new_value": "UA-5487-SFO-MRY", "reason": "Flight UA-5487 is scheduled to depart from SFO instead of MRY.", "row": 62 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "58", "act_arr_time": "4:54 p.m.", "act_dep_time": "11:41 a.m.", "flight": "AA-1544-SAN-ORD", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "59" }, { "_row": "59", "act_arr_time": "11:11 a.m.", "act_dep_time": "10:53 a.m.", "flight": "UA-5487-SFO-MRY", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "60" }, { "_row": "60", "act_arr_time": "12/2/11 6:35 a.m.", "act_dep_time": "11:29 p.m.", "flight": "AA-204-LAX-MCO", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "61" }, { "_row": "61", "act_arr_time": "7:04 a.m.", "act_dep_time": "6:10 a.m.", "flight": "UA-414-LAX-SFO", "sched_arr_time": "", "sched_dep_time": "", "src": "helloflight", "tuple_id": "62" } ], "task_id": "DataForge-Bench-light-verified:flights:10067:0022", "task_index": 23, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1325,\"column\":\"abv\",\"new_value\":\"0.053%\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1326,\"column\":\"abv\",\"new_value\":\"0.053\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1327,\"column\":\"abv\",\"new_value\":\"0.071\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1328,\"column\":\"abv\",\"new_value\":\"0.053\",\"reason\":\"The new value for abv is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 3, "wrong_value": 1 }, "fn": 5, "fp": 4, "ground_truth_count": 5, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.053%", "reason": "The new value for abv is incorrect.", "row": 1325 }, { "column": "abv", "new_value": "0.053", "reason": "The new value for abv is incorrect.", "row": 1326 }, { "column": "abv", "new_value": "0.071", "reason": "The new value for abv is incorrect.", "row": 1327 }, { "column": "abv", "new_value": "0.053", "reason": "The new value for abv is incorrect.", "row": 1328 } ], "recall": 0.0, "schema_case_errors": 0, "target_rows": [ { "_row": "1325", "abv": "0.053%", "beer-name": "Shark Bait", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "11", "id": "1508", "index": "1326", "ounces": "12.0 oz", "state": "FL", "style": "Fruit / Vegetable Beer" }, { "_row": "1326", "abv": "0.053", "beer-name": "Gator Tail Brown Ale", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "30", "id": "1507", "index": "1327", "ounces": "12.0 ounce", "state": "FL", "style": "American Brown Ale" }, { "_row": "1327", "abv": "0.071", "beer-name": "Miami Vice IPA", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "62", "id": "1506", "index": "1328", "ounces": "12.0 oz.", "state": "FL", "style": "American IPA" }, { "_row": "1328", "abv": "0.053", "beer-name": "Big Rod Coconut Ale", "brewery-name": "Miami Brewing Company", "brewery_id": "393", "city": "Miami", "ibu": "16", "id": "1325", "index": "1329", "ounces": "12.0 oz", "state": "FL", "style": "American Blonde Ale" } ], "task_id": "DataForge-Bench-light-verified:beers:10071:0023", "task_index": 24, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":286,\"column\":\"Index\",\"new_value\":\"10016\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":287,\"column\":\"Index\",\"new_value\":\"10016\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":288,\"column\":\"Index\",\"new_value\":\"10016\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":289,\"column\":\"Index\",\"new_value\":\"10016\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"}]}", "f1": 0.0, "failure_taxonomy": { "schema_case_error": 4 }, "fn": 0, "fp": 0, "ground_truth_count": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "10016", "reason": "The patient's condition is surgical infection prevention.", "row": 286 }, { "column": "Index", "new_value": "10016", "reason": "The patient's condition is surgical infection prevention.", "row": 287 }, { "column": "Index", "new_value": "10016", "reason": "The patient's condition is surgical infection prevention.", "row": 288 }, { "column": "Index", "new_value": "10016", "reason": "The patient's condition is surgical infection prevention.", "row": 289 } ], "recall": 0.0, "schema_case_errors": 4, "target_rows": [ { "Address1": "1000 first street north", "Address2": "empty", "Address3": "empty", "City": "alabaster", "Condition": "surgical infection prevention", "CountyName": "shelby", "EmergencyService": "yes", "HospitalName": "shelby baptist medical center", "HospitalOwner": "voluntary non-profit - church", "HospitalType": "acute care hospitals", "MeasureCode": "scip-card-2", "MeasureName": "surgery patients who were taking heart drugs called beta blockers before coming to the hospital who were kept on the beta blockers during the period just before and after their surgery", "PhoneNumber": "2056208100", "ProviderNumber": "10016", "Sample": "96 patients", "Score": "89%", "State": "al", "Stateavg": "al_scip-card-2", "ZipCode": "35007", "_row": "286", "index": "287" }, { "Address1": "1000 first street north", "Address2": "empty", "Address3": "empty", "City": "alabaster", "Condition": "surgical infection prevention", "CountyName": "shelby", "EmergencyService": "yes", "HospitalName": "shelby baptist medical center", "HospitalOwner": "voluntary non-profit - church", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-1", "MeasureName": "surgery patients who were given an antibiotic at the right time (within one hour before surgery) to help prevent infection", "PhoneNumber": "2056208100", "ProviderNumber": "10016", "Sample": "368 patients", "Score": "98%", "State": "al", "Stateavg": "al_scip-inf-1", "ZipCode": "35007", "_row": "287", "index": "288" }, { "Address1": "1000 first street north", "Address2": "empty", "Address3": "empty", "City": "alabaster", "Condition": "surgical infection prevention", "CountyName": "shelby", "EmergencyService": "yes", "HospitalName": "shelby baptist medical center", "HospitalOwner": "voluntary non-profit - church", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-2", "MeasureName": "surgery patients who were given the right kind of antibiotic to help prevent infection", "PhoneNumber": "2056208100", "ProviderNumber": "10016", "Sample": "370 patients", "Score": "99%", "State": "al", "Stateavg": "al_scip-inf-2", "ZipCode": "35007", "_row": "288", "index": "289" }, { "Address1": "1000 first street north", "Address2": "empty", "Address3": "empty", "City": "alabaster", "Condition": "surgical infection prevention", "CountyName": "shelby", "EmergencyService": "yes", "HospitalName": "shelby baptist medical center", "HospitalOwner": "voluntary non-profit - church", "HospitalType": "acute care hospitals", "MeasureCode": "scip-inf-3", "MeasureName": "surgery patients whose preventive antibiotics were stopped at the right time (within 24 hours after surgery)", "PhoneNumber": "2056208100", "ProviderNumber": "10016", "Sample": "331 patients", "Score": "91%", "State": "al", "Stateavg": "al_scip-inf-3", "ZipCode": "35007", "_row": "289", "index": "290" } ], "task_id": "DataForge-Bench-light-verified:hospital:10072:0024", "task_index": 25, "tp": 0 } ], "model_label": "sft", "runtime_seconds": 645.014, "task_scores": [ { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":55,\"column\":\"ProviderNumber\",\"new_value\":\"10005\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to any hospital records.\"},{\"row\":56,\"column\":\"ProviderNumber\",\"new_value\":\"10005\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to any hospital records.\"},{\"row\":57,\"column\":\"ProviderNumber\",\"new_value\":\"10005\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to any hospital records.\"},{\"row\":58,\"column\":\"ProviderNumber\",\"new_value\":\"10005\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to any hospital records.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 4 }, "fn": 4, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "10005", "reason": "The patient's address does not match any hospital records.", "row": 55 }, { "column": "ProviderNumber", "new_value": "10005", "reason": "The patient's address does not match any hospital records.", "row": 56 }, { "column": "ProviderNumber", "new_value": "10005", "reason": "The patient's address does not match any hospital records.", "row": 57 }, { "column": "ProviderNumber", "new_value": "10005", "reason": "The patient's address does not match any hospital records.", "row": 58 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10000:0000", "task_index": 1, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1651,\"column\":\"act_arr_time\",\"new_value\":\"11:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1652,\"column\":\"act_arr_time\",\"new_value\":\"11:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1653,\"column\":\"act_arr_time\",\"new_value\":\"11:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1654,\"column\":\"act_arr_time\",\"new_value\":\"11:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "wrong_value": 4 }, "fn": 12, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "11:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1651 }, { "column": "act_arr_time", "new_value": "11:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1652 }, { "column": "act_arr_time", "new_value": "11:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1653 }, { "column": "act_arr_time", "new_value": "11:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1654 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10004:0001", "task_index": 2, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":657,\"column\":\"abv\",\"new_value\":\"0.052\",\"reason\":\"The new value for abv is 0.052, which does not exist in the beer records.\"},{\"row\":658,\"column\":\"abv\",\"new_value\":\"0.06\",\"reason\":\"The new value for abv is 0.06, which does not exist in the beer records.\"},{\"row\":659,\"column\":\"abv\",\"new_value\":\"0.052\",\"reason\":\"The new value for abv is 0.052, which exists but has a different meaning than the original value.\"},{\"row\":660,\"column\":\"abv\",\"new_value\":\"0.065\",\"reason\":\"The new value for abv is 0.065, which does not exist in the beer records.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 4 }, "fn": 5, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.052", "reason": "The new value for abv is 0.052, which does not exist in the beer records.", "row": 657 }, { "column": "abv", "new_value": "0.06", "reason": "The new value for abv is 0.06, which does not exist in the beer records.", "row": 658 }, { "column": "abv", "new_value": "0.052", "reason": "The new value for abv is 0.052, which exists but has a different meaning than the original value.", "row": 659 }, { "column": "abv", "new_value": "0.065", "reason": "The new value for abv is 0.065, which does not exist in the beer records.", "row": 660 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10008:0002", "task_index": 3, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":161,\"column\":\"Index\",\"new_value\":\"10009\",\"reason\":\"The patient's name matches the expected value.\",\"_type\":\"patient\"},{\"row\":162,\"column\":\"Index\",\"new_value\":\"10009\",\"reason\":\"The patient's name matches the expected value.\",\"_type\":\"patient\"},{\"row\":163,\"column\":\"Index\",\"new_value\":\"10009\",\"reason\":\"The patient's name matches the expected value.\",\"_type\":\"patient\"},{\"row\":164,\"column\":\"Index\",\"new_value\":\"10009\",\"reason\":\"The patient's name matches the expected value.\",\"_type\":\"patient\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "schema_case_error": 4 }, "fn": 4, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "10009", "reason": "The patient's name matches the expected value.", "row": 161 }, { "column": "Index", "new_value": "10009", "reason": "The patient's name matches the expected value.", "row": 162 }, { "column": "Index", "new_value": "10009", "reason": "The patient's name matches the expected value.", "row": 163 }, { "column": "Index", "new_value": "10009", "reason": "The patient's name matches the expected value.", "row": 164 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10009:0003", "task_index": 4, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1433,\"column\":\"act_arr_time\",\"new_value\":\"3:31 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1434,\"column\":\"act_arr_time\",\"new_value\":\"5:26 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1435,\"column\":\"act_arr_time\",\"new_value\":\"7:10 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1436,\"column\":\"act_arr_time\",\"new_value\":\"12:09 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "3:31 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1433 }, { "column": "act_arr_time", "new_value": "5:26 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1434 }, { "column": "act_arr_time", "new_value": "7:10 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1435 }, { "column": "act_arr_time", "new_value": "12:09 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1436 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10013:0004", "task_index": 5, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1592,\"column\":\"style\",\"new_value\":\"Scottish Ale\",\"reason\":\"The beer name has been changed to 'Scottish Ale' which is a different style than the previous one.\"},{\"row\":1591,\"column\":\"style\",\"new_value\":\"American Pale Ale (APA)\",\"reason\":\"The beer name has been changed to 'American Pale Ale (APA)' which is a different style than the previous one.\"},{\"row\":1590,\"column\":\"style\",\"new_value\":\"Scottish Ale\",\"reason\":\"The beer name has been changed to 'Scottish Ale' which is a different style than the previous one.\"},{\"row\":1589,\"column\":\"style\",\"new_value\":\"American Pale Ale (APA)\",\"reason\":\"The beer name has been changed to 'American Pale Ale (APA)' which is a different style than the previous one.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 4 }, "fn": 7, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "style", "new_value": "Scottish Ale", "reason": "The beer name has been changed to 'Scottish Ale' which is a different style than the previous one.", "row": 1592 }, { "column": "style", "new_value": "American Pale Ale (APA)", "reason": "The beer name has been changed to 'American Pale Ale (APA)' which is a different style than the previous one.", "row": 1591 }, { "column": "style", "new_value": "Scottish Ale", "reason": "The beer name has been changed to 'Scottish Ale' which is a different style than the previous one.", "row": 1590 }, { "column": "style", "new_value": "American Pale Ale (APA)", "reason": "The beer name has been changed to 'American Pale Ale (APA)' which is a different style than the previous one.", "row": 1589 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10017:0005", "task_index": 6, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":512,\"column\":\"ProviderNumber\",\"new_value\":\"10022\",\"reason\":\"The patient's condition requires immediate attention and may require urgent care.\",\"why\":\"The proposed new value '10022' aligns with the emergency service requirement for this specific patient.\"},{\"row\":513,\"column\":\"ProviderNumber\",\"new_value\":\"10022\",\"reason\":\"The patient's condition requires immediate attention and may require urgent care.\",\"why\":\"The proposed new value '10022' aligns with the emergency service requirement for this specific patient.\"},{\"row\":514,\"column\":\"ProviderNumber\",\"new_value\":\"10022\",\"reason\":\"The patient's condition requires immediate attention and may require urgent care.\",\"why\":\"The proposed new value '10022' aligns with the emergency service requirement for this specific patient.\"},{\"row\":515,\"column\":\"ProviderNumber\",\"new_value\":\"10022\",\"reason\":\"The patient's condition requires immediate attention and may require urgent care.\",\"why\":\"The proposed new value '10022' aligns with the emergency service requirement for this specific patient.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "overrepair": 4 }, "fn": 2, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "10022", "reason": "The patient's condition requires immediate attention and may require urgent care.", "row": 512 }, { "column": "ProviderNumber", "new_value": "10022", "reason": "The patient's condition requires immediate attention and may require urgent care.", "row": 513 }, { "column": "ProviderNumber", "new_value": "10022", "reason": "The patient's condition requires immediate attention and may require urgent care.", "row": 514 }, { "column": "ProviderNumber", "new_value": "10022", "reason": "The patient's condition requires immediate attention and may require urgent care.", "row": 515 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10018:0006", "task_index": 7, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":324,\"column\":\"act_arr_time\",\"new_value\":\"2:49 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":325,\"column\":\"act_arr_time\",\"new_value\":\"1:23 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":326,\"column\":\"act_arr_time\",\"new_value\":\"10:40 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":327,\"column\":\"act_arr_time\",\"new_value\":\"\",\"reason\":\"The new value for act_arr_time is empty.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "wrong_value": 4 }, "fn": 8, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "2:49 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 324 }, { "column": "act_arr_time", "new_value": "1:23 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 325 }, { "column": "act_arr_time", "new_value": "10:40 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 326 }, { "column": "act_arr_time", "new_value": "", "reason": "The new value for act_arr_time is empty.", "row": 327 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10022:0007", "task_index": 8, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":379,\"column\":\"abv\",\"new_value\":\"0.045\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":380,\"column\":\"abv\",\"new_value\":\"0.062%\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":381,\"column\":\"abv\",\"new_value\":\"0.054\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":382,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The new value for ABV is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 3, "wrong_value": 1 }, "fn": 8, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.045", "reason": "The new value for ABV is incorrect.", "row": 379 }, { "column": "abv", "new_value": "0.062%", "reason": "The new value for ABV is incorrect.", "row": 380 }, { "column": "abv", "new_value": "0.054", "reason": "The new value for ABV is incorrect.", "row": 381 }, { "column": "abv", "new_value": "0.05", "reason": "The new value for ABV is incorrect.", "row": 382 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10026:0008", "task_index": 9, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":52,\"column\":\"Address1\",\"new_value\":\"2505 u s highway 431 north\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be 2505 U.S. Highway 431 North.\"},{\"row\":53,\"column\":\"Address1\",\"new_value\":\"2505 u s highway 431 north\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be 2505 U.S. Highway 431 North.\"},{\"row\":54,\"column\":\"Address1\",\"new_value\":\"2505 u s highway 431 north\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be 2505 U.S. Highway 431 North.\"},{\"row\":55,\"column\":\"Address1\",\"new_value\":\"2505 u s highway 431 north\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be 2505 U.S. Highway 431 North.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "2505 u s highway 431 north", "reason": "The address is incorrect.", "row": 52 }, { "column": "Address1", "new_value": "2505 u s highway 431 north", "reason": "The address is incorrect.", "row": 53 }, { "column": "Address1", "new_value": "2505 u s highway 431 north", "reason": "The address is incorrect.", "row": 54 }, { "column": "Address1", "new_value": "2505 u s highway 431 north", "reason": "The address is incorrect.", "row": 55 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10027:0009", "task_index": 10, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":655,\"column\":\"act_arr_time\",\"new_value\":\"9:05 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":656,\"column\":\"act_arr_time\",\"new_value\":\"5:03 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":657,\"column\":\"act_arr_time\",\"new_value\":\"10:27 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":658,\"column\":\"act_arr_time\",\"new_value\":\"10:56 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 3, "wrong_value": 1 }, "fn": 9, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "9:05 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 655 }, { "column": "act_arr_time", "new_value": "5:03 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 656 }, { "column": "act_arr_time", "new_value": "10:27 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 657 }, { "column": "act_arr_time", "new_value": "10:56 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 658 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10031:0010", "task_index": 11, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2211,\"column\":\"beer-name\",\"new_value\":\"Triangle Belgian Golden Ale\",\"reason\":\"The beer name has been changed to 'Triangle Belgian Golden Ale'.\"},{\"row\":2212,\"column\":\"beer-name\",\"new_value\":\"Troegenator\",\"reason\":\"The beer name has been changed to 'Troegenator'.\"},{\"row\":2213,\"column\":\"beer-name\",\"new_value\":\"Nugget Nectar\",\"reason\":\"The beer name has been changed to 'Nugget Nectar'.\"},{\"row\":2214,\"column\":\"beer-name\",\"new_value\":\"Triangle Belgian Golden Ale\",\"reason\":\"The beer name has been changed to 'Triangle Belgian Golden Ale'.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 4 }, "fn": 8, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "beer-name", "new_value": "Triangle Belgian Golden Ale", "reason": "The beer name has been changed to 'Triangle Belgian Golden Ale'.", "row": 2211 }, { "column": "beer-name", "new_value": "Troegenator", "reason": "The beer name has been changed to 'Troegenator'.", "row": 2212 }, { "column": "beer-name", "new_value": "Nugget Nectar", "reason": "The beer name has been changed to 'Nugget Nectar'.", "row": 2213 }, { "column": "beer-name", "new_value": "Triangle Belgian Golden Ale", "reason": "The beer name has been changed to 'Triangle Belgian Golden Ale'.", "row": 2214 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10035:0011", "task_index": 12, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":502,\"column\":\"ProviderNumber\",\"new_value\":\"Voluntary Non-Profit - Private\",\"reason\":\"The Provider number is already assigned.\",\"}}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "truncated_json", "parse_ok": false, "precision": 0.0, "predicted_repairs": [], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10036:0012", "task_index": 13, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":679,\"column\":\"act_arr_time\",\"new_value\":\"7:53 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":680,\"column\":\"act_arr_time\",\"new_value\":\"6:35 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":681,\"column\":\"act_arr_time\",\"new_value\":\"7:36 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":682,\"column\":\"act_arr_time\",\"new_value\":\"9:49 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 1, "wrong_value": 3 }, "fn": 10, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "7:53 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 679 }, { "column": "act_arr_time", "new_value": "6:35 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 680 }, { "column": "act_arr_time", "new_value": "7:36 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 681 }, { "column": "act_arr_time", "new_value": "9:49 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 682 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10040:0013", "task_index": 14, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":739,\"column\":\"abv\",\"new_value\":\"0.043\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":740,\"column\":\"ibu\",\"new_value\":\"12\",\"reason\":\"The new value for ibu is incorrect.\"},{\"row\":741,\"column\":\"ounces\",\"new_value\":\"12.0 ounce\",\"reason\":\"The new value for ounces is incorrect.\"},{\"row\":742,\"column\":\"beer-name\",\"new_value\":\"Dundee Summer Wheat Beer\",\"reason\":\"The new value for beer-name is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 3, "wrong_value": 1 }, "fn": 7, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.043", "reason": "The new value for abv is incorrect.", "row": 739 }, { "column": "ibu", "new_value": "12", "reason": "The new value for ibu is incorrect.", "row": 740 }, { "column": "ounces", "new_value": "12.0 ounce", "reason": "The new value for ounces is incorrect.", "row": 741 }, { "column": "beer-name", "new_value": "Dundee Summer Wheat Beer", "reason": "The new value for beer-name is incorrect.", "row": 742 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10044:0014", "task_index": 15, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":384,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's provider number is missing.\",\"why\":\"This will prevent us from accurately updating their health information.\"},{\"row\":385,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's provider number is missing.\",\"why\":\"This will prevent us from accurately updating their health information.\"},{\"row\":386,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's provider number is missing.\",\"why\":\"This will prevent us from accurately updating their health information.\"},{\"row\":387,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's provider number is missing.\",\"why\":\"This will prevent us from accurately updating their health information.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 4 }, "fn": 7, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's provider number is missing.", "row": 384 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's provider number is missing.", "row": 385 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's provider number is missing.", "row": 386 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's provider number is missing.", "row": 387 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10045:0015", "task_index": 16, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1215,\"column\":\"act_arr_time\",\"new_value\":\"3:13 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1216,\"column\":\"act_arr_time\",\"new_value\":\"1:30 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1217,\"column\":\"act_arr_time\",\"new_value\":\"6:36 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1218,\"column\":\"act_arr_time\",\"new_value\":\"7:35 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "overrepair": 2, "wrong_value": 2 }, "fn": 4, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "3:13 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1215 }, { "column": "act_arr_time", "new_value": "1:30 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1216 }, { "column": "act_arr_time", "new_value": "6:36 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1217 }, { "column": "act_arr_time", "new_value": "7:35 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1218 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10049:0016", "task_index": 17, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":123,\"column\":\"abv\",\"new_value\":\"0.05%\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":124,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":125,\"column\":\"abv\",\"new_value\":\"0.068\",\"reason\":\"The new value for ABV is incorrect.\"},{\"row\":126,\"column\":\"abv\",\"new_value\":\"0.06\",\"reason\":\"The new value for ABV is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 3, "wrong_value": 1 }, "fn": 9, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.05%", "reason": "The new value for ABV is incorrect.", "row": 123 }, { "column": "abv", "new_value": "0.05", "reason": "The new value for ABV is incorrect.", "row": 124 }, { "column": "abv", "new_value": "0.068", "reason": "The new value for ABV is incorrect.", "row": 125 }, { "column": "abv", "new_value": "0.06", "reason": "The new value for ABV is incorrect.", "row": 126 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10053:0017", "task_index": 18, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":82,\"column\":\"Index\",\"new_value\":\"83\",\"reason\":\"The patient's condition has changed from heart failure to pneumonia.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "schema_case_error": 1 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "83", "reason": "The patient's condition has changed from heart failure to pneumonia.", "row": 82 } ], "recall": 0.0, "schema_case_errors": 1, "task_id": "DataForge-Bench-light-verified:hospital:10054:0018", "task_index": 19, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1732,\"column\":\"flight\",\"new_value\":\"UA-3050-PHX-CLT\",\"reason\":\"The flight is currently on a different schedule than the other flights listed.\"},{\"row\":1731,\"column\":\"flight\",\"new_value\":\"UA-5487-SFO-MRY\",\"reason\":\"The flight is currently on a different schedule than the other flights listed.\"},{\"row\":1730,\"column\":\"flight\",\"new_value\":\"UA-2830-MCO-CLT\",\"reason\":\"The flight is currently on a different schedule than the other flights listed.\"},{\"row\":1729,\"column\":\"flight\",\"new_value\":\"UA-2726-FLL-PHL\",\"reason\":\"The flight is currently on a different schedule than the other flights listed.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "UA-3050-PHX-CLT", "reason": "The flight is currently on a different schedule than the other flights listed.", "row": 1732 }, { "column": "flight", "new_value": "UA-5487-SFO-MRY", "reason": "The flight is currently on a different schedule than the other flights listed.", "row": 1731 }, { "column": "flight", "new_value": "UA-2830-MCO-CLT", "reason": "The flight is currently on a different schedule than the other flights listed.", "row": 1730 }, { "column": "flight", "new_value": "UA-2726-FLL-PHL", "reason": "The flight is currently on a different schedule than the other flights listed.", "row": 1729 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10058:0019", "task_index": 20, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":909,\"column\":\"abv\",\"new_value\":\"0.093\",\"reason\":\"The new value for 'abv' is incorrect.\"},{\"row\":910,\"column\":\"beer-name\",\"new_value\":\"Coffee Oatmeal Stout\",\"reason\":\"The new value for 'beer-name' is incorrect.\"},{\"row\":911,\"column\":\"abv\",\"new_value\":\"0.06\",\"reason\":\"The new value for 'abv' is incorrect.\"},{\"row\":912,\"column\":\"beer-name\",\"new_value\":\"Good People American Brown Ale\",\"reason\":\"The new value for 'beer-name' is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 4 }, "fn": 5, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.093", "reason": "The new value for 'abv' is incorrect.", "row": 909 }, { "column": "beer-name", "new_value": "Coffee Oatmeal Stout", "reason": "The new value for 'beer-name' is incorrect.", "row": 910 }, { "column": "abv", "new_value": "0.06", "reason": "The new value for 'abv' is incorrect.", "row": 911 }, { "column": "beer-name", "new_value": "Good People American Brown Ale", "reason": "The new value for 'beer-name' is incorrect.", "row": 912 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10062:0020", "task_index": 21, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":429,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":430,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":431,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":432,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1, "overrepair": 4 }, "fn": 1, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 429 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 430 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 431 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 432 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10063:0021", "task_index": 22, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":61,\"column\":\"flight\",\"new_value\":\"UA-414-LAX-SFO\",\"reason\":\"Flight UA-414 is scheduled to arrive at SFO instead of LAX.\"},{\"row\":62,\"column\":\"flight\",\"new_value\":\"UA-5487-SFO-MRY\",\"reason\":\"Flight UA-5487 is scheduled to depart from SFO instead of MRY.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 16, "overrepair": 1, "wrong_cell": 1 }, "fn": 16, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_row", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "UA-414-LAX-SFO", "reason": "Flight UA-414 is scheduled to arrive at SFO instead of LAX.", "row": 61 }, { "column": "flight", "new_value": "UA-5487-SFO-MRY", "reason": "Flight UA-5487 is scheduled to depart from SFO instead of MRY.", "row": 62 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10067:0022", "task_index": 23, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1325,\"column\":\"abv\",\"new_value\":\"0.053%\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1326,\"column\":\"abv\",\"new_value\":\"0.053\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1327,\"column\":\"abv\",\"new_value\":\"0.071\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1328,\"column\":\"abv\",\"new_value\":\"0.053\",\"reason\":\"The new value for abv is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 3, "wrong_value": 1 }, "fn": 5, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.053%", "reason": "The new value for abv is incorrect.", "row": 1325 }, { "column": "abv", "new_value": "0.053", "reason": "The new value for abv is incorrect.", "row": 1326 }, { "column": "abv", "new_value": "0.071", "reason": "The new value for abv is incorrect.", "row": 1327 }, { "column": "abv", "new_value": "0.053", "reason": "The new value for abv is incorrect.", "row": 1328 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10071:0023", "task_index": 24, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":286,\"column\":\"Index\",\"new_value\":\"10016\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":287,\"column\":\"Index\",\"new_value\":\"10016\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":288,\"column\":\"Index\",\"new_value\":\"10016\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":289,\"column\":\"Index\",\"new_value\":\"10016\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"}]}", "f1": 0.0, "failure_taxonomy": { "schema_case_error": 4 }, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "10016", "reason": "The patient's condition is surgical infection prevention.", "row": 286 }, { "column": "Index", "new_value": "10016", "reason": "The patient's condition is surgical infection prevention.", "row": 287 }, { "column": "Index", "new_value": "10016", "reason": "The patient's condition is surgical infection prevention.", "row": 288 }, { "column": "Index", "new_value": "10016", "reason": "The patient's condition is surgical infection prevention.", "row": 289 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10072:0024", "task_index": 25, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1523,\"column\":\"act_arr_time\",\"new_value\":\"1:11 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1524,\"column\":\"act_arr_time\",\"new_value\":\"1:11 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1525,\"column\":\"act_arr_time\",\"new_value\":\"2:39 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1526,\"column\":\"act_arr_time\",\"new_value\":\"2:39 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "wrong_value": 4 }, "fn": 12, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "1:11 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1523 }, { "column": "act_arr_time", "new_value": "1:11 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1524 }, { "column": "act_arr_time", "new_value": "2:39 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1525 }, { "column": "act_arr_time", "new_value": "2:39 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1526 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10076:0025", "task_index": 26, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":299,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The beer name is incorrect.\"},{\"row\":300,\"column\":\"abv\",\"new_value\":\"0.064\",\"reason\":\"The beer name is incorrect.\"},{\"row\":301,\"column\":\"abv\",\"new_value\":\"0.045\",\"reason\":\"The beer name is incorrect.\"},{\"row\":302,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The beer name is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 4 }, "fn": 8, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.05", "reason": "The beer name is incorrect.", "row": 299 }, { "column": "abv", "new_value": "0.064", "reason": "The beer name is incorrect.", "row": 300 }, { "column": "abv", "new_value": "0.045", "reason": "The beer name is incorrect.", "row": 301 }, { "column": "abv", "new_value": "0.05", "reason": "The beer name is incorrect.", "row": 302 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10080:0026", "task_index": 27, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":70,\"column\":\"Index\",\"new_value\":\"71\",\"reason\":\"The patient's address has been updated to 205 Marengo Street.\"},{\"row\":71,\"column\":\"Index\",\"new_value\":\"72\",\"reason\":\"The patient's address has been updated to 205 Marengo Street.\"},{\"row\":72,\"column\":\"Index\",\"new_value\":\"73\",\"reason\":\"The patient's address has been updated to 205 Marengo Street.\"},{\"row\":73,\"column\":\"Index\",\"new_value\":\"74\",\"reason\":\"The patient's address has been updated to 205 Marengo Street.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3, "schema_case_error": 4 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "71", "reason": "The patient's address has been updated to 205 Marengo Street.", "row": 70 }, { "column": "Index", "new_value": "72", "reason": "The patient's address has been updated to 205 Marengo Street.", "row": 71 }, { "column": "Index", "new_value": "73", "reason": "The patient's address has been updated to 205 Marengo Street.", "row": 72 }, { "column": "Index", "new_value": "74", "reason": "The patient's address has been updated to 205 Marengo Street.", "row": 73 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10081:0027", "task_index": 28, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":521,\"column\":\"act_arr_time\",\"new_value\":\"10:10 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":522,\"column\":\"sched_arr_time\",\"new_value\":\"2:35 p.m.\",\"reason\":\"The new value for sched_arr_time is incorrect.\"},{\"row\":523,\"column\":\"act_dep_time\",\"new_value\":\"6:43 p.m.\",\"reason\":\"The new value for act_dep_time is incorrect.\"},{\"row\":524,\"column\":\"sched_dep_time\",\"new_value\":\"7:57 a.m.\",\"reason\":\"The new value for sched_dep_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 1, "wrong_value": 3 }, "fn": 7, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "10:10 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 521 }, { "column": "sched_arr_time", "new_value": "2:35 p.m.", "reason": "The new value for sched_arr_time is incorrect.", "row": 522 }, { "column": "act_dep_time", "new_value": "6:43 p.m.", "reason": "The new value for act_dep_time is incorrect.", "row": 523 }, { "column": "sched_dep_time", "new_value": "7:57 a.m.", "reason": "The new value for sched_dep_time is incorrect.", "row": 524 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10085:0028", "task_index": 29, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1713,\"column\":\"abv\",\"new_value\":\"0.064\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1714,\"column\":\"abv\",\"new_value\":\"0.064\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1715,\"column\":\"abv\",\"new_value\":\"0.064\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1716,\"column\":\"abv\",\"new_value\":\"0.064\",\"reason\":\"The new value for abv is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 10, "overrepair": 4 }, "fn": 10, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.064", "reason": "The new value for abv is incorrect.", "row": 1713 }, { "column": "abv", "new_value": "0.064", "reason": "The new value for abv is incorrect.", "row": 1714 }, { "column": "abv", "new_value": "0.064", "reason": "The new value for abv is incorrect.", "row": 1715 }, { "column": "abv", "new_value": "0.064", "reason": "The new value for abv is incorrect.", "row": 1716 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10089:0029", "task_index": 30, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":530,\"column\":\"ProviderNumber\",\"new_value\":\"10023\",\"reason\":\"The ProviderNumber should be unique for each hospital.\"},{\"row\":531,\"column\":\"ProviderNumber\",\"new_value\":\"10023\",\"reason\":\"The ProviderNumber should be unique for each hospital.\"},{\"row\":532,\"column\":\"ProviderNumber\",\"new_value\":\"10023\",\"reason\":\"The ProviderNumber should be unique for each hospital.\"},{\"row\":533,\"column\":\"ProviderNumber\",\"new_value\":\"10023\",\"reason\":\"The ProviderNumber should be unique for each hospital.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 4 }, "fn": 4, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "10023", "reason": "The ProviderNumber should be unique for each hospital.", "row": 530 }, { "column": "ProviderNumber", "new_value": "10023", "reason": "The ProviderNumber should be unique for each hospital.", "row": 531 }, { "column": "ProviderNumber", "new_value": "10023", "reason": "The ProviderNumber should be unique for each hospital.", "row": 532 }, { "column": "ProviderNumber", "new_value": "10023", "reason": "The ProviderNumber should be unique for each hospital.", "row": 533 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10090:0030", "task_index": 31, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":445,\"column\":\"act_arr_time\",\"new_value\":\"3:31 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":446,\"column\":\"act_arr_time\",\"new_value\":\"11:21 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":447,\"column\":\"act_arr_time\",\"new_value\":\"11:21 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":448,\"column\":\"act_arr_time\",\"new_value\":\"7:17 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 11, "overrepair": 3, "wrong_value": 1 }, "fn": 12, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "3:31 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 445 }, { "column": "act_arr_time", "new_value": "11:21 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 446 }, { "column": "act_arr_time", "new_value": "11:21 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 447 }, { "column": "act_arr_time", "new_value": "7:17 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 448 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10094:0031", "task_index": 32, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":979,\"column\":\"abv\",\"new_value\":\"0.05%\",\"reason\":\"The beer name is incorrect.\"},{\"row\":980,\"column\":\"abv\",\"new_value\":\"0.057\",\"reason\":\"The beer name is incorrect.\"},{\"row\":981,\"column\":\"abv\",\"new_value\":\"0.067\",\"reason\":\"The beer name is incorrect.\"},{\"row\":982,\"column\":\"abv\",\"new_value\":\"0.058\",\"reason\":\"The beer name is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 3, "wrong_value": 1 }, "fn": 6, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.05%", "reason": "The beer name is incorrect.", "row": 979 }, { "column": "abv", "new_value": "0.057", "reason": "The beer name is incorrect.", "row": 980 }, { "column": "abv", "new_value": "0.067", "reason": "The beer name is incorrect.", "row": 981 }, { "column": "abv", "new_value": "0.058", "reason": "The beer name is incorrect.", "row": 982 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10098:0032", "task_index": 33, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":241,\"column\":\"ProviderNumber\",\"new_value\":\"x00x5\",\"reason\":\"The patient's name does not match any existing records.\",\"why\":\"This action is necessary as the current record for the patient with ProviderNumber x00x5 was found in the 'target_rows' array.\"},{\"row\":242,\"column\":\"ProviderNumber\",\"new_value\":\"x00x5\",\"reason\":\"The patient's name does not match any existing records.\",\"why\":\"This action is necessary as the current record for the patient with ProviderNumber x00x5 was found in the 'target_rows' array.\"},{\"row\":243,\"column\":\"ProviderNumber\",\"new_value\":\"x00x5\",\"reason\":\"The patient's name does not match any existing records.\",\"why\":\"This action is necessary as the current record for the patient with ProviderNumber x00x5 was found in the 'target_rows' array.\"},{\"row\":244,\"column\":\"ProviderNumber\",\"new_value\":\"x00x5\",\"reason\":\"The patient's name does not match any existing records.\",\"why\":\"This action is necessary as the current record for the patient with ProviderNumber x00x5 was found in the 'target_rows' array.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "overrepair": 3, "wrong_value": 1 }, "fn": 3, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "x00x5", "reason": "The patient's name does not match any existing records.", "row": 241 }, { "column": "ProviderNumber", "new_value": "x00x5", "reason": "The patient's name does not match any existing records.", "row": 242 }, { "column": "ProviderNumber", "new_value": "x00x5", "reason": "The patient's name does not match any existing records.", "row": 243 }, { "column": "ProviderNumber", "new_value": "x00x5", "reason": "The patient's name does not match any existing records.", "row": 244 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10099:0033", "task_index": 34, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":612,\"column\":\"act_arr_time\",\"new_value\":\"7:10 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":613,\"column\":\"act_arr_time\",\"new_value\":\"3:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":614,\"column\":\"act_arr_time\",\"new_value\":\"5:43 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":615,\"column\":\"act_arr_time\",\"new_value\":\"11:06 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "7:10 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 612 }, { "column": "act_arr_time", "new_value": "3:43 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 613 }, { "column": "act_arr_time", "new_value": "5:43 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 614 }, { "column": "act_arr_time", "new_value": "11:06 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 615 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10103:0034", "task_index": 35, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1815,\"column\":\"abv\",\"new_value\":\"0.055\",\"reason\":\"The beer name 'Freestyle Pilsner' does not exist in the database.\"},{\"row\":1816,\"column\":\"abv\",\"new_value\":\"0.066\",\"reason\":\"The beer name 'Happy Camper IPA' does not exist in the database.\"},{\"row\":1817,\"column\":\"abv\",\"new_value\":\"0.05%\",\"reason\":\"The beer name 'Oval Beach Blonde Ale' does not exist in the database.\"},{\"row\":1818,\"column\":\"abv\",\"new_value\":\"0.065\",\"reason\":\"The beer name 'Oak Aged Cider' does not exist in the database.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 3, "wrong_value": 1 }, "fn": 8, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.055", "reason": "The beer name 'Freestyle Pilsner' does not exist in the database.", "row": 1815 }, { "column": "abv", "new_value": "0.066", "reason": "The beer name 'Happy Camper IPA' does not exist in the database.", "row": 1816 }, { "column": "abv", "new_value": "0.05%", "reason": "The beer name 'Oval Beach Blonde Ale' does not exist in the database.", "row": 1817 }, { "column": "abv", "new_value": "0.065", "reason": "The beer name 'Oak Aged Cider' does not exist in the database.", "row": 1818 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10107:0035", "task_index": 36, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":175,\"column\":\"Address1\",\"new_value\":\"8000 alabama xigxway 69\",\"reason\":\"The address does not match any allowed column value.\",\"why\":\"This address is not associated with a specific hospital or provider.\"},{\"row\":176,\"column\":\"Address2\",\"new_value\":\"empty\",\"reason\":\"The address does not match any allowed column value.\",\"why\":\"This address is empty and cannot be used as a reference point for repairs.\"},{\"row\":177,\"column\":\"Address3\",\"new_value\":\"empty\",\"reason\":\"The address does not match any allowed column value.\",\"why\":\"This address is empty and cannot be used as a reference point for repairs.\"},{\"row\":178,\"column\":\"Address1\",\"new_value\":\"8000 alabama highway 69\",\"reason\":\"The address does not match any allowed column value.\",\"why\":\"This address is not associated with a specific hospital or provider.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1, "overrepair": 3, "wrong_value": 1 }, "fn": 2, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "8000 alabama xigxway 69", "reason": "The address does not match any allowed column value.", "row": 175 }, { "column": "Address2", "new_value": "empty", "reason": "The address does not match any allowed column value.", "row": 176 }, { "column": "Address3", "new_value": "empty", "reason": "The address does not match any allowed column value.", "row": 177 }, { "column": "Address1", "new_value": "8000 alabama highway 69", "reason": "The address does not match any allowed column value.", "row": 178 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10108:0036", "task_index": 37, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1716,\"column\":\"act_arr_time\",\"new_value\":\"1:42 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1717,\"column\":\"act_arr_time\",\"new_value\":\"2:49 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1718,\"column\":\"act_arr_time\",\"new_value\":\"6:30 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1719,\"column\":\"act_arr_time\",\"new_value\":\"1:19 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "wrong_value": 4 }, "fn": 9, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "1:42 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1716 }, { "column": "act_arr_time", "new_value": "2:49 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1717 }, { "column": "act_arr_time", "new_value": "6:30 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1718 }, { "column": "act_arr_time", "new_value": "1:19 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1719 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10112:0037", "task_index": 38, "tp": 0 }, { "canonicalized_f1": 0.2, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":914,\"column\":\"abv\",\"new_value\":\"0.059\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":915,\"column\":\"abv\",\"new_value\":\"0.054\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":916,\"column\":\"abv\",\"new_value\":\"0.054000000000000006%\", \"reason\":\"The new value for abv is incorrect.\"},{\"row\":917,\"column\":\"abv\",\"new_value\":\"0.042\",\"reason\":\"The new value for abv is incorrect.\"}]}", "f1": 0.2, "failure_taxonomy": { "missed_repair": 4, "overrepair": 2, "wrong_value": 1 }, "fn": 5, "fp": 3, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.25, "predicted_repairs": [ { "column": "abv", "new_value": "0.059", "reason": "The new value for abv is incorrect.", "row": 914 }, { "column": "abv", "new_value": "0.054", "reason": "The new value for abv is incorrect.", "row": 915 }, { "column": "abv", "new_value": "0.054000000000000006%", "reason": "The new value for abv is incorrect.", "row": 916 }, { "column": "abv", "new_value": "0.042", "reason": "The new value for abv is incorrect.", "row": 917 } ], "recall": 0.1667, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10116:0038", "task_index": 39, "tp": 1 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":738,\"column\":\"ProviderNumber\",\"new_value\":\"10035\",\"reason\":\"The patient's address does not match any known hospital records.\",\"why\":\"This is an error as the address provided does not correspond to any hospital records.\"},{\"row\":739,\"column\":\"ProviderNumber\",\"new_value\":\"10035\",\"reason\":\"The patient's address does not match any known hospital records.\",\"why\":\"This is an error as the address provided does not correspond to any hospital records.\"},{\"row\":740,\"column\":\"ProviderNumber\",\"new_value\":\"10035\",\"reason\":\"The patient's address does not match any known hospital records.\",\"why\":\"This is an error as the address provided does not correspond to any hospital records.\"},{\"row\":741,\"column\":\"ProviderNumber\",\"new_value\":\"10036\",\"reason\":\"The patient's address does not match any known hospital records.\",\"why\":\"This is an error as the address provided does not correspond to any hospital records.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "10035", "reason": "The patient's address does not match any known hospital records.", "row": 738 }, { "column": "ProviderNumber", "new_value": "10035", "reason": "The patient's address does not match any known hospital records.", "row": 739 }, { "column": "ProviderNumber", "new_value": "10035", "reason": "The patient's address does not match any known hospital records.", "row": 740 }, { "column": "ProviderNumber", "new_value": "10036", "reason": "The patient's address does not match any known hospital records.", "row": 741 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10117:0039", "task_index": 40, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":757,\"column\":\"act_arr_time\",\"new_value\":\"3:26 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":758,\"column\":\"act_arr_time\",\"new_value\":\"10:34 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":759,\"column\":\"act_arr_time\",\"new_value\":\"8:38 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":760,\"column\":\"act_arr_time\",\"new_value\":\"3:04 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 11, "wrong_value": 4 }, "fn": 15, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "3:26 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 757 }, { "column": "act_arr_time", "new_value": "10:34 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 758 }, { "column": "act_arr_time", "new_value": "8:38 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 759 }, { "column": "act_arr_time", "new_value": "3:04 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 760 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10121:0040", "task_index": 41, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1374,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The beer name is incorrect.\"},{\"row\":1375,\"column\":\"abv\",\"new_value\":\"0.068\",\"reason\":\"The beer name is incorrect.\"},{\"row\":1376,\"column\":\"abv\",\"new_value\":\"0.048\",\"reason\":\"The beer name is incorrect.\"},{\"row\":1377,\"column\":\"abv\",\"new_value\":\"0.092\",\"reason\":\"The beer name is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9, "overrepair": 4 }, "fn": 9, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.05", "reason": "The beer name is incorrect.", "row": 1374 }, { "column": "abv", "new_value": "0.068", "reason": "The beer name is incorrect.", "row": 1375 }, { "column": "abv", "new_value": "0.048", "reason": "The beer name is incorrect.", "row": 1376 }, { "column": "abv", "new_value": "0.092", "reason": "The beer name is incorrect.", "row": 1377 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10125:0041", "task_index": 42, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":242,\"column\":\"Index\",\"new_value\":\"200 med center drive\",\"reason\":\"The address '200 med center drive' does not exist in the dataset.\"},{\"row\":243,\"column\":\"Index\",\"new_value\":\"200 med center drive\",\"reason\":\"The address '200 med center drive' does not exist in the dataset.\"},{\"row\":244,\"column\":\"Index\",\"new_value\":\"33700 highway 43\",\"reason\":\"The address '33700 highway 43' does not exist in the dataset.\"},{\"row\":245,\"column\":\"Index\",\"new_value\":\"33700 highway 43\",\"reason\":\"The address '33700 highway 43' does not exist in the dataset.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3, "schema_case_error": 4 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "200 med center drive", "reason": "The address '200 med center drive' does not exist in the dataset.", "row": 242 }, { "column": "Index", "new_value": "200 med center drive", "reason": "The address '200 med center drive' does not exist in the dataset.", "row": 243 }, { "column": "Index", "new_value": "33700 highway 43", "reason": "The address '33700 highway 43' does not exist in the dataset.", "row": 244 }, { "column": "Index", "new_value": "33700 highway 43", "reason": "The address '33700 highway 43' does not exist in the dataset.", "row": 245 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10126:0042", "task_index": 43, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2088,\"column\":\"act_arr_time\",\"new_value\":\"8:56 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":2089,\"column\":\"act_arr_time\",\"new_value\":\"2:46 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":2090,\"column\":\"act_arr_time\",\"new_value\":\"5:34 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":2091,\"column\":\"act_arr_time\",\"new_value\":\"3:33 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 3, "wrong_value": 1 }, "fn": 7, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "8:56 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 2088 }, { "column": "act_arr_time", "new_value": "2:46 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 2089 }, { "column": "act_arr_time", "new_value": "5:34 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 2090 }, { "column": "act_arr_time", "new_value": "3:33 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 2091 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10130:0043", "task_index": 44, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2140,\"column\":\"abv\",\"new_value\":\"0.06\",\"reason\":\"Corrected beer ABV to 0.06%\"},{\"row\":2141,\"column\":\"abv\",\"new_value\":\"0.096%\",\"reason\":\"Incorrectly set abv value for Beer Name 'The Crusher'.\"},{\"row\":2142,\"column\":\"abv\",\"new_value\":\"0.08\",\"reason\":\"Incorrectly set abv value for Beer Name 'Beelzebub'.\"},{\"row\":2143,\"column\":\"abv\",\"new_value\":\"0.07\",\"reason\":\"Incorrectly set abv value for Beer Name 'Focal Banger'.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 10, "overrepair": 3, "wrong_value": 1 }, "fn": 11, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.06", "reason": "Corrected beer ABV to 0.06%", "row": 2140 }, { "column": "abv", "new_value": "0.096%", "reason": "Incorrectly set abv value for Beer Name 'The Crusher'.", "row": 2141 }, { "column": "abv", "new_value": "0.08", "reason": "Incorrectly set abv value for Beer Name 'Beelzebub'.", "row": 2142 }, { "column": "abv", "new_value": "0.07", "reason": "Incorrectly set abv value for Beer Name 'Focal Banger'.", "row": 2143 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10134:0044", "task_index": 45, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":254,\"column\":\"ProviderNumber\",\"new_value\":\"10015\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to an existing hospital record.\"},{\"row\":255,\"column\":\"ProviderNumber\",\"new_value\":\"10015\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to an existing hospital record.\"},{\"row\":256,\"column\":\"ProviderNumber\",\"new_value\":\"10015\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to an existing hospital record.\"},{\"row\":257,\"column\":\"ProviderNumber\",\"new_value\":\"10015\",\"reason\":\"The patient's address does not match any hospital records.\",\"why\":\"This is a potential error as the address provided does not correspond to an existing hospital record.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3, "overrepair": 4 }, "fn": 3, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "10015", "reason": "The patient's address does not match any hospital records.", "row": 254 }, { "column": "ProviderNumber", "new_value": "10015", "reason": "The patient's address does not match any hospital records.", "row": 255 }, { "column": "ProviderNumber", "new_value": "10015", "reason": "The patient's address does not match any hospital records.", "row": 256 }, { "column": "ProviderNumber", "new_value": "10015", "reason": "The patient's address does not match any hospital records.", "row": 257 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10135:0045", "task_index": 46, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":421,\"column\":\"act_arr_time\",\"new_value\":\"10:58 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":422,\"column\":\"act_arr_time\",\"new_value\":\"11:53 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":423,\"column\":\"act_arr_time\",\"new_value\":\"2:49 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":424,\"column\":\"act_arr_time\",\"new_value\":\"10:12 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 10, "wrong_value": 4 }, "fn": 14, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "10:58 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 421 }, { "column": "act_arr_time", "new_value": "11:53 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 422 }, { "column": "act_arr_time", "new_value": "2:49 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 423 }, { "column": "act_arr_time", "new_value": "10:12 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 424 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10139:0046", "task_index": 47, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1042,\"column\":\"abv\",\"new_value\":\"0.06\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1043,\"column\":\"abv\",\"new_value\":\"0.063%\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1044,\"column\":\"abv\",\"new_value\":\"0.08%\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1045,\"column\":\"abv\",\"new_value\":\"0.099\",\"reason\":\"The new value for abv is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 2, "wrong_value": 2 }, "fn": 7, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.06", "reason": "The new value for abv is incorrect.", "row": 1042 }, { "column": "abv", "new_value": "0.063%", "reason": "The new value for abv is incorrect.", "row": 1043 }, { "column": "abv", "new_value": "0.08%", "reason": "The new value for abv is incorrect.", "row": 1044 }, { "column": "abv", "new_value": "0.099", "reason": "The new value for abv is incorrect.", "row": 1045 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10143:0047", "task_index": 48, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":666,\"column\":\"Index\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address does not exist.\",\"__type\":\"cell_replacement\"},{\"row\":667,\"column\":\"Index\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address does not exist.\",\"__type\":\"cell_replacement\"},{\"row\":668,\"column\":\"Index\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address does not exist.\",\"__type\":\"cell_replacement\"},{\"row\":669,\"column\":\"Index\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address does not exist.\",\"__type\":\"cell_replacement\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "schema_case_error": 4 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "619 south 19th street", "reason": "The address does not exist.", "row": 666 }, { "column": "Index", "new_value": "619 south 19th street", "reason": "The address does not exist.", "row": 667 }, { "column": "Index", "new_value": "619 south 19th street", "reason": "The address does not exist.", "row": 668 }, { "column": "Index", "new_value": "619 south 19th street", "reason": "The address does not exist.", "row": 669 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10144:0048", "task_index": 49, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1909,\"column\":\"act_arr_time\",\"new_value\":\"11:06 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1910,\"column\":\"act_arr_time\",\"new_value\":\"6:38 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1911,\"column\":\"act_arr_time\",\"new_value\":\"7:36 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1912,\"column\":\"act_arr_time\",\"new_value\":\"9:53 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "11:06 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1909 }, { "column": "act_arr_time", "new_value": "6:38 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1910 }, { "column": "act_arr_time", "new_value": "7:36 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1911 }, { "column": "act_arr_time", "new_value": "9:53 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1912 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10148:0049", "task_index": 50, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1217,\"column\":\"beer-name\",\"new_value\":\"Sweet Josie\",\"reason\":\"The beer name is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 1 }, "fn": 6, "fp": 1, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "beer-name", "new_value": "Sweet Josie", "reason": "The beer name is incorrect.", "row": 1217 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10152:0050", "task_index": 51, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":125,\"column\":\"Index\",\"new_value\":\"101 hospital circle\",\"reason\":\"The address '101 hospital circle' does not exist in the dataset.\",\"why\":\"This is a potential error as the address does not exist.\"},{\"row\":126,\"column\":\"Index\",\"new_value\":\"101 hospital circle\",\"reason\":\"The address '101 hospital circle' does not exist in the dataset.\",\"why\":\"This is a potential error as the address does not exist.\"},{\"row\":127,\"column\":\"Index\",\"new_value\":\"101 hospital circle\",\"reason\":\"The address '101 hospital circle' does not exist in the dataset.\",\"why\":\"This is a potential error as the address does not exist.\"},{\"row\":128,\"column\":\"Index\",\"new_value\":\"101 hospital circle\",\"reason\":\"The address '101 hospital circle' does not exist in the dataset.\",\"why\":\"This is a potential error as the address does not exist.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1, "schema_case_error": 4 }, "fn": 1, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "101 hospital circle", "reason": "The address '101 hospital circle' does not exist in the dataset.", "row": 125 }, { "column": "Index", "new_value": "101 hospital circle", "reason": "The address '101 hospital circle' does not exist in the dataset.", "row": 126 }, { "column": "Index", "new_value": "101 hospital circle", "reason": "The address '101 hospital circle' does not exist in the dataset.", "row": 127 }, { "column": "Index", "new_value": "101 hospital circle", "reason": "The address '101 hospital circle' does not exist in the dataset.", "row": 128 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10153:0051", "task_index": 52, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1563,\"column\":\"act_arr_time\",\"new_value\":\"9:12 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1564,\"column\":\"act_arr_time\",\"new_value\":\"4:22 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1565,\"column\":\"act_arr_time\",\"new_value\":\"4:22 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1566,\"column\":\"act_arr_time\",\"new_value\":\"7:03 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "wrong_value": 4 }, "fn": 12, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "9:12 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1563 }, { "column": "act_arr_time", "new_value": "4:22 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1564 }, { "column": "act_arr_time", "new_value": "4:22 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1565 }, { "column": "act_arr_time", "new_value": "7:03 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1566 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10157:0052", "task_index": 53, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1295,\"column\":\"abv\",\"new_value\":\"0.047%\",\"reason\":\"The new value for abv is incorrect.\",\"why\":\"The new value of 0.047% does not match the expected value of 0.05.\"},{\"row\":1296,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The new value for abv is incorrect.\",\"why\":\"The new value of 0.05 does not match the expected value of 0.047%.\"},{\"row\":1297,\"column\":\"abv\",\"new_value\":\"0.06%\",\"reason\":\"The new value for abv is incorrect.\",\"why\":\"The new value of 0.06% does not match the expected value of 0.047%.\"},{\"row\":1298,\"column\":\"abv\",\"new_value\":\"0.047\",\"reason\":\"The new value for abv is incorrect.\",\"why\":\"The new value of 0.047% does not match the expected value of 0.05.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 2, "wrong_value": 2 }, "fn": 9, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.047%", "reason": "The new value for abv is incorrect.", "row": 1295 }, { "column": "abv", "new_value": "0.05", "reason": "The new value for abv is incorrect.", "row": 1296 }, { "column": "abv", "new_value": "0.06%", "reason": "The new value for abv is incorrect.", "row": 1297 }, { "column": "abv", "new_value": "0.047", "reason": "The new value for abv is incorrect.", "row": 1298 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10161:0053", "task_index": 54, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":478,\"column\":\"Address1\",\"new_value\":\"126 hospital ave\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '126 hospital ave' does not exist in the dataset.\"},{\"row\":479,\"column\":\"Address1\",\"new_value\":\"126 hospital ave\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '126 hospital ave' does not exist in the dataset.\"},{\"row\":480,\"column\":\"Address1\",\"new_value\":\"126 hospital ave\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '126 hospital ave' does not exist in the dataset.\"},{\"row\":481,\"column\":\"Address1\",\"new_value\":\"126 hospital ave\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '126 hospital ave' does not exist in the dataset.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "126 hospital ave", "reason": "The address is incorrect.", "row": 478 }, { "column": "Address1", "new_value": "126 hospital ave", "reason": "The address is incorrect.", "row": 479 }, { "column": "Address1", "new_value": "126 hospital ave", "reason": "The address is incorrect.", "row": 480 }, { "column": "Address1", "new_value": "126 hospital ave", "reason": "The address is incorrect.", "row": 481 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10162:0054", "task_index": 55, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":172,\"column\":\"flight\",\"new_value\":\"CO-1561-PHX-IAH\",\"reason\":\"The flight CO-1561-PHX-IAH is not listed in the contract version.\"},{\"row\":171,\"column\":\"flight\",\"new_value\":\"AA-1886-BOS-MIA\",\"reason\":\"The flight AA-1886-BOS-MIA is not listed in the contract version.\"},{\"row\":170,\"column\":\"flight\",\"new_value\":\"AA-484-DFW-MIA\",\"reason\":\"The flight AA-484-DFW-MIA is not listed in the contract version.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 13, "overrepair": 3 }, "fn": 13, "fp": 3, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "CO-1561-PHX-IAH", "reason": "The flight CO-1561-PHX-IAH is not listed in the contract version.", "row": 172 }, { "column": "flight", "new_value": "AA-1886-BOS-MIA", "reason": "The flight AA-1886-BOS-MIA is not listed in the contract version.", "row": 171 }, { "column": "flight", "new_value": "AA-484-DFW-MIA", "reason": "The flight AA-484-DFW-MIA is not listed in the contract version.", "row": 170 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10166:0055", "task_index": 56, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":777,\"column\":\"abv\",\"new_value\":\"0.05%\",\"reason\":\"The beer name is incorrect.\"},{\"row\":778,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The beer name is incorrect.\"},{\"row\":779,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The beer name is incorrect.\"},{\"row\":780,\"column\":\"abv\",\"new_value\":\"0.07%\",\"reason\":\"The abv value is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 2, "wrong_value": 2 }, "fn": 8, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.05%", "reason": "The beer name is incorrect.", "row": 777 }, { "column": "abv", "new_value": "0.05", "reason": "The beer name is incorrect.", "row": 778 }, { "column": "abv", "new_value": "0.05", "reason": "The beer name is incorrect.", "row": 779 }, { "column": "abv", "new_value": "0.07%", "reason": "The abv value is incorrect.", "row": 780 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10170:0056", "task_index": 57, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":553,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":554,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":555,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"},{\"row\":556,\"column\":\"ProviderNumber\",\"new_value\":\"none\",\"reason\":\"The patient's address does not have a valid zip code.\",\"why\":\"The zip code is missing or incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3, "overrepair": 4 }, "fn": 3, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 553 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 554 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 555 }, { "column": "ProviderNumber", "new_value": "none", "reason": "The patient's address does not have a valid zip code.", "row": 556 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10171:0057", "task_index": 58, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2177,\"column\":\"act_arr_time\",\"new_value\":\"11:32 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect; it should be 11:32 a.m.\"},{\"row\":2176,\"column\":\"act_arr_time\",\"new_value\":\"7:10 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect; it should be 7:10 p.m.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9, "overrepair": 2 }, "fn": 9, "fp": 2, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "11:32 a.m.", "reason": "The new value for act_arr_time is incorrect; it should be 11:32 a.m.", "row": 2177 }, { "column": "act_arr_time", "new_value": "7:10 p.m.", "reason": "The new value for act_arr_time is incorrect; it should be 7:10 p.m.", "row": 2176 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10175:0058", "task_index": 59, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1062,\"column\":\"abv\",\"new_value\":\"0.063\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1063,\"column\":\"abv\",\"new_value\":\"0.049\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1064,\"column\":\"abv\",\"new_value\":\"0.048\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1065,\"column\":\"abv\",\"new_value\":\"0.065\",\"reason\":\"The new value for abv is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 4 }, "fn": 4, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.063", "reason": "The new value for abv is incorrect.", "row": 1062 }, { "column": "abv", "new_value": "0.049", "reason": "The new value for abv is incorrect.", "row": 1063 }, { "column": "abv", "new_value": "0.048", "reason": "The new value for abv is incorrect.", "row": 1064 }, { "column": "abv", "new_value": "0.065", "reason": "The new value for abv is incorrect.", "row": 1065 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10179:0059", "task_index": 60, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":672,\"column\":\"Index\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address does not exist.\",\"_type\":\"invalid_address\"},{\"row\":673,\"column\":\"Index\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address does not exist.\",\"_type\":\"invalid_address\"},{\"row\":674,\"column\":\"Index\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address does not exist.\",\"_type\":\"invalid_address\"},{\"row\":675,\"column\":\"Index\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address does not exist.\",\"_type\":\"invalid_address\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "schema_case_error": 4 }, "fn": 5, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "619 south 19th street", "reason": "The address does not exist.", "row": 672 }, { "column": "Index", "new_value": "619 south 19th street", "reason": "The address does not exist.", "row": 673 }, { "column": "Index", "new_value": "619 south 19th street", "reason": "The address does not exist.", "row": 674 }, { "column": "Index", "new_value": "619 south 19th street", "reason": "The address does not exist.", "row": 675 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10180:0060", "task_index": 61, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2241,\"column\":\"act_arr_time\",\"new_value\":\"7:57 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":2242,\"column\":\"act_arr_time\",\"new_value\":\"4:56 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":2243,\"column\":\"act_arr_time\",\"new_value\":\"6:09 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":2244,\"column\":\"act_arr_time\",\"new_value\":\"6:09 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 2, "wrong_value": 2 }, "fn": 6, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "7:57 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 2241 }, { "column": "act_arr_time", "new_value": "4:56 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 2242 }, { "column": "act_arr_time", "new_value": "6:09 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 2243 }, { "column": "act_arr_time", "new_value": "6:09 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 2244 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10184:0061", "task_index": 62, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1383,\"column\":\"abv\",\"new_value\":\"0.056\",\"reason\":\"The new value for abv is 0.056 which does not match any of the allowed values.\"},{\"row\":1384,\"column\":\"abv\",\"new_value\":\"0.042\",\"reason\":\"The new value for abv is 0.042 which does not match any of the allowed values.\"},{\"row\":1385,\"column\":\"abv\",\"new_value\":\"0.075\",\"reason\":\"The new value for abv is 0.075 which does not match any of the allowed values.\"},{\"row\":1386,\"column\":\"abv\",\"new_value\":\"0.068\",\"reason\":\"The new value for abv is 0.068 which does not match any of the allowed values.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 4 }, "fn": 6, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.056", "reason": "The new value for abv is 0.056 which does not match any of the allowed values.", "row": 1383 }, { "column": "abv", "new_value": "0.042", "reason": "The new value for abv is 0.042 which does not match any of the allowed values.", "row": 1384 }, { "column": "abv", "new_value": "0.075", "reason": "The new value for abv is 0.075 which does not match any of the allowed values.", "row": 1385 }, { "column": "abv", "new_value": "0.068", "reason": "The new value for abv is 0.068 which does not match any of the allowed values.", "row": 1386 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10188:0062", "task_index": 63, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":293,\"column\":\"Address1\",\"new_value\":\"1000 first street north\",\"reason\":\"The address is incorrect.\",\"why\":\"This address does not match any of the allowed addresses.\"},{\"row\":294,\"column\":\"Address1\",\"new_value\":\"1720 university blvd\",\"reason\":\"The address is incorrect.\",\"why\":\"This address does not match any of the allowed addresses.\"},{\"row\":295,\"column\":\"Address1\",\"new_value\":\"1720 univxrsity blvd\",\"reason\":\"The address is incorrect.\",\"why\":\"This address does not match any of the allowed addresses.\"},{\"row\":296,\"column\":\"Address1\",\"new_value\":\"1720 university blvd\",\"reason\":\"The address is incorrect.\",\"why\":\"This address does not match any of the allowed addresses.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1, "overrepair": 3, "wrong_value": 1 }, "fn": 2, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "1000 first street north", "reason": "The address is incorrect.", "row": 293 }, { "column": "Address1", "new_value": "1720 university blvd", "reason": "The address is incorrect.", "row": 294 }, { "column": "Address1", "new_value": "1720 univxrsity blvd", "reason": "The address is incorrect.", "row": 295 }, { "column": "Address1", "new_value": "1720 university blvd", "reason": "The address is incorrect.", "row": 296 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10189:0063", "task_index": 64, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":563,\"column\":\"act_arr_time\",\"new_value\":\"10:27 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":564,\"column\":\"act_arr_time\",\"new_value\":\"10:56 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":565,\"column\":\"act_arr_time\",\"new_value\":\"11:01 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":566,\"column\":\"act_arr_time\",\"new_value\":\"11:56 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "10:27 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 563 }, { "column": "act_arr_time", "new_value": "10:56 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 564 }, { "column": "act_arr_time", "new_value": "11:01 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 565 }, { "column": "act_arr_time", "new_value": "11:56 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 566 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10193:0064", "task_index": 65, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":763,\"column\":\"abv\",\"new_value\":\"0.052\",\"reason\":\"The beer name is incorrect.\"},{\"row\":764,\"column\":\"abv\",\"new_value\":\"0.048\",\"reason\":\"The beer name is incorrect.\"},{\"row\":765,\"column\":\"abv\",\"new_value\":\"0.054\",\"reason\":\"The beer name is incorrect.\"},{\"row\":766,\"column\":\"abv\",\"new_value\":\"0.04%\",\"reason\":\"The beer name is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 3, "wrong_value": 1 }, "fn": 8, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.052", "reason": "The beer name is incorrect.", "row": 763 }, { "column": "abv", "new_value": "0.048", "reason": "The beer name is incorrect.", "row": 764 }, { "column": "abv", "new_value": "0.054", "reason": "The beer name is incorrect.", "row": 765 }, { "column": "abv", "new_value": "0.04%", "reason": "The beer name is incorrect.", "row": 766 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10197:0065", "task_index": 66, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":86,\"column\":\"Index\",\"new_value\":\"87\",\"reason\":\"The patient's address has been updated to '205 Marengo Street' instead of '205 Marengo Street', which does not match any allowed column name.\"},{\"row\":87,\"column\":\"Index\",\"new_value\":\"88\",\"reason\":\"The patient's address has been updated to '205 Marengo Street' instead of '205 Marengo Street', which does not match any allowed column name.\"},{\"row\":88,\"column\":\"Index\",\"new_value\":\"89\",\"reason\":\"The patient's address has been updated to '205 Marengo Street' instead of '205 Marengo Street', which does not match any allowed column name.\"},{\"row\":89,\"column\":\"Index\",\"new_value\":\"90\",\"reason\":\"The patient's address has been updated to '205 Marengo Street' instead of '205 Marengo Street', which does not match any allowed column name.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1, "schema_case_error": 4 }, "fn": 1, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "87", "reason": "The patient's address has been updated to '205 Marengo Street' instead of '205 Marengo Street', which does not match any allowed column name.", "row": 86 }, { "column": "Index", "new_value": "88", "reason": "The patient's address has been updated to '205 Marengo Street' instead of '205 Marengo Street', which does not match any allowed column name.", "row": 87 }, { "column": "Index", "new_value": "89", "reason": "The patient's address has been updated to '205 Marengo Street' instead of '205 Marengo Street', which does not match any allowed column name.", "row": 88 }, { "column": "Index", "new_value": "90", "reason": "The patient's address has been updated to '205 Marengo Street' instead of '205 Marengo Street', which does not match any allowed column name.", "row": 89 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10198:0066", "task_index": 67, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1051,\"column\":\"flight\",\"new_value\":\"UA-62-IAH-EWR\",\"reason\":\"The flight is now operated by United Airlines (UA). This change will affect the departure time from IAH to EWR.\"},{\"row\":1052,\"column\":\"flight\",\"new_value\":\"UA-62-IAH-EWR\",\"reason\":\"The flight is now operated by United Airlines (UA). This change will affect the arrival time from IAH to EWR.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 1, "wrong_cell": 1 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_row", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "UA-62-IAH-EWR", "reason": "The flight is now operated by United Airlines (UA). This change will affect the departure time from IAH to EWR.", "row": 1051 }, { "column": "flight", "new_value": "UA-62-IAH-EWR", "reason": "The flight is now operated by United Airlines (UA). This change will affect the arrival time from IAH to EWR.", "row": 1052 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10202:0067", "task_index": 68, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1692,\"column\":\"abv\",\"new_value\":\"0.049\",\"reason\":\"The new value for abv is 0.049 which does not match any existing value in the dataset.\"},{\"row\":1693,\"column\":\"abv\",\"new_value\":\"0.051\",\"reason\":\"The new value for abv is 0.051 which does not match any existing value in the dataset.\"},{\"row\":1694,\"column\":\"abv\",\"new_value\":\"0.055\",\"reason\":\"The new value for abv is 0.055 which does not match any existing value in the dataset.\"},{\"row\":1695,\"column\":\"abv\",\"new_value\":\"0.07%\",\"reason\":\"The new value for abv is 0.07% which does not match any existing value in the dataset.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 3, "wrong_value": 1 }, "fn": 7, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.049", "reason": "The new value for abv is 0.049 which does not match any existing value in the dataset.", "row": 1692 }, { "column": "abv", "new_value": "0.051", "reason": "The new value for abv is 0.051 which does not match any existing value in the dataset.", "row": 1693 }, { "column": "abv", "new_value": "0.055", "reason": "The new value for abv is 0.055 which does not match any existing value in the dataset.", "row": 1694 }, { "column": "abv", "new_value": "0.07%", "reason": "The new value for abv is 0.07% which does not match any existing value in the dataset.", "row": 1695 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10206:0068", "task_index": 69, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":296,\"column\":\"Address1\",\"new_value\":\"1720 university blvd\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '1720 university blvd' does not exist in the dataset.\"},{\"row\":297,\"column\":\"Address1\",\"new_value\":\"1720 university blvd\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '1720 university blvd' does not exist in the dataset.\"},{\"row\":298,\"column\":\"Address1\",\"new_value\":\"1720 university blvd\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '1720 university blvd' does not exist in the dataset.\"},{\"row\":299,\"column\":\"Address1\",\"new_value\":\"1720 university blvd\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '1720 university blvd' does not exist in the dataset.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "1720 university blvd", "reason": "The address is incorrect.", "row": 296 }, { "column": "Address1", "new_value": "1720 university blvd", "reason": "The address is incorrect.", "row": 297 }, { "column": "Address1", "new_value": "1720 university blvd", "reason": "The address is incorrect.", "row": 298 }, { "column": "Address1", "new_value": "1720 university blvd", "reason": "The address is incorrect.", "row": 299 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10207:0069", "task_index": 70, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1517,\"column\":\"tuple_id\",\"new_value\":\"1518\",\"reason\":\"The flight is now scheduled to depart at 11:50 a.m.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 12, "overrepair": 1 }, "fn": 12, "fp": 1, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "tuple_id", "new_value": "1518", "reason": "The flight is now scheduled to depart at 11:50 a.m.", "row": 1517 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10211:0070", "task_index": 71, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1331,\"column\":\"abv\",\"new_value\":\"0.063\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1332,\"column\":\"abv\",\"new_value\":\"0.048\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1333,\"column\":\"abv\",\"new_value\":\"0.05%\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":1334,\"column\":\"abv\",\"new_value\":\"0.057%\",\"reason\":\"The new value for abv is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 4, "overrepair": 2, "wrong_value": 2 }, "fn": 6, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.063", "reason": "The new value for abv is incorrect.", "row": 1331 }, { "column": "abv", "new_value": "0.048", "reason": "The new value for abv is incorrect.", "row": 1332 }, { "column": "abv", "new_value": "0.05%", "reason": "The new value for abv is incorrect.", "row": 1333 }, { "column": "abv", "new_value": "0.057%", "reason": "The new value for abv is incorrect.", "row": 1334 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10215:0071", "task_index": 72, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":747,\"column\":\"Address1\",\"new_value\":\"849 south three notch street\",\"reason\":\"The address does not exist.\",\"why\":\"This is a potential error as the address does not exist.\"},{\"row\":748,\"column\":\"Address1\",\"new_value\":\"849 south three notch street\",\"reason\":\"The address does not exist.\",\"why\":\"This is a potential error as the address does not exist.\"},{\"row\":749,\"column\":\"Address1\",\"new_value\":\"849 south three notch street\",\"reason\":\"The address does not exist.\",\"why\":\"This is a potential error as the address does not exist.\"},{\"row\":750,\"column\":\"Address1\",\"new_value\":\"849 south three notch street\",\"reason\":\"The address does not exist.\",\"why\":\"This is a potential error as the address does not exist.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "overrepair": 4 }, "fn": 2, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "849 south three notch street", "reason": "The address does not exist.", "row": 747 }, { "column": "Address1", "new_value": "849 south three notch street", "reason": "The address does not exist.", "row": 748 }, { "column": "Address1", "new_value": "849 south three notch street", "reason": "The address does not exist.", "row": 749 }, { "column": "Address1", "new_value": "849 south three notch street", "reason": "The address does not exist.", "row": 750 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10216:0072", "task_index": 73, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1002,\"column\":\"flight\",\"new_value\":\"UA-2726-FLL-PHL\",\"reason\":\"The flight is currently booked for UA-2726-FLL-PHL, which does not exist in our database.\"},{\"row\":1003,\"column\":\"flight\",\"new_value\":\"AA-3468-CVG-MIA\",\"reason\":\"The flight is currently booked for AA-3468-CVG-MIA, which does not exist in our database.\"},{\"row\":1004,\"column\":\"flight\",\"new_value\":\"UA-3515-IAD-MSP\",\"reason\":\"The flight is currently booked for UA-3515-IAD-MSP, which does not exist in our database.\"},{\"row\":1005,\"column\":\"flight\",\"new_value\":\"AA-484-DFW-MIA\",\"reason\":\"The flight is currently booked for AA-484-DFW-MIA, which does not exist in our database.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 3, "wrong_cell": 1 }, "fn": 7, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_row", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "UA-2726-FLL-PHL", "reason": "The flight is currently booked for UA-2726-FLL-PHL, which does not exist in our database.", "row": 1002 }, { "column": "flight", "new_value": "AA-3468-CVG-MIA", "reason": "The flight is currently booked for AA-3468-CVG-MIA, which does not exist in our database.", "row": 1003 }, { "column": "flight", "new_value": "UA-3515-IAD-MSP", "reason": "The flight is currently booked for UA-3515-IAD-MSP, which does not exist in our database.", "row": 1004 }, { "column": "flight", "new_value": "AA-484-DFW-MIA", "reason": "The flight is currently booked for AA-484-DFW-MIA, which does not exist in our database.", "row": 1005 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10220:0073", "task_index": 74, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1960,\"column\":\"abv\",\"new_value\":\"0.072\",\"reason\":\"The beer name has been changed to Old Potentate.\"},{\"row\":1961,\"column\":\"abv\",\"new_value\":\"0.05\",\"reason\":\"The beer name has been changed to Bombshell Blonde.\"},{\"row\":1962,\"column\":\"abv\",\"new_value\":\"0.099\",\"reason\":\"The beer name has been changed to PRO-AM (2012) (2012).\"},{\"row\":1963,\"column\":\"abv\",\"new_value\":\"0.055\",\"reason\":\"The beer name has been changed to Walloon (2014).\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 4 }, "fn": 5, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.072", "reason": "The beer name has been changed to Old Potentate.", "row": 1960 }, { "column": "abv", "new_value": "0.05", "reason": "The beer name has been changed to Bombshell Blonde.", "row": 1961 }, { "column": "abv", "new_value": "0.099", "reason": "The beer name has been changed to PRO-AM (2012) (2012).", "row": 1962 }, { "column": "abv", "new_value": "0.055", "reason": "The beer name has been changed to Walloon (2014).", "row": 1963 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10224:0074", "task_index": 75, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":208,\"column\":\"Index\",\"new_value\":\"10011\",\"reason\":\"The patient's index matches the expected value.\",\"_type\":\"patient\"},{\"row\":209,\"column\":\"Index\",\"new_value\":\"10011\",\"reason\":\"The patient's index matches the expected value.\",\"_type\":\"patient\"},{\"row\":210,\"column\":\"Index\",\"new_value\":\"10011\",\"reason\":\"The patient's index matches the expected value.\",\"_type\":\"patient\"},{\"row\":211,\"column\":\"Index\",\"new_value\":\"10011\",\"reason\":\"The patient's index matches the expected value.\",\"_type\":\"patient\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "schema_case_error": 4 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "10011", "reason": "The patient's index matches the expected value.", "row": 208 }, { "column": "Index", "new_value": "10011", "reason": "The patient's index matches the expected value.", "row": 209 }, { "column": "Index", "new_value": "10011", "reason": "The patient's index matches the expected value.", "row": 210 }, { "column": "Index", "new_value": "10011", "reason": "The patient's index matches the expected value.", "row": 211 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10225:0075", "task_index": 76, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1138,\"column\":\"act_arr_time\",\"new_value\":\"4:59 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1139,\"column\":\"act_arr_time\",\"new_value\":\"8:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1140,\"column\":\"act_arr_time\",\"new_value\":\"8:45 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1141,\"column\":\"act_arr_time\",\"new_value\":\"CO-4888-IAH-DAL\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 9, "overrepair": 2, "wrong_value": 2 }, "fn": 11, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "4:59 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1138 }, { "column": "act_arr_time", "new_value": "8:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1139 }, { "column": "act_arr_time", "new_value": "8:45 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1140 }, { "column": "act_arr_time", "new_value": "CO-4888-IAH-DAL", "reason": "The new value for act_arr_time is incorrect.", "row": 1141 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10229:0076", "task_index": 77, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2137,\"column\":\"beer-name\",\"new_value\":\"Petit Mutant\",\"reason\":\"This beer has a different name than the other beers in the dataset.\"},{\"row\":2138,\"column\":\"beer-name\",\"new_value\":\"Broken Bridge\",\"reason\":\"This beer has a different name than the other beers in the dataset.\"},{\"row\":2139,\"column\":\"beer-name\",\"new_value\":\"Battle LIne\",\"reason\":\"This beer has a different name than the other beers in the dataset.\"},{\"row\":2140,\"column\":\"beer-name\",\"new_value\":\"Brutus\",\"reason\":\"This beer has a different name than the other beers in the dataset.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 4 }, "fn": 5, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "beer-name", "new_value": "Petit Mutant", "reason": "This beer has a different name than the other beers in the dataset.", "row": 2137 }, { "column": "beer-name", "new_value": "Broken Bridge", "reason": "This beer has a different name than the other beers in the dataset.", "row": 2138 }, { "column": "beer-name", "new_value": "Battle LIne", "reason": "This beer has a different name than the other beers in the dataset.", "row": 2139 }, { "column": "beer-name", "new_value": "Brutus", "reason": "This beer has a different name than the other beers in the dataset.", "row": 2140 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10233:0077", "task_index": 78, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":690,\"column\":\"Address1\",\"new_value\":\"619 south 19th street\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be '619 South 19th Street'.\"},{\"row\":691,\"column\":\"Address1\",\"new_value\":\"805 Friendship Road\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be '805 Friendship Road'.\"},{\"row\":692,\"column\":\"Address1\",\"new_value\":\"805 Friendship Road\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be '805 Friendship Road'.\"},{\"row\":693,\"column\":\"Address1\",\"new_value\":\"805 Friendship Road\",\"reason\":\"The address is incorrect.\",\"why\":\"The correct address should be '805 Friendship Road'.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3, "overrepair": 4 }, "fn": 3, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "619 south 19th street", "reason": "The address is incorrect.", "row": 690 }, { "column": "Address1", "new_value": "805 Friendship Road", "reason": "The address is incorrect.", "row": 691 }, { "column": "Address1", "new_value": "805 Friendship Road", "reason": "The address is incorrect.", "row": 692 }, { "column": "Address1", "new_value": "805 Friendship Road", "reason": "The address is incorrect.", "row": 693 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10234:0078", "task_index": 79, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":27,\"column\":\"flight\",\"new_value\":\"AA-1007-MIA-PHX\",\"reason\":\"This flight is currently on a different route or has an alternate departure time that does not align with the current repair schedule.\"},{\"row\":26,\"column\":\"flight\",\"new_value\":\"AA-643-MIA-ORD\",\"reason\":\"This flight is currently on a different route or has an alternate departure time that does not align with the current repair schedule.\"},{\"row\":25,\"column\":\"flight\",\"new_value\":\"AA-3842-MSP-ORD\",\"reason\":\"This flight is currently on a different route or has an alternate departure time that does not align with the current repair schedule.\"},{\"row\":24,\"column\":\"flight\",\"new_value\":\"AA-3-JFK-LAX\",\"reason\":\"This flight is currently on a different route or has an alternate departure time that does not align with the current repair schedule.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "AA-1007-MIA-PHX", "reason": "This flight is currently on a different route or has an alternate departure time that does not align with the current repair schedule.", "row": 27 }, { "column": "flight", "new_value": "AA-643-MIA-ORD", "reason": "This flight is currently on a different route or has an alternate departure time that does not align with the current repair schedule.", "row": 26 }, { "column": "flight", "new_value": "AA-3842-MSP-ORD", "reason": "This flight is currently on a different route or has an alternate departure time that does not align with the current repair schedule.", "row": 25 }, { "column": "flight", "new_value": "AA-3-JFK-LAX", "reason": "This flight is currently on a different route or has an alternate departure time that does not align with the current repair schedule.", "row": 24 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10238:0079", "task_index": 80, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1187,\"column\":\"beer-name\",\"new_value\":\"Elevated IPA\",\"reason\":\"The beer name has been changed to 'Elevated IPA' as per the contract.\"},{\"row\":1188,\"column\":\"beer-name\",\"new_value\":\"Rumspringa Golden Bock\",\"reason\":\"The beer name has been changed to 'Rumspringa Golden Bock' as per the contract.\"},{\"row\":1189,\"column\":\"beer-name\",\"new_value\":\"Lancaster German Style K\\u00f6lsch\",\"reason\":\"The beer name has been changed to 'Lancaster German Style K\\u00f6lsch' as per the contract.\"},{\"row\":1190,\"column\":\"beer-name\",\"new_value\":\"Elevated IPA\",\"reason\":\"The beer name has been changed to 'Elevated IPA' as per the contract.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 4 }, "fn": 8, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "beer-name", "new_value": "Elevated IPA", "reason": "The beer name has been changed to 'Elevated IPA' as per the contract.", "row": 1187 }, { "column": "beer-name", "new_value": "Rumspringa Golden Bock", "reason": "The beer name has been changed to 'Rumspringa Golden Bock' as per the contract.", "row": 1188 }, { "column": "beer-name", "new_value": "Lancaster German Style K\u00f6lsch", "reason": "The beer name has been changed to 'Lancaster German Style K\u00f6lsch' as per the contract.", "row": 1189 }, { "column": "beer-name", "new_value": "Elevated IPA", "reason": "The beer name has been changed to 'Elevated IPA' as per the contract.", "row": 1190 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10242:0080", "task_index": 81, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":194,\"column\":\"Address1\",\"new_value\":\"50 medical park east drive\",\"reason\":\"The address is incorrect.\",\"why\":\"The new address does not match any existing hospital addresses.\"},{\"row\":195,\"column\":\"Address1\",\"new_value\":\"50 medical park east drive\",\"reason\":\"The address is incorrect.\",\"why\":\"The new address does not match any existing hospital addresses.\"},{\"row\":196,\"column\":\"Address1\",\"new_value\":\"50 medical park east drive\",\"reason\":\"The address is incorrect.\",\"why\":\"The new address does not match any existing hospital addresses.\"},{\"row\":197,\"column\":\"Address1\",\"new_value\":\"50 medical park east drive\",\"reason\":\"The address is incorrect.\",\"why\":\"The new address does not match any existing hospital addresses.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 1, "overrepair": 4 }, "fn": 1, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "Address1", "new_value": "50 medical park east drive", "reason": "The address is incorrect.", "row": 194 }, { "column": "Address1", "new_value": "50 medical park east drive", "reason": "The address is incorrect.", "row": 195 }, { "column": "Address1", "new_value": "50 medical park east drive", "reason": "The address is incorrect.", "row": 196 }, { "column": "Address1", "new_value": "50 medical park east drive", "reason": "The address is incorrect.", "row": 197 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10243:0081", "task_index": 82, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1980,\"column\":\"act_arr_time\",\"new_value\":\"11:50 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1981,\"column\":\"act_arr_time\",\"new_value\":\"3:19 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1982,\"column\":\"act_arr_time\",\"new_value\":\"1:30 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1983,\"column\":\"act_arr_time\",\"new_value\":\"3:21 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 2, "wrong_value": 2 }, "fn": 9, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "11:50 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1980 }, { "column": "act_arr_time", "new_value": "3:19 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1981 }, { "column": "act_arr_time", "new_value": "1:30 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1982 }, { "column": "act_arr_time", "new_value": "3:21 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1983 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10247:0082", "task_index": 83, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2160,\"column\":\"abv\",\"new_value\":\"0.099\",\"reason\":\"Corrected beer ABV to 0.099%.\"},{\"row\":2161,\"column\":\"ibu\",\"new_value\":\"101\",\"reason\":\"Corrected IBU to 101%.\"},{\"row\":2162,\"column\":\"style\",\"new_value\":\"American Double / Imperial IPA\",\"reason\":\"Changed style to American Double/Imperial IPA.\"},{\"row\":2163,\"column\":\"style\",\"new_value\":\"American Pilsner\",\"reason\":\"Changed style to American Pilsner.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 4 }, "fn": 6, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.099", "reason": "Corrected beer ABV to 0.099%.", "row": 2160 }, { "column": "ibu", "new_value": "101", "reason": "Corrected IBU to 101%.", "row": 2161 }, { "column": "style", "new_value": "American Double / Imperial IPA", "reason": "Changed style to American Double/Imperial IPA.", "row": 2162 }, { "column": "style", "new_value": "American Pilsner", "reason": "Changed style to American Pilsner.", "row": 2163 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10251:0083", "task_index": 84, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":141,\"column\":\"Index\",\"new_value\":\"10008\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":142,\"column\":\"Index\",\"new_value\":\"10008\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":143,\"column\":\"Index\",\"new_value\":\"10008\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"},{\"row\":144,\"column\":\"Index\",\"new_value\":\"10008\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"This cell already contains the correct value for the 'ProviderNumber' column.\"}]}", "f1": 0.0, "failure_taxonomy": { "schema_case_error": 4 }, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "10008", "reason": "The patient's condition is surgical infection prevention.", "row": 141 }, { "column": "Index", "new_value": "10008", "reason": "The patient's condition is surgical infection prevention.", "row": 142 }, { "column": "Index", "new_value": "10008", "reason": "The patient's condition is surgical infection prevention.", "row": 143 }, { "column": "Index", "new_value": "10008", "reason": "The patient's condition is surgical infection prevention.", "row": 144 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10252:0084", "task_index": 85, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1342,\"column\":\"flight\",\"new_value\":\"AA-484-DFW-MIA\",\"reason\":\"This flight is currently on a different route or has an incorrect departure time.\"},{\"row\":1343,\"column\":\"flight\",\"new_value\":\"AA-616-DFW-DTW\",\"reason\":\"This flight is currently on a different route or has an incorrect arrival time.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 11, "overrepair": 1, "wrong_cell": 1 }, "fn": 11, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_row", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "AA-484-DFW-MIA", "reason": "This flight is currently on a different route or has an incorrect departure time.", "row": 1342 }, { "column": "flight", "new_value": "AA-616-DFW-DTW", "reason": "This flight is currently on a different route or has an incorrect arrival time.", "row": 1343 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10256:0085", "task_index": 86, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":2050,\"column\":\"abv\",\"new_value\":\"0.063\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":2051,\"column\":\"abv\",\"new_value\":\"0.063\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":2052,\"column\":\"abv\",\"new_value\":\"0.063\",\"reason\":\"The new value for abv is incorrect.\"},{\"row\":2053,\"column\":\"abv\",\"new_value\":\"0.063\",\"reason\":\"The new value for abv is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 3, "wrong_value": 1 }, "fn": 7, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.063", "reason": "The new value for abv is incorrect.", "row": 2050 }, { "column": "abv", "new_value": "0.063", "reason": "The new value for abv is incorrect.", "row": 2051 }, { "column": "abv", "new_value": "0.063", "reason": "The new value for abv is incorrect.", "row": 2052 }, { "column": "abv", "new_value": "0.063", "reason": "The new value for abv is incorrect.", "row": 2053 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10260:0086", "task_index": 87, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":236,\"column\":\"Index\",\"new_value\":\"200 med center drive\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '200 med center drive' does not exist in the dataset.\"},{\"row\":237,\"column\":\"Index\",\"new_value\":\"200 med center drive\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '200 med center drive' does not exist in the dataset.\"},{\"row\":238,\"column\":\"Index\",\"new_value\":\"200 med center drive\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '200 med center drive' does not exist in the dataset.\"},{\"row\":239,\"column\":\"Index\",\"new_value\":\"200 med center drive\",\"reason\":\"The address is incorrect.\",\"why\":\"The address '200 med center drive' does not exist in the dataset.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "schema_case_error": 4 }, "fn": 6, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "200 med center drive", "reason": "The address is incorrect.", "row": 236 }, { "column": "Index", "new_value": "200 med center drive", "reason": "The address is incorrect.", "row": 237 }, { "column": "Index", "new_value": "200 med center drive", "reason": "The address is incorrect.", "row": 238 }, { "column": "Index", "new_value": "200 med center drive", "reason": "The address is incorrect.", "row": 239 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10261:0087", "task_index": 88, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1375,\"column\":\"act_arr_time\",\"new_value\":\"7:36 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1376,\"column\":\"act_arr_time\",\"new_value\":\"9:53 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1377,\"column\":\"act_arr_time\",\"new_value\":\"6:34 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1378,\"column\":\"act_arr_time\",\"new_value\":\"10:47 a Dec 1\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 6, "overrepair": 3, "wrong_value": 1 }, "fn": 7, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "7:36 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1375 }, { "column": "act_arr_time", "new_value": "9:53 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1376 }, { "column": "act_arr_time", "new_value": "6:34 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1377 }, { "column": "act_arr_time", "new_value": "10:47 a Dec 1", "reason": "The new value for act_arr_time is incorrect.", "row": 1378 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10265:0088", "task_index": 89, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":319,\"column\":\"abv\",\"new_value\":\"0.07\",\"reason\":\"The beer name is incorrect.\"},{\"row\":320,\"column\":\"abv\",\"new_value\":\"0.06%\",\"reason\":\"The abv value is incorrect.\"},{\"row\":321,\"column\":\"abv\",\"new_value\":\"0.044\",\"reason\":\"The abv value is incorrect.\"},{\"row\":322,\"column\":\"abv\",\"new_value\":\"0.055\",\"reason\":\"The abv value is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 5, "overrepair": 3, "wrong_value": 1 }, "fn": 6, "fp": 4, "inferability": "deterministic_normalization", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.07", "reason": "The beer name is incorrect.", "row": 319 }, { "column": "abv", "new_value": "0.06%", "reason": "The abv value is incorrect.", "row": 320 }, { "column": "abv", "new_value": "0.044", "reason": "The abv value is incorrect.", "row": 321 }, { "column": "abv", "new_value": "0.055", "reason": "The abv value is incorrect.", "row": 322 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10269:0089", "task_index": 90, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":557,\"column\":\"Index\",\"new_value\":\"10024\",\"reason\":\"The patient's condition is pneumonia.\",\"why\":\"The patient needs to receive a pneumococcal vaccine as part of their routine health check-up.\"},{\"row\":558,\"column\":\"Index\",\"new_value\":\"10024\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"The patient needs to take antibiotics to prevent infection during surgery.\"},{\"row\":559,\"column\":\"Index\",\"new_value\":\"10024\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"The patient needs to take antibiotics to prevent infection during surgery.\"},{\"row\":560,\"column\":\"Index\",\"new_value\":\"10024\",\"reason\":\"The patient's condition is surgical infection prevention.\",\"why\":\"The patient needs to take antibiotics to prevent infection during surgery.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 3, "schema_case_error": 4 }, "fn": 3, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "10024", "reason": "The patient's condition is pneumonia.", "row": 557 }, { "column": "Index", "new_value": "10024", "reason": "The patient's condition is surgical infection prevention.", "row": 558 }, { "column": "Index", "new_value": "10024", "reason": "The patient's condition is surgical infection prevention.", "row": 559 }, { "column": "Index", "new_value": "10024", "reason": "The patient's condition is surgical infection prevention.", "row": 560 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10270:0090", "task_index": 91, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":127,\"column\":\"flight\",\"new_value\":\"UA-2708-EWR-CLT\",\"reason\":\"The flight is scheduled to depart at 4:22 PM instead of 4:22 AM.\"},{\"row\":126,\"column\":\"flight\",\"new_value\":\"UA-2704-DTW-PHX\",\"reason\":\"The flight is scheduled to arrive at 11:47 AM instead of 11:47 a.m.\"},{\"row\":125,\"column\":\"flight\",\"new_value\":\"UA-858-PVG-SFO\",\"reason\":\"The flight is scheduled to arrive at 1:50 PM instead of 1:50 p.m.\"},{\"row\":124,\"column\":\"flight\",\"new_value\":\"UA-858-PVG-SFO\",\"reason\":\"The flight is scheduled to depart at 7:50 AM instead of 7:50 a.m.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 16, "overrepair": 4 }, "fn": 16, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "flight", "new_value": "UA-2708-EWR-CLT", "reason": "The flight is scheduled to depart at 4:22 PM instead of 4:22 AM.", "row": 127 }, { "column": "flight", "new_value": "UA-2704-DTW-PHX", "reason": "The flight is scheduled to arrive at 11:47 AM instead of 11:47 a.m.", "row": 126 }, { "column": "flight", "new_value": "UA-858-PVG-SFO", "reason": "The flight is scheduled to arrive at 1:50 PM instead of 1:50 p.m.", "row": 125 }, { "column": "flight", "new_value": "UA-858-PVG-SFO", "reason": "The flight is scheduled to depart at 7:50 AM instead of 7:50 a.m.", "row": 124 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10274:0091", "task_index": 92, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":862,\"column\":\"abv\",\"new_value\":\"0.07\",\"reason\":\"The beer name has been changed to 'Big Star White IPA' which is a different style from the original 'Big Star IPA'. This change may affect the customer experience or flavor profile.\"},{\"row\":863,\"column\":\"abv\",\"new_value\":\"0.055999999999999994%\", \"reason\":\"The beer name has been changed to 'Old Detroit' which is a different brewery name. This change may affect the customer experience or flavor profile.\"},{\"row\":864,\"column\":\"abv\",\"new_value\":\"0.069\",\"reason\":\"The beer name has been changed to 'Batch 69 IPA' which is a different style from the original 'Big Star IPA'. This change may affect the customer experience or flavor profile.\"},{\"row\":865,\"column\":\"abv\",\"new_value\":\"0.055\",\"reason\":\"The beer name has been changed to 'Twisted Helles Summer Lager' which is a different style from the original 'Big Star IPA'. This change may affect the customer experience or flavor profile.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 3, "wrong_value": 1 }, "fn": 8, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.07", "reason": "The beer name has been changed to 'Big Star White IPA' which is a different style from the original 'Big Star IPA'. This change may affect the customer experience or flavor profile.", "row": 862 }, { "column": "abv", "new_value": "0.055999999999999994%", "reason": "The beer name has been changed to 'Old Detroit' which is a different brewery name. This change may affect the customer experience or flavor profile.", "row": 863 }, { "column": "abv", "new_value": "0.069", "reason": "The beer name has been changed to 'Batch 69 IPA' which is a different style from the original 'Big Star IPA'. This change may affect the customer experience or flavor profile.", "row": 864 }, { "column": "abv", "new_value": "0.055", "reason": "The beer name has been changed to 'Twisted Helles Summer Lager' which is a different style from the original 'Big Star IPA'. This change may affect the customer experience or flavor profile.", "row": 865 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10278:0092", "task_index": 93, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":475,\"column\":\"Index\",\"new_value\":\"10021\",\"reason\":\"The patient's index matches the expected value.\",\"_type\":\"patient\"},{\"row\":476,\"column\":\"Index\",\"new_value\":\"10021\",\"reason\":\"The patient's index matches the expected value.\",\"_type\":\"patient\"},{\"row\":477,\"column\":\"Index\",\"new_value\":\"10021\",\"reason\":\"The patient's index matches the expected value.\",\"_type\":\"patient\"},{\"row\":478,\"column\":\"Index\",\"new_value\":\"10021\",\"reason\":\"The patient's index matches the expected value.\",\"_type\":\"patient\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 2, "schema_case_error": 4 }, "fn": 2, "fp": 0, "inferability": "external_reference_required", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "10021", "reason": "The patient's index matches the expected value.", "row": 475 }, { "column": "Index", "new_value": "10021", "reason": "The patient's index matches the expected value.", "row": 476 }, { "column": "Index", "new_value": "10021", "reason": "The patient's index matches the expected value.", "row": 477 }, { "column": "Index", "new_value": "10021", "reason": "The patient's index matches the expected value.", "row": 478 } ], "recall": 0.0, "schema_case_errors": 4, "task_id": "DataForge-Bench-light-verified:hospital:10279:0093", "task_index": 94, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":560,\"column\":\"act_arr_time\",\"new_value\":\"1:14 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":561,\"column\":\"act_arr_time\",\"new_value\":\"9:05 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":562,\"column\":\"act_arr_time\",\"new_value\":\"5:05 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":563,\"column\":\"act_arr_time\",\"new_value\":\"10:27 a.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "overrepair": 4 }, "fn": 0, "fp": 4, "inferability": "not_inferable_from_prompt", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "1:14 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 560 }, { "column": "act_arr_time", "new_value": "9:05 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 561 }, { "column": "act_arr_time", "new_value": "5:05 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 562 }, { "column": "act_arr_time", "new_value": "10:27 a.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 563 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10283:0094", "task_index": 95, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":428,\"column\":\"abv\",\"new_value\":\"0.073\",\"reason\":\"The beer name is incorrect.\"},{\"row\":429,\"column\":\"abv\",\"new_value\":\"0.055999999999999994%\", \"reason\":\"The value does not match the expected format (%.2f)\"},{\"row\":430,\"column\":\"abv\",\"new_value\":\"0.093\",\"reason\":\"The beer name is incorrect.\"},{\"row\":431,\"column\":\"abv\",\"new_value\":\"0.065\",\"reason\":\"The value does not match the expected format (%.2f)\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 3, "wrong_value": 1 }, "fn": 9, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.073", "reason": "The beer name is incorrect.", "row": 428 }, { "column": "abv", "new_value": "0.055999999999999994%", "reason": "The value does not match the expected format (%.2f)", "row": 429 }, { "column": "abv", "new_value": "0.093", "reason": "The beer name is incorrect.", "row": 430 }, { "column": "abv", "new_value": "0.065", "reason": "The value does not match the expected format (%.2f)", "row": 431 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10287:0095", "task_index": 96, "tp": 0 }, { "canonicalized_f1": 0.3333, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":637,\"column\":\"ProviderNumber\",\"new_value\":\"10029\",\"reason\":\"The patient's condition requires surgical infection prevention measures.\"},{\"row\":638,\"column\":\"ProviderNumber\",\"new_value\":\"10029\",\"reason\":\"The patient's condition requires surgical infection prevention measures.\"},{\"row\":639,\"column\":\"ProviderNumber\",\"new_value\":\"10029\",\"reason\":\"The patient's condition requires surgical infection prevention measures.\"},{\"row\":640,\"column\":\"ProviderNumber\",\"new_value\":\"10029\",\"reason\":\"The patient's condition requires surgical infection prevention measures.\"}]}", "f1": 0.3333, "failure_taxonomy": { "missed_repair": 1, "overrepair": 3 }, "fn": 1, "fp": 3, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.25, "predicted_repairs": [ { "column": "ProviderNumber", "new_value": "10029", "reason": "The patient's condition requires surgical infection prevention measures.", "row": 637 }, { "column": "ProviderNumber", "new_value": "10029", "reason": "The patient's condition requires surgical infection prevention measures.", "row": 638 }, { "column": "ProviderNumber", "new_value": "10029", "reason": "The patient's condition requires surgical infection prevention measures.", "row": 639 }, { "column": "ProviderNumber", "new_value": "10029", "reason": "The patient's condition requires surgical infection prevention measures.", "row": 640 } ], "recall": 0.5, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:hospital:10288:0096", "task_index": 97, "tp": 1 }, { "canonicalized_f1": 0.0, "dataset": "flights", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":1033,\"column\":\"act_arr_time\",\"new_value\":\"3:04 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1034,\"column\":\"act_arr_time\",\"new_value\":\"4:23 p.m.\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1035,\"column\":\"act_arr_time\",\"new_value\":\"Dec 02 Not Available\",\"reason\":\"The new value for act_arr_time is incorrect.\"},{\"row\":1036,\"column\":\"act_arr_time\",\"new_value\":\"Not Available\",\"reason\":\"The new value for act_arr_time is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 7, "overrepair": 2, "wrong_value": 2 }, "fn": 9, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "act_arr_time", "new_value": "3:04 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1033 }, { "column": "act_arr_time", "new_value": "4:23 p.m.", "reason": "The new value for act_arr_time is incorrect.", "row": 1034 }, { "column": "act_arr_time", "new_value": "Dec 02 Not Available", "reason": "The new value for act_arr_time is incorrect.", "row": 1035 }, { "column": "act_arr_time", "new_value": "Not Available", "reason": "The new value for act_arr_time is incorrect.", "row": 1036 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:flights:10292:0097", "task_index": 98, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "beers", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":802,\"column\":\"abv\",\"new_value\":\"0.055\",\"reason\":\"The new value for Abv is incorrect.\"},{\"row\":803,\"column\":\"abv\",\"new_value\":\"0.047%\",\"reason\":\"The new value for Abv is incorrect.\"},{\"row\":804,\"column\":\"abv\",\"new_value\":\"0.045\",\"reason\":\"The new value for Abv is incorrect.\"},{\"row\":805,\"column\":\"abv\",\"new_value\":\"0.075\",\"reason\":\"The new value for Abv is incorrect.\"}]}", "f1": 0.0, "failure_taxonomy": { "missed_repair": 8, "overrepair": 3, "wrong_value": 1 }, "fn": 9, "fp": 4, "inferability": "external_reference_required", "parse_error_kind": null, "parse_ok": true, "precision": 0.0, "predicted_repairs": [ { "column": "abv", "new_value": "0.055", "reason": "The new value for Abv is incorrect.", "row": 802 }, { "column": "abv", "new_value": "0.047%", "reason": "The new value for Abv is incorrect.", "row": 803 }, { "column": "abv", "new_value": "0.045", "reason": "The new value for Abv is incorrect.", "row": 804 }, { "column": "abv", "new_value": "0.075", "reason": "The new value for Abv is incorrect.", "row": 805 } ], "recall": 0.0, "schema_case_errors": 0, "task_id": "DataForge-Bench-light-verified:beers:10296:0098", "task_index": 99, "tp": 0 }, { "canonicalized_f1": 0.0, "dataset": "hospital", "decoded_preview": "{\"action\":\"submit_repairs\",\"repairs\":[{\"row\":421,\"column\":\"Index\",\"new_value\":\"422\",\"reason\":\"The patient's address does not match any hospital records.\"},{\"row\":422,\"column\":\"Index\",\"new_value\":\"423\",\"reason\":\"The patient's address does not match any hospital records.\"},{\"row\":424,\"column\":\"Index\",\"new_value\":\"425\",\"reason\":\"The patient's address does not match any hospital records.\"}]}", "f1": 0.0, "failure_taxonomy": { "schema_case_error": 3 }, "fn": 0, "fp": 0, "inferability": "not_inferable_from_prompt", "parse_error_kind": "invalid_column", "parse_ok": false, "precision": 0.0, "predicted_repairs": [ { "column": "Index", "new_value": "422", "reason": "The patient's address does not match any hospital records.", "row": 421 }, { "column": "Index", "new_value": "423", "reason": "The patient's address does not match any hospital records.", "row": 422 }, { "column": "Index", "new_value": "425", "reason": "The patient's address does not match any hospital records.", "row": 424 } ], "recall": 0.0, "schema_case_errors": 3, "task_id": "DataForge-Bench-light-verified:hospital:10297:0099", "task_index": 100, "tp": 0 } ] }, "sft_eval": { "canonicalized_macro_f1": 0.0053, "dataset_f1": { "beers": 0.0061, "flights": 0.0, "hospital": 0.0098 }, "failure_taxonomy": { "invalid_column": 15, "invalid_row": 4, "missed_repair": 500, "overrepair": 241, "schema_case_error": 56, "truncated_json": 1, "wrong_cell": 4, "wrong_value": 74 }, "macro_f1": 0.0053, "mean_f1": 0.0053, "parse_success_rate": 0.8, "schema_case_error_count": 56, "slice_scores": { "deterministic_normalization": { "macro_f1": 0.0, "parse_success_rate": 1.0, "schema_case_error_count": 0, "tasks": 20 }, "external_reference_required": { "macro_f1": 0.0081, "parse_success_rate": 0.7424, "schema_case_error_count": 45, "tasks": 66 }, "not_inferable_from_prompt": { "macro_f1": 0.0, "parse_success_rate": 0.7857, "schema_case_error_count": 11, "tasks": 14 } }, "tasks": 100 }, "source_audit": { "blockers": [], "datasets": [ { "clean_sha256": "ea3ee44998455c0b491750c348509de176c758a3bbf58e4530c0a136bb248b4b", "columns": 20, "dataset": "hospital", "dirty_sha256": "dbc5575b915fe8b5e0ac6dc6172f38ba91e611fdb76d09a8f4a81cb7ea9925ac", "errors": [], "ground_truth_cells": 509, "rows": 1000, "source_revision": "7be1334b8c7bbdac3f47ef514fb3e1e8c5fc181c", "status": "pass" }, { "clean_sha256": "0acfcfd8985b06fdd363965c9e8d9522c43e7589a93d79ae7dc311e1c37fdf3b", "columns": 7, "dataset": "flights", "dirty_sha256": "1b5c1afa10aa0e7c20fd7e14d05c56772715b2771aa0f5fa67ed1709e1eecd46", "errors": [], "ground_truth_cells": 4920, "rows": 2376, "source_revision": "7be1334b8c7bbdac3f47ef514fb3e1e8c5fc181c", "status": "pass" }, { "clean_sha256": "373227df59ad197e154dd5149125789e415019535c7223355e9486ee1b3b93de", "columns": 11, "dataset": "beers", "dirty_sha256": "7110bf4931a9445a1675e544d6c996817c739136239f8a2b02e088c7ec0a1f68", "errors": [], "ground_truth_cells": 4362, "rows": 2410, "source_revision": "7be1334b8c7bbdac3f47ef514fb3e1e8c5fc181c", "status": "pass" } ], "ok": true, "schema_version": "dataforge_real_world_source_audit_v1", "status": "pass" }, "task_manifest_sha256": "f3172aa87316da98b6230d0432e5efdcf82e220c642fd5091bca000a73020a5e" }