Buckets:
| { | |
| "paper": { | |
| "title": "Optimal Fair Aggregation of Crowdsourced Noisy Labels using Demographic Parity Constraints", | |
| "orid": "niQUth28zn", | |
| "arxiv": "2601.23221" | |
| }, | |
| "claims": [ | |
| "The demographic parity gap of majority-vote aggregated labels is bounded above by an exponentially decaying term in the number of annotators R, of the form sum over groups a of E[e^{-R K_phi(a,X)}] (Proposition 3.2).", | |
| "Under interpretable conditions on annotator skill, both Majority Vote and Bayesian aggregation are shown to achieve asymptotically unbiased fairness gaps relative to ground-truth labels as crowd size grows (Theorem 3.4).", | |
| "For majority voting with R annotators, the aggregated fairness gap is bounded by epsilon(R) times the sum of individual annotators' fairness gaps, showing majority voting can amplify rather than average out bias (Proposition 3.6).", | |
| "Convergence of the fairness gap to the ground-truth value requires a majority of annotators to have skill exceeding 0.5 plus a margin epsilon; without this, majority voting fails to converge (Section 3, following Proposition 3.6).", | |
| "FairCrowd, a post-processing method, enforces a strict epsilon-fairness constraint on the output of any aggregation rule while provably bounding the resulting accuracy loss (Theorem 4.1)." | |
| ], | |
| "source_audit": { | |
| "paper_tex_sha256": "438c5312734b573e455c1b8b05657e18742805f2e6144bf1346b53354abb9427", | |
| "official_code_link_found": false, | |
| "prop_3_2_label_found": true, | |
| "theorem_3_4_label_found": true, | |
| "prop_3_6_label_found": true, | |
| "faircrowd_theorem_label_found": true, | |
| "note": "No official GitHub repository is linked in the paper source." | |
| }, | |
| "claim_1": { | |
| "exact_configurations": 18, | |
| "rows": [ | |
| { | |
| "R": 3, | |
| "mv_gap": 0.12554999999999994, | |
| "bayes_gap": 0.12555, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.07445000000000002, | |
| "bayes_abs_bias": 0.07444999999999996, | |
| "mv_exponential_bound": 1.4193917695909102, | |
| "bayes_exponential_bound": 1.4193917695909102, | |
| "mv_bound_slack": 1.34494176959091, | |
| "bayes_bound_slack": 1.3449417695909103 | |
| }, | |
| { | |
| "R": 5, | |
| "mv_gap": 0.14668087499999993, | |
| "bayes_gap": 0.14668087499999993, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.05331912500000002, | |
| "bayes_abs_bias": 0.05331912500000002, | |
| "mv_exponential_bound": 1.133832371700915, | |
| "bayes_exponential_bound": 1.133832371700915, | |
| "mv_bound_slack": 1.080513246700915, | |
| "bayes_bound_slack": 1.080513246700915 | |
| }, | |
| { | |
| "R": 9, | |
| "mv_gap": 0.17045280657421863, | |
| "bayes_gap": 0.1704528065742189, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.029547193425781326, | |
| "bayes_abs_bias": 0.02954719342578105, | |
| "mv_exponential_bound": 0.730322489126292, | |
| "bayes_exponential_bound": 0.730322489126292, | |
| "mv_bound_slack": 0.7007752957005107, | |
| "bayes_bound_slack": 0.700775295700511 | |
| }, | |
| { | |
| "R": 15, | |
| "mv_gap": 0.1865375243164199, | |
| "bayes_gap": 0.18653752431641968, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.013462475683580055, | |
| "bayes_abs_bias": 0.013462475683580277, | |
| "mv_exponential_bound": 0.38605520688436606, | |
| "bayes_exponential_bound": 0.38605520688436606, | |
| "mv_bound_slack": 0.372592731200786, | |
| "bayes_bound_slack": 0.3725927312007858 | |
| }, | |
| { | |
| "R": 25, | |
| "mv_gap": 0.19583196228101485, | |
| "bayes_gap": 0.19583196228101485, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.004168037718985107, | |
| "bayes_abs_bias": 0.004168037718985107, | |
| "mv_exponential_bound": 0.1405399402409523, | |
| "bayes_exponential_bound": 0.1405399402409523, | |
| "mv_bound_slack": 0.13637190252196718, | |
| "bayes_bound_slack": 0.13637190252196718 | |
| }, | |
| { | |
| "R": 41, | |
| "mv_gap": 0.1992311696838257, | |
| "bayes_gap": 0.19923116968382532, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.0007688303161742516, | |
| "bayes_abs_bias": 0.0007688303161746401, | |
| "mv_exponential_bound": 0.030782951306858288, | |
| "bayes_exponential_bound": 0.030782951306858288, | |
| "mv_bound_slack": 0.030014120990684037, | |
| "bayes_bound_slack": 0.030014120990683648 | |
| }, | |
| { | |
| "R": 61, | |
| "mv_gap": 0.19989167433888377, | |
| "bayes_gap": 0.19989167433888338, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.00010832566111618203, | |
| "bayes_abs_bias": 0.0001083256611165706, | |
| "mv_exponential_bound": 0.005058292628266722, | |
| "bayes_exponential_bound": 0.005058292628266722, | |
| "mv_bound_slack": 0.00494996696715054, | |
| "bayes_bound_slack": 0.004949966967150152 | |
| }, | |
| { | |
| "R": 101, | |
| "mv_gap": 0.19999740497309426, | |
| "bayes_gap": 0.19999740497309454, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 2.59502690569402e-06, | |
| "bayes_abs_bias": 2.5950269054164643e-06, | |
| "mv_exponential_bound": 0.0001504948309636977, | |
| "bayes_exponential_bound": 0.0001504948309636977, | |
| "mv_bound_slack": 0.0001478998040580037, | |
| "bayes_bound_slack": 0.00014789980405828125 | |
| }, | |
| { | |
| "R": 161, | |
| "mv_gap": 0.199999988827116, | |
| "bayes_gap": 0.19999998882711573, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 1.117288395136029e-08, | |
| "bayes_abs_bias": 1.1172884228916047e-08, | |
| "mv_exponential_bound": 8.026568576087593e-07, | |
| "bayes_exponential_bound": 8.026568576087593e-07, | |
| "mv_bound_slack": 7.914839736573991e-07, | |
| "bayes_bound_slack": 7.914839733798433e-07 | |
| } | |
| ], | |
| "max_mv_bound_violation": -7.914839736573991e-07, | |
| "max_bayes_bound_violation": -7.914839733798433e-07, | |
| "all_bounds_hold": true | |
| }, | |
| "claim_2": { | |
| "true_dp_gap": 0.19999999999999996, | |
| "mv_final_abs_bias": 1.117288395136029e-08, | |
| "bayes_final_abs_bias": 1.1172884228916047e-08, | |
| "mv_log_bias_slope_per_worker": -0.09382906315241858, | |
| "bayes_log_bias_slope_per_worker": -0.0938290629787273, | |
| "majority_condition_margin": 0.19999999999999996, | |
| "bayes_divergence_sum_at_final_R": [ | |
| 6.439999999999997, | |
| 10.0625 | |
| ], | |
| "supported": true, | |
| "rows": [ | |
| { | |
| "R": 3, | |
| "mv_gap": 0.12554999999999994, | |
| "bayes_gap": 0.12555, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.07445000000000002, | |
| "bayes_abs_bias": 0.07444999999999996, | |
| "mv_exponential_bound": 1.4193917695909102, | |
| "bayes_exponential_bound": 1.4193917695909102, | |
| "mv_bound_slack": 1.34494176959091, | |
| "bayes_bound_slack": 1.3449417695909103 | |
| }, | |
| { | |
| "R": 5, | |
| "mv_gap": 0.14668087499999993, | |
| "bayes_gap": 0.14668087499999993, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.05331912500000002, | |
| "bayes_abs_bias": 0.05331912500000002, | |
| "mv_exponential_bound": 1.133832371700915, | |
| "bayes_exponential_bound": 1.133832371700915, | |
| "mv_bound_slack": 1.080513246700915, | |
| "bayes_bound_slack": 1.080513246700915 | |
| }, | |
| { | |
| "R": 9, | |
| "mv_gap": 0.17045280657421863, | |
| "bayes_gap": 0.1704528065742189, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.029547193425781326, | |
| "bayes_abs_bias": 0.02954719342578105, | |
| "mv_exponential_bound": 0.730322489126292, | |
| "bayes_exponential_bound": 0.730322489126292, | |
| "mv_bound_slack": 0.7007752957005107, | |
| "bayes_bound_slack": 0.700775295700511 | |
| }, | |
| { | |
| "R": 15, | |
| "mv_gap": 0.1865375243164199, | |
| "bayes_gap": 0.18653752431641968, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.013462475683580055, | |
| "bayes_abs_bias": 0.013462475683580277, | |
| "mv_exponential_bound": 0.38605520688436606, | |
| "bayes_exponential_bound": 0.38605520688436606, | |
| "mv_bound_slack": 0.372592731200786, | |
| "bayes_bound_slack": 0.3725927312007858 | |
| }, | |
| { | |
| "R": 25, | |
| "mv_gap": 0.19583196228101485, | |
| "bayes_gap": 0.19583196228101485, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.004168037718985107, | |
| "bayes_abs_bias": 0.004168037718985107, | |
| "mv_exponential_bound": 0.1405399402409523, | |
| "bayes_exponential_bound": 0.1405399402409523, | |
| "mv_bound_slack": 0.13637190252196718, | |
| "bayes_bound_slack": 0.13637190252196718 | |
| }, | |
| { | |
| "R": 41, | |
| "mv_gap": 0.1992311696838257, | |
| "bayes_gap": 0.19923116968382532, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.0007688303161742516, | |
| "bayes_abs_bias": 0.0007688303161746401, | |
| "mv_exponential_bound": 0.030782951306858288, | |
| "bayes_exponential_bound": 0.030782951306858288, | |
| "mv_bound_slack": 0.030014120990684037, | |
| "bayes_bound_slack": 0.030014120990683648 | |
| }, | |
| { | |
| "R": 61, | |
| "mv_gap": 0.19989167433888377, | |
| "bayes_gap": 0.19989167433888338, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 0.00010832566111618203, | |
| "bayes_abs_bias": 0.0001083256611165706, | |
| "mv_exponential_bound": 0.005058292628266722, | |
| "bayes_exponential_bound": 0.005058292628266722, | |
| "mv_bound_slack": 0.00494996696715054, | |
| "bayes_bound_slack": 0.004949966967150152 | |
| }, | |
| { | |
| "R": 101, | |
| "mv_gap": 0.19999740497309426, | |
| "bayes_gap": 0.19999740497309454, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 2.59502690569402e-06, | |
| "bayes_abs_bias": 2.5950269054164643e-06, | |
| "mv_exponential_bound": 0.0001504948309636977, | |
| "bayes_exponential_bound": 0.0001504948309636977, | |
| "mv_bound_slack": 0.0001478998040580037, | |
| "bayes_bound_slack": 0.00014789980405828125 | |
| }, | |
| { | |
| "R": 161, | |
| "mv_gap": 0.199999988827116, | |
| "bayes_gap": 0.19999998882711573, | |
| "true_gap": 0.19999999999999996, | |
| "mv_abs_bias": 1.117288395136029e-08, | |
| "bayes_abs_bias": 1.1172884228916047e-08, | |
| "mv_exponential_bound": 8.026568576087593e-07, | |
| "bayes_exponential_bound": 8.026568576087593e-07, | |
| "mv_bound_slack": 7.914839736573991e-07, | |
| "bayes_bound_slack": 7.914839733798433e-07 | |
| } | |
| ] | |
| }, | |
| "claim_3": { | |
| "eta": 0.46882235549939, | |
| "eta_argmax_lambda": 0.3949889069928964, | |
| "paper_eta": 0.4688, | |
| "paper_eta_abs_error": 2.2355499390003253e-05, | |
| "rows": [ | |
| { | |
| "R": 3, | |
| "individual_dp_gap": 0.09000000000000002, | |
| "aggregate_dp_gap": 0.12554999999999994, | |
| "epsilon_R": 0.5440766446997511, | |
| "bound": 0.14690069406893286, | |
| "slack": 0.02135069406893292 | |
| }, | |
| { | |
| "R": 5, | |
| "individual_dp_gap": 0.09000000000000002, | |
| "aggregate_dp_gap": 0.14668087499999993, | |
| "epsilon_R": 0.4214399567965093, | |
| "bound": 0.18964798055842924, | |
| "slack": 0.04296710555842931 | |
| }, | |
| { | |
| "R": 9, | |
| "individual_dp_gap": 0.09000000000000002, | |
| "aggregate_dp_gap": 0.17045280657421863, | |
| "epsilon_R": 0.3141227972771897, | |
| "bound": 0.25443946579452376, | |
| "slack": 0.08398665922030513 | |
| }, | |
| { | |
| "R": 15, | |
| "individual_dp_gap": 0.09000000000000002, | |
| "aggregate_dp_gap": 0.1865375243164199, | |
| "epsilon_R": 0.24331847250372887, | |
| "bound": 0.32847993788003405, | |
| "slack": 0.14194241356361414 | |
| }, | |
| { | |
| "R": 25, | |
| "individual_dp_gap": 0.09000000000000002, | |
| "aggregate_dp_gap": 0.19583196228101485, | |
| "epsilon_R": 0.18847367836631382, | |
| "bound": 0.42406577632420617, | |
| "slack": 0.22823381404319132 | |
| }, | |
| { | |
| "R": 41, | |
| "individual_dp_gap": 0.09000000000000002, | |
| "aggregate_dp_gap": 0.1992311696838257, | |
| "epsilon_R": 0.14717321683725212, | |
| "bound": 0.5430691701294604, | |
| "slack": 0.34383800044563473 | |
| }, | |
| { | |
| "R": 81, | |
| "individual_dp_gap": 0.09000000000000002, | |
| "aggregate_dp_gap": 0.19998353345434094, | |
| "epsilon_R": 0.10470759909239656, | |
| "bound": 0.7633183973835711, | |
| "slack": 0.5633348639292302 | |
| }, | |
| { | |
| "R": 161, | |
| "individual_dp_gap": 0.09000000000000002, | |
| "aggregate_dp_gap": 0.199999988827116, | |
| "epsilon_R": 0.07426903361132658, | |
| "bound": 1.0761582970281225, | |
| "slack": 0.8761583082010065 | |
| } | |
| ], | |
| "all_bounds_hold": true, | |
| "bound_loglog_slope": 0.5000000000000001, | |
| "target_amplification_slope": 0.5, | |
| "destructive_control": { | |
| "finite_grid_atom_mass_times_sigma_sup": 0.42242531084522794, | |
| "argmax_n_p": [ | |
| 25, | |
| 0.02 | |
| ], | |
| "wrong_asymptotic_constant": 0.3989422804014327, | |
| "wrong_constant_violated": true | |
| } | |
| }, | |
| "claim_4": { | |
| "rows": [ | |
| { | |
| "skill": 0.5, | |
| "R": 5, | |
| "gap": 0.0, | |
| "abs_bias": 0.19999999999999996, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.5, | |
| "R": 15, | |
| "gap": 0.0, | |
| "abs_bias": 0.19999999999999996, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.5, | |
| "R": 41, | |
| "gap": 0.0, | |
| "abs_bias": 0.19999999999999996, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.5, | |
| "R": 101, | |
| "gap": 0.0, | |
| "abs_bias": 0.19999999999999996, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.5, | |
| "R": 401, | |
| "gap": 0.0, | |
| "abs_bias": 0.19999999999999996, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.7, | |
| "R": 5, | |
| "gap": 0.134768, | |
| "abs_bias": 0.06523199999999996, | |
| "fraction_above_half_plus_margin": 1.0 | |
| }, | |
| { | |
| "skill": 0.7, | |
| "R": 15, | |
| "gap": 0.17999498397848945, | |
| "abs_bias": 0.020005016021510502, | |
| "fraction_above_half_plus_margin": 1.0 | |
| }, | |
| { | |
| "skill": 0.7, | |
| "R": 41, | |
| "gap": 0.19857203870411033, | |
| "abs_bias": 0.0014279612958896237, | |
| "fraction_above_half_plus_margin": 1.0 | |
| }, | |
| { | |
| "skill": 0.7, | |
| "R": 101, | |
| "gap": 0.19999482297826598, | |
| "abs_bias": 5.1770217339708324e-06, | |
| "fraction_above_half_plus_margin": 1.0 | |
| }, | |
| { | |
| "skill": 0.7, | |
| "R": 401, | |
| "gap": 0.19999999999999996, | |
| "abs_bias": 0.0, | |
| "fraction_above_half_plus_margin": 1.0 | |
| }, | |
| { | |
| "skill": 0.25, | |
| "R": 5, | |
| "gap": 0.15859374999999987, | |
| "abs_bias": 0.04140625000000009, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.25, | |
| "R": 15, | |
| "gap": 0.19308006465435024, | |
| "abs_bias": 0.006919935345649719, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.25, | |
| "R": 41, | |
| "gap": 0.19989030066354102, | |
| "abs_bias": 0.00010969933645893493, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.25, | |
| "R": 101, | |
| "gap": 0.1999999869679226, | |
| "abs_bias": 1.3032077361696537e-08, | |
| "fraction_above_half_plus_margin": 0.0 | |
| }, | |
| { | |
| "skill": 0.25, | |
| "R": 401, | |
| "gap": 0.19999999999999996, | |
| "abs_bias": 0.0, | |
| "fraction_above_half_plus_margin": 0.0 | |
| } | |
| ], | |
| "no_signal_persistent_bias": 0.19999999999999996, | |
| "competent_final_bias": 0.0, | |
| "adversarial_final_bias": 0.0, | |
| "source_condition_interpretation": "Equation (7) is explicitly introduced as a sufficient condition, not a necessary condition.", | |
| "literal_registered_claim_counterexample": "A homogeneous adversarial crowd with skill 0.25 has no annotator above 0.5+epsilon, yet majority vote converges to 1-Y. Because the demographic-parity gap uses an absolute difference, Delta_DP(1-Y) equals Delta_DP(Y), so the fairness gap still converges.", | |
| "verdict": "partially_supported_but_necessity_wording_is_false" | |
| }, | |
| "claim_5": { | |
| "finite_cells": 16, | |
| "lp_formulation": "Independent primal risk minimization over randomized predictions g(w,a) in [0,1], with both signs of the demographic-parity constraint.", | |
| "rows": [ | |
| { | |
| "epsilon": 0.02, | |
| "dp_gap": 0.020000000000000018, | |
| "risk": 0.12401093368055566, | |
| "bayes_dp_gap": 0.16068147187499976, | |
| "bayes_risk": 0.09274838437500016, | |
| "accuracy_loss_vs_bayes": 0.0312625493055555, | |
| "active_constraint": true, | |
| "fair": true, | |
| "fractional_cells": 1, | |
| "lp_status": 0 | |
| }, | |
| { | |
| "epsilon": 0.05, | |
| "dp_gap": 0.050000000000000044, | |
| "risk": 0.117344267013889, | |
| "bayes_dp_gap": 0.16068147187499976, | |
| "bayes_risk": 0.09274838437500016, | |
| "accuracy_loss_vs_bayes": 0.024595882638888844, | |
| "active_constraint": true, | |
| "fair": true, | |
| "fractional_cells": 1, | |
| "lp_status": 0 | |
| }, | |
| { | |
| "epsilon": 0.1, | |
| "dp_gap": 0.10000000000000006, | |
| "risk": 0.10623315590277788, | |
| "bayes_dp_gap": 0.16068147187499976, | |
| "bayes_risk": 0.09274838437500016, | |
| "accuracy_loss_vs_bayes": 0.013484771527777717, | |
| "active_constraint": true, | |
| "fair": true, | |
| "fractional_cells": 1, | |
| "lp_status": 0 | |
| }, | |
| { | |
| "epsilon": 0.15, | |
| "dp_gap": 0.15000000000000005, | |
| "risk": 0.09512204479166675, | |
| "bayes_dp_gap": 0.16068147187499976, | |
| "bayes_risk": 0.09274838437500016, | |
| "accuracy_loss_vs_bayes": 0.0023736604166665898, | |
| "active_constraint": true, | |
| "fair": true, | |
| "fractional_cells": 1, | |
| "lp_status": 0 | |
| }, | |
| { | |
| "epsilon": 0.2, | |
| "dp_gap": 0.16068147187499976, | |
| "risk": 0.09274838437500016, | |
| "bayes_dp_gap": 0.16068147187499976, | |
| "bayes_risk": 0.09274838437500016, | |
| "accuracy_loss_vs_bayes": 0.0, | |
| "active_constraint": false, | |
| "fair": true, | |
| "fractional_cells": 0, | |
| "lp_status": 0 | |
| }, | |
| { | |
| "epsilon": 0.3, | |
| "dp_gap": 0.16068147187499976, | |
| "risk": 0.09274838437500016, | |
| "bayes_dp_gap": 0.16068147187499976, | |
| "bayes_risk": 0.09274838437500016, | |
| "accuracy_loss_vs_bayes": 0.0, | |
| "active_constraint": false, | |
| "fair": true, | |
| "fractional_cells": 0, | |
| "lp_status": 0 | |
| } | |
| ], | |
| "all_fair": true, | |
| "monotone_risk": true, | |
| "destructive_control": { | |
| "unconstrained_bayes_dp_gap": 0.16068147187499976, | |
| "violates_eps_0_05": true | |
| } | |
| }, | |
| "verdicts": { | |
| "claim_1": "supported_exact_finite_domain_audit", | |
| "claim_2": "supported_exact_convergence_audit_not_proof", | |
| "claim_3": "supported_bound_and_destructive_control", | |
| "claim_4": "partially_supported_registered_necessity_wording_overstates_source", | |
| "claim_5": "supported_independent_primal_lp_audit" | |
| }, | |
| "scope": "Clean-room X=empty binary finite-domain audit. Real crowdsourcing datasets, Dawid-Skene estimation, and multiclass extension not rerun.", | |
| "wall_time_seconds": 0.59849395999845 | |
| } |
Xet Storage Details
- Size:
- 19.4 kB
- Xet hash:
- d740f644b3fa11bc28628d2fd278d7cd190716304674d9a9579f527a46fe4fff
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.