File size: 4,135 Bytes
186a89f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | {
"schema_version": 2,
"title": "Efficient Bayesian Inference from Noisy Pairwise Comparisons",
"emoji": "🎯",
"space_id": "SabaPivot/repro-efficient-bayesian-inference-from-noisy-pairwise-comparisons",
"paper": {
"arxiv_id": "2510.09333"
},
"tags": [
"icml2026-repro",
"paper-NinueNAODD"
],
"updated_at": "2026-08-02T16:31:40.600613+00:00",
"root": {
"slug": "index",
"title": "Efficient Bayesian Inference from Noisy Pairwise Comparisons",
"file": "pages/index.md",
"children": [
{
"slug": "executive-summary",
"title": "Executive summary",
"file": "pages/executive-summary/page.md",
"children": []
},
{
"slug": "claim-1-em-monotonic-convergence-vs-gradient-based-crowd-bt",
"title": "Claim 1: BBQ's EM algorithm has closed-form updates that guarantee monotonic improvement of the likelihood and convergence to a stationary point, in contrast to gradient-based methods like Crowd-BT which lack such convergence guarantees (Section 3.3).",
"file": "pages/claim-1-em-monotonic-convergence-vs-gradient-based-crowd-bt/page.md",
"children": []
},
{
"slug": "claim-2-runtime-seconds-for-bbq-vs-15-min-crowd-bt-on-humaine",
"title": "Claim 2: BBQ converges within seconds on all tested datasets, whereas Crowd-BT takes approximately 15 minutes on the HUMAINE dataset (105,220 comparisons) despite BBQ using an unoptimized plain NumPy implementation (Figure 3).",
"file": "pages/claim-2-runtime-seconds-for-bbq-vs-15-min-crowd-bt-on-humaine/page.md",
"children": []
},
{
"slug": "claim-3-ihq-unscreened-top-1-agreement-61-92-vs-33-15-vs-24-32",
"title": "Claim 3: On the unscreened IHQ dataset, BBQ achieves 61.92% top-1 agreement with final rankings, compared to 33.15% for Crowd-BT and 24.32% for Bayes-BT (Table 1).",
"file": "pages/claim-3-ihq-unscreened-top-1-agreement-61-92-vs-33-15-vs-24-32/page.md",
"children": []
},
{
"slug": "claim-4-kendall-tau-first-on-5-of-8-datasets-and-100-top-1-on-three",
"title": "Claim 4: BBQ ranks first in Kendall's Tau agreement on 5 of 8 evaluated datasets and second on the remaining three, and achieves 100% top-1 agreement on the MT-Bench, WD, and HiFiC datasets (Table 1).",
"file": "pages/claim-4-kendall-tau-first-on-5-of-8-datasets-and-100-top-1-on-three/page.md",
"children": []
},
{
"slug": "claim-5-type-i-error-1-for-bbq-and-crowd-bt-vs-0-1-bayes-bt",
"title": "Claim 5: At the 99% confidence level, BBQ and Crowd-BT both show well-calibrated Type I error rates of about 1%, while Bayes-BT is overly conservative at about 0.1% (Appendix G, Figure 6).",
"file": "pages/claim-5-type-i-error-1-for-bbq-and-crowd-bt-vs-0-1-bayes-bt/page.md",
"children": []
},
{
"slug": "claim-6-rater-quality-q-r-correlates-r-0-724-with-rater-agreement",
"title": "Claim 6: BBQ's rater-quality parameter (mixture weight q_r modeling whether a rater follows Bradley-Terry or guesses randomly) correlates with Pearson r=0.724 with rater agreement to final rankings on the unscreened IHQ data, enabling identification of unreliable crowdsourced raters without a separate screening procedure (Figure 2, Equation 2).",
"file": "pages/claim-6-rater-quality-q-r-correlates-r-0-724-with-rater-agreement/page.md",
"children": []
},
{
"slug": "claim-99-fresh-independent-cpu-audit",
"title": "Fresh independent CPU audit",
"file": "pages/claim-99-fresh-independent-cpu-audit/page.md",
"children": []
},
{
"slug": "conclusion",
"title": "Conclusion",
"file": "pages/conclusion/page.md",
"children": []
}
]
},
"traces": [],
"workspace": {
"file": "workspace.json",
"file_count": 0,
"total_size": 0,
"bucket_id": null
},
"agent_view_tokens": 10975,
"trace_view_tokens": 10,
"workspace_view_tokens": 8,
"revision": "7af58f963242c9ed30fa"
}
|