| { |
| "all_scientific_gates_pass": true, |
| "claim_2_val5": { |
| "absolute_gain": 0.06, |
| "absolute_gain_over_majority_vote": 0.15000000000000002, |
| "largest_reproduced_absolute_gain_dataset": "Val5", |
| "majority_vote": 0.31, |
| "ptbcc": 0.46, |
| "relative_gain": 0.15, |
| "strongest_reproduced_baseline": 0.4 |
| }, |
| "claim_3_falsification": { |
| "absolute_shortfall": 0.02355063891614273, |
| "aircr_accuracy_required_to_reach_reported_macro": 0.9827063891614269, |
| "measured_ten_dataset_ptbcc": 0.7236493610838572, |
| "reported_ptbcc": 0.7472 |
| }, |
| "claim_4_falsification": { |
| "macro_by_prototypes": { |
| "2": 0.7236493610838572, |
| "3": 0.7250569980641843, |
| "4": 0.725806675063574 |
| }, |
| "peak": 4 |
| }, |
| "claim_5_cost": { |
| "interpretation": "This is an exact comparison of learned confusion-structure parameters, the computational-work term reduced by prototype sharing. Environment-dependent wall-clock time is deliberately not used as scored evidence.", |
| "pooled_ibcc_free_parameters": 71650, |
| "pooled_ptbcc_free_parameters": 4354, |
| "ratio": 0.060767620376831824, |
| "reduction": 0.9392323796231682, |
| "registered_dataset_rows": 11 |
| }, |
| "claims": [ |
| { |
| "claim": 1, |
| "literal_claim": "PTBCC (Prototype-driven Bayesian Classifier Combination) models annotators via a shared set of prototype confusion matrices rather than learning one confusion matrix per annotator (Section on method overview)." |
| }, |
| { |
| "claim": 2, |
| "literal_claim": "PTBCC achieves up to 15% accuracy improvement over the best baseline in its best-case dataset (Val5) (Table 4)." |
| }, |
| { |
| "claim": 3, |
| "literal_claim": "Across 11 real-world crowdsourcing datasets, PTBCC attains an average accuracy of 0.7472, versus 0.7175 for FGBCC, 0.7132 for BWA, and 0.6986 for majority voting (Table 4)." |
| }, |
| { |
| "claim": 4, |
| "literal_claim": "PTBCC's ablation over prototype set size |S| shows accuracy peaking at |S|=2 (0.7472) and degrading to 0.7300 at |S|=3 and 0.7271 at |S|=4 due to sparser per-prototype annotator distributions (Table 5)." |
| }, |
| { |
| "claim": 5, |
| "literal_claim": "PTBCC uses less than 10% of the computational cost of confusion-matrix-based baselines while matching or exceeding their accuracy (Section on computational efficiency)." |
| } |
| ], |
| "dataset_count": 10, |
| "macros": { |
| "BWA": 0.701003255458328, |
| "DS": 0.7016899769890417, |
| "IBCC": 0.6935982786099606, |
| "MV": 0.6937261736117043, |
| "PTBCC_S2": 0.7236493610838572, |
| "PTBCC_S3_mean_3_seeds": 0.7250569980641843, |
| "PTBCC_S4_mean_3_seeds": 0.725806675063574 |
| }, |
| "mechanism": { |
| "dominant_prototype_recall_mean": 0.9993333333333333, |
| "dominant_prototype_recall_min": 0.98, |
| "label_shuffle_accuracy_drop": 0.772111111111111, |
| "label_shuffled_ptbcc_accuracy_mean": 0.2081111111111111, |
| "majority_vote_accuracy_mean": 0.9724444444444444, |
| "mean_max_annotator_weight": 0.5953221954840179, |
| "prototype_mae_max": 0.07200504499687686, |
| "prototype_mae_mean": 0.04819687934303456, |
| "ptbcc_accuracy_mean": 0.9802222222222221, |
| "ptbcc_beats_mv_seeds": 29, |
| "seeds": 30 |
| }, |
| "paper_orid": "KJq0iScNM6", |
| "per_dataset": { |
| "Adult": { |
| "iterations": { |
| "BWA": [ |
| 41, |
| 18, |
| 9, |
| 13 |
| ], |
| "DS": 47, |
| "IBCC": 100, |
| "PTBCC_S2": 22, |
| "PTBCC_S3": [ |
| 42, |
| 26, |
| 28 |
| ], |
| "PTBCC_S4": [ |
| 37, |
| 54, |
| 55 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.7417417417417418, |
| "DS": 0.7627627627627628, |
| "IBCC": 0.7447447447447447, |
| "MV": 0.7597597597597597, |
| "PTBCC_S2": 0.7687687687687688, |
| "PTBCC_S3_mean_3_seeds": 0.7727727727727727, |
| "PTBCC_S4_mean_3_seeds": 0.7747747747747749 |
| }, |
| "statistics": { |
| "annotators": 825, |
| "classes": 4, |
| "labels": 89799, |
| "source_sha256": { |
| "truth-inference-at-scale/data/crowd_truth_inference/s5_AdultContent/label.csv": "d72988724f7cdd628ce21fa1aaabecc4d03aef21bf59644d9e4033b774ac9a32", |
| "truth-inference-at-scale/data/crowd_truth_inference/s5_AdultContent/truth.csv": "1c52c5d528d16e031360d87ce384a80611bf23dedd5c7cfddf2c8ffc058772cb" |
| }, |
| "tasks": 11040, |
| "truths": 333 |
| } |
| }, |
| "CF": { |
| "iterations": { |
| "BWA": [ |
| 5, |
| 5, |
| 5, |
| 5, |
| 6 |
| ], |
| "DS": 60, |
| "IBCC": 18, |
| "PTBCC_S2": 103, |
| "PTBCC_S3": [ |
| 38, |
| 66, |
| 32 |
| ], |
| "PTBCC_S4": [ |
| 38, |
| 31, |
| 25 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.8933333333333333, |
| "DS": 0.7866666666666666, |
| "IBCC": 0.8833333333333333, |
| "MV": 0.9, |
| "PTBCC_S2": 0.8833333333333333, |
| "PTBCC_S3_mean_3_seeds": 0.8755555555555556, |
| "PTBCC_S4_mean_3_seeds": 0.8844444444444445 |
| }, |
| "statistics": { |
| "annotators": 461, |
| "classes": 5, |
| "labels": 1720, |
| "source_sha256": { |
| "truth-inference-at-scale/data/active-crowd-toolkit/CF/label.csv": "b0f98ddc9afefffcaf9670f591a3441b1bfc3eab70f8e4c35746310962920f06", |
| "truth-inference-at-scale/data/active-crowd-toolkit/CF/truth.csv": "8d8f1b773b310fa91e550658ef88baedc4c522ccd395a1ffa9c185327b88080f" |
| }, |
| "tasks": 300, |
| "truths": 300 |
| } |
| }, |
| "Dog": { |
| "iterations": { |
| "BWA": [ |
| 9, |
| 9, |
| 9, |
| 9 |
| ], |
| "DS": 15, |
| "IBCC": 25, |
| "PTBCC_S2": 33, |
| "PTBCC_S3": [ |
| 30, |
| 49, |
| 42 |
| ], |
| "PTBCC_S4": [ |
| 35, |
| 38, |
| 51 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.8314745972738538, |
| "DS": 0.8426270136307311, |
| "IBCC": 0.838909541511772, |
| "MV": 0.8178438661710037, |
| "PTBCC_S2": 0.8240396530359355, |
| "PTBCC_S3_mean_3_seeds": 0.8244527054935977, |
| "PTBCC_S4_mean_3_seeds": 0.8244527054935977 |
| }, |
| "statistics": { |
| "annotators": 109, |
| "classes": 4, |
| "labels": 8070, |
| "source_sha256": { |
| "truth-inference-at-scale/data/crowd_truth_inference/s4_Dog data/label.csv": "c240c0efc442e4936b35f28424d66fb04e687c0f29b254af94c26e15a71aa820", |
| "truth-inference-at-scale/data/crowd_truth_inference/s4_Dog data/truth.csv": "b299494a7aba3cf5abcd3b0e002f6e899f3e1f1d864a26703325db355555cb76" |
| }, |
| "tasks": 807, |
| "truths": 807 |
| } |
| }, |
| "Face": { |
| "iterations": { |
| "BWA": [ |
| 6, |
| 7, |
| 7, |
| 10 |
| ], |
| "DS": 20, |
| "IBCC": 19, |
| "PTBCC_S2": 17, |
| "PTBCC_S3": [ |
| 19, |
| 20, |
| 18 |
| ], |
| "PTBCC_S4": [ |
| 27, |
| 23, |
| 20 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.6181506849315068, |
| "DS": 0.6421232876712328, |
| "IBCC": 0.6404109589041096, |
| "MV": 0.6301369863013698, |
| "PTBCC_S2": 0.6523972602739726, |
| "PTBCC_S3_mean_3_seeds": 0.6529680365296804, |
| "PTBCC_S4_mean_3_seeds": 0.6523972602739726 |
| }, |
| "statistics": { |
| "annotators": 27, |
| "classes": 4, |
| "labels": 5242, |
| "source_sha256": { |
| "truth-inference-at-scale/data/crowd_truth_inference/s4_Face Sentiment Identification/label.csv": "fc10f625183432f6ed82d783ec0afb4a2e79009843a1c20757baad5c4e469686", |
| "truth-inference-at-scale/data/crowd_truth_inference/s4_Face Sentiment Identification/truth.csv": "503987498b22b586a34bd9211cce88bf4fbb436396ccae3fd5e19a64a5a8dee9" |
| }, |
| "tasks": 584, |
| "truths": 584 |
| } |
| }, |
| "Fact": { |
| "iterations": { |
| "BWA": [ |
| 21, |
| 22, |
| 14 |
| ], |
| "DS": 59, |
| "IBCC": 18, |
| "PTBCC_S2": 16, |
| "PTBCC_S3": [ |
| 14, |
| 16, |
| 17 |
| ], |
| "PTBCC_S4": [ |
| 22, |
| 16, |
| 15 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.8871527777777778, |
| "DS": 0.8524305555555556, |
| "IBCC": 0.8767361111111112, |
| "MV": 0.9010416666666666, |
| "PTBCC_S2": 0.9010416666666666, |
| "PTBCC_S3_mean_3_seeds": 0.9010416666666666, |
| "PTBCC_S4_mean_3_seeds": 0.9010416666666666 |
| }, |
| "statistics": { |
| "annotators": 57, |
| "classes": 3, |
| "labels": 214915, |
| "source_sha256": { |
| "truth-inference-at-scale/data/crowdscale2013/fact_eval/label.csv": "c8be134ea9adb9fe5984657dcd1894dce89ba93a539a600a62654c801d6e6d3a", |
| "truth-inference-at-scale/data/crowdscale2013/fact_eval/truth.csv": "5e0ea8af9d225cd4d03e175d279e5ab00958c6f74c3ebf85b068a271a83ad741" |
| }, |
| "tasks": 42624, |
| "truths": 576 |
| } |
| }, |
| "MS": { |
| "iterations": { |
| "BWA": [ |
| 12, |
| 9, |
| 9, |
| 22, |
| 9, |
| 11, |
| 12, |
| 7, |
| 8, |
| 9 |
| ], |
| "DS": 30, |
| "IBCC": 42, |
| "PTBCC_S2": 32, |
| "PTBCC_S3": [ |
| 43, |
| 58, |
| 75 |
| ], |
| "PTBCC_S4": [ |
| 116, |
| 49, |
| 94 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.7857142857142857, |
| "DS": 0.7742857142857142, |
| "IBCC": 0.79, |
| "MV": 0.71, |
| "PTBCC_S2": 0.7885714285714286, |
| "PTBCC_S3_mean_3_seeds": 0.787142857142857, |
| "PTBCC_S4_mean_3_seeds": 0.7890476190476191 |
| }, |
| "statistics": { |
| "annotators": 44, |
| "classes": 10, |
| "labels": 2945, |
| "source_sha256": { |
| "truth-inference-at-scale/data/active-crowd-toolkit/MS/label.csv": "4e59b88dbe48a0cd545a745c9718f3955729205d29da1bf883adc64bf7a3e378", |
| "truth-inference-at-scale/data/active-crowd-toolkit/MS/truth.csv": "981c635d5cde03cd903bf319b030178420bc8aa3b6a3b7846abea4e974bc2294" |
| }, |
| "tasks": 700, |
| "truths": 700 |
| } |
| }, |
| "Senti": { |
| "iterations": { |
| "BWA": [ |
| 14, |
| 16, |
| 12, |
| 16, |
| 29 |
| ], |
| "DS": 81, |
| "IBCC": 107, |
| "PTBCC_S2": 17, |
| "PTBCC_S3": [ |
| 24, |
| 21, |
| 33 |
| ], |
| "PTBCC_S4": [ |
| 33, |
| 41, |
| 30 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.89, |
| "DS": 0.816, |
| "IBCC": 0.831, |
| "MV": 0.902, |
| "PTBCC_S2": 0.88, |
| "PTBCC_S3_mean_3_seeds": 0.8810000000000001, |
| "PTBCC_S4_mean_3_seeds": 0.882 |
| }, |
| "statistics": { |
| "annotators": 1960, |
| "classes": 5, |
| "labels": 569274, |
| "source_sha256": { |
| "truth-inference-at-scale/data/crowdscale2013/sentiment/label.csv": "d336fabd935d27081a3a769d455ea6deb60b9312cc9f024c061c04059caa3b60", |
| "truth-inference-at-scale/data/crowdscale2013/sentiment/truth.csv": "26b6526a8ff541c86abbf57fa8736e0b4aea007c16bea4b0170cfd23ac5866e5" |
| }, |
| "tasks": 98980, |
| "truths": 1000 |
| } |
| }, |
| "Val5": { |
| "iterations": { |
| "BWA": [ |
| 6, |
| 7, |
| 5, |
| 7, |
| 6 |
| ], |
| "DS": 14, |
| "IBCC": 17, |
| "PTBCC_S2": 31, |
| "PTBCC_S3": [ |
| 60, |
| 109, |
| 156 |
| ], |
| "PTBCC_S4": [ |
| 51, |
| 131, |
| 106 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.32, |
| "DS": 0.4, |
| "IBCC": 0.34, |
| "MV": 0.31, |
| "PTBCC_S2": 0.46, |
| "PTBCC_S3_mean_3_seeds": 0.4633333333333333, |
| "PTBCC_S4_mean_3_seeds": 0.4466666666666666 |
| }, |
| "statistics": { |
| "annotators": 38, |
| "classes": 5, |
| "labels": 1000, |
| "source_sha256": { |
| "CrowdTI/truth_inference_crowd/datasets/f201_Emotion_FULL/answer.csv": "8a33684760a7fc51980ddc2d19d157f9d0197bcc3611cc4a2c8cdb65fdaf1d1b", |
| "CrowdTI/truth_inference_crowd/datasets/f201_Emotion_FULL/truth.csv": "0fdfa983818a3524ba572c7eb7a3fd6b314844105ae9449b630425faa3176be4" |
| }, |
| "tasks": 100, |
| "truths": 100 |
| } |
| }, |
| "Val7": { |
| "iterations": { |
| "BWA": [ |
| 6, |
| 7, |
| 6, |
| 5, |
| 7, |
| 6, |
| 6 |
| ], |
| "DS": 16, |
| "IBCC": 23, |
| "PTBCC_S2": 89, |
| "PTBCC_S3": [ |
| 130, |
| 73, |
| 58 |
| ], |
| "PTBCC_S4": [ |
| 151, |
| 96, |
| 61 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.22, |
| "DS": 0.31, |
| "IBCC": 0.24, |
| "MV": 0.23, |
| "PTBCC_S2": 0.28, |
| "PTBCC_S3_mean_3_seeds": 0.2933333333333333, |
| "PTBCC_S4_mean_3_seeds": 0.3 |
| }, |
| "statistics": { |
| "annotators": 38, |
| "classes": 7, |
| "labels": 1000, |
| "source_sha256": { |
| "CrowdTI/truth_inference_crowd/datasets/f201_Emotion_FULL/answer.csv": "8a33684760a7fc51980ddc2d19d157f9d0197bcc3611cc4a2c8cdb65fdaf1d1b", |
| "CrowdTI/truth_inference_crowd/datasets/f201_Emotion_FULL/truth.csv": "0fdfa983818a3524ba572c7eb7a3fd6b314844105ae9449b630425faa3176be4" |
| }, |
| "tasks": 100, |
| "truths": 100 |
| } |
| }, |
| "Web": { |
| "iterations": { |
| "BWA": [ |
| 17, |
| 13, |
| 8, |
| 11, |
| 20 |
| ], |
| "DS": 71, |
| "IBCC": 115, |
| "PTBCC_S2": 71, |
| "PTBCC_S3": [ |
| 38, |
| 46, |
| 61 |
| ], |
| "PTBCC_S4": [ |
| 39, |
| 31, |
| 62 |
| ] |
| }, |
| "scores": { |
| "BWA": 0.8224651338107802, |
| "DS": 0.8300037693177534, |
| "IBCC": 0.7508480964945344, |
| "MV": 0.7764794572182435, |
| "PTBCC_S2": 0.7983415001884658, |
| "PTBCC_S3_mean_3_seeds": 0.798969719814047, |
| "PTBCC_S4_mean_3_seeds": 0.8032416132679985 |
| }, |
| "statistics": { |
| "annotators": 177, |
| "classes": 5, |
| "labels": 15567, |
| "source_sha256": { |
| "truth-inference-at-scale/data/SpectralMethodsMeetEM/web/label.csv": "90b6e66284288079fab48593c61fb7949f964d007fce077b13d21d0b7ab43211", |
| "truth-inference-at-scale/data/SpectralMethodsMeetEM/web/truth.csv": "1b215fe642b88107fc07c0f4b9169b70e178f3a65f76926ea1fbeb86fd688c4d" |
| }, |
| "tasks": 2665, |
| "truths": 2653 |
| } |
| } |
| }, |
| "schema": "icml-ptbcc-native-v1", |
| "scientific_gates": { |
| "all_ten_registered_scales_exact": true, |
| "baseline_bwa_within_0_02": true, |
| "baseline_mv_within_0_015": true, |
| "label_shuffle_destroys_at_least_0_60_accuracy": true, |
| "mechanism_beats_mv_at_least_25_of_30": true, |
| "mechanism_dominant_recall_above_0_85": true, |
| "mechanism_prototype_mae_below_0_12": true, |
| "missing_aircr_required_accuracy_implausible": true, |
| "pooled_parameter_work_below_10_percent": true, |
| "ptbcc_headline_gap_at_least_0_015": true, |
| "ptbcc_matches_or_exceeds_best_reproduced_macro": true, |
| "s3_exceeds_s2": true, |
| "val5_absolute_gain_over_mv_is_15_points": true, |
| "val5_is_largest_reproduced_absolute_gain": true, |
| "val5_relative_gain_brackets_15_percent": true |
| }, |
| "source_commits": { |
| "crowdti": "429a11bee1480ab01784fd00633167ca76efd954", |
| "truth_inference_at_scale": "621789b2d57324d3559dc973b2613d2296d73f55" |
| } |
| } |
|
|
