| { |
| "result_dir": "/HDD2/disheng/Spatial_VLM_Stereo/June/Spatial_VLM_Stereo/01_3_prelimilary_intervention_different_baseline/result", |
| "runs": [ |
| { |
| "model_id": "Qwen/Qwen2.5-VL-3B-Instruct", |
| "task": "vsr", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 2195, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 2195, |
| "accuracy": 0.7712984054669704, |
| "num_failures": 502, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 2195, |
| "accuracy": 0.7676537585421412, |
| "num_failures": 510, |
| "flip_rate_vs_reference": 0.07015945330296128, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 2195, |
| "accuracy": 0.7594533029612757, |
| "num_failures": 528, |
| "flip_rate_vs_reference": 0.07927107061503418, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 2195, |
| "accuracy": 0.7580865603644646, |
| "num_failures": 531, |
| "flip_rate_vs_reference": 0.08792710706150342, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 2195, |
| "accuracy": 0.752619589977221, |
| "num_failures": 543, |
| "flip_rate_vs_reference": 0.09977220956719818, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 2195, |
| "accuracy": 0.7480637813211846, |
| "num_failures": 553, |
| "flip_rate_vs_reference": 0.10888382687927108, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.02323462414578581, |
| "stability": { |
| "always_correct": 1434, |
| "always_wrong": 332, |
| "unstable": 429, |
| "always_correct_rate": 0.6533029612756264, |
| "always_wrong_rate": 0.15125284738041003, |
| "unstable_rate": 0.19544419134396354 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 502, |
| "failures_b": 510, |
| "failures_shared": 429, |
| "failure_jaccard": 0.7358490566037735, |
| "failure_recall_a_in_b": 0.8545816733067729, |
| "successes_a": 1693, |
| "successes_b": 1685, |
| "successes_shared": 1612, |
| "success_jaccard": 0.912797281993205, |
| "answer_agreement": 0.9298405466970388 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 502, |
| "failures_b": 528, |
| "failures_shared": 428, |
| "failure_jaccard": 0.7109634551495017, |
| "failure_recall_a_in_b": 0.852589641434263, |
| "successes_a": 1693, |
| "successes_b": 1667, |
| "successes_shared": 1593, |
| "success_jaccard": 0.9015280135823429, |
| "answer_agreement": 0.9207289293849659 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 502, |
| "failures_b": 531, |
| "failures_shared": 420, |
| "failure_jaccard": 0.6851549755301795, |
| "failure_recall_a_in_b": 0.8366533864541833, |
| "successes_a": 1693, |
| "successes_b": 1664, |
| "successes_shared": 1582, |
| "success_jaccard": 0.8912676056338028, |
| "answer_agreement": 0.9120728929384966 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 502, |
| "failures_b": 543, |
| "failures_shared": 413, |
| "failure_jaccard": 0.6534810126582279, |
| "failure_recall_a_in_b": 0.8227091633466136, |
| "successes_a": 1693, |
| "successes_b": 1652, |
| "successes_shared": 1563, |
| "success_jaccard": 0.877104377104377, |
| "answer_agreement": 0.9002277904328019 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 502, |
| "failures_b": 553, |
| "failures_shared": 408, |
| "failure_jaccard": 0.6306027820710973, |
| "failure_recall_a_in_b": 0.8127490039840638, |
| "successes_a": 1693, |
| "successes_b": 1642, |
| "successes_shared": 1548, |
| "success_jaccard": 0.8662562954672636, |
| "answer_agreement": 0.8911161731207289 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 510, |
| "failures_b": 528, |
| "failures_shared": 433, |
| "failure_jaccard": 0.715702479338843, |
| "failure_recall_a_in_b": 0.8490196078431372, |
| "successes_a": 1685, |
| "successes_b": 1667, |
| "successes_shared": 1590, |
| "success_jaccard": 0.9023836549375709, |
| "answer_agreement": 0.9216400911161731 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 510, |
| "failures_b": 531, |
| "failures_shared": 434, |
| "failure_jaccard": 0.71499176276771, |
| "failure_recall_a_in_b": 0.8509803921568627, |
| "successes_a": 1685, |
| "successes_b": 1664, |
| "successes_shared": 1588, |
| "success_jaccard": 0.9017603634298694, |
| "answer_agreement": 0.9211845102505695 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 510, |
| "failures_b": 543, |
| "failures_shared": 429, |
| "failure_jaccard": 0.6875, |
| "failure_recall_a_in_b": 0.8411764705882353, |
| "successes_a": 1685, |
| "successes_b": 1652, |
| "successes_shared": 1571, |
| "success_jaccard": 0.8895809739524349, |
| "answer_agreement": 0.9111617312072893 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 510, |
| "failures_b": 553, |
| "failures_shared": 423, |
| "failure_jaccard": 0.6609375, |
| "failure_recall_a_in_b": 0.8294117647058824, |
| "successes_a": 1685, |
| "successes_b": 1642, |
| "successes_shared": 1555, |
| "success_jaccard": 0.8775395033860045, |
| "answer_agreement": 0.9011389521640091 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 528, |
| "failures_b": 531, |
| "failures_shared": 445, |
| "failure_jaccard": 0.7247557003257329, |
| "failure_recall_a_in_b": 0.8428030303030303, |
| "successes_a": 1667, |
| "successes_b": 1664, |
| "successes_shared": 1581, |
| "success_jaccard": 0.9034285714285715, |
| "answer_agreement": 0.9230068337129841 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 528, |
| "failures_b": 543, |
| "failures_shared": 441, |
| "failure_jaccard": 0.7, |
| "failure_recall_a_in_b": 0.8352272727272727, |
| "successes_a": 1667, |
| "successes_b": 1652, |
| "successes_shared": 1565, |
| "success_jaccard": 0.8922462941847207, |
| "answer_agreement": 0.9138952164009112 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 528, |
| "failures_b": 553, |
| "failures_shared": 430, |
| "failure_jaccard": 0.6605222734254992, |
| "failure_recall_a_in_b": 0.8143939393939394, |
| "successes_a": 1667, |
| "successes_b": 1642, |
| "successes_shared": 1544, |
| "success_jaccard": 0.8747875354107649, |
| "answer_agreement": 0.8993166287015946 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 531, |
| "failures_b": 543, |
| "failures_shared": 447, |
| "failure_jaccard": 0.7129186602870813, |
| "failure_recall_a_in_b": 0.8418079096045198, |
| "successes_a": 1664, |
| "successes_b": 1652, |
| "successes_shared": 1568, |
| "success_jaccard": 0.897025171624714, |
| "answer_agreement": 0.9179954441913439 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 531, |
| "failures_b": 553, |
| "failures_shared": 445, |
| "failure_jaccard": 0.6964006259780907, |
| "failure_recall_a_in_b": 0.8380414312617702, |
| "successes_a": 1664, |
| "successes_b": 1642, |
| "successes_shared": 1556, |
| "success_jaccard": 0.8891428571428571, |
| "answer_agreement": 0.911617312072893 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 543, |
| "failures_b": 553, |
| "failures_shared": 460, |
| "failure_jaccard": 0.7232704402515723, |
| "failure_recall_a_in_b": 0.8471454880294659, |
| "successes_a": 1652, |
| "successes_b": 1642, |
| "successes_shared": 1559, |
| "success_jaccard": 0.8985590778097983, |
| "answer_agreement": 0.9198177676537586 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen2.5-VL-3B-Instruct", |
| "task": "youtube_level1", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 821, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 821, |
| "accuracy": 0.3995127892813642, |
| "num_failures": 493, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 821, |
| "accuracy": 0.4056029232643118, |
| "num_failures": 488, |
| "flip_rate_vs_reference": 0.14859926918392205, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 821, |
| "accuracy": 0.41169305724725946, |
| "num_failures": 483, |
| "flip_rate_vs_reference": 0.1559074299634592, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 821, |
| "accuracy": 0.4104750304506699, |
| "num_failures": 484, |
| "flip_rate_vs_reference": 0.16321559074299635, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 821, |
| "accuracy": 0.3995127892813642, |
| "num_failures": 493, |
| "flip_rate_vs_reference": 0.1814859926918392, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 821, |
| "accuracy": 0.4007308160779537, |
| "num_failures": 492, |
| "flip_rate_vs_reference": 0.19366626065773446, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.012180267965895275, |
| "stability": { |
| "always_correct": 247, |
| "always_wrong": 391, |
| "unstable": 183, |
| "always_correct_rate": 0.30085261875761266, |
| "always_wrong_rate": 0.4762484774665043, |
| "unstable_rate": 0.22289890377588306 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 493, |
| "failures_b": 488, |
| "failures_shared": 450, |
| "failure_jaccard": 0.847457627118644, |
| "failure_recall_a_in_b": 0.9127789046653144, |
| "successes_a": 328, |
| "successes_b": 333, |
| "successes_shared": 290, |
| "success_jaccard": 0.7816711590296496, |
| "answer_agreement": 0.8514007308160779 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 493, |
| "failures_b": 483, |
| "failures_shared": 451, |
| "failure_jaccard": 0.8590476190476191, |
| "failure_recall_a_in_b": 0.9148073022312373, |
| "successes_a": 328, |
| "successes_b": 338, |
| "successes_shared": 296, |
| "success_jaccard": 0.8, |
| "answer_agreement": 0.8440925700365408 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 493, |
| "failures_b": 484, |
| "failures_shared": 450, |
| "failure_jaccard": 0.8538899430740038, |
| "failure_recall_a_in_b": 0.9127789046653144, |
| "successes_a": 328, |
| "successes_b": 337, |
| "successes_shared": 294, |
| "success_jaccard": 0.7924528301886793, |
| "answer_agreement": 0.8367844092570037 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 493, |
| "failures_b": 493, |
| "failures_shared": 445, |
| "failure_jaccard": 0.822550831792976, |
| "failure_recall_a_in_b": 0.9026369168356998, |
| "successes_a": 328, |
| "successes_b": 328, |
| "successes_shared": 280, |
| "success_jaccard": 0.7446808510638298, |
| "answer_agreement": 0.8185140073081608 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 493, |
| "failures_b": 492, |
| "failures_shared": 441, |
| "failure_jaccard": 0.8106617647058824, |
| "failure_recall_a_in_b": 0.8945233265720081, |
| "successes_a": 328, |
| "successes_b": 329, |
| "successes_shared": 277, |
| "success_jaccard": 0.7289473684210527, |
| "answer_agreement": 0.8063337393422655 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 488, |
| "failures_b": 483, |
| "failures_shared": 449, |
| "failure_jaccard": 0.8601532567049809, |
| "failure_recall_a_in_b": 0.9200819672131147, |
| "successes_a": 333, |
| "successes_b": 338, |
| "successes_shared": 299, |
| "success_jaccard": 0.803763440860215, |
| "answer_agreement": 0.8440925700365408 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 488, |
| "failures_b": 484, |
| "failures_shared": 447, |
| "failure_jaccard": 0.8514285714285714, |
| "failure_recall_a_in_b": 0.9159836065573771, |
| "successes_a": 333, |
| "successes_b": 337, |
| "successes_shared": 296, |
| "success_jaccard": 0.7914438502673797, |
| "answer_agreement": 0.8392204628501827 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 488, |
| "failures_b": 493, |
| "failures_shared": 454, |
| "failure_jaccard": 0.8614800759013282, |
| "failure_recall_a_in_b": 0.930327868852459, |
| "successes_a": 333, |
| "successes_b": 328, |
| "successes_shared": 294, |
| "success_jaccard": 0.8010899182561307, |
| "answer_agreement": 0.833130328867235 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 488, |
| "failures_b": 492, |
| "failures_shared": 442, |
| "failure_jaccard": 0.8215613382899628, |
| "failure_recall_a_in_b": 0.9057377049180327, |
| "successes_a": 333, |
| "successes_b": 329, |
| "successes_shared": 283, |
| "success_jaccard": 0.7467018469656992, |
| "answer_agreement": 0.8063337393422655 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 483, |
| "failures_b": 484, |
| "failures_shared": 446, |
| "failure_jaccard": 0.8560460652591171, |
| "failure_recall_a_in_b": 0.9233954451345756, |
| "successes_a": 338, |
| "successes_b": 337, |
| "successes_shared": 300, |
| "success_jaccard": 0.8, |
| "answer_agreement": 0.8453105968331304 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 483, |
| "failures_b": 493, |
| "failures_shared": 450, |
| "failure_jaccard": 0.8555133079847909, |
| "failure_recall_a_in_b": 0.9316770186335404, |
| "successes_a": 338, |
| "successes_b": 328, |
| "successes_shared": 295, |
| "success_jaccard": 0.7951482479784366, |
| "answer_agreement": 0.8550548112058465 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 483, |
| "failures_b": 492, |
| "failures_shared": 445, |
| "failure_jaccard": 0.839622641509434, |
| "failure_recall_a_in_b": 0.9213250517598344, |
| "successes_a": 338, |
| "successes_b": 329, |
| "successes_shared": 291, |
| "success_jaccard": 0.773936170212766, |
| "answer_agreement": 0.8270401948842875 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 484, |
| "failures_b": 493, |
| "failures_shared": 452, |
| "failure_jaccard": 0.8609523809523809, |
| "failure_recall_a_in_b": 0.9338842975206612, |
| "successes_a": 337, |
| "successes_b": 328, |
| "successes_shared": 296, |
| "success_jaccard": 0.8021680216802168, |
| "answer_agreement": 0.8514007308160779 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 484, |
| "failures_b": 492, |
| "failures_shared": 441, |
| "failure_jaccard": 0.8242990654205608, |
| "failure_recall_a_in_b": 0.9111570247933884, |
| "successes_a": 337, |
| "successes_b": 329, |
| "successes_shared": 286, |
| "success_jaccard": 0.7526315789473684, |
| "answer_agreement": 0.8136419001218027 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 493, |
| "failures_b": 492, |
| "failures_shared": 459, |
| "failure_jaccard": 0.8726235741444867, |
| "failure_recall_a_in_b": 0.9310344827586207, |
| "successes_a": 328, |
| "successes_b": 329, |
| "successes_shared": 295, |
| "success_jaccard": 0.8149171270718232, |
| "answer_agreement": 0.8660170523751523 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen2.5-VL-3B-Instruct", |
| "task": "youtube_level2", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 764, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 764, |
| "accuracy": 0.362565445026178, |
| "num_failures": 487, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 764, |
| "accuracy": 0.3743455497382199, |
| "num_failures": 478, |
| "flip_rate_vs_reference": 0.1387434554973822, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 764, |
| "accuracy": 0.3782722513089005, |
| "num_failures": 475, |
| "flip_rate_vs_reference": 0.17277486910994763, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 764, |
| "accuracy": 0.3795811518324607, |
| "num_failures": 474, |
| "flip_rate_vs_reference": 0.1793193717277487, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 764, |
| "accuracy": 0.38089005235602097, |
| "num_failures": 473, |
| "flip_rate_vs_reference": 0.17408376963350786, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 764, |
| "accuracy": 0.3704188481675393, |
| "num_failures": 481, |
| "flip_rate_vs_reference": 0.2107329842931937, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.01832460732984298, |
| "stability": { |
| "always_correct": 198, |
| "always_wrong": 392, |
| "unstable": 174, |
| "always_correct_rate": 0.2591623036649215, |
| "always_wrong_rate": 0.5130890052356021, |
| "unstable_rate": 0.22774869109947643 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 487, |
| "failures_b": 478, |
| "failures_shared": 450, |
| "failure_jaccard": 0.8737864077669902, |
| "failure_recall_a_in_b": 0.9240246406570842, |
| "successes_a": 277, |
| "successes_b": 286, |
| "successes_shared": 249, |
| "success_jaccard": 0.7929936305732485, |
| "answer_agreement": 0.8612565445026178 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 487, |
| "failures_b": 475, |
| "failures_shared": 441, |
| "failure_jaccard": 0.8464491362763915, |
| "failure_recall_a_in_b": 0.9055441478439425, |
| "successes_a": 277, |
| "successes_b": 289, |
| "successes_shared": 243, |
| "success_jaccard": 0.7523219814241486, |
| "answer_agreement": 0.8272251308900523 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 487, |
| "failures_b": 474, |
| "failures_shared": 437, |
| "failure_jaccard": 0.833969465648855, |
| "failure_recall_a_in_b": 0.8973305954825462, |
| "successes_a": 277, |
| "successes_b": 290, |
| "successes_shared": 240, |
| "success_jaccard": 0.7339449541284404, |
| "answer_agreement": 0.8206806282722513 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 487, |
| "failures_b": 473, |
| "failures_shared": 437, |
| "failure_jaccard": 0.8355640535372849, |
| "failure_recall_a_in_b": 0.8973305954825462, |
| "successes_a": 277, |
| "successes_b": 291, |
| "successes_shared": 241, |
| "success_jaccard": 0.7370030581039755, |
| "answer_agreement": 0.8259162303664922 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 487, |
| "failures_b": 481, |
| "failures_shared": 435, |
| "failure_jaccard": 0.8161350844277674, |
| "failure_recall_a_in_b": 0.893223819301848, |
| "successes_a": 277, |
| "successes_b": 283, |
| "successes_shared": 231, |
| "success_jaccard": 0.7021276595744681, |
| "answer_agreement": 0.7892670157068062 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 478, |
| "failures_b": 475, |
| "failures_shared": 444, |
| "failure_jaccard": 0.8722986247544204, |
| "failure_recall_a_in_b": 0.9288702928870293, |
| "successes_a": 286, |
| "successes_b": 289, |
| "successes_shared": 255, |
| "success_jaccard": 0.796875, |
| "answer_agreement": 0.8547120418848168 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 478, |
| "failures_b": 474, |
| "failures_shared": 438, |
| "failure_jaccard": 0.8521400778210116, |
| "failure_recall_a_in_b": 0.9163179916317992, |
| "successes_a": 286, |
| "successes_b": 290, |
| "successes_shared": 250, |
| "success_jaccard": 0.7668711656441718, |
| "answer_agreement": 0.8416230366492147 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 478, |
| "failures_b": 473, |
| "failures_shared": 441, |
| "failure_jaccard": 0.8647058823529412, |
| "failure_recall_a_in_b": 0.9225941422594143, |
| "successes_a": 286, |
| "successes_b": 291, |
| "successes_shared": 254, |
| "success_jaccard": 0.7863777089783281, |
| "answer_agreement": 0.8586387434554974 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 478, |
| "failures_b": 481, |
| "failures_shared": 435, |
| "failure_jaccard": 0.8301526717557252, |
| "failure_recall_a_in_b": 0.9100418410041841, |
| "successes_a": 286, |
| "successes_b": 283, |
| "successes_shared": 240, |
| "success_jaccard": 0.729483282674772, |
| "answer_agreement": 0.8141361256544503 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 475, |
| "failures_b": 474, |
| "failures_shared": 436, |
| "failure_jaccard": 0.8499025341130604, |
| "failure_recall_a_in_b": 0.9178947368421052, |
| "successes_a": 289, |
| "successes_b": 290, |
| "successes_shared": 251, |
| "success_jaccard": 0.7652439024390244, |
| "answer_agreement": 0.8390052356020943 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 475, |
| "failures_b": 473, |
| "failures_shared": 435, |
| "failure_jaccard": 0.847953216374269, |
| "failure_recall_a_in_b": 0.9157894736842105, |
| "successes_a": 289, |
| "successes_b": 291, |
| "successes_shared": 251, |
| "success_jaccard": 0.7629179331306991, |
| "answer_agreement": 0.8390052356020943 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 475, |
| "failures_b": 481, |
| "failures_shared": 436, |
| "failure_jaccard": 0.8384615384615385, |
| "failure_recall_a_in_b": 0.9178947368421052, |
| "successes_a": 289, |
| "successes_b": 283, |
| "successes_shared": 244, |
| "success_jaccard": 0.7439024390243902, |
| "answer_agreement": 0.8180628272251309 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 474, |
| "failures_b": 473, |
| "failures_shared": 436, |
| "failure_jaccard": 0.8532289628180039, |
| "failure_recall_a_in_b": 0.919831223628692, |
| "successes_a": 290, |
| "successes_b": 291, |
| "successes_shared": 253, |
| "success_jaccard": 0.7713414634146342, |
| "answer_agreement": 0.8534031413612565 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 474, |
| "failures_b": 481, |
| "failures_shared": 438, |
| "failure_jaccard": 0.8471953578336557, |
| "failure_recall_a_in_b": 0.9240506329113924, |
| "successes_a": 290, |
| "successes_b": 283, |
| "successes_shared": 247, |
| "success_jaccard": 0.7576687116564417, |
| "answer_agreement": 0.837696335078534 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 473, |
| "failures_b": 481, |
| "failures_shared": 439, |
| "failure_jaccard": 0.8524271844660194, |
| "failure_recall_a_in_b": 0.9281183932346723, |
| "successes_a": 291, |
| "successes_b": 283, |
| "successes_shared": 249, |
| "success_jaccard": 0.7661538461538462, |
| "answer_agreement": 0.8298429319371727 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen2.5-VL-3B-Instruct", |
| "task": "youtube_level3", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 836, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 836, |
| "accuracy": 0.26674641148325356, |
| "num_failures": 613, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 836, |
| "accuracy": 0.2452153110047847, |
| "num_failures": 631, |
| "flip_rate_vs_reference": 0.2452153110047847, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 836, |
| "accuracy": 0.23923444976076555, |
| "num_failures": 636, |
| "flip_rate_vs_reference": 0.23803827751196172, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 836, |
| "accuracy": 0.25239234449760767, |
| "num_failures": 625, |
| "flip_rate_vs_reference": 0.26435406698564595, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 836, |
| "accuracy": 0.2595693779904306, |
| "num_failures": 619, |
| "flip_rate_vs_reference": 0.25478468899521534, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 836, |
| "accuracy": 0.26555023923444976, |
| "num_failures": 614, |
| "flip_rate_vs_reference": 0.27392344497607657, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.027511961722488015, |
| "stability": { |
| "always_correct": 111, |
| "always_wrong": 502, |
| "unstable": 223, |
| "always_correct_rate": 0.13277511961722488, |
| "always_wrong_rate": 0.6004784688995215, |
| "unstable_rate": 0.26674641148325356 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 613, |
| "failures_b": 631, |
| "failures_shared": 569, |
| "failure_jaccard": 0.8429629629629629, |
| "failure_recall_a_in_b": 0.9282218597063622, |
| "successes_a": 223, |
| "successes_b": 205, |
| "successes_shared": 161, |
| "success_jaccard": 0.602996254681648, |
| "answer_agreement": 0.7547846889952153 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 613, |
| "failures_b": 636, |
| "failures_shared": 570, |
| "failure_jaccard": 0.8394698085419735, |
| "failure_recall_a_in_b": 0.9298531810766721, |
| "successes_a": 223, |
| "successes_b": 200, |
| "successes_shared": 157, |
| "success_jaccard": 0.5902255639097744, |
| "answer_agreement": 0.7619617224880383 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 613, |
| "failures_b": 625, |
| "failures_shared": 562, |
| "failure_jaccard": 0.8313609467455622, |
| "failure_recall_a_in_b": 0.9168026101141925, |
| "successes_a": 223, |
| "successes_b": 211, |
| "successes_shared": 160, |
| "success_jaccard": 0.583941605839416, |
| "answer_agreement": 0.7356459330143541 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 613, |
| "failures_b": 619, |
| "failures_shared": 561, |
| "failure_jaccard": 0.8360655737704918, |
| "failure_recall_a_in_b": 0.9151712887438825, |
| "successes_a": 223, |
| "successes_b": 217, |
| "successes_shared": 165, |
| "success_jaccard": 0.6, |
| "answer_agreement": 0.7452153110047847 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 613, |
| "failures_b": 614, |
| "failures_shared": 551, |
| "failure_jaccard": 0.8150887573964497, |
| "failure_recall_a_in_b": 0.8988580750407831, |
| "successes_a": 223, |
| "successes_b": 222, |
| "successes_shared": 160, |
| "success_jaccard": 0.5614035087719298, |
| "answer_agreement": 0.7260765550239234 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 631, |
| "failures_b": 636, |
| "failures_shared": 588, |
| "failure_jaccard": 0.865979381443299, |
| "failure_recall_a_in_b": 0.9318541996830428, |
| "successes_a": 205, |
| "successes_b": 200, |
| "successes_shared": 157, |
| "success_jaccard": 0.6330645161290323, |
| "answer_agreement": 0.7811004784688995 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 631, |
| "failures_b": 625, |
| "failures_shared": 580, |
| "failure_jaccard": 0.8579881656804734, |
| "failure_recall_a_in_b": 0.919175911251981, |
| "successes_a": 205, |
| "successes_b": 211, |
| "successes_shared": 160, |
| "success_jaccard": 0.625, |
| "answer_agreement": 0.7667464114832536 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 631, |
| "failures_b": 619, |
| "failures_shared": 571, |
| "failure_jaccard": 0.8409425625920471, |
| "failure_recall_a_in_b": 0.9049128367670365, |
| "successes_a": 205, |
| "successes_b": 217, |
| "successes_shared": 157, |
| "success_jaccard": 0.5924528301886792, |
| "answer_agreement": 0.7667464114832536 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 631, |
| "failures_b": 614, |
| "failures_shared": 569, |
| "failure_jaccard": 0.841715976331361, |
| "failure_recall_a_in_b": 0.901743264659271, |
| "successes_a": 205, |
| "successes_b": 222, |
| "successes_shared": 160, |
| "success_jaccard": 0.599250936329588, |
| "answer_agreement": 0.7464114832535885 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 636, |
| "failures_b": 625, |
| "failures_shared": 582, |
| "failure_jaccard": 0.8571428571428571, |
| "failure_recall_a_in_b": 0.9150943396226415, |
| "successes_a": 200, |
| "successes_b": 211, |
| "successes_shared": 157, |
| "success_jaccard": 0.6181102362204725, |
| "answer_agreement": 0.7739234449760766 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 636, |
| "failures_b": 619, |
| "failures_shared": 582, |
| "failure_jaccard": 0.8647845468053492, |
| "failure_recall_a_in_b": 0.9150943396226415, |
| "successes_a": 200, |
| "successes_b": 217, |
| "successes_shared": 163, |
| "success_jaccard": 0.6417322834645669, |
| "answer_agreement": 0.7918660287081339 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 636, |
| "failures_b": 614, |
| "failures_shared": 578, |
| "failure_jaccard": 0.8601190476190477, |
| "failure_recall_a_in_b": 0.9088050314465409, |
| "successes_a": 200, |
| "successes_b": 222, |
| "successes_shared": 164, |
| "success_jaccard": 0.6356589147286822, |
| "answer_agreement": 0.7715311004784688 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 625, |
| "failures_b": 619, |
| "failures_shared": 574, |
| "failure_jaccard": 0.8567164179104477, |
| "failure_recall_a_in_b": 0.9184, |
| "successes_a": 211, |
| "successes_b": 217, |
| "successes_shared": 166, |
| "success_jaccard": 0.6335877862595419, |
| "answer_agreement": 0.7799043062200957 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 625, |
| "failures_b": 614, |
| "failures_shared": 573, |
| "failure_jaccard": 0.8603603603603603, |
| "failure_recall_a_in_b": 0.9168, |
| "successes_a": 211, |
| "successes_b": 222, |
| "successes_shared": 170, |
| "success_jaccard": 0.6463878326996197, |
| "answer_agreement": 0.7811004784688995 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 619, |
| "failures_b": 614, |
| "failures_shared": 578, |
| "failure_jaccard": 0.8824427480916031, |
| "failure_recall_a_in_b": 0.9337641357027464, |
| "successes_a": 217, |
| "successes_b": 222, |
| "successes_shared": 181, |
| "success_jaccard": 0.7015503875968992, |
| "answer_agreement": 0.8014354066985646 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen2.5-VL-7B-Instruct", |
| "task": "vsr", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 2195, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 2195, |
| "accuracy": 0.8223234624145785, |
| "num_failures": 390, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 2195, |
| "accuracy": 0.8077448747152619, |
| "num_failures": 422, |
| "flip_rate_vs_reference": 0.06013667425968109, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 2195, |
| "accuracy": 0.8027334851936219, |
| "num_failures": 433, |
| "flip_rate_vs_reference": 0.07334851936218678, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 2195, |
| "accuracy": 0.8068337129840547, |
| "num_failures": 424, |
| "flip_rate_vs_reference": 0.06560364464692484, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 2195, |
| "accuracy": 0.8022779043280183, |
| "num_failures": 434, |
| "flip_rate_vs_reference": 0.08109339407744874, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 2195, |
| "accuracy": 0.7881548974943052, |
| "num_failures": 465, |
| "flip_rate_vs_reference": 0.09339407744874716, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.034168564920273314, |
| "stability": { |
| "always_correct": 1579, |
| "always_wrong": 268, |
| "unstable": 348, |
| "always_correct_rate": 0.7193621867881549, |
| "always_wrong_rate": 0.12209567198177676, |
| "unstable_rate": 0.15854214123006835 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 390, |
| "failures_b": 422, |
| "failures_shared": 340, |
| "failure_jaccard": 0.7203389830508474, |
| "failure_recall_a_in_b": 0.8717948717948718, |
| "successes_a": 1805, |
| "successes_b": 1773, |
| "successes_shared": 1723, |
| "success_jaccard": 0.9288409703504044, |
| "answer_agreement": 0.939863325740319 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 390, |
| "failures_b": 433, |
| "failures_shared": 331, |
| "failure_jaccard": 0.6727642276422764, |
| "failure_recall_a_in_b": 0.8487179487179487, |
| "successes_a": 1805, |
| "successes_b": 1762, |
| "successes_shared": 1703, |
| "success_jaccard": 0.9136266094420601, |
| "answer_agreement": 0.9266514806378132 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 390, |
| "failures_b": 424, |
| "failures_shared": 335, |
| "failure_jaccard": 0.6993736951983298, |
| "failure_recall_a_in_b": 0.8589743589743589, |
| "successes_a": 1805, |
| "successes_b": 1771, |
| "successes_shared": 1716, |
| "success_jaccard": 0.9225806451612903, |
| "answer_agreement": 0.9343963553530752 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 390, |
| "failures_b": 434, |
| "failures_shared": 323, |
| "failure_jaccard": 0.6447105788423154, |
| "failure_recall_a_in_b": 0.8282051282051283, |
| "successes_a": 1805, |
| "successes_b": 1761, |
| "successes_shared": 1694, |
| "success_jaccard": 0.9049145299145299, |
| "answer_agreement": 0.9189066059225512 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 390, |
| "failures_b": 465, |
| "failures_shared": 325, |
| "failure_jaccard": 0.6132075471698113, |
| "failure_recall_a_in_b": 0.8333333333333334, |
| "successes_a": 1805, |
| "successes_b": 1730, |
| "successes_shared": 1665, |
| "success_jaccard": 0.8903743315508021, |
| "answer_agreement": 0.9066059225512528 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 422, |
| "failures_b": 433, |
| "failures_shared": 362, |
| "failure_jaccard": 0.7342799188640974, |
| "failure_recall_a_in_b": 0.8578199052132701, |
| "successes_a": 1773, |
| "successes_b": 1762, |
| "successes_shared": 1702, |
| "success_jaccard": 0.928532460447354, |
| "answer_agreement": 0.9403189066059225 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 422, |
| "failures_b": 424, |
| "failures_shared": 355, |
| "failure_jaccard": 0.7230142566191446, |
| "failure_recall_a_in_b": 0.8412322274881516, |
| "successes_a": 1773, |
| "successes_b": 1771, |
| "successes_shared": 1704, |
| "success_jaccard": 0.9260869565217391, |
| "answer_agreement": 0.9380410022779043 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 422, |
| "failures_b": 434, |
| "failures_shared": 345, |
| "failure_jaccard": 0.675146771037182, |
| "failure_recall_a_in_b": 0.8175355450236966, |
| "successes_a": 1773, |
| "successes_b": 1761, |
| "successes_shared": 1684, |
| "success_jaccard": 0.9102702702702703, |
| "answer_agreement": 0.924373576309795 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 422, |
| "failures_b": 465, |
| "failures_shared": 348, |
| "failure_jaccard": 0.6456400742115028, |
| "failure_recall_a_in_b": 0.8246445497630331, |
| "successes_a": 1773, |
| "successes_b": 1730, |
| "successes_shared": 1656, |
| "success_jaccard": 0.8965890633459664, |
| "answer_agreement": 0.9129840546697039 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 433, |
| "failures_b": 424, |
| "failures_shared": 359, |
| "failure_jaccard": 0.7208835341365462, |
| "failure_recall_a_in_b": 0.8290993071593533, |
| "successes_a": 1762, |
| "successes_b": 1771, |
| "successes_shared": 1697, |
| "success_jaccard": 0.9242919389978214, |
| "answer_agreement": 0.9366742596810934 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 433, |
| "failures_b": 434, |
| "failures_shared": 357, |
| "failure_jaccard": 0.7, |
| "failure_recall_a_in_b": 0.8244803695150116, |
| "successes_a": 1762, |
| "successes_b": 1761, |
| "successes_shared": 1685, |
| "success_jaccard": 0.9167573449401524, |
| "answer_agreement": 0.9302961275626423 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 433, |
| "failures_b": 465, |
| "failures_shared": 360, |
| "failure_jaccard": 0.6691449814126395, |
| "failure_recall_a_in_b": 0.8314087759815243, |
| "successes_a": 1762, |
| "successes_b": 1730, |
| "successes_shared": 1657, |
| "success_jaccard": 0.9029972752043597, |
| "answer_agreement": 0.9189066059225512 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 424, |
| "failures_b": 434, |
| "failures_shared": 361, |
| "failure_jaccard": 0.7263581488933601, |
| "failure_recall_a_in_b": 0.8514150943396226, |
| "successes_a": 1771, |
| "successes_b": 1761, |
| "successes_shared": 1698, |
| "success_jaccard": 0.925845147219193, |
| "answer_agreement": 0.9380410022779043 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 424, |
| "failures_b": 465, |
| "failures_shared": 363, |
| "failure_jaccard": 0.6901140684410646, |
| "failure_recall_a_in_b": 0.8561320754716981, |
| "successes_a": 1771, |
| "successes_b": 1730, |
| "successes_shared": 1669, |
| "success_jaccard": 0.9110262008733624, |
| "answer_agreement": 0.9257403189066059 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 434, |
| "failures_b": 465, |
| "failures_shared": 376, |
| "failure_jaccard": 0.7189292543021033, |
| "failure_recall_a_in_b": 0.8663594470046083, |
| "successes_a": 1761, |
| "successes_b": 1730, |
| "successes_shared": 1672, |
| "success_jaccard": 0.9191863661352392, |
| "answer_agreement": 0.9330296127562643 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen2.5-VL-7B-Instruct", |
| "task": "youtube_level1", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 821, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 821, |
| "accuracy": 0.4470158343483557, |
| "num_failures": 454, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 821, |
| "accuracy": 0.4482338611449452, |
| "num_failures": 453, |
| "flip_rate_vs_reference": 0.1218026796589525, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 821, |
| "accuracy": 0.45554202192448234, |
| "num_failures": 447, |
| "flip_rate_vs_reference": 0.146163215590743, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 821, |
| "accuracy": 0.46650426309378806, |
| "num_failures": 438, |
| "flip_rate_vs_reference": 0.14981729598051158, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 821, |
| "accuracy": 0.44214372716199757, |
| "num_failures": 458, |
| "flip_rate_vs_reference": 0.17295980511571254, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 821, |
| "accuracy": 0.4518879415347138, |
| "num_failures": 450, |
| "flip_rate_vs_reference": 0.20219244823386115, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.024360535931790495, |
| "stability": { |
| "always_correct": 284, |
| "always_wrong": 366, |
| "unstable": 171, |
| "always_correct_rate": 0.3459196102314251, |
| "always_wrong_rate": 0.4457978075517661, |
| "unstable_rate": 0.20828258221680876 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 454, |
| "failures_b": 453, |
| "failures_shared": 417, |
| "failure_jaccard": 0.8510204081632653, |
| "failure_recall_a_in_b": 0.9185022026431718, |
| "successes_a": 367, |
| "successes_b": 368, |
| "successes_shared": 331, |
| "success_jaccard": 0.8193069306930693, |
| "answer_agreement": 0.8781973203410475 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 454, |
| "failures_b": 447, |
| "failures_shared": 413, |
| "failure_jaccard": 0.8463114754098361, |
| "failure_recall_a_in_b": 0.9096916299559471, |
| "successes_a": 367, |
| "successes_b": 374, |
| "successes_shared": 333, |
| "success_jaccard": 0.8161764705882353, |
| "answer_agreement": 0.853836784409257 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 454, |
| "failures_b": 438, |
| "failures_shared": 405, |
| "failure_jaccard": 0.8316221765913757, |
| "failure_recall_a_in_b": 0.8920704845814978, |
| "successes_a": 367, |
| "successes_b": 383, |
| "successes_shared": 334, |
| "success_jaccard": 0.8028846153846154, |
| "answer_agreement": 0.8501827040194885 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 454, |
| "failures_b": 458, |
| "failures_shared": 409, |
| "failure_jaccard": 0.8131212723658051, |
| "failure_recall_a_in_b": 0.9008810572687225, |
| "successes_a": 367, |
| "successes_b": 363, |
| "successes_shared": 318, |
| "success_jaccard": 0.7718446601941747, |
| "answer_agreement": 0.8270401948842875 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 454, |
| "failures_b": 450, |
| "failures_shared": 394, |
| "failure_jaccard": 0.7725490196078432, |
| "failure_recall_a_in_b": 0.8678414096916299, |
| "successes_a": 367, |
| "successes_b": 371, |
| "successes_shared": 311, |
| "success_jaccard": 0.7283372365339579, |
| "answer_agreement": 0.7978075517661388 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 453, |
| "failures_b": 447, |
| "failures_shared": 419, |
| "failure_jaccard": 0.8711018711018711, |
| "failure_recall_a_in_b": 0.9249448123620309, |
| "successes_a": 368, |
| "successes_b": 374, |
| "successes_shared": 340, |
| "success_jaccard": 0.845771144278607, |
| "answer_agreement": 0.8830694275274056 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 453, |
| "failures_b": 438, |
| "failures_shared": 412, |
| "failure_jaccard": 0.860125260960334, |
| "failure_recall_a_in_b": 0.9094922737306843, |
| "successes_a": 368, |
| "successes_b": 383, |
| "successes_shared": 342, |
| "success_jaccard": 0.8361858190709046, |
| "answer_agreement": 0.8842874543239951 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 453, |
| "failures_b": 458, |
| "failures_shared": 416, |
| "failure_jaccard": 0.8404040404040404, |
| "failure_recall_a_in_b": 0.9183222958057395, |
| "successes_a": 368, |
| "successes_b": 363, |
| "successes_shared": 326, |
| "success_jaccard": 0.8049382716049382, |
| "answer_agreement": 0.8623629719853837 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 453, |
| "failures_b": 450, |
| "failures_shared": 408, |
| "failure_jaccard": 0.8242424242424242, |
| "failure_recall_a_in_b": 0.9006622516556292, |
| "successes_a": 368, |
| "successes_b": 371, |
| "successes_shared": 326, |
| "success_jaccard": 0.7893462469733656, |
| "answer_agreement": 0.8428745432399513 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 447, |
| "failures_b": 438, |
| "failures_shared": 414, |
| "failure_jaccard": 0.8789808917197452, |
| "failure_recall_a_in_b": 0.9261744966442953, |
| "successes_a": 374, |
| "successes_b": 383, |
| "successes_shared": 350, |
| "success_jaccard": 0.85995085995086, |
| "answer_agreement": 0.8842874543239951 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 447, |
| "failures_b": 458, |
| "failures_shared": 420, |
| "failure_jaccard": 0.865979381443299, |
| "failure_recall_a_in_b": 0.9395973154362416, |
| "successes_a": 374, |
| "successes_b": 363, |
| "successes_shared": 336, |
| "success_jaccard": 0.8379052369077307, |
| "answer_agreement": 0.8721071863580999 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 447, |
| "failures_b": 450, |
| "failures_shared": 411, |
| "failure_jaccard": 0.845679012345679, |
| "failure_recall_a_in_b": 0.9194630872483222, |
| "successes_a": 374, |
| "successes_b": 371, |
| "successes_shared": 335, |
| "success_jaccard": 0.8170731707317073, |
| "answer_agreement": 0.8501827040194885 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 438, |
| "failures_b": 458, |
| "failures_shared": 416, |
| "failure_jaccard": 0.8666666666666667, |
| "failure_recall_a_in_b": 0.9497716894977168, |
| "successes_a": 383, |
| "successes_b": 363, |
| "successes_shared": 341, |
| "success_jaccard": 0.8419753086419753, |
| "answer_agreement": 0.8952496954933008 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 438, |
| "failures_b": 450, |
| "failures_shared": 406, |
| "failure_jaccard": 0.8423236514522822, |
| "failure_recall_a_in_b": 0.9269406392694064, |
| "successes_a": 383, |
| "successes_b": 371, |
| "successes_shared": 339, |
| "success_jaccard": 0.8168674698795181, |
| "answer_agreement": 0.8611449451887941 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 458, |
| "failures_b": 450, |
| "failures_shared": 423, |
| "failure_jaccard": 0.8721649484536083, |
| "failure_recall_a_in_b": 0.9235807860262009, |
| "successes_a": 363, |
| "successes_b": 371, |
| "successes_shared": 336, |
| "success_jaccard": 0.8442211055276382, |
| "answer_agreement": 0.879415347137637 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen2.5-VL-7B-Instruct", |
| "task": "youtube_level2", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 764, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 764, |
| "accuracy": 0.4031413612565445, |
| "num_failures": 456, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 764, |
| "accuracy": 0.39397905759162305, |
| "num_failures": 463, |
| "flip_rate_vs_reference": 0.14397905759162305, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 764, |
| "accuracy": 0.39790575916230364, |
| "num_failures": 460, |
| "flip_rate_vs_reference": 0.13219895287958114, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 764, |
| "accuracy": 0.4018324607329843, |
| "num_failures": 457, |
| "flip_rate_vs_reference": 0.17277486910994763, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 764, |
| "accuracy": 0.3769633507853403, |
| "num_failures": 476, |
| "flip_rate_vs_reference": 0.18979057591623036, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 764, |
| "accuracy": 0.3769633507853403, |
| "num_failures": 476, |
| "flip_rate_vs_reference": 0.20026178010471204, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.02617801047120416, |
| "stability": { |
| "always_correct": 221, |
| "always_wrong": 382, |
| "unstable": 161, |
| "always_correct_rate": 0.2892670157068063, |
| "always_wrong_rate": 0.5, |
| "unstable_rate": 0.2107329842931937 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 456, |
| "failures_b": 463, |
| "failures_shared": 429, |
| "failure_jaccard": 0.8755102040816326, |
| "failure_recall_a_in_b": 0.9407894736842105, |
| "successes_a": 308, |
| "successes_b": 301, |
| "successes_shared": 274, |
| "success_jaccard": 0.817910447761194, |
| "answer_agreement": 0.856020942408377 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 456, |
| "failures_b": 460, |
| "failures_shared": 428, |
| "failure_jaccard": 0.8770491803278688, |
| "failure_recall_a_in_b": 0.9385964912280702, |
| "successes_a": 308, |
| "successes_b": 304, |
| "successes_shared": 276, |
| "success_jaccard": 0.8214285714285714, |
| "answer_agreement": 0.8678010471204188 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 456, |
| "failures_b": 457, |
| "failures_shared": 415, |
| "failure_jaccard": 0.8333333333333334, |
| "failure_recall_a_in_b": 0.9100877192982456, |
| "successes_a": 308, |
| "successes_b": 307, |
| "successes_shared": 266, |
| "success_jaccard": 0.7621776504297995, |
| "answer_agreement": 0.8272251308900523 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 456, |
| "failures_b": 476, |
| "failures_shared": 421, |
| "failure_jaccard": 0.8238747553816047, |
| "failure_recall_a_in_b": 0.9232456140350878, |
| "successes_a": 308, |
| "successes_b": 288, |
| "successes_shared": 253, |
| "success_jaccard": 0.7376093294460642, |
| "answer_agreement": 0.8102094240837696 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 456, |
| "failures_b": 476, |
| "failures_shared": 418, |
| "failure_jaccard": 0.8132295719844358, |
| "failure_recall_a_in_b": 0.9166666666666666, |
| "successes_a": 308, |
| "successes_b": 288, |
| "successes_shared": 250, |
| "success_jaccard": 0.7225433526011561, |
| "answer_agreement": 0.7997382198952879 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 463, |
| "failures_b": 460, |
| "failures_shared": 436, |
| "failure_jaccard": 0.8952772073921971, |
| "failure_recall_a_in_b": 0.9416846652267818, |
| "successes_a": 301, |
| "successes_b": 304, |
| "successes_shared": 277, |
| "success_jaccard": 0.8445121951219512, |
| "answer_agreement": 0.8835078534031413 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 463, |
| "failures_b": 457, |
| "failures_shared": 423, |
| "failure_jaccard": 0.8511066398390342, |
| "failure_recall_a_in_b": 0.9136069114470843, |
| "successes_a": 301, |
| "successes_b": 307, |
| "successes_shared": 267, |
| "success_jaccard": 0.782991202346041, |
| "answer_agreement": 0.8481675392670157 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 463, |
| "failures_b": 476, |
| "failures_shared": 439, |
| "failure_jaccard": 0.878, |
| "failure_recall_a_in_b": 0.9481641468682506, |
| "successes_a": 301, |
| "successes_b": 288, |
| "successes_shared": 264, |
| "success_jaccard": 0.8123076923076923, |
| "answer_agreement": 0.8586387434554974 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 463, |
| "failures_b": 476, |
| "failures_shared": 433, |
| "failure_jaccard": 0.8557312252964426, |
| "failure_recall_a_in_b": 0.9352051835853131, |
| "successes_a": 301, |
| "successes_b": 288, |
| "successes_shared": 258, |
| "success_jaccard": 0.7794561933534743, |
| "answer_agreement": 0.849476439790576 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 460, |
| "failures_b": 457, |
| "failures_shared": 426, |
| "failure_jaccard": 0.8676171079429735, |
| "failure_recall_a_in_b": 0.9260869565217391, |
| "successes_a": 304, |
| "successes_b": 307, |
| "successes_shared": 273, |
| "success_jaccard": 0.8076923076923077, |
| "answer_agreement": 0.8691099476439791 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 460, |
| "failures_b": 476, |
| "failures_shared": 439, |
| "failure_jaccard": 0.8832997987927566, |
| "failure_recall_a_in_b": 0.9543478260869566, |
| "successes_a": 304, |
| "successes_b": 288, |
| "successes_shared": 267, |
| "success_jaccard": 0.8215384615384616, |
| "answer_agreement": 0.8717277486910995 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 460, |
| "failures_b": 476, |
| "failures_shared": 434, |
| "failure_jaccard": 0.8645418326693227, |
| "failure_recall_a_in_b": 0.9434782608695652, |
| "successes_a": 304, |
| "successes_b": 288, |
| "successes_shared": 262, |
| "success_jaccard": 0.793939393939394, |
| "answer_agreement": 0.8455497382198953 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 457, |
| "failures_b": 476, |
| "failures_shared": 430, |
| "failure_jaccard": 0.8548707753479126, |
| "failure_recall_a_in_b": 0.9409190371991247, |
| "successes_a": 307, |
| "successes_b": 288, |
| "successes_shared": 261, |
| "success_jaccard": 0.781437125748503, |
| "answer_agreement": 0.8638743455497382 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 457, |
| "failures_b": 476, |
| "failures_shared": 427, |
| "failure_jaccard": 0.8438735177865613, |
| "failure_recall_a_in_b": 0.9343544857768052, |
| "successes_a": 307, |
| "successes_b": 288, |
| "successes_shared": 258, |
| "success_jaccard": 0.7655786350148368, |
| "answer_agreement": 0.8337696335078534 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 476, |
| "failures_b": 476, |
| "failures_shared": 451, |
| "failure_jaccard": 0.9001996007984032, |
| "failure_recall_a_in_b": 0.9474789915966386, |
| "successes_a": 288, |
| "successes_b": 288, |
| "successes_shared": 263, |
| "success_jaccard": 0.8402555910543131, |
| "answer_agreement": 0.8887434554973822 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen2.5-VL-7B-Instruct", |
| "task": "youtube_level3", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 836, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 836, |
| "accuracy": 0.2799043062200957, |
| "num_failures": 602, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 836, |
| "accuracy": 0.28708133971291866, |
| "num_failures": 596, |
| "flip_rate_vs_reference": 0.19736842105263158, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 836, |
| "accuracy": 0.2679425837320574, |
| "num_failures": 612, |
| "flip_rate_vs_reference": 0.20454545454545456, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 836, |
| "accuracy": 0.2822966507177033, |
| "num_failures": 600, |
| "flip_rate_vs_reference": 0.2284688995215311, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 836, |
| "accuracy": 0.27631578947368424, |
| "num_failures": 605, |
| "flip_rate_vs_reference": 0.2452153110047847, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 836, |
| "accuracy": 0.27392344497607657, |
| "num_failures": 607, |
| "flip_rate_vs_reference": 0.23205741626794257, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.019138755980861233, |
| "stability": { |
| "always_correct": 143, |
| "always_wrong": 503, |
| "unstable": 190, |
| "always_correct_rate": 0.17105263157894737, |
| "always_wrong_rate": 0.6016746411483254, |
| "unstable_rate": 0.22727272727272727 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 602, |
| "failures_b": 596, |
| "failures_shared": 555, |
| "failure_jaccard": 0.8631415241057543, |
| "failure_recall_a_in_b": 0.9219269102990033, |
| "successes_a": 234, |
| "successes_b": 240, |
| "successes_shared": 193, |
| "success_jaccard": 0.6868327402135231, |
| "answer_agreement": 0.8026315789473685 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 602, |
| "failures_b": 612, |
| "failures_shared": 567, |
| "failure_jaccard": 0.8763523956723338, |
| "failure_recall_a_in_b": 0.9418604651162791, |
| "successes_a": 234, |
| "successes_b": 224, |
| "successes_shared": 189, |
| "success_jaccard": 0.7026022304832714, |
| "answer_agreement": 0.7954545454545454 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 602, |
| "failures_b": 600, |
| "failures_shared": 556, |
| "failure_jaccard": 0.8606811145510835, |
| "failure_recall_a_in_b": 0.9235880398671097, |
| "successes_a": 234, |
| "successes_b": 236, |
| "successes_shared": 190, |
| "success_jaccard": 0.6785714285714286, |
| "answer_agreement": 0.7715311004784688 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 602, |
| "failures_b": 605, |
| "failures_shared": 551, |
| "failure_jaccard": 0.8399390243902439, |
| "failure_recall_a_in_b": 0.915282392026578, |
| "successes_a": 234, |
| "successes_b": 231, |
| "successes_shared": 180, |
| "success_jaccard": 0.631578947368421, |
| "answer_agreement": 0.7547846889952153 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 602, |
| "failures_b": 607, |
| "failures_shared": 556, |
| "failure_jaccard": 0.8514548238897397, |
| "failure_recall_a_in_b": 0.9235880398671097, |
| "successes_a": 234, |
| "successes_b": 229, |
| "successes_shared": 183, |
| "success_jaccard": 0.6535714285714286, |
| "answer_agreement": 0.7679425837320574 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 596, |
| "failures_b": 612, |
| "failures_shared": 561, |
| "failure_jaccard": 0.8670788253477589, |
| "failure_recall_a_in_b": 0.9412751677852349, |
| "successes_a": 240, |
| "successes_b": 224, |
| "successes_shared": 189, |
| "success_jaccard": 0.6872727272727273, |
| "answer_agreement": 0.8205741626794258 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 596, |
| "failures_b": 600, |
| "failures_shared": 559, |
| "failure_jaccard": 0.8775510204081632, |
| "failure_recall_a_in_b": 0.9379194630872483, |
| "successes_a": 240, |
| "successes_b": 236, |
| "successes_shared": 199, |
| "success_jaccard": 0.7184115523465704, |
| "answer_agreement": 0.8098086124401914 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 596, |
| "failures_b": 605, |
| "failures_shared": 554, |
| "failure_jaccard": 0.8562596599690881, |
| "failure_recall_a_in_b": 0.9295302013422819, |
| "successes_a": 240, |
| "successes_b": 231, |
| "successes_shared": 189, |
| "success_jaccard": 0.6702127659574468, |
| "answer_agreement": 0.7954545454545454 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 596, |
| "failures_b": 607, |
| "failures_shared": 552, |
| "failure_jaccard": 0.847926267281106, |
| "failure_recall_a_in_b": 0.9261744966442953, |
| "successes_a": 240, |
| "successes_b": 229, |
| "successes_shared": 185, |
| "success_jaccard": 0.6514084507042254, |
| "answer_agreement": 0.7715311004784688 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 612, |
| "failures_b": 600, |
| "failures_shared": 565, |
| "failure_jaccard": 0.8732612055641422, |
| "failure_recall_a_in_b": 0.923202614379085, |
| "successes_a": 224, |
| "successes_b": 236, |
| "successes_shared": 189, |
| "success_jaccard": 0.6974169741697417, |
| "answer_agreement": 0.8169856459330144 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 612, |
| "failures_b": 605, |
| "failures_shared": 566, |
| "failure_jaccard": 0.869431643625192, |
| "failure_recall_a_in_b": 0.9248366013071896, |
| "successes_a": 224, |
| "successes_b": 231, |
| "successes_shared": 185, |
| "success_jaccard": 0.6851851851851852, |
| "answer_agreement": 0.8169856459330144 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 612, |
| "failures_b": 607, |
| "failures_shared": 566, |
| "failure_jaccard": 0.8667687595712098, |
| "failure_recall_a_in_b": 0.9248366013071896, |
| "successes_a": 224, |
| "successes_b": 229, |
| "successes_shared": 183, |
| "success_jaccard": 0.6777777777777778, |
| "answer_agreement": 0.8002392344497608 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 600, |
| "failures_b": 605, |
| "failures_shared": 565, |
| "failure_jaccard": 0.8828125, |
| "failure_recall_a_in_b": 0.9416666666666667, |
| "successes_a": 236, |
| "successes_b": 231, |
| "successes_shared": 196, |
| "success_jaccard": 0.7232472324723247, |
| "answer_agreement": 0.8217703349282297 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 600, |
| "failures_b": 607, |
| "failures_shared": 560, |
| "failure_jaccard": 0.865533230293663, |
| "failure_recall_a_in_b": 0.9333333333333333, |
| "successes_a": 236, |
| "successes_b": 229, |
| "successes_shared": 189, |
| "success_jaccard": 0.6847826086956522, |
| "answer_agreement": 0.7942583732057417 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 605, |
| "failures_b": 607, |
| "failures_shared": 564, |
| "failure_jaccard": 0.8703703703703703, |
| "failure_recall_a_in_b": 0.9322314049586777, |
| "successes_a": 231, |
| "successes_b": 229, |
| "successes_shared": 188, |
| "success_jaccard": 0.6911764705882353, |
| "answer_agreement": 0.8014354066985646 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen3.5-4B", |
| "task": "vsr", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 2195, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 2195, |
| "accuracy": 0.8214123006833713, |
| "num_failures": 392, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 2195, |
| "accuracy": 0.8173120728929385, |
| "num_failures": 401, |
| "flip_rate_vs_reference": 0.04510250569476082, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 2195, |
| "accuracy": 0.8186788154897494, |
| "num_failures": 398, |
| "flip_rate_vs_reference": 0.04738041002277904, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 2195, |
| "accuracy": 0.8113895216400911, |
| "num_failures": 414, |
| "flip_rate_vs_reference": 0.06287015945330296, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 2195, |
| "accuracy": 0.8072892938496583, |
| "num_failures": 423, |
| "flip_rate_vs_reference": 0.07425968109339408, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 2195, |
| "accuracy": 0.8045558086560365, |
| "num_failures": 429, |
| "flip_rate_vs_reference": 0.07608200455580866, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.016856492027334857, |
| "stability": { |
| "always_correct": 1635, |
| "always_wrong": 283, |
| "unstable": 277, |
| "always_correct_rate": 0.744874715261959, |
| "always_wrong_rate": 0.12892938496583142, |
| "unstable_rate": 0.12619589977220957 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 392, |
| "failures_b": 401, |
| "failures_shared": 347, |
| "failure_jaccard": 0.7780269058295964, |
| "failure_recall_a_in_b": 0.8852040816326531, |
| "successes_a": 1803, |
| "successes_b": 1794, |
| "successes_shared": 1749, |
| "success_jaccard": 0.9464285714285714, |
| "answer_agreement": 0.9548974943052392 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 392, |
| "failures_b": 398, |
| "failures_shared": 343, |
| "failure_jaccard": 0.767337807606264, |
| "failure_recall_a_in_b": 0.875, |
| "successes_a": 1803, |
| "successes_b": 1797, |
| "successes_shared": 1748, |
| "success_jaccard": 0.9438444924406048, |
| "answer_agreement": 0.952619589977221 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 392, |
| "failures_b": 414, |
| "failures_shared": 334, |
| "failure_jaccard": 0.7076271186440678, |
| "failure_recall_a_in_b": 0.8520408163265306, |
| "successes_a": 1803, |
| "successes_b": 1781, |
| "successes_shared": 1723, |
| "success_jaccard": 0.9258463191832348, |
| "answer_agreement": 0.937129840546697 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 392, |
| "failures_b": 423, |
| "failures_shared": 326, |
| "failure_jaccard": 0.6666666666666666, |
| "failure_recall_a_in_b": 0.8316326530612245, |
| "successes_a": 1803, |
| "successes_b": 1772, |
| "successes_shared": 1706, |
| "success_jaccard": 0.9127875869448903, |
| "answer_agreement": 0.9257403189066059 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 392, |
| "failures_b": 429, |
| "failures_shared": 327, |
| "failure_jaccard": 0.6619433198380567, |
| "failure_recall_a_in_b": 0.8341836734693877, |
| "successes_a": 1803, |
| "successes_b": 1766, |
| "successes_shared": 1701, |
| "success_jaccard": 0.9105995717344754, |
| "answer_agreement": 0.9239179954441914 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 401, |
| "failures_b": 398, |
| "failures_shared": 356, |
| "failure_jaccard": 0.8036117381489842, |
| "failure_recall_a_in_b": 0.8877805486284289, |
| "successes_a": 1794, |
| "successes_b": 1797, |
| "successes_shared": 1752, |
| "success_jaccard": 0.9526916802610114, |
| "answer_agreement": 0.9603644646924829 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 401, |
| "failures_b": 414, |
| "failures_shared": 350, |
| "failure_jaccard": 0.7526881720430108, |
| "failure_recall_a_in_b": 0.8728179551122195, |
| "successes_a": 1794, |
| "successes_b": 1781, |
| "successes_shared": 1730, |
| "success_jaccard": 0.9376693766937669, |
| "answer_agreement": 0.9476082004555809 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 401, |
| "failures_b": 423, |
| "failures_shared": 341, |
| "failure_jaccard": 0.7060041407867494, |
| "failure_recall_a_in_b": 0.8503740648379052, |
| "successes_a": 1794, |
| "successes_b": 1772, |
| "successes_shared": 1712, |
| "success_jaccard": 0.9234088457389428, |
| "answer_agreement": 0.9353075170842825 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 401, |
| "failures_b": 429, |
| "failures_shared": 337, |
| "failure_jaccard": 0.6835699797160243, |
| "failure_recall_a_in_b": 0.8403990024937655, |
| "successes_a": 1794, |
| "successes_b": 1766, |
| "successes_shared": 1702, |
| "success_jaccard": 0.9160387513455328, |
| "answer_agreement": 0.9289293849658314 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 398, |
| "failures_b": 414, |
| "failures_shared": 355, |
| "failure_jaccard": 0.7768052516411379, |
| "failure_recall_a_in_b": 0.8919597989949749, |
| "successes_a": 1797, |
| "successes_b": 1781, |
| "successes_shared": 1738, |
| "success_jaccard": 0.9445652173913044, |
| "answer_agreement": 0.9535307517084283 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 398, |
| "failures_b": 423, |
| "failures_shared": 350, |
| "failure_jaccard": 0.7430997876857749, |
| "failure_recall_a_in_b": 0.8793969849246231, |
| "successes_a": 1797, |
| "successes_b": 1772, |
| "successes_shared": 1724, |
| "success_jaccard": 0.9344173441734417, |
| "answer_agreement": 0.944874715261959 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 398, |
| "failures_b": 429, |
| "failures_shared": 341, |
| "failure_jaccard": 0.7016460905349794, |
| "failure_recall_a_in_b": 0.8567839195979899, |
| "successes_a": 1797, |
| "successes_b": 1766, |
| "successes_shared": 1709, |
| "success_jaccard": 0.9217907227615966, |
| "answer_agreement": 0.9339407744874715 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 414, |
| "failures_b": 423, |
| "failures_shared": 359, |
| "failure_jaccard": 0.7510460251046025, |
| "failure_recall_a_in_b": 0.8671497584541062, |
| "successes_a": 1781, |
| "successes_b": 1772, |
| "successes_shared": 1717, |
| "success_jaccard": 0.9351851851851852, |
| "answer_agreement": 0.9457858769931663 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 414, |
| "failures_b": 429, |
| "failures_shared": 354, |
| "failure_jaccard": 0.7239263803680982, |
| "failure_recall_a_in_b": 0.855072463768116, |
| "successes_a": 1781, |
| "successes_b": 1766, |
| "successes_shared": 1706, |
| "success_jaccard": 0.9266702878870179, |
| "answer_agreement": 0.9384965831435079 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 423, |
| "failures_b": 429, |
| "failures_shared": 372, |
| "failure_jaccard": 0.775, |
| "failure_recall_a_in_b": 0.8794326241134752, |
| "successes_a": 1772, |
| "successes_b": 1766, |
| "successes_shared": 1715, |
| "success_jaccard": 0.9407569939659901, |
| "answer_agreement": 0.9507972665148063 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen3.5-4B", |
| "task": "youtube_level1", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 821, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 821, |
| "accuracy": 0.5274056029232643, |
| "num_failures": 388, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0036540803897685747 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 821, |
| "accuracy": 0.5213154689403167, |
| "num_failures": 393, |
| "flip_rate_vs_reference": 0.10353227771010962, |
| "unparsed_rate": 0.007308160779537149 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 821, |
| "accuracy": 0.5188794153471377, |
| "num_failures": 395, |
| "flip_rate_vs_reference": 0.12545676004872108, |
| "unparsed_rate": 0.001218026796589525 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 821, |
| "accuracy": 0.5408038976857491, |
| "num_failures": 377, |
| "flip_rate_vs_reference": 0.14007308160779536, |
| "unparsed_rate": 0.0048721071863581 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 821, |
| "accuracy": 0.5347137637028014, |
| "num_failures": 382, |
| "flip_rate_vs_reference": 0.15956151035322777, |
| "unparsed_rate": 0.001218026796589525 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 821, |
| "accuracy": 0.5347137637028014, |
| "num_failures": 382, |
| "flip_rate_vs_reference": 0.1656516443361754, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.02192448233861144, |
| "stability": { |
| "always_correct": 343, |
| "always_wrong": 296, |
| "unstable": 182, |
| "always_correct_rate": 0.4177831912302071, |
| "always_wrong_rate": 0.36053593179049936, |
| "unstable_rate": 0.22168087697929353 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 388, |
| "failures_b": 393, |
| "failures_shared": 361, |
| "failure_jaccard": 0.8595238095238096, |
| "failure_recall_a_in_b": 0.9304123711340206, |
| "successes_a": 433, |
| "successes_b": 428, |
| "successes_shared": 401, |
| "success_jaccard": 0.8717391304347826, |
| "answer_agreement": 0.8964677222898904 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 388, |
| "failures_b": 395, |
| "failures_shared": 354, |
| "failure_jaccard": 0.8251748251748252, |
| "failure_recall_a_in_b": 0.9123711340206185, |
| "successes_a": 433, |
| "successes_b": 426, |
| "successes_shared": 392, |
| "success_jaccard": 0.8394004282655246, |
| "answer_agreement": 0.8745432399512789 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 388, |
| "failures_b": 377, |
| "failures_shared": 342, |
| "failure_jaccard": 0.8085106382978723, |
| "failure_recall_a_in_b": 0.8814432989690721, |
| "successes_a": 433, |
| "successes_b": 444, |
| "successes_shared": 398, |
| "success_jaccard": 0.8308977035490606, |
| "answer_agreement": 0.8599269183922046 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 388, |
| "failures_b": 382, |
| "failures_shared": 339, |
| "failure_jaccard": 0.7865429234338747, |
| "failure_recall_a_in_b": 0.8737113402061856, |
| "successes_a": 433, |
| "successes_b": 439, |
| "successes_shared": 390, |
| "success_jaccard": 0.8091286307053942, |
| "answer_agreement": 0.8404384896467723 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 388, |
| "failures_b": 382, |
| "failures_shared": 338, |
| "failure_jaccard": 0.7824074074074074, |
| "failure_recall_a_in_b": 0.8711340206185567, |
| "successes_a": 433, |
| "successes_b": 439, |
| "successes_shared": 389, |
| "success_jaccard": 0.8053830227743272, |
| "answer_agreement": 0.8343483556638246 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 393, |
| "failures_b": 395, |
| "failures_shared": 363, |
| "failure_jaccard": 0.8541176470588235, |
| "failure_recall_a_in_b": 0.9236641221374046, |
| "successes_a": 428, |
| "successes_b": 426, |
| "successes_shared": 396, |
| "success_jaccard": 0.8646288209606987, |
| "answer_agreement": 0.8855054811205847 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 393, |
| "failures_b": 377, |
| "failures_shared": 342, |
| "failure_jaccard": 0.7990654205607477, |
| "failure_recall_a_in_b": 0.8702290076335878, |
| "successes_a": 428, |
| "successes_b": 444, |
| "successes_shared": 393, |
| "success_jaccard": 0.8204592901878914, |
| "answer_agreement": 0.8635809987819733 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 393, |
| "failures_b": 382, |
| "failures_shared": 343, |
| "failure_jaccard": 0.7939814814814815, |
| "failure_recall_a_in_b": 0.8727735368956743, |
| "successes_a": 428, |
| "successes_b": 439, |
| "successes_shared": 389, |
| "success_jaccard": 0.8138075313807531, |
| "answer_agreement": 0.8453105968331304 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 393, |
| "failures_b": 382, |
| "failures_shared": 338, |
| "failure_jaccard": 0.7734553775743707, |
| "failure_recall_a_in_b": 0.8600508905852418, |
| "successes_a": 428, |
| "successes_b": 439, |
| "successes_shared": 384, |
| "success_jaccard": 0.7950310559006211, |
| "answer_agreement": 0.8258221680876979 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 395, |
| "failures_b": 377, |
| "failures_shared": 349, |
| "failure_jaccard": 0.8250591016548463, |
| "failure_recall_a_in_b": 0.8835443037974684, |
| "successes_a": 426, |
| "successes_b": 444, |
| "successes_shared": 398, |
| "success_jaccard": 0.8432203389830508, |
| "answer_agreement": 0.8781973203410475 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 395, |
| "failures_b": 382, |
| "failures_shared": 350, |
| "failure_jaccard": 0.819672131147541, |
| "failure_recall_a_in_b": 0.8860759493670886, |
| "successes_a": 426, |
| "successes_b": 439, |
| "successes_shared": 394, |
| "success_jaccard": 0.8365180467091295, |
| "answer_agreement": 0.8684531059683313 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 395, |
| "failures_b": 382, |
| "failures_shared": 347, |
| "failure_jaccard": 0.8069767441860465, |
| "failure_recall_a_in_b": 0.8784810126582279, |
| "successes_a": 426, |
| "successes_b": 439, |
| "successes_shared": 391, |
| "success_jaccard": 0.8248945147679325, |
| "answer_agreement": 0.8514007308160779 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 377, |
| "failures_b": 382, |
| "failures_shared": 341, |
| "failure_jaccard": 0.8157894736842105, |
| "failure_recall_a_in_b": 0.9045092838196287, |
| "successes_a": 444, |
| "successes_b": 439, |
| "successes_shared": 403, |
| "success_jaccard": 0.8395833333333333, |
| "answer_agreement": 0.8721071863580999 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 377, |
| "failures_b": 382, |
| "failures_shared": 338, |
| "failure_jaccard": 0.8028503562945368, |
| "failure_recall_a_in_b": 0.896551724137931, |
| "successes_a": 444, |
| "successes_b": 439, |
| "successes_shared": 400, |
| "success_jaccard": 0.8281573498964804, |
| "answer_agreement": 0.8611449451887941 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 382, |
| "failures_b": 382, |
| "failures_shared": 346, |
| "failure_jaccard": 0.8277511961722488, |
| "failure_recall_a_in_b": 0.9057591623036649, |
| "successes_a": 439, |
| "successes_b": 439, |
| "successes_shared": 403, |
| "success_jaccard": 0.848421052631579, |
| "answer_agreement": 0.8781973203410475 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen3.5-4B", |
| "task": "youtube_level2", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 764, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 764, |
| "accuracy": 0.3913612565445026, |
| "num_failures": 465, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 764, |
| "accuracy": 0.3900523560209424, |
| "num_failures": 466, |
| "flip_rate_vs_reference": 0.12172774869109948, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 764, |
| "accuracy": 0.38219895287958117, |
| "num_failures": 472, |
| "flip_rate_vs_reference": 0.14136125654450263, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 764, |
| "accuracy": 0.387434554973822, |
| "num_failures": 468, |
| "flip_rate_vs_reference": 0.16361256544502617, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 764, |
| "accuracy": 0.3795811518324607, |
| "num_failures": 474, |
| "flip_rate_vs_reference": 0.17146596858638743, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 764, |
| "accuracy": 0.38481675392670156, |
| "num_failures": 470, |
| "flip_rate_vs_reference": 0.18717277486910994, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.011780104712041883, |
| "stability": { |
| "always_correct": 230, |
| "always_wrong": 401, |
| "unstable": 133, |
| "always_correct_rate": 0.3010471204188482, |
| "always_wrong_rate": 0.524869109947644, |
| "unstable_rate": 0.17408376963350786 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 465, |
| "failures_b": 466, |
| "failures_shared": 438, |
| "failure_jaccard": 0.8884381338742393, |
| "failure_recall_a_in_b": 0.9419354838709677, |
| "successes_a": 299, |
| "successes_b": 298, |
| "successes_shared": 271, |
| "success_jaccard": 0.8312883435582822, |
| "answer_agreement": 0.8782722513089005 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 465, |
| "failures_b": 472, |
| "failures_shared": 438, |
| "failure_jaccard": 0.8777555110220441, |
| "failure_recall_a_in_b": 0.9419354838709677, |
| "successes_a": 299, |
| "successes_b": 292, |
| "successes_shared": 265, |
| "success_jaccard": 0.8128834355828221, |
| "answer_agreement": 0.8586387434554974 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 465, |
| "failures_b": 468, |
| "failures_shared": 433, |
| "failure_jaccard": 0.866, |
| "failure_recall_a_in_b": 0.9311827956989247, |
| "successes_a": 299, |
| "successes_b": 296, |
| "successes_shared": 264, |
| "success_jaccard": 0.797583081570997, |
| "answer_agreement": 0.8363874345549738 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 465, |
| "failures_b": 474, |
| "failures_shared": 434, |
| "failure_jaccard": 0.8594059405940594, |
| "failure_recall_a_in_b": 0.9333333333333333, |
| "successes_a": 299, |
| "successes_b": 290, |
| "successes_shared": 259, |
| "success_jaccard": 0.7848484848484848, |
| "answer_agreement": 0.8285340314136126 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 465, |
| "failures_b": 470, |
| "failures_shared": 423, |
| "failure_jaccard": 0.826171875, |
| "failure_recall_a_in_b": 0.9096774193548387, |
| "successes_a": 299, |
| "successes_b": 294, |
| "successes_shared": 252, |
| "success_jaccard": 0.7390029325513197, |
| "answer_agreement": 0.81282722513089 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 466, |
| "failures_b": 472, |
| "failures_shared": 441, |
| "failure_jaccard": 0.8873239436619719, |
| "failure_recall_a_in_b": 0.9463519313304721, |
| "successes_a": 298, |
| "successes_b": 292, |
| "successes_shared": 267, |
| "success_jaccard": 0.826625386996904, |
| "answer_agreement": 0.8821989528795812 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 466, |
| "failures_b": 468, |
| "failures_shared": 438, |
| "failure_jaccard": 0.8830645161290323, |
| "failure_recall_a_in_b": 0.9399141630901288, |
| "successes_a": 298, |
| "successes_b": 296, |
| "successes_shared": 268, |
| "success_jaccard": 0.8220858895705522, |
| "answer_agreement": 0.8664921465968587 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 466, |
| "failures_b": 474, |
| "failures_shared": 437, |
| "failure_jaccard": 0.8687872763419483, |
| "failure_recall_a_in_b": 0.9377682403433476, |
| "successes_a": 298, |
| "successes_b": 290, |
| "successes_shared": 261, |
| "success_jaccard": 0.7981651376146789, |
| "answer_agreement": 0.856020942408377 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 466, |
| "failures_b": 470, |
| "failures_shared": 432, |
| "failure_jaccard": 0.8571428571428571, |
| "failure_recall_a_in_b": 0.927038626609442, |
| "successes_a": 298, |
| "successes_b": 294, |
| "successes_shared": 260, |
| "success_jaccard": 0.7831325301204819, |
| "answer_agreement": 0.8599476439790575 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 472, |
| "failures_b": 468, |
| "failures_shared": 445, |
| "failure_jaccard": 0.898989898989899, |
| "failure_recall_a_in_b": 0.9427966101694916, |
| "successes_a": 292, |
| "successes_b": 296, |
| "successes_shared": 269, |
| "success_jaccard": 0.8432601880877743, |
| "answer_agreement": 0.8821989528795812 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 472, |
| "failures_b": 474, |
| "failures_shared": 447, |
| "failure_jaccard": 0.8957915831663327, |
| "failure_recall_a_in_b": 0.9470338983050848, |
| "successes_a": 292, |
| "successes_b": 290, |
| "successes_shared": 265, |
| "success_jaccard": 0.8359621451104101, |
| "answer_agreement": 0.8678010471204188 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 472, |
| "failures_b": 470, |
| "failures_shared": 438, |
| "failure_jaccard": 0.8690476190476191, |
| "failure_recall_a_in_b": 0.9279661016949152, |
| "successes_a": 292, |
| "successes_b": 294, |
| "successes_shared": 260, |
| "success_jaccard": 0.7975460122699386, |
| "answer_agreement": 0.856020942408377 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 468, |
| "failures_b": 474, |
| "failures_shared": 446, |
| "failure_jaccard": 0.8991935483870968, |
| "failure_recall_a_in_b": 0.9529914529914529, |
| "successes_a": 296, |
| "successes_b": 290, |
| "successes_shared": 268, |
| "success_jaccard": 0.8427672955974843, |
| "answer_agreement": 0.8704188481675392 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 468, |
| "failures_b": 470, |
| "failures_shared": 440, |
| "failure_jaccard": 0.8835341365461847, |
| "failure_recall_a_in_b": 0.9401709401709402, |
| "successes_a": 296, |
| "successes_b": 294, |
| "successes_shared": 266, |
| "success_jaccard": 0.8209876543209876, |
| "answer_agreement": 0.8704188481675392 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 474, |
| "failures_b": 470, |
| "failures_shared": 443, |
| "failure_jaccard": 0.8842315369261478, |
| "failure_recall_a_in_b": 0.9345991561181435, |
| "successes_a": 290, |
| "successes_b": 294, |
| "successes_shared": 263, |
| "success_jaccard": 0.8193146417445483, |
| "answer_agreement": 0.8520942408376964 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen3.5-4B", |
| "task": "youtube_level3", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 836, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 836, |
| "accuracy": 0.2930622009569378, |
| "num_failures": 591, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 836, |
| "accuracy": 0.29904306220095694, |
| "num_failures": 586, |
| "flip_rate_vs_reference": 0.23803827751196172, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 836, |
| "accuracy": 0.2966507177033493, |
| "num_failures": 588, |
| "flip_rate_vs_reference": 0.2595693779904306, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 836, |
| "accuracy": 0.3038277511961722, |
| "num_failures": 582, |
| "flip_rate_vs_reference": 0.2595693779904306, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 836, |
| "accuracy": 0.28827751196172247, |
| "num_failures": 595, |
| "flip_rate_vs_reference": 0.27870813397129185, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 836, |
| "accuracy": 0.2930622009569378, |
| "num_failures": 591, |
| "flip_rate_vs_reference": 0.29545454545454547, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.015550239234449759, |
| "stability": { |
| "always_correct": 144, |
| "always_wrong": 474, |
| "unstable": 218, |
| "always_correct_rate": 0.1722488038277512, |
| "always_wrong_rate": 0.5669856459330144, |
| "unstable_rate": 0.2607655502392344 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 591, |
| "failures_b": 586, |
| "failures_shared": 539, |
| "failure_jaccard": 0.8448275862068966, |
| "failure_recall_a_in_b": 0.9120135363790186, |
| "successes_a": 245, |
| "successes_b": 250, |
| "successes_shared": 198, |
| "success_jaccard": 0.6666666666666666, |
| "answer_agreement": 0.7619617224880383 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 591, |
| "failures_b": 588, |
| "failures_shared": 534, |
| "failure_jaccard": 0.827906976744186, |
| "failure_recall_a_in_b": 0.9035532994923858, |
| "successes_a": 245, |
| "successes_b": 248, |
| "successes_shared": 191, |
| "success_jaccard": 0.6324503311258278, |
| "answer_agreement": 0.7404306220095693 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 591, |
| "failures_b": 582, |
| "failures_shared": 530, |
| "failure_jaccard": 0.8242612752721618, |
| "failure_recall_a_in_b": 0.8967851099830795, |
| "successes_a": 245, |
| "successes_b": 254, |
| "successes_shared": 193, |
| "success_jaccard": 0.630718954248366, |
| "answer_agreement": 0.7404306220095693 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 591, |
| "failures_b": 595, |
| "failures_shared": 531, |
| "failure_jaccard": 0.8106870229007633, |
| "failure_recall_a_in_b": 0.8984771573604061, |
| "successes_a": 245, |
| "successes_b": 241, |
| "successes_shared": 181, |
| "success_jaccard": 0.5934426229508196, |
| "answer_agreement": 0.7212918660287081 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 591, |
| "failures_b": 591, |
| "failures_shared": 525, |
| "failure_jaccard": 0.7990867579908676, |
| "failure_recall_a_in_b": 0.8883248730964467, |
| "successes_a": 245, |
| "successes_b": 245, |
| "successes_shared": 179, |
| "success_jaccard": 0.5755627009646302, |
| "answer_agreement": 0.7045454545454546 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 586, |
| "failures_b": 588, |
| "failures_shared": 546, |
| "failure_jaccard": 0.8694267515923567, |
| "failure_recall_a_in_b": 0.931740614334471, |
| "successes_a": 250, |
| "successes_b": 248, |
| "successes_shared": 208, |
| "success_jaccard": 0.7172413793103448, |
| "answer_agreement": 0.8229665071770335 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 586, |
| "failures_b": 582, |
| "failures_shared": 540, |
| "failure_jaccard": 0.8598726114649682, |
| "failure_recall_a_in_b": 0.9215017064846417, |
| "successes_a": 250, |
| "successes_b": 254, |
| "successes_shared": 208, |
| "success_jaccard": 0.7027027027027027, |
| "answer_agreement": 0.7918660287081339 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 586, |
| "failures_b": 595, |
| "failures_shared": 546, |
| "failure_jaccard": 0.8598425196850393, |
| "failure_recall_a_in_b": 0.931740614334471, |
| "successes_a": 250, |
| "successes_b": 241, |
| "successes_shared": 201, |
| "success_jaccard": 0.6931034482758621, |
| "answer_agreement": 0.7811004784688995 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 586, |
| "failures_b": 591, |
| "failures_shared": 541, |
| "failure_jaccard": 0.85062893081761, |
| "failure_recall_a_in_b": 0.9232081911262798, |
| "successes_a": 250, |
| "successes_b": 245, |
| "successes_shared": 200, |
| "success_jaccard": 0.6779661016949152, |
| "answer_agreement": 0.7822966507177034 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 588, |
| "failures_b": 582, |
| "failures_shared": 544, |
| "failure_jaccard": 0.8690095846645367, |
| "failure_recall_a_in_b": 0.9251700680272109, |
| "successes_a": 248, |
| "successes_b": 254, |
| "successes_shared": 210, |
| "success_jaccard": 0.7191780821917808, |
| "answer_agreement": 0.8145933014354066 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 588, |
| "failures_b": 595, |
| "failures_shared": 555, |
| "failure_jaccard": 0.8837579617834395, |
| "failure_recall_a_in_b": 0.9438775510204082, |
| "successes_a": 248, |
| "successes_b": 241, |
| "successes_shared": 208, |
| "success_jaccard": 0.7402135231316725, |
| "answer_agreement": 0.8050239234449761 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 588, |
| "failures_b": 591, |
| "failures_shared": 538, |
| "failure_jaccard": 0.8393135725429017, |
| "failure_recall_a_in_b": 0.9149659863945578, |
| "successes_a": 248, |
| "successes_b": 245, |
| "successes_shared": 195, |
| "success_jaccard": 0.6543624161073825, |
| "answer_agreement": 0.7870813397129187 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 582, |
| "failures_b": 595, |
| "failures_shared": 551, |
| "failure_jaccard": 0.8801916932907349, |
| "failure_recall_a_in_b": 0.9467353951890034, |
| "successes_a": 254, |
| "successes_b": 241, |
| "successes_shared": 210, |
| "success_jaccard": 0.7368421052631579, |
| "answer_agreement": 0.819377990430622 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 582, |
| "failures_b": 591, |
| "failures_shared": 539, |
| "failure_jaccard": 0.8501577287066246, |
| "failure_recall_a_in_b": 0.9261168384879725, |
| "successes_a": 254, |
| "successes_b": 245, |
| "successes_shared": 202, |
| "success_jaccard": 0.6801346801346801, |
| "answer_agreement": 0.7930622009569378 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 595, |
| "failures_b": 591, |
| "failures_shared": 550, |
| "failure_jaccard": 0.8647798742138365, |
| "failure_recall_a_in_b": 0.9243697478991597, |
| "successes_a": 241, |
| "successes_b": 245, |
| "successes_shared": 200, |
| "success_jaccard": 0.6993006993006993, |
| "answer_agreement": 0.7918660287081339 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen3.5-4B-Base", |
| "task": "vsr", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 2195, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 2195, |
| "accuracy": 0.8318906605922551, |
| "num_failures": 369, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 2195, |
| "accuracy": 0.8314350797266514, |
| "num_failures": 370, |
| "flip_rate_vs_reference": 0.04601366742596811, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 2195, |
| "accuracy": 0.8323462414578587, |
| "num_failures": 368, |
| "flip_rate_vs_reference": 0.05330296127562642, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 2195, |
| "accuracy": 0.8277904328018223, |
| "num_failures": 378, |
| "flip_rate_vs_reference": 0.05968109339407745, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 2195, |
| "accuracy": 0.8241457858769932, |
| "num_failures": 386, |
| "flip_rate_vs_reference": 0.07881548974943052, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 2195, |
| "accuracy": 0.8164009111617312, |
| "num_failures": 403, |
| "flip_rate_vs_reference": 0.0856492027334852, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.015945330296127547, |
| "stability": { |
| "always_correct": 1653, |
| "always_wrong": 240, |
| "unstable": 302, |
| "always_correct_rate": 0.7530751708428246, |
| "always_wrong_rate": 0.10933940774487472, |
| "unstable_rate": 0.13758542141230068 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 369, |
| "failures_b": 370, |
| "failures_shared": 319, |
| "failure_jaccard": 0.7595238095238095, |
| "failure_recall_a_in_b": 0.8644986449864499, |
| "successes_a": 1826, |
| "successes_b": 1825, |
| "successes_shared": 1775, |
| "success_jaccard": 0.9461620469083155, |
| "answer_agreement": 0.9539863325740319 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 369, |
| "failures_b": 368, |
| "failures_shared": 310, |
| "failure_jaccard": 0.7259953161592506, |
| "failure_recall_a_in_b": 0.8401084010840109, |
| "successes_a": 1826, |
| "successes_b": 1827, |
| "successes_shared": 1768, |
| "success_jaccard": 0.9379310344827586, |
| "answer_agreement": 0.9466970387243736 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 369, |
| "failures_b": 378, |
| "failures_shared": 308, |
| "failure_jaccard": 0.7015945330296127, |
| "failure_recall_a_in_b": 0.8346883468834688, |
| "successes_a": 1826, |
| "successes_b": 1817, |
| "successes_shared": 1756, |
| "success_jaccard": 0.9305776364599894, |
| "answer_agreement": 0.9403189066059225 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 369, |
| "failures_b": 386, |
| "failures_shared": 291, |
| "failure_jaccard": 0.6271551724137931, |
| "failure_recall_a_in_b": 0.7886178861788617, |
| "successes_a": 1826, |
| "successes_b": 1809, |
| "successes_shared": 1731, |
| "success_jaccard": 0.9091386554621849, |
| "answer_agreement": 0.9211845102505695 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 369, |
| "failures_b": 403, |
| "failures_shared": 292, |
| "failure_jaccard": 0.6083333333333333, |
| "failure_recall_a_in_b": 0.7913279132791328, |
| "successes_a": 1826, |
| "successes_b": 1792, |
| "successes_shared": 1715, |
| "success_jaccard": 0.9012086179716238, |
| "answer_agreement": 0.9143507972665148 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 370, |
| "failures_b": 368, |
| "failures_shared": 319, |
| "failure_jaccard": 0.7613365155131265, |
| "failure_recall_a_in_b": 0.8621621621621621, |
| "successes_a": 1825, |
| "successes_b": 1827, |
| "successes_shared": 1776, |
| "success_jaccard": 0.9466950959488273, |
| "answer_agreement": 0.9544419134396356 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 370, |
| "failures_b": 378, |
| "failures_shared": 310, |
| "failure_jaccard": 0.7077625570776256, |
| "failure_recall_a_in_b": 0.8378378378378378, |
| "successes_a": 1825, |
| "successes_b": 1817, |
| "successes_shared": 1757, |
| "success_jaccard": 0.9320954907161804, |
| "answer_agreement": 0.9416856492027335 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 370, |
| "failures_b": 386, |
| "failures_shared": 302, |
| "failure_jaccard": 0.6651982378854625, |
| "failure_recall_a_in_b": 0.8162162162162162, |
| "successes_a": 1825, |
| "successes_b": 1809, |
| "successes_shared": 1741, |
| "success_jaccard": 0.9197041732699419, |
| "answer_agreement": 0.9307517084282461 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 370, |
| "failures_b": 403, |
| "failures_shared": 298, |
| "failure_jaccard": 0.6273684210526316, |
| "failure_recall_a_in_b": 0.8054054054054054, |
| "successes_a": 1825, |
| "successes_b": 1792, |
| "successes_shared": 1720, |
| "success_jaccard": 0.9066947812335266, |
| "answer_agreement": 0.9193621867881548 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 368, |
| "failures_b": 378, |
| "failures_shared": 310, |
| "failure_jaccard": 0.7110091743119266, |
| "failure_recall_a_in_b": 0.842391304347826, |
| "successes_a": 1827, |
| "successes_b": 1817, |
| "successes_shared": 1759, |
| "success_jaccard": 0.9331564986737401, |
| "answer_agreement": 0.9425968109339408 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 368, |
| "failures_b": 386, |
| "failures_shared": 316, |
| "failure_jaccard": 0.7214611872146118, |
| "failure_recall_a_in_b": 0.8586956521739131, |
| "successes_a": 1827, |
| "successes_b": 1809, |
| "successes_shared": 1757, |
| "success_jaccard": 0.9350718467269824, |
| "answer_agreement": 0.9444191343963554 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 368, |
| "failures_b": 403, |
| "failures_shared": 310, |
| "failure_jaccard": 0.6724511930585684, |
| "failure_recall_a_in_b": 0.842391304347826, |
| "successes_a": 1827, |
| "successes_b": 1792, |
| "successes_shared": 1734, |
| "success_jaccard": 0.919893899204244, |
| "answer_agreement": 0.9312072892938497 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 378, |
| "failures_b": 386, |
| "failures_shared": 323, |
| "failure_jaccard": 0.7324263038548753, |
| "failure_recall_a_in_b": 0.8544973544973545, |
| "successes_a": 1817, |
| "successes_b": 1809, |
| "successes_shared": 1754, |
| "success_jaccard": 0.936965811965812, |
| "answer_agreement": 0.9462414578587699 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 378, |
| "failures_b": 403, |
| "failures_shared": 325, |
| "failure_jaccard": 0.7127192982456141, |
| "failure_recall_a_in_b": 0.8597883597883598, |
| "successes_a": 1817, |
| "successes_b": 1792, |
| "successes_shared": 1739, |
| "success_jaccard": 0.9299465240641711, |
| "answer_agreement": 0.9403189066059225 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 386, |
| "failures_b": 403, |
| "failures_shared": 336, |
| "failure_jaccard": 0.7417218543046358, |
| "failure_recall_a_in_b": 0.8704663212435233, |
| "successes_a": 1809, |
| "successes_b": 1792, |
| "successes_shared": 1742, |
| "success_jaccard": 0.9370629370629371, |
| "answer_agreement": 0.9466970387243736 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen3.5-4B-Base", |
| "task": "youtube_level1", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 821, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 821, |
| "accuracy": 0.5420219244823387, |
| "num_failures": 376, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 821, |
| "accuracy": 0.5456760048721072, |
| "num_failures": 373, |
| "flip_rate_vs_reference": 0.10840438489646773, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 821, |
| "accuracy": 0.5602923264311814, |
| "num_failures": 361, |
| "flip_rate_vs_reference": 0.11693057247259439, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 821, |
| "accuracy": 0.5724725943970768, |
| "num_failures": 351, |
| "flip_rate_vs_reference": 0.13276492082825822, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 821, |
| "accuracy": 0.5663824604141291, |
| "num_failures": 356, |
| "flip_rate_vs_reference": 0.14981729598051158, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 821, |
| "accuracy": 0.5554202192448234, |
| "num_failures": 365, |
| "flip_rate_vs_reference": 0.15712545676004872, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.030450669914738104, |
| "stability": { |
| "always_correct": 364, |
| "always_wrong": 286, |
| "unstable": 171, |
| "always_correct_rate": 0.44336175395858707, |
| "always_wrong_rate": 0.34835566382460414, |
| "unstable_rate": 0.20828258221680876 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 376, |
| "failures_b": 373, |
| "failures_shared": 341, |
| "failure_jaccard": 0.8357843137254902, |
| "failure_recall_a_in_b": 0.9069148936170213, |
| "successes_a": 445, |
| "successes_b": 448, |
| "successes_shared": 413, |
| "success_jaccard": 0.8604166666666667, |
| "answer_agreement": 0.8915956151035322 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 376, |
| "failures_b": 361, |
| "failures_shared": 332, |
| "failure_jaccard": 0.8197530864197531, |
| "failure_recall_a_in_b": 0.8829787234042553, |
| "successes_a": 445, |
| "successes_b": 460, |
| "successes_shared": 416, |
| "success_jaccard": 0.8507157464212679, |
| "answer_agreement": 0.8830694275274056 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 376, |
| "failures_b": 351, |
| "failures_shared": 322, |
| "failure_jaccard": 0.7950617283950617, |
| "failure_recall_a_in_b": 0.8563829787234043, |
| "successes_a": 445, |
| "successes_b": 470, |
| "successes_shared": 416, |
| "success_jaccard": 0.8336673346693386, |
| "answer_agreement": 0.8672350791717418 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 376, |
| "failures_b": 356, |
| "failures_shared": 319, |
| "failure_jaccard": 0.7723970944309927, |
| "failure_recall_a_in_b": 0.848404255319149, |
| "successes_a": 445, |
| "successes_b": 465, |
| "successes_shared": 408, |
| "success_jaccard": 0.8127490039840638, |
| "answer_agreement": 0.8501827040194885 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 376, |
| "failures_b": 365, |
| "failures_shared": 320, |
| "failure_jaccard": 0.7600950118764845, |
| "failure_recall_a_in_b": 0.851063829787234, |
| "successes_a": 445, |
| "successes_b": 456, |
| "successes_shared": 400, |
| "success_jaccard": 0.7984031936127745, |
| "answer_agreement": 0.8428745432399513 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 373, |
| "failures_b": 361, |
| "failures_shared": 338, |
| "failure_jaccard": 0.8535353535353535, |
| "failure_recall_a_in_b": 0.9061662198391421, |
| "successes_a": 448, |
| "successes_b": 460, |
| "successes_shared": 425, |
| "success_jaccard": 0.8799171842650103, |
| "answer_agreement": 0.9037758830694276 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 373, |
| "failures_b": 351, |
| "failures_shared": 321, |
| "failure_jaccard": 0.7965260545905707, |
| "failure_recall_a_in_b": 0.8605898123324397, |
| "successes_a": 448, |
| "successes_b": 470, |
| "successes_shared": 418, |
| "success_jaccard": 0.836, |
| "answer_agreement": 0.8672350791717418 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 373, |
| "failures_b": 356, |
| "failures_shared": 318, |
| "failure_jaccard": 0.7737226277372263, |
| "failure_recall_a_in_b": 0.8525469168900804, |
| "successes_a": 448, |
| "successes_b": 465, |
| "successes_shared": 410, |
| "success_jaccard": 0.8151093439363817, |
| "answer_agreement": 0.8489646772228989 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 373, |
| "failures_b": 365, |
| "failures_shared": 319, |
| "failure_jaccard": 0.7613365155131265, |
| "failure_recall_a_in_b": 0.8552278820375335, |
| "successes_a": 448, |
| "successes_b": 456, |
| "successes_shared": 402, |
| "success_jaccard": 0.8007968127490039, |
| "answer_agreement": 0.8453105968331304 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 361, |
| "failures_b": 351, |
| "failures_shared": 318, |
| "failure_jaccard": 0.8071065989847716, |
| "failure_recall_a_in_b": 0.8808864265927978, |
| "successes_a": 460, |
| "successes_b": 470, |
| "successes_shared": 427, |
| "success_jaccard": 0.8489065606361829, |
| "answer_agreement": 0.876979293544458 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 361, |
| "failures_b": 356, |
| "failures_shared": 325, |
| "failure_jaccard": 0.8290816326530612, |
| "failure_recall_a_in_b": 0.9002770083102493, |
| "successes_a": 460, |
| "successes_b": 465, |
| "successes_shared": 429, |
| "success_jaccard": 0.8649193548387096, |
| "answer_agreement": 0.8855054811205847 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 361, |
| "failures_b": 365, |
| "failures_shared": 318, |
| "failure_jaccard": 0.7794117647058824, |
| "failure_recall_a_in_b": 0.8808864265927978, |
| "successes_a": 460, |
| "successes_b": 456, |
| "successes_shared": 413, |
| "success_jaccard": 0.8210735586481114, |
| "answer_agreement": 0.8587088915956151 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 351, |
| "failures_b": 356, |
| "failures_shared": 320, |
| "failure_jaccard": 0.8268733850129198, |
| "failure_recall_a_in_b": 0.9116809116809117, |
| "successes_a": 470, |
| "successes_b": 465, |
| "successes_shared": 434, |
| "success_jaccard": 0.8662674650698603, |
| "answer_agreement": 0.8842874543239951 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 351, |
| "failures_b": 365, |
| "failures_shared": 325, |
| "failure_jaccard": 0.8312020460358056, |
| "failure_recall_a_in_b": 0.9259259259259259, |
| "successes_a": 470, |
| "successes_b": 456, |
| "successes_shared": 430, |
| "success_jaccard": 0.8669354838709677, |
| "answer_agreement": 0.8964677222898904 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 356, |
| "failures_b": 365, |
| "failures_shared": 324, |
| "failure_jaccard": 0.8161209068010076, |
| "failure_recall_a_in_b": 0.9101123595505618, |
| "successes_a": 465, |
| "successes_b": 456, |
| "successes_shared": 424, |
| "success_jaccard": 0.8531187122736419, |
| "answer_agreement": 0.8891595615103532 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen3.5-4B-Base", |
| "task": "youtube_level2", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 764, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 764, |
| "accuracy": 0.4424083769633508, |
| "num_failures": 426, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 764, |
| "accuracy": 0.45287958115183247, |
| "num_failures": 418, |
| "flip_rate_vs_reference": 0.13089005235602094, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 764, |
| "accuracy": 0.450261780104712, |
| "num_failures": 420, |
| "flip_rate_vs_reference": 0.13743455497382198, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 764, |
| "accuracy": 0.45418848167539266, |
| "num_failures": 417, |
| "flip_rate_vs_reference": 0.14790575916230367, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 764, |
| "accuracy": 0.450261780104712, |
| "num_failures": 420, |
| "flip_rate_vs_reference": 0.17801047120418848, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 764, |
| "accuracy": 0.4397905759162304, |
| "num_failures": 428, |
| "flip_rate_vs_reference": 0.18848167539267016, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.014397905759162277, |
| "stability": { |
| "always_correct": 264, |
| "always_wrong": 350, |
| "unstable": 150, |
| "always_correct_rate": 0.34554973821989526, |
| "always_wrong_rate": 0.4581151832460733, |
| "unstable_rate": 0.19633507853403143 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 426, |
| "failures_b": 418, |
| "failures_shared": 389, |
| "failure_jaccard": 0.8549450549450549, |
| "failure_recall_a_in_b": 0.9131455399061033, |
| "successes_a": 338, |
| "successes_b": 346, |
| "successes_shared": 309, |
| "success_jaccard": 0.824, |
| "answer_agreement": 0.8691099476439791 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 426, |
| "failures_b": 420, |
| "failures_shared": 388, |
| "failure_jaccard": 0.8471615720524017, |
| "failure_recall_a_in_b": 0.9107981220657277, |
| "successes_a": 338, |
| "successes_b": 344, |
| "successes_shared": 306, |
| "success_jaccard": 0.8138297872340425, |
| "answer_agreement": 0.862565445026178 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 426, |
| "failures_b": 417, |
| "failures_shared": 384, |
| "failure_jaccard": 0.8366013071895425, |
| "failure_recall_a_in_b": 0.9014084507042254, |
| "successes_a": 338, |
| "successes_b": 347, |
| "successes_shared": 305, |
| "success_jaccard": 0.8026315789473685, |
| "answer_agreement": 0.8520942408376964 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 426, |
| "failures_b": 420, |
| "failures_shared": 382, |
| "failure_jaccard": 0.8232758620689655, |
| "failure_recall_a_in_b": 0.8967136150234741, |
| "successes_a": 338, |
| "successes_b": 344, |
| "successes_shared": 300, |
| "success_jaccard": 0.7853403141361257, |
| "answer_agreement": 0.8219895287958116 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 426, |
| "failures_b": 428, |
| "failures_shared": 381, |
| "failure_jaccard": 0.8054968287526427, |
| "failure_recall_a_in_b": 0.8943661971830986, |
| "successes_a": 338, |
| "successes_b": 336, |
| "successes_shared": 291, |
| "success_jaccard": 0.7597911227154047, |
| "answer_agreement": 0.8115183246073299 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 418, |
| "failures_b": 420, |
| "failures_shared": 388, |
| "failure_jaccard": 0.8622222222222222, |
| "failure_recall_a_in_b": 0.9282296650717703, |
| "successes_a": 346, |
| "successes_b": 344, |
| "successes_shared": 314, |
| "success_jaccard": 0.8351063829787234, |
| "answer_agreement": 0.8808900523560209 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 418, |
| "failures_b": 417, |
| "failures_shared": 390, |
| "failure_jaccard": 0.8764044943820225, |
| "failure_recall_a_in_b": 0.9330143540669856, |
| "successes_a": 346, |
| "successes_b": 347, |
| "successes_shared": 319, |
| "success_jaccard": 0.8529411764705882, |
| "answer_agreement": 0.8795811518324608 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 418, |
| "failures_b": 420, |
| "failures_shared": 384, |
| "failure_jaccard": 0.8458149779735683, |
| "failure_recall_a_in_b": 0.9186602870813397, |
| "successes_a": 346, |
| "successes_b": 344, |
| "successes_shared": 310, |
| "success_jaccard": 0.8157894736842105, |
| "answer_agreement": 0.8416230366492147 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 418, |
| "failures_b": 428, |
| "failures_shared": 383, |
| "failure_jaccard": 0.8272138228941684, |
| "failure_recall_a_in_b": 0.916267942583732, |
| "successes_a": 346, |
| "successes_b": 336, |
| "successes_shared": 301, |
| "success_jaccard": 0.7900262467191601, |
| "answer_agreement": 0.8350785340314136 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 420, |
| "failures_b": 417, |
| "failures_shared": 397, |
| "failure_jaccard": 0.9022727272727272, |
| "failure_recall_a_in_b": 0.9452380952380952, |
| "successes_a": 344, |
| "successes_b": 347, |
| "successes_shared": 324, |
| "success_jaccard": 0.8828337874659401, |
| "answer_agreement": 0.9070680628272252 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 420, |
| "failures_b": 420, |
| "failures_shared": 391, |
| "failure_jaccard": 0.8708240534521158, |
| "failure_recall_a_in_b": 0.930952380952381, |
| "successes_a": 344, |
| "successes_b": 344, |
| "successes_shared": 315, |
| "success_jaccard": 0.8445040214477212, |
| "answer_agreement": 0.8638743455497382 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 420, |
| "failures_b": 428, |
| "failures_shared": 386, |
| "failure_jaccard": 0.8354978354978355, |
| "failure_recall_a_in_b": 0.919047619047619, |
| "successes_a": 344, |
| "successes_b": 336, |
| "successes_shared": 302, |
| "success_jaccard": 0.798941798941799, |
| "answer_agreement": 0.8520942408376964 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 417, |
| "failures_b": 420, |
| "failures_shared": 393, |
| "failure_jaccard": 0.8851351351351351, |
| "failure_recall_a_in_b": 0.9424460431654677, |
| "successes_a": 347, |
| "successes_b": 344, |
| "successes_shared": 320, |
| "success_jaccard": 0.862533692722372, |
| "answer_agreement": 0.8808900523560209 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 417, |
| "failures_b": 428, |
| "failures_shared": 391, |
| "failure_jaccard": 0.8612334801762115, |
| "failure_recall_a_in_b": 0.9376498800959233, |
| "successes_a": 347, |
| "successes_b": 336, |
| "successes_shared": 310, |
| "success_jaccard": 0.8310991957104558, |
| "answer_agreement": 0.8730366492146597 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 420, |
| "failures_b": 428, |
| "failures_shared": 395, |
| "failure_jaccard": 0.8719646799116998, |
| "failure_recall_a_in_b": 0.9404761904761905, |
| "successes_a": 344, |
| "successes_b": 336, |
| "successes_shared": 311, |
| "success_jaccard": 0.8428184281842819, |
| "answer_agreement": 0.8848167539267016 |
| } |
| ] |
| }, |
| { |
| "model_id": "Qwen/Qwen3.5-4B-Base", |
| "task": "youtube_level3", |
| "baselines": [ |
| "0.00", |
| "0.05", |
| "0.10", |
| "0.15", |
| "0.20", |
| "0.25" |
| ], |
| "reference_baseline": "0.00", |
| "num_common_samples": 836, |
| "dropped_per_baseline": { |
| "0.00": 0, |
| "0.05": 0, |
| "0.10": 0, |
| "0.15": 0, |
| "0.20": 0, |
| "0.25": 0 |
| }, |
| "per_baseline": [ |
| { |
| "baseline": "0.00", |
| "num_samples": 836, |
| "accuracy": 0.3026315789473684, |
| "num_failures": 583, |
| "flip_rate_vs_reference": 0.0, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.05", |
| "num_samples": 836, |
| "accuracy": 0.2715311004784689, |
| "num_failures": 609, |
| "flip_rate_vs_reference": 0.24760765550239233, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.10", |
| "num_samples": 836, |
| "accuracy": 0.27751196172248804, |
| "num_failures": 604, |
| "flip_rate_vs_reference": 0.2452153110047847, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.15", |
| "num_samples": 836, |
| "accuracy": 0.26674641148325356, |
| "num_failures": 613, |
| "flip_rate_vs_reference": 0.26555023923444976, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.20", |
| "num_samples": 836, |
| "accuracy": 0.2727272727272727, |
| "num_failures": 608, |
| "flip_rate_vs_reference": 0.27631578947368424, |
| "unparsed_rate": 0.0 |
| }, |
| { |
| "baseline": "0.25", |
| "num_samples": 836, |
| "accuracy": 0.2822966507177033, |
| "num_failures": 600, |
| "flip_rate_vs_reference": 0.284688995215311, |
| "unparsed_rate": 0.0 |
| } |
| ], |
| "accuracy_spread": 0.03588516746411485, |
| "stability": { |
| "always_correct": 137, |
| "always_wrong": 482, |
| "unstable": 217, |
| "always_correct_rate": 0.1638755980861244, |
| "always_wrong_rate": 0.5765550239234449, |
| "unstable_rate": 0.2595693779904306 |
| }, |
| "pairwise": [ |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.05", |
| "failures_a": 583, |
| "failures_b": 609, |
| "failures_shared": 545, |
| "failure_jaccard": 0.8423493044822257, |
| "failure_recall_a_in_b": 0.934819897084048, |
| "successes_a": 253, |
| "successes_b": 227, |
| "successes_shared": 189, |
| "success_jaccard": 0.6494845360824743, |
| "answer_agreement": 0.7523923444976076 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.10", |
| "failures_a": 583, |
| "failures_b": 604, |
| "failures_shared": 541, |
| "failure_jaccard": 0.8374613003095975, |
| "failure_recall_a_in_b": 0.9279588336192109, |
| "successes_a": 253, |
| "successes_b": 232, |
| "successes_shared": 190, |
| "success_jaccard": 0.6440677966101694, |
| "answer_agreement": 0.7547846889952153 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.15", |
| "failures_a": 583, |
| "failures_b": 613, |
| "failures_shared": 542, |
| "failure_jaccard": 0.8287461773700305, |
| "failure_recall_a_in_b": 0.9296740994854202, |
| "successes_a": 253, |
| "successes_b": 223, |
| "successes_shared": 182, |
| "success_jaccard": 0.6190476190476191, |
| "answer_agreement": 0.7344497607655502 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.20", |
| "failures_a": 583, |
| "failures_b": 608, |
| "failures_shared": 534, |
| "failure_jaccard": 0.8127853881278538, |
| "failure_recall_a_in_b": 0.9159519725557461, |
| "successes_a": 253, |
| "successes_b": 228, |
| "successes_shared": 179, |
| "success_jaccard": 0.5927152317880795, |
| "answer_agreement": 0.7236842105263158 |
| }, |
| { |
| "baseline_a": "0.00", |
| "baseline_b": "0.25", |
| "failures_a": 583, |
| "failures_b": 600, |
| "failures_shared": 528, |
| "failure_jaccard": 0.8061068702290076, |
| "failure_recall_a_in_b": 0.9056603773584906, |
| "successes_a": 253, |
| "successes_b": 236, |
| "successes_shared": 181, |
| "success_jaccard": 0.5876623376623377, |
| "answer_agreement": 0.715311004784689 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.10", |
| "failures_a": 609, |
| "failures_b": 604, |
| "failures_shared": 563, |
| "failure_jaccard": 0.8661538461538462, |
| "failure_recall_a_in_b": 0.9244663382594417, |
| "successes_a": 227, |
| "successes_b": 232, |
| "successes_shared": 186, |
| "success_jaccard": 0.6813186813186813, |
| "answer_agreement": 0.7906698564593302 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.15", |
| "failures_a": 609, |
| "failures_b": 613, |
| "failures_shared": 567, |
| "failure_jaccard": 0.8656488549618321, |
| "failure_recall_a_in_b": 0.9310344827586207, |
| "successes_a": 227, |
| "successes_b": 223, |
| "successes_shared": 181, |
| "success_jaccard": 0.6728624535315985, |
| "answer_agreement": 0.7727272727272727 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.20", |
| "failures_a": 609, |
| "failures_b": 608, |
| "failures_shared": 562, |
| "failure_jaccard": 0.8580152671755725, |
| "failure_recall_a_in_b": 0.922824302134647, |
| "successes_a": 227, |
| "successes_b": 228, |
| "successes_shared": 181, |
| "success_jaccard": 0.6605839416058394, |
| "answer_agreement": 0.7691387559808612 |
| }, |
| { |
| "baseline_a": "0.05", |
| "baseline_b": "0.25", |
| "failures_a": 609, |
| "failures_b": 600, |
| "failures_shared": 550, |
| "failure_jaccard": 0.834597875569044, |
| "failure_recall_a_in_b": 0.90311986863711, |
| "successes_a": 227, |
| "successes_b": 236, |
| "successes_shared": 177, |
| "success_jaccard": 0.6188811188811189, |
| "answer_agreement": 0.7559808612440191 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.15", |
| "failures_a": 604, |
| "failures_b": 613, |
| "failures_shared": 569, |
| "failure_jaccard": 0.8780864197530864, |
| "failure_recall_a_in_b": 0.9420529801324503, |
| "successes_a": 232, |
| "successes_b": 223, |
| "successes_shared": 188, |
| "success_jaccard": 0.704119850187266, |
| "answer_agreement": 0.819377990430622 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.20", |
| "failures_a": 604, |
| "failures_b": 608, |
| "failures_shared": 566, |
| "failure_jaccard": 0.8761609907120743, |
| "failure_recall_a_in_b": 0.9370860927152318, |
| "successes_a": 232, |
| "successes_b": 228, |
| "successes_shared": 190, |
| "success_jaccard": 0.7037037037037037, |
| "answer_agreement": 0.8038277511961722 |
| }, |
| { |
| "baseline_a": "0.10", |
| "baseline_b": "0.25", |
| "failures_a": 604, |
| "failures_b": 600, |
| "failures_shared": 560, |
| "failure_jaccard": 0.8695652173913043, |
| "failure_recall_a_in_b": 0.9271523178807947, |
| "successes_a": 232, |
| "successes_b": 236, |
| "successes_shared": 192, |
| "success_jaccard": 0.6956521739130435, |
| "answer_agreement": 0.7954545454545454 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.20", |
| "failures_a": 613, |
| "failures_b": 608, |
| "failures_shared": 570, |
| "failure_jaccard": 0.8755760368663594, |
| "failure_recall_a_in_b": 0.9298531810766721, |
| "successes_a": 223, |
| "successes_b": 228, |
| "successes_shared": 185, |
| "success_jaccard": 0.6954887218045113, |
| "answer_agreement": 0.8098086124401914 |
| }, |
| { |
| "baseline_a": "0.15", |
| "baseline_b": "0.25", |
| "failures_a": 613, |
| "failures_b": 600, |
| "failures_shared": 563, |
| "failure_jaccard": 0.8661538461538462, |
| "failure_recall_a_in_b": 0.9184339314845025, |
| "successes_a": 223, |
| "successes_b": 236, |
| "successes_shared": 186, |
| "success_jaccard": 0.6813186813186813, |
| "answer_agreement": 0.7894736842105263 |
| }, |
| { |
| "baseline_a": "0.20", |
| "baseline_b": "0.25", |
| "failures_a": 608, |
| "failures_b": 600, |
| "failures_shared": 560, |
| "failure_jaccard": 0.8641975308641975, |
| "failure_recall_a_in_b": 0.9210526315789473, |
| "successes_a": 228, |
| "successes_b": 236, |
| "successes_shared": 188, |
| "success_jaccard": 0.6811594202898551, |
| "answer_agreement": 0.7978468899521531 |
| } |
| ] |
| } |
| ] |
| } |
