{ "result_dir": "/HDD2/disheng/Spatial_VLM_Stereo/June/Spatial_VLM_Stereo/01_3_prelimilary_intervention_different_baseline/result", "runs": [ { "model_id": "Qwen/Qwen2.5-VL-3B-Instruct", "task": "vsr", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 2195, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 2195, "accuracy": 0.7712984054669704, "num_failures": 502, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 2195, "accuracy": 0.7676537585421412, "num_failures": 510, "flip_rate_vs_reference": 0.07015945330296128, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 2195, "accuracy": 0.7594533029612757, "num_failures": 528, "flip_rate_vs_reference": 0.07927107061503418, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 2195, "accuracy": 0.7580865603644646, "num_failures": 531, "flip_rate_vs_reference": 0.08792710706150342, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 2195, "accuracy": 0.752619589977221, "num_failures": 543, "flip_rate_vs_reference": 0.09977220956719818, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 2195, "accuracy": 0.7480637813211846, "num_failures": 553, "flip_rate_vs_reference": 0.10888382687927108, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.02323462414578581, "stability": { "always_correct": 1434, "always_wrong": 332, "unstable": 429, "always_correct_rate": 0.6533029612756264, "always_wrong_rate": 0.15125284738041003, "unstable_rate": 0.19544419134396354 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 502, "failures_b": 510, "failures_shared": 429, "failure_jaccard": 0.7358490566037735, "failure_recall_a_in_b": 0.8545816733067729, "successes_a": 1693, "successes_b": 1685, "successes_shared": 1612, "success_jaccard": 0.912797281993205, "answer_agreement": 0.9298405466970388 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 502, "failures_b": 528, "failures_shared": 428, "failure_jaccard": 0.7109634551495017, "failure_recall_a_in_b": 0.852589641434263, "successes_a": 1693, "successes_b": 1667, "successes_shared": 1593, "success_jaccard": 0.9015280135823429, "answer_agreement": 0.9207289293849659 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 502, "failures_b": 531, "failures_shared": 420, "failure_jaccard": 0.6851549755301795, "failure_recall_a_in_b": 0.8366533864541833, "successes_a": 1693, "successes_b": 1664, "successes_shared": 1582, "success_jaccard": 0.8912676056338028, "answer_agreement": 0.9120728929384966 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 502, "failures_b": 543, "failures_shared": 413, "failure_jaccard": 0.6534810126582279, "failure_recall_a_in_b": 0.8227091633466136, "successes_a": 1693, "successes_b": 1652, "successes_shared": 1563, "success_jaccard": 0.877104377104377, "answer_agreement": 0.9002277904328019 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 502, "failures_b": 553, "failures_shared": 408, "failure_jaccard": 0.6306027820710973, "failure_recall_a_in_b": 0.8127490039840638, "successes_a": 1693, "successes_b": 1642, "successes_shared": 1548, "success_jaccard": 0.8662562954672636, "answer_agreement": 0.8911161731207289 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 510, "failures_b": 528, "failures_shared": 433, "failure_jaccard": 0.715702479338843, "failure_recall_a_in_b": 0.8490196078431372, "successes_a": 1685, "successes_b": 1667, "successes_shared": 1590, "success_jaccard": 0.9023836549375709, "answer_agreement": 0.9216400911161731 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 510, "failures_b": 531, "failures_shared": 434, "failure_jaccard": 0.71499176276771, "failure_recall_a_in_b": 0.8509803921568627, "successes_a": 1685, "successes_b": 1664, "successes_shared": 1588, "success_jaccard": 0.9017603634298694, "answer_agreement": 0.9211845102505695 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 510, "failures_b": 543, "failures_shared": 429, "failure_jaccard": 0.6875, "failure_recall_a_in_b": 0.8411764705882353, "successes_a": 1685, "successes_b": 1652, "successes_shared": 1571, "success_jaccard": 0.8895809739524349, "answer_agreement": 0.9111617312072893 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 510, "failures_b": 553, "failures_shared": 423, "failure_jaccard": 0.6609375, "failure_recall_a_in_b": 0.8294117647058824, "successes_a": 1685, "successes_b": 1642, "successes_shared": 1555, "success_jaccard": 0.8775395033860045, "answer_agreement": 0.9011389521640091 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 528, "failures_b": 531, "failures_shared": 445, "failure_jaccard": 0.7247557003257329, "failure_recall_a_in_b": 0.8428030303030303, "successes_a": 1667, "successes_b": 1664, "successes_shared": 1581, "success_jaccard": 0.9034285714285715, "answer_agreement": 0.9230068337129841 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 528, "failures_b": 543, "failures_shared": 441, "failure_jaccard": 0.7, "failure_recall_a_in_b": 0.8352272727272727, "successes_a": 1667, "successes_b": 1652, "successes_shared": 1565, "success_jaccard": 0.8922462941847207, "answer_agreement": 0.9138952164009112 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 528, "failures_b": 553, "failures_shared": 430, "failure_jaccard": 0.6605222734254992, "failure_recall_a_in_b": 0.8143939393939394, "successes_a": 1667, "successes_b": 1642, "successes_shared": 1544, "success_jaccard": 0.8747875354107649, "answer_agreement": 0.8993166287015946 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 531, "failures_b": 543, "failures_shared": 447, "failure_jaccard": 0.7129186602870813, "failure_recall_a_in_b": 0.8418079096045198, "successes_a": 1664, "successes_b": 1652, "successes_shared": 1568, "success_jaccard": 0.897025171624714, "answer_agreement": 0.9179954441913439 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 531, "failures_b": 553, "failures_shared": 445, "failure_jaccard": 0.6964006259780907, "failure_recall_a_in_b": 0.8380414312617702, "successes_a": 1664, "successes_b": 1642, "successes_shared": 1556, "success_jaccard": 0.8891428571428571, "answer_agreement": 0.911617312072893 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 543, "failures_b": 553, "failures_shared": 460, "failure_jaccard": 0.7232704402515723, "failure_recall_a_in_b": 0.8471454880294659, "successes_a": 1652, "successes_b": 1642, "successes_shared": 1559, "success_jaccard": 0.8985590778097983, "answer_agreement": 0.9198177676537586 } ] }, { "model_id": "Qwen/Qwen2.5-VL-3B-Instruct", "task": "youtube_level1", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 821, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 821, "accuracy": 0.3995127892813642, "num_failures": 493, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 821, "accuracy": 0.4056029232643118, "num_failures": 488, "flip_rate_vs_reference": 0.14859926918392205, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 821, "accuracy": 0.41169305724725946, "num_failures": 483, "flip_rate_vs_reference": 0.1559074299634592, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 821, "accuracy": 0.4104750304506699, "num_failures": 484, "flip_rate_vs_reference": 0.16321559074299635, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 821, "accuracy": 0.3995127892813642, "num_failures": 493, "flip_rate_vs_reference": 0.1814859926918392, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 821, "accuracy": 0.4007308160779537, "num_failures": 492, "flip_rate_vs_reference": 0.19366626065773446, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.012180267965895275, "stability": { "always_correct": 247, "always_wrong": 391, "unstable": 183, "always_correct_rate": 0.30085261875761266, "always_wrong_rate": 0.4762484774665043, "unstable_rate": 0.22289890377588306 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 493, "failures_b": 488, "failures_shared": 450, "failure_jaccard": 0.847457627118644, "failure_recall_a_in_b": 0.9127789046653144, "successes_a": 328, "successes_b": 333, "successes_shared": 290, "success_jaccard": 0.7816711590296496, "answer_agreement": 0.8514007308160779 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 493, "failures_b": 483, "failures_shared": 451, "failure_jaccard": 0.8590476190476191, "failure_recall_a_in_b": 0.9148073022312373, "successes_a": 328, "successes_b": 338, "successes_shared": 296, "success_jaccard": 0.8, "answer_agreement": 0.8440925700365408 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 493, "failures_b": 484, "failures_shared": 450, "failure_jaccard": 0.8538899430740038, "failure_recall_a_in_b": 0.9127789046653144, "successes_a": 328, "successes_b": 337, "successes_shared": 294, "success_jaccard": 0.7924528301886793, "answer_agreement": 0.8367844092570037 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 493, "failures_b": 493, "failures_shared": 445, "failure_jaccard": 0.822550831792976, "failure_recall_a_in_b": 0.9026369168356998, "successes_a": 328, "successes_b": 328, "successes_shared": 280, "success_jaccard": 0.7446808510638298, "answer_agreement": 0.8185140073081608 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 493, "failures_b": 492, "failures_shared": 441, "failure_jaccard": 0.8106617647058824, "failure_recall_a_in_b": 0.8945233265720081, "successes_a": 328, "successes_b": 329, "successes_shared": 277, "success_jaccard": 0.7289473684210527, "answer_agreement": 0.8063337393422655 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 488, "failures_b": 483, "failures_shared": 449, "failure_jaccard": 0.8601532567049809, "failure_recall_a_in_b": 0.9200819672131147, "successes_a": 333, "successes_b": 338, "successes_shared": 299, "success_jaccard": 0.803763440860215, "answer_agreement": 0.8440925700365408 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 488, "failures_b": 484, "failures_shared": 447, "failure_jaccard": 0.8514285714285714, "failure_recall_a_in_b": 0.9159836065573771, "successes_a": 333, "successes_b": 337, "successes_shared": 296, "success_jaccard": 0.7914438502673797, "answer_agreement": 0.8392204628501827 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 488, "failures_b": 493, "failures_shared": 454, "failure_jaccard": 0.8614800759013282, "failure_recall_a_in_b": 0.930327868852459, "successes_a": 333, "successes_b": 328, "successes_shared": 294, "success_jaccard": 0.8010899182561307, "answer_agreement": 0.833130328867235 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 488, "failures_b": 492, "failures_shared": 442, "failure_jaccard": 0.8215613382899628, "failure_recall_a_in_b": 0.9057377049180327, "successes_a": 333, "successes_b": 329, "successes_shared": 283, "success_jaccard": 0.7467018469656992, "answer_agreement": 0.8063337393422655 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 483, "failures_b": 484, "failures_shared": 446, "failure_jaccard": 0.8560460652591171, "failure_recall_a_in_b": 0.9233954451345756, "successes_a": 338, "successes_b": 337, "successes_shared": 300, "success_jaccard": 0.8, "answer_agreement": 0.8453105968331304 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 483, "failures_b": 493, "failures_shared": 450, "failure_jaccard": 0.8555133079847909, "failure_recall_a_in_b": 0.9316770186335404, "successes_a": 338, "successes_b": 328, "successes_shared": 295, "success_jaccard": 0.7951482479784366, "answer_agreement": 0.8550548112058465 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 483, "failures_b": 492, "failures_shared": 445, "failure_jaccard": 0.839622641509434, "failure_recall_a_in_b": 0.9213250517598344, "successes_a": 338, "successes_b": 329, "successes_shared": 291, "success_jaccard": 0.773936170212766, "answer_agreement": 0.8270401948842875 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 484, "failures_b": 493, "failures_shared": 452, "failure_jaccard": 0.8609523809523809, "failure_recall_a_in_b": 0.9338842975206612, "successes_a": 337, "successes_b": 328, "successes_shared": 296, "success_jaccard": 0.8021680216802168, "answer_agreement": 0.8514007308160779 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 484, "failures_b": 492, "failures_shared": 441, "failure_jaccard": 0.8242990654205608, "failure_recall_a_in_b": 0.9111570247933884, "successes_a": 337, "successes_b": 329, "successes_shared": 286, "success_jaccard": 0.7526315789473684, "answer_agreement": 0.8136419001218027 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 493, "failures_b": 492, "failures_shared": 459, "failure_jaccard": 0.8726235741444867, "failure_recall_a_in_b": 0.9310344827586207, "successes_a": 328, "successes_b": 329, "successes_shared": 295, "success_jaccard": 0.8149171270718232, "answer_agreement": 0.8660170523751523 } ] }, { "model_id": "Qwen/Qwen2.5-VL-3B-Instruct", "task": "youtube_level2", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 764, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 764, "accuracy": 0.362565445026178, "num_failures": 487, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 764, "accuracy": 0.3743455497382199, "num_failures": 478, "flip_rate_vs_reference": 0.1387434554973822, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 764, "accuracy": 0.3782722513089005, "num_failures": 475, "flip_rate_vs_reference": 0.17277486910994763, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 764, "accuracy": 0.3795811518324607, "num_failures": 474, "flip_rate_vs_reference": 0.1793193717277487, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 764, "accuracy": 0.38089005235602097, "num_failures": 473, "flip_rate_vs_reference": 0.17408376963350786, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 764, "accuracy": 0.3704188481675393, "num_failures": 481, "flip_rate_vs_reference": 0.2107329842931937, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.01832460732984298, "stability": { "always_correct": 198, "always_wrong": 392, "unstable": 174, "always_correct_rate": 0.2591623036649215, "always_wrong_rate": 0.5130890052356021, "unstable_rate": 0.22774869109947643 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 487, "failures_b": 478, "failures_shared": 450, "failure_jaccard": 0.8737864077669902, "failure_recall_a_in_b": 0.9240246406570842, "successes_a": 277, "successes_b": 286, "successes_shared": 249, "success_jaccard": 0.7929936305732485, "answer_agreement": 0.8612565445026178 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 487, "failures_b": 475, "failures_shared": 441, "failure_jaccard": 0.8464491362763915, "failure_recall_a_in_b": 0.9055441478439425, "successes_a": 277, "successes_b": 289, "successes_shared": 243, "success_jaccard": 0.7523219814241486, "answer_agreement": 0.8272251308900523 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 487, "failures_b": 474, "failures_shared": 437, "failure_jaccard": 0.833969465648855, "failure_recall_a_in_b": 0.8973305954825462, "successes_a": 277, "successes_b": 290, "successes_shared": 240, "success_jaccard": 0.7339449541284404, "answer_agreement": 0.8206806282722513 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 487, "failures_b": 473, "failures_shared": 437, "failure_jaccard": 0.8355640535372849, "failure_recall_a_in_b": 0.8973305954825462, "successes_a": 277, "successes_b": 291, "successes_shared": 241, "success_jaccard": 0.7370030581039755, "answer_agreement": 0.8259162303664922 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 487, "failures_b": 481, "failures_shared": 435, "failure_jaccard": 0.8161350844277674, "failure_recall_a_in_b": 0.893223819301848, "successes_a": 277, "successes_b": 283, "successes_shared": 231, "success_jaccard": 0.7021276595744681, "answer_agreement": 0.7892670157068062 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 478, "failures_b": 475, "failures_shared": 444, "failure_jaccard": 0.8722986247544204, "failure_recall_a_in_b": 0.9288702928870293, "successes_a": 286, "successes_b": 289, "successes_shared": 255, "success_jaccard": 0.796875, "answer_agreement": 0.8547120418848168 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 478, "failures_b": 474, "failures_shared": 438, "failure_jaccard": 0.8521400778210116, "failure_recall_a_in_b": 0.9163179916317992, "successes_a": 286, "successes_b": 290, "successes_shared": 250, "success_jaccard": 0.7668711656441718, "answer_agreement": 0.8416230366492147 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 478, "failures_b": 473, "failures_shared": 441, "failure_jaccard": 0.8647058823529412, "failure_recall_a_in_b": 0.9225941422594143, "successes_a": 286, "successes_b": 291, "successes_shared": 254, "success_jaccard": 0.7863777089783281, "answer_agreement": 0.8586387434554974 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 478, "failures_b": 481, "failures_shared": 435, "failure_jaccard": 0.8301526717557252, "failure_recall_a_in_b": 0.9100418410041841, "successes_a": 286, "successes_b": 283, "successes_shared": 240, "success_jaccard": 0.729483282674772, "answer_agreement": 0.8141361256544503 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 475, "failures_b": 474, "failures_shared": 436, "failure_jaccard": 0.8499025341130604, "failure_recall_a_in_b": 0.9178947368421052, "successes_a": 289, "successes_b": 290, "successes_shared": 251, "success_jaccard": 0.7652439024390244, "answer_agreement": 0.8390052356020943 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 475, "failures_b": 473, "failures_shared": 435, "failure_jaccard": 0.847953216374269, "failure_recall_a_in_b": 0.9157894736842105, "successes_a": 289, "successes_b": 291, "successes_shared": 251, "success_jaccard": 0.7629179331306991, "answer_agreement": 0.8390052356020943 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 475, "failures_b": 481, "failures_shared": 436, "failure_jaccard": 0.8384615384615385, "failure_recall_a_in_b": 0.9178947368421052, "successes_a": 289, "successes_b": 283, "successes_shared": 244, "success_jaccard": 0.7439024390243902, "answer_agreement": 0.8180628272251309 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 474, "failures_b": 473, "failures_shared": 436, "failure_jaccard": 0.8532289628180039, "failure_recall_a_in_b": 0.919831223628692, "successes_a": 290, "successes_b": 291, "successes_shared": 253, "success_jaccard": 0.7713414634146342, "answer_agreement": 0.8534031413612565 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 474, "failures_b": 481, "failures_shared": 438, "failure_jaccard": 0.8471953578336557, "failure_recall_a_in_b": 0.9240506329113924, "successes_a": 290, "successes_b": 283, "successes_shared": 247, "success_jaccard": 0.7576687116564417, "answer_agreement": 0.837696335078534 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 473, "failures_b": 481, "failures_shared": 439, "failure_jaccard": 0.8524271844660194, "failure_recall_a_in_b": 0.9281183932346723, "successes_a": 291, "successes_b": 283, "successes_shared": 249, "success_jaccard": 0.7661538461538462, "answer_agreement": 0.8298429319371727 } ] }, { "model_id": "Qwen/Qwen2.5-VL-3B-Instruct", "task": "youtube_level3", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 836, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 836, "accuracy": 0.26674641148325356, "num_failures": 613, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 836, "accuracy": 0.2452153110047847, "num_failures": 631, "flip_rate_vs_reference": 0.2452153110047847, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 836, "accuracy": 0.23923444976076555, "num_failures": 636, "flip_rate_vs_reference": 0.23803827751196172, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 836, "accuracy": 0.25239234449760767, "num_failures": 625, "flip_rate_vs_reference": 0.26435406698564595, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 836, "accuracy": 0.2595693779904306, "num_failures": 619, "flip_rate_vs_reference": 0.25478468899521534, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 836, "accuracy": 0.26555023923444976, "num_failures": 614, "flip_rate_vs_reference": 0.27392344497607657, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.027511961722488015, "stability": { "always_correct": 111, "always_wrong": 502, "unstable": 223, "always_correct_rate": 0.13277511961722488, "always_wrong_rate": 0.6004784688995215, "unstable_rate": 0.26674641148325356 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 613, "failures_b": 631, "failures_shared": 569, "failure_jaccard": 0.8429629629629629, "failure_recall_a_in_b": 0.9282218597063622, "successes_a": 223, "successes_b": 205, "successes_shared": 161, "success_jaccard": 0.602996254681648, "answer_agreement": 0.7547846889952153 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 613, "failures_b": 636, "failures_shared": 570, "failure_jaccard": 0.8394698085419735, "failure_recall_a_in_b": 0.9298531810766721, "successes_a": 223, "successes_b": 200, "successes_shared": 157, "success_jaccard": 0.5902255639097744, "answer_agreement": 0.7619617224880383 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 613, "failures_b": 625, "failures_shared": 562, "failure_jaccard": 0.8313609467455622, "failure_recall_a_in_b": 0.9168026101141925, "successes_a": 223, "successes_b": 211, "successes_shared": 160, "success_jaccard": 0.583941605839416, "answer_agreement": 0.7356459330143541 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 613, "failures_b": 619, "failures_shared": 561, "failure_jaccard": 0.8360655737704918, "failure_recall_a_in_b": 0.9151712887438825, "successes_a": 223, "successes_b": 217, "successes_shared": 165, "success_jaccard": 0.6, "answer_agreement": 0.7452153110047847 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 613, "failures_b": 614, "failures_shared": 551, "failure_jaccard": 0.8150887573964497, "failure_recall_a_in_b": 0.8988580750407831, "successes_a": 223, "successes_b": 222, "successes_shared": 160, "success_jaccard": 0.5614035087719298, "answer_agreement": 0.7260765550239234 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 631, "failures_b": 636, "failures_shared": 588, "failure_jaccard": 0.865979381443299, "failure_recall_a_in_b": 0.9318541996830428, "successes_a": 205, "successes_b": 200, "successes_shared": 157, "success_jaccard": 0.6330645161290323, "answer_agreement": 0.7811004784688995 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 631, "failures_b": 625, "failures_shared": 580, "failure_jaccard": 0.8579881656804734, "failure_recall_a_in_b": 0.919175911251981, "successes_a": 205, "successes_b": 211, "successes_shared": 160, "success_jaccard": 0.625, "answer_agreement": 0.7667464114832536 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 631, "failures_b": 619, "failures_shared": 571, "failure_jaccard": 0.8409425625920471, "failure_recall_a_in_b": 0.9049128367670365, "successes_a": 205, "successes_b": 217, "successes_shared": 157, "success_jaccard": 0.5924528301886792, "answer_agreement": 0.7667464114832536 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 631, "failures_b": 614, "failures_shared": 569, "failure_jaccard": 0.841715976331361, "failure_recall_a_in_b": 0.901743264659271, "successes_a": 205, "successes_b": 222, "successes_shared": 160, "success_jaccard": 0.599250936329588, "answer_agreement": 0.7464114832535885 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 636, "failures_b": 625, "failures_shared": 582, "failure_jaccard": 0.8571428571428571, "failure_recall_a_in_b": 0.9150943396226415, "successes_a": 200, "successes_b": 211, "successes_shared": 157, "success_jaccard": 0.6181102362204725, "answer_agreement": 0.7739234449760766 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 636, "failures_b": 619, "failures_shared": 582, "failure_jaccard": 0.8647845468053492, "failure_recall_a_in_b": 0.9150943396226415, "successes_a": 200, "successes_b": 217, "successes_shared": 163, "success_jaccard": 0.6417322834645669, "answer_agreement": 0.7918660287081339 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 636, "failures_b": 614, "failures_shared": 578, "failure_jaccard": 0.8601190476190477, "failure_recall_a_in_b": 0.9088050314465409, "successes_a": 200, "successes_b": 222, "successes_shared": 164, "success_jaccard": 0.6356589147286822, "answer_agreement": 0.7715311004784688 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 625, "failures_b": 619, "failures_shared": 574, "failure_jaccard": 0.8567164179104477, "failure_recall_a_in_b": 0.9184, "successes_a": 211, "successes_b": 217, "successes_shared": 166, "success_jaccard": 0.6335877862595419, "answer_agreement": 0.7799043062200957 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 625, "failures_b": 614, "failures_shared": 573, "failure_jaccard": 0.8603603603603603, "failure_recall_a_in_b": 0.9168, "successes_a": 211, "successes_b": 222, "successes_shared": 170, "success_jaccard": 0.6463878326996197, "answer_agreement": 0.7811004784688995 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 619, "failures_b": 614, "failures_shared": 578, "failure_jaccard": 0.8824427480916031, "failure_recall_a_in_b": 0.9337641357027464, "successes_a": 217, "successes_b": 222, "successes_shared": 181, "success_jaccard": 0.7015503875968992, "answer_agreement": 0.8014354066985646 } ] }, { "model_id": "Qwen/Qwen2.5-VL-7B-Instruct", "task": "vsr", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 2195, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 2195, "accuracy": 0.8223234624145785, "num_failures": 390, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 2195, "accuracy": 0.8077448747152619, "num_failures": 422, "flip_rate_vs_reference": 0.06013667425968109, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 2195, "accuracy": 0.8027334851936219, "num_failures": 433, "flip_rate_vs_reference": 0.07334851936218678, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 2195, "accuracy": 0.8068337129840547, "num_failures": 424, "flip_rate_vs_reference": 0.06560364464692484, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 2195, "accuracy": 0.8022779043280183, "num_failures": 434, "flip_rate_vs_reference": 0.08109339407744874, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 2195, "accuracy": 0.7881548974943052, "num_failures": 465, "flip_rate_vs_reference": 0.09339407744874716, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.034168564920273314, "stability": { "always_correct": 1579, "always_wrong": 268, "unstable": 348, "always_correct_rate": 0.7193621867881549, "always_wrong_rate": 0.12209567198177676, "unstable_rate": 0.15854214123006835 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 390, "failures_b": 422, "failures_shared": 340, "failure_jaccard": 0.7203389830508474, "failure_recall_a_in_b": 0.8717948717948718, "successes_a": 1805, "successes_b": 1773, "successes_shared": 1723, "success_jaccard": 0.9288409703504044, "answer_agreement": 0.939863325740319 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 390, "failures_b": 433, "failures_shared": 331, "failure_jaccard": 0.6727642276422764, "failure_recall_a_in_b": 0.8487179487179487, "successes_a": 1805, "successes_b": 1762, "successes_shared": 1703, "success_jaccard": 0.9136266094420601, "answer_agreement": 0.9266514806378132 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 390, "failures_b": 424, "failures_shared": 335, "failure_jaccard": 0.6993736951983298, "failure_recall_a_in_b": 0.8589743589743589, "successes_a": 1805, "successes_b": 1771, "successes_shared": 1716, "success_jaccard": 0.9225806451612903, "answer_agreement": 0.9343963553530752 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 390, "failures_b": 434, "failures_shared": 323, "failure_jaccard": 0.6447105788423154, "failure_recall_a_in_b": 0.8282051282051283, "successes_a": 1805, "successes_b": 1761, "successes_shared": 1694, "success_jaccard": 0.9049145299145299, "answer_agreement": 0.9189066059225512 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 390, "failures_b": 465, "failures_shared": 325, "failure_jaccard": 0.6132075471698113, "failure_recall_a_in_b": 0.8333333333333334, "successes_a": 1805, "successes_b": 1730, "successes_shared": 1665, "success_jaccard": 0.8903743315508021, "answer_agreement": 0.9066059225512528 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 422, "failures_b": 433, "failures_shared": 362, "failure_jaccard": 0.7342799188640974, "failure_recall_a_in_b": 0.8578199052132701, "successes_a": 1773, "successes_b": 1762, "successes_shared": 1702, "success_jaccard": 0.928532460447354, "answer_agreement": 0.9403189066059225 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 422, "failures_b": 424, "failures_shared": 355, "failure_jaccard": 0.7230142566191446, "failure_recall_a_in_b": 0.8412322274881516, "successes_a": 1773, "successes_b": 1771, "successes_shared": 1704, "success_jaccard": 0.9260869565217391, "answer_agreement": 0.9380410022779043 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 422, "failures_b": 434, "failures_shared": 345, "failure_jaccard": 0.675146771037182, "failure_recall_a_in_b": 0.8175355450236966, "successes_a": 1773, "successes_b": 1761, "successes_shared": 1684, "success_jaccard": 0.9102702702702703, "answer_agreement": 0.924373576309795 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 422, "failures_b": 465, "failures_shared": 348, "failure_jaccard": 0.6456400742115028, "failure_recall_a_in_b": 0.8246445497630331, "successes_a": 1773, "successes_b": 1730, "successes_shared": 1656, "success_jaccard": 0.8965890633459664, "answer_agreement": 0.9129840546697039 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 433, "failures_b": 424, "failures_shared": 359, "failure_jaccard": 0.7208835341365462, "failure_recall_a_in_b": 0.8290993071593533, "successes_a": 1762, "successes_b": 1771, "successes_shared": 1697, "success_jaccard": 0.9242919389978214, "answer_agreement": 0.9366742596810934 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 433, "failures_b": 434, "failures_shared": 357, "failure_jaccard": 0.7, "failure_recall_a_in_b": 0.8244803695150116, "successes_a": 1762, "successes_b": 1761, "successes_shared": 1685, "success_jaccard": 0.9167573449401524, "answer_agreement": 0.9302961275626423 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 433, "failures_b": 465, "failures_shared": 360, "failure_jaccard": 0.6691449814126395, "failure_recall_a_in_b": 0.8314087759815243, "successes_a": 1762, "successes_b": 1730, "successes_shared": 1657, "success_jaccard": 0.9029972752043597, "answer_agreement": 0.9189066059225512 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 424, "failures_b": 434, "failures_shared": 361, "failure_jaccard": 0.7263581488933601, "failure_recall_a_in_b": 0.8514150943396226, "successes_a": 1771, "successes_b": 1761, "successes_shared": 1698, "success_jaccard": 0.925845147219193, "answer_agreement": 0.9380410022779043 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 424, "failures_b": 465, "failures_shared": 363, "failure_jaccard": 0.6901140684410646, "failure_recall_a_in_b": 0.8561320754716981, "successes_a": 1771, "successes_b": 1730, "successes_shared": 1669, "success_jaccard": 0.9110262008733624, "answer_agreement": 0.9257403189066059 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 434, "failures_b": 465, "failures_shared": 376, "failure_jaccard": 0.7189292543021033, "failure_recall_a_in_b": 0.8663594470046083, "successes_a": 1761, "successes_b": 1730, "successes_shared": 1672, "success_jaccard": 0.9191863661352392, "answer_agreement": 0.9330296127562643 } ] }, { "model_id": "Qwen/Qwen2.5-VL-7B-Instruct", "task": "youtube_level1", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 821, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 821, "accuracy": 0.4470158343483557, "num_failures": 454, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 821, "accuracy": 0.4482338611449452, "num_failures": 453, "flip_rate_vs_reference": 0.1218026796589525, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 821, "accuracy": 0.45554202192448234, "num_failures": 447, "flip_rate_vs_reference": 0.146163215590743, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 821, "accuracy": 0.46650426309378806, "num_failures": 438, "flip_rate_vs_reference": 0.14981729598051158, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 821, "accuracy": 0.44214372716199757, "num_failures": 458, "flip_rate_vs_reference": 0.17295980511571254, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 821, "accuracy": 0.4518879415347138, "num_failures": 450, "flip_rate_vs_reference": 0.20219244823386115, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.024360535931790495, "stability": { "always_correct": 284, "always_wrong": 366, "unstable": 171, "always_correct_rate": 0.3459196102314251, "always_wrong_rate": 0.4457978075517661, "unstable_rate": 0.20828258221680876 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 454, "failures_b": 453, "failures_shared": 417, "failure_jaccard": 0.8510204081632653, "failure_recall_a_in_b": 0.9185022026431718, "successes_a": 367, "successes_b": 368, "successes_shared": 331, "success_jaccard": 0.8193069306930693, "answer_agreement": 0.8781973203410475 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 454, "failures_b": 447, "failures_shared": 413, "failure_jaccard": 0.8463114754098361, "failure_recall_a_in_b": 0.9096916299559471, "successes_a": 367, "successes_b": 374, "successes_shared": 333, "success_jaccard": 0.8161764705882353, "answer_agreement": 0.853836784409257 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 454, "failures_b": 438, "failures_shared": 405, "failure_jaccard": 0.8316221765913757, "failure_recall_a_in_b": 0.8920704845814978, "successes_a": 367, "successes_b": 383, "successes_shared": 334, "success_jaccard": 0.8028846153846154, "answer_agreement": 0.8501827040194885 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 454, "failures_b": 458, "failures_shared": 409, "failure_jaccard": 0.8131212723658051, "failure_recall_a_in_b": 0.9008810572687225, "successes_a": 367, "successes_b": 363, "successes_shared": 318, "success_jaccard": 0.7718446601941747, "answer_agreement": 0.8270401948842875 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 454, "failures_b": 450, "failures_shared": 394, "failure_jaccard": 0.7725490196078432, "failure_recall_a_in_b": 0.8678414096916299, "successes_a": 367, "successes_b": 371, "successes_shared": 311, "success_jaccard": 0.7283372365339579, "answer_agreement": 0.7978075517661388 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 453, "failures_b": 447, "failures_shared": 419, "failure_jaccard": 0.8711018711018711, "failure_recall_a_in_b": 0.9249448123620309, "successes_a": 368, "successes_b": 374, "successes_shared": 340, "success_jaccard": 0.845771144278607, "answer_agreement": 0.8830694275274056 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 453, "failures_b": 438, "failures_shared": 412, "failure_jaccard": 0.860125260960334, "failure_recall_a_in_b": 0.9094922737306843, "successes_a": 368, "successes_b": 383, "successes_shared": 342, "success_jaccard": 0.8361858190709046, "answer_agreement": 0.8842874543239951 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 453, "failures_b": 458, "failures_shared": 416, "failure_jaccard": 0.8404040404040404, "failure_recall_a_in_b": 0.9183222958057395, "successes_a": 368, "successes_b": 363, "successes_shared": 326, "success_jaccard": 0.8049382716049382, "answer_agreement": 0.8623629719853837 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 453, "failures_b": 450, "failures_shared": 408, "failure_jaccard": 0.8242424242424242, "failure_recall_a_in_b": 0.9006622516556292, "successes_a": 368, "successes_b": 371, "successes_shared": 326, "success_jaccard": 0.7893462469733656, "answer_agreement": 0.8428745432399513 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 447, "failures_b": 438, "failures_shared": 414, "failure_jaccard": 0.8789808917197452, "failure_recall_a_in_b": 0.9261744966442953, "successes_a": 374, "successes_b": 383, "successes_shared": 350, "success_jaccard": 0.85995085995086, "answer_agreement": 0.8842874543239951 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 447, "failures_b": 458, "failures_shared": 420, "failure_jaccard": 0.865979381443299, "failure_recall_a_in_b": 0.9395973154362416, "successes_a": 374, "successes_b": 363, "successes_shared": 336, "success_jaccard": 0.8379052369077307, "answer_agreement": 0.8721071863580999 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 447, "failures_b": 450, "failures_shared": 411, "failure_jaccard": 0.845679012345679, "failure_recall_a_in_b": 0.9194630872483222, "successes_a": 374, "successes_b": 371, "successes_shared": 335, "success_jaccard": 0.8170731707317073, "answer_agreement": 0.8501827040194885 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 438, "failures_b": 458, "failures_shared": 416, "failure_jaccard": 0.8666666666666667, "failure_recall_a_in_b": 0.9497716894977168, "successes_a": 383, "successes_b": 363, "successes_shared": 341, "success_jaccard": 0.8419753086419753, "answer_agreement": 0.8952496954933008 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 438, "failures_b": 450, "failures_shared": 406, "failure_jaccard": 0.8423236514522822, "failure_recall_a_in_b": 0.9269406392694064, "successes_a": 383, "successes_b": 371, "successes_shared": 339, "success_jaccard": 0.8168674698795181, "answer_agreement": 0.8611449451887941 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 458, "failures_b": 450, "failures_shared": 423, "failure_jaccard": 0.8721649484536083, "failure_recall_a_in_b": 0.9235807860262009, "successes_a": 363, "successes_b": 371, "successes_shared": 336, "success_jaccard": 0.8442211055276382, "answer_agreement": 0.879415347137637 } ] }, { "model_id": "Qwen/Qwen2.5-VL-7B-Instruct", "task": "youtube_level2", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 764, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 764, "accuracy": 0.4031413612565445, "num_failures": 456, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 764, "accuracy": 0.39397905759162305, "num_failures": 463, "flip_rate_vs_reference": 0.14397905759162305, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 764, "accuracy": 0.39790575916230364, "num_failures": 460, "flip_rate_vs_reference": 0.13219895287958114, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 764, "accuracy": 0.4018324607329843, "num_failures": 457, "flip_rate_vs_reference": 0.17277486910994763, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 764, "accuracy": 0.3769633507853403, "num_failures": 476, "flip_rate_vs_reference": 0.18979057591623036, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 764, "accuracy": 0.3769633507853403, "num_failures": 476, "flip_rate_vs_reference": 0.20026178010471204, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.02617801047120416, "stability": { "always_correct": 221, "always_wrong": 382, "unstable": 161, "always_correct_rate": 0.2892670157068063, "always_wrong_rate": 0.5, "unstable_rate": 0.2107329842931937 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 456, "failures_b": 463, "failures_shared": 429, "failure_jaccard": 0.8755102040816326, "failure_recall_a_in_b": 0.9407894736842105, "successes_a": 308, "successes_b": 301, "successes_shared": 274, "success_jaccard": 0.817910447761194, "answer_agreement": 0.856020942408377 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 456, "failures_b": 460, "failures_shared": 428, "failure_jaccard": 0.8770491803278688, "failure_recall_a_in_b": 0.9385964912280702, "successes_a": 308, "successes_b": 304, "successes_shared": 276, "success_jaccard": 0.8214285714285714, "answer_agreement": 0.8678010471204188 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 456, "failures_b": 457, "failures_shared": 415, "failure_jaccard": 0.8333333333333334, "failure_recall_a_in_b": 0.9100877192982456, "successes_a": 308, "successes_b": 307, "successes_shared": 266, "success_jaccard": 0.7621776504297995, "answer_agreement": 0.8272251308900523 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 456, "failures_b": 476, "failures_shared": 421, "failure_jaccard": 0.8238747553816047, "failure_recall_a_in_b": 0.9232456140350878, "successes_a": 308, "successes_b": 288, "successes_shared": 253, "success_jaccard": 0.7376093294460642, "answer_agreement": 0.8102094240837696 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 456, "failures_b": 476, "failures_shared": 418, "failure_jaccard": 0.8132295719844358, "failure_recall_a_in_b": 0.9166666666666666, "successes_a": 308, "successes_b": 288, "successes_shared": 250, "success_jaccard": 0.7225433526011561, "answer_agreement": 0.7997382198952879 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 463, "failures_b": 460, "failures_shared": 436, "failure_jaccard": 0.8952772073921971, "failure_recall_a_in_b": 0.9416846652267818, "successes_a": 301, "successes_b": 304, "successes_shared": 277, "success_jaccard": 0.8445121951219512, "answer_agreement": 0.8835078534031413 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 463, "failures_b": 457, "failures_shared": 423, "failure_jaccard": 0.8511066398390342, "failure_recall_a_in_b": 0.9136069114470843, "successes_a": 301, "successes_b": 307, "successes_shared": 267, "success_jaccard": 0.782991202346041, "answer_agreement": 0.8481675392670157 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 463, "failures_b": 476, "failures_shared": 439, "failure_jaccard": 0.878, "failure_recall_a_in_b": 0.9481641468682506, "successes_a": 301, "successes_b": 288, "successes_shared": 264, "success_jaccard": 0.8123076923076923, "answer_agreement": 0.8586387434554974 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 463, "failures_b": 476, "failures_shared": 433, "failure_jaccard": 0.8557312252964426, "failure_recall_a_in_b": 0.9352051835853131, "successes_a": 301, "successes_b": 288, "successes_shared": 258, "success_jaccard": 0.7794561933534743, "answer_agreement": 0.849476439790576 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 460, "failures_b": 457, "failures_shared": 426, "failure_jaccard": 0.8676171079429735, "failure_recall_a_in_b": 0.9260869565217391, "successes_a": 304, "successes_b": 307, "successes_shared": 273, "success_jaccard": 0.8076923076923077, "answer_agreement": 0.8691099476439791 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 460, "failures_b": 476, "failures_shared": 439, "failure_jaccard": 0.8832997987927566, "failure_recall_a_in_b": 0.9543478260869566, "successes_a": 304, "successes_b": 288, "successes_shared": 267, "success_jaccard": 0.8215384615384616, "answer_agreement": 0.8717277486910995 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 460, "failures_b": 476, "failures_shared": 434, "failure_jaccard": 0.8645418326693227, "failure_recall_a_in_b": 0.9434782608695652, "successes_a": 304, "successes_b": 288, "successes_shared": 262, "success_jaccard": 0.793939393939394, "answer_agreement": 0.8455497382198953 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 457, "failures_b": 476, "failures_shared": 430, "failure_jaccard": 0.8548707753479126, "failure_recall_a_in_b": 0.9409190371991247, "successes_a": 307, "successes_b": 288, "successes_shared": 261, "success_jaccard": 0.781437125748503, "answer_agreement": 0.8638743455497382 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 457, "failures_b": 476, "failures_shared": 427, "failure_jaccard": 0.8438735177865613, "failure_recall_a_in_b": 0.9343544857768052, "successes_a": 307, "successes_b": 288, "successes_shared": 258, "success_jaccard": 0.7655786350148368, "answer_agreement": 0.8337696335078534 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 476, "failures_b": 476, "failures_shared": 451, "failure_jaccard": 0.9001996007984032, "failure_recall_a_in_b": 0.9474789915966386, "successes_a": 288, "successes_b": 288, "successes_shared": 263, "success_jaccard": 0.8402555910543131, "answer_agreement": 0.8887434554973822 } ] }, { "model_id": "Qwen/Qwen2.5-VL-7B-Instruct", "task": "youtube_level3", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 836, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 836, "accuracy": 0.2799043062200957, "num_failures": 602, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 836, "accuracy": 0.28708133971291866, "num_failures": 596, "flip_rate_vs_reference": 0.19736842105263158, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 836, "accuracy": 0.2679425837320574, "num_failures": 612, "flip_rate_vs_reference": 0.20454545454545456, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 836, "accuracy": 0.2822966507177033, "num_failures": 600, "flip_rate_vs_reference": 0.2284688995215311, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 836, "accuracy": 0.27631578947368424, "num_failures": 605, "flip_rate_vs_reference": 0.2452153110047847, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 836, "accuracy": 0.27392344497607657, "num_failures": 607, "flip_rate_vs_reference": 0.23205741626794257, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.019138755980861233, "stability": { "always_correct": 143, "always_wrong": 503, "unstable": 190, "always_correct_rate": 0.17105263157894737, "always_wrong_rate": 0.6016746411483254, "unstable_rate": 0.22727272727272727 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 602, "failures_b": 596, "failures_shared": 555, "failure_jaccard": 0.8631415241057543, "failure_recall_a_in_b": 0.9219269102990033, "successes_a": 234, "successes_b": 240, "successes_shared": 193, "success_jaccard": 0.6868327402135231, "answer_agreement": 0.8026315789473685 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 602, "failures_b": 612, "failures_shared": 567, "failure_jaccard": 0.8763523956723338, "failure_recall_a_in_b": 0.9418604651162791, "successes_a": 234, "successes_b": 224, "successes_shared": 189, "success_jaccard": 0.7026022304832714, "answer_agreement": 0.7954545454545454 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 602, "failures_b": 600, "failures_shared": 556, "failure_jaccard": 0.8606811145510835, "failure_recall_a_in_b": 0.9235880398671097, "successes_a": 234, "successes_b": 236, "successes_shared": 190, "success_jaccard": 0.6785714285714286, "answer_agreement": 0.7715311004784688 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 602, "failures_b": 605, "failures_shared": 551, "failure_jaccard": 0.8399390243902439, "failure_recall_a_in_b": 0.915282392026578, "successes_a": 234, "successes_b": 231, "successes_shared": 180, "success_jaccard": 0.631578947368421, "answer_agreement": 0.7547846889952153 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 602, "failures_b": 607, "failures_shared": 556, "failure_jaccard": 0.8514548238897397, "failure_recall_a_in_b": 0.9235880398671097, "successes_a": 234, "successes_b": 229, "successes_shared": 183, "success_jaccard": 0.6535714285714286, "answer_agreement": 0.7679425837320574 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 596, "failures_b": 612, "failures_shared": 561, "failure_jaccard": 0.8670788253477589, "failure_recall_a_in_b": 0.9412751677852349, "successes_a": 240, "successes_b": 224, "successes_shared": 189, "success_jaccard": 0.6872727272727273, "answer_agreement": 0.8205741626794258 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 596, "failures_b": 600, "failures_shared": 559, "failure_jaccard": 0.8775510204081632, "failure_recall_a_in_b": 0.9379194630872483, "successes_a": 240, "successes_b": 236, "successes_shared": 199, "success_jaccard": 0.7184115523465704, "answer_agreement": 0.8098086124401914 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 596, "failures_b": 605, "failures_shared": 554, "failure_jaccard": 0.8562596599690881, "failure_recall_a_in_b": 0.9295302013422819, "successes_a": 240, "successes_b": 231, "successes_shared": 189, "success_jaccard": 0.6702127659574468, "answer_agreement": 0.7954545454545454 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 596, "failures_b": 607, "failures_shared": 552, "failure_jaccard": 0.847926267281106, "failure_recall_a_in_b": 0.9261744966442953, "successes_a": 240, "successes_b": 229, "successes_shared": 185, "success_jaccard": 0.6514084507042254, "answer_agreement": 0.7715311004784688 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 612, "failures_b": 600, "failures_shared": 565, "failure_jaccard": 0.8732612055641422, "failure_recall_a_in_b": 0.923202614379085, "successes_a": 224, "successes_b": 236, "successes_shared": 189, "success_jaccard": 0.6974169741697417, "answer_agreement": 0.8169856459330144 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 612, "failures_b": 605, "failures_shared": 566, "failure_jaccard": 0.869431643625192, "failure_recall_a_in_b": 0.9248366013071896, "successes_a": 224, "successes_b": 231, "successes_shared": 185, "success_jaccard": 0.6851851851851852, "answer_agreement": 0.8169856459330144 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 612, "failures_b": 607, "failures_shared": 566, "failure_jaccard": 0.8667687595712098, "failure_recall_a_in_b": 0.9248366013071896, "successes_a": 224, "successes_b": 229, "successes_shared": 183, "success_jaccard": 0.6777777777777778, "answer_agreement": 0.8002392344497608 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 600, "failures_b": 605, "failures_shared": 565, "failure_jaccard": 0.8828125, "failure_recall_a_in_b": 0.9416666666666667, "successes_a": 236, "successes_b": 231, "successes_shared": 196, "success_jaccard": 0.7232472324723247, "answer_agreement": 0.8217703349282297 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 600, "failures_b": 607, "failures_shared": 560, "failure_jaccard": 0.865533230293663, "failure_recall_a_in_b": 0.9333333333333333, "successes_a": 236, "successes_b": 229, "successes_shared": 189, "success_jaccard": 0.6847826086956522, "answer_agreement": 0.7942583732057417 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 605, "failures_b": 607, "failures_shared": 564, "failure_jaccard": 0.8703703703703703, "failure_recall_a_in_b": 0.9322314049586777, "successes_a": 231, "successes_b": 229, "successes_shared": 188, "success_jaccard": 0.6911764705882353, "answer_agreement": 0.8014354066985646 } ] }, { "model_id": "Qwen/Qwen3.5-4B", "task": "vsr", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 2195, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 2195, "accuracy": 0.8214123006833713, "num_failures": 392, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 2195, "accuracy": 0.8173120728929385, "num_failures": 401, "flip_rate_vs_reference": 0.04510250569476082, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 2195, "accuracy": 0.8186788154897494, "num_failures": 398, "flip_rate_vs_reference": 0.04738041002277904, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 2195, "accuracy": 0.8113895216400911, "num_failures": 414, "flip_rate_vs_reference": 0.06287015945330296, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 2195, "accuracy": 0.8072892938496583, "num_failures": 423, "flip_rate_vs_reference": 0.07425968109339408, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 2195, "accuracy": 0.8045558086560365, "num_failures": 429, "flip_rate_vs_reference": 0.07608200455580866, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.016856492027334857, "stability": { "always_correct": 1635, "always_wrong": 283, "unstable": 277, "always_correct_rate": 0.744874715261959, "always_wrong_rate": 0.12892938496583142, "unstable_rate": 0.12619589977220957 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 392, "failures_b": 401, "failures_shared": 347, "failure_jaccard": 0.7780269058295964, "failure_recall_a_in_b": 0.8852040816326531, "successes_a": 1803, "successes_b": 1794, "successes_shared": 1749, "success_jaccard": 0.9464285714285714, "answer_agreement": 0.9548974943052392 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 392, "failures_b": 398, "failures_shared": 343, "failure_jaccard": 0.767337807606264, "failure_recall_a_in_b": 0.875, "successes_a": 1803, "successes_b": 1797, "successes_shared": 1748, "success_jaccard": 0.9438444924406048, "answer_agreement": 0.952619589977221 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 392, "failures_b": 414, "failures_shared": 334, "failure_jaccard": 0.7076271186440678, "failure_recall_a_in_b": 0.8520408163265306, "successes_a": 1803, "successes_b": 1781, "successes_shared": 1723, "success_jaccard": 0.9258463191832348, "answer_agreement": 0.937129840546697 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 392, "failures_b": 423, "failures_shared": 326, "failure_jaccard": 0.6666666666666666, "failure_recall_a_in_b": 0.8316326530612245, "successes_a": 1803, "successes_b": 1772, "successes_shared": 1706, "success_jaccard": 0.9127875869448903, "answer_agreement": 0.9257403189066059 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 392, "failures_b": 429, "failures_shared": 327, "failure_jaccard": 0.6619433198380567, "failure_recall_a_in_b": 0.8341836734693877, "successes_a": 1803, "successes_b": 1766, "successes_shared": 1701, "success_jaccard": 0.9105995717344754, "answer_agreement": 0.9239179954441914 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 401, "failures_b": 398, "failures_shared": 356, "failure_jaccard": 0.8036117381489842, "failure_recall_a_in_b": 0.8877805486284289, "successes_a": 1794, "successes_b": 1797, "successes_shared": 1752, "success_jaccard": 0.9526916802610114, "answer_agreement": 0.9603644646924829 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 401, "failures_b": 414, "failures_shared": 350, "failure_jaccard": 0.7526881720430108, "failure_recall_a_in_b": 0.8728179551122195, "successes_a": 1794, "successes_b": 1781, "successes_shared": 1730, "success_jaccard": 0.9376693766937669, "answer_agreement": 0.9476082004555809 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 401, "failures_b": 423, "failures_shared": 341, "failure_jaccard": 0.7060041407867494, "failure_recall_a_in_b": 0.8503740648379052, "successes_a": 1794, "successes_b": 1772, "successes_shared": 1712, "success_jaccard": 0.9234088457389428, "answer_agreement": 0.9353075170842825 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 401, "failures_b": 429, "failures_shared": 337, "failure_jaccard": 0.6835699797160243, "failure_recall_a_in_b": 0.8403990024937655, "successes_a": 1794, "successes_b": 1766, "successes_shared": 1702, "success_jaccard": 0.9160387513455328, "answer_agreement": 0.9289293849658314 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 398, "failures_b": 414, "failures_shared": 355, "failure_jaccard": 0.7768052516411379, "failure_recall_a_in_b": 0.8919597989949749, "successes_a": 1797, "successes_b": 1781, "successes_shared": 1738, "success_jaccard": 0.9445652173913044, "answer_agreement": 0.9535307517084283 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 398, "failures_b": 423, "failures_shared": 350, "failure_jaccard": 0.7430997876857749, "failure_recall_a_in_b": 0.8793969849246231, "successes_a": 1797, "successes_b": 1772, "successes_shared": 1724, "success_jaccard": 0.9344173441734417, "answer_agreement": 0.944874715261959 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 398, "failures_b": 429, "failures_shared": 341, "failure_jaccard": 0.7016460905349794, "failure_recall_a_in_b": 0.8567839195979899, "successes_a": 1797, "successes_b": 1766, "successes_shared": 1709, "success_jaccard": 0.9217907227615966, "answer_agreement": 0.9339407744874715 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 414, "failures_b": 423, "failures_shared": 359, "failure_jaccard": 0.7510460251046025, "failure_recall_a_in_b": 0.8671497584541062, "successes_a": 1781, "successes_b": 1772, "successes_shared": 1717, "success_jaccard": 0.9351851851851852, "answer_agreement": 0.9457858769931663 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 414, "failures_b": 429, "failures_shared": 354, "failure_jaccard": 0.7239263803680982, "failure_recall_a_in_b": 0.855072463768116, "successes_a": 1781, "successes_b": 1766, "successes_shared": 1706, "success_jaccard": 0.9266702878870179, "answer_agreement": 0.9384965831435079 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 423, "failures_b": 429, "failures_shared": 372, "failure_jaccard": 0.775, "failure_recall_a_in_b": 0.8794326241134752, "successes_a": 1772, "successes_b": 1766, "successes_shared": 1715, "success_jaccard": 0.9407569939659901, "answer_agreement": 0.9507972665148063 } ] }, { "model_id": "Qwen/Qwen3.5-4B", "task": "youtube_level1", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 821, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 821, "accuracy": 0.5274056029232643, "num_failures": 388, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0036540803897685747 }, { "baseline": "0.05", "num_samples": 821, "accuracy": 0.5213154689403167, "num_failures": 393, "flip_rate_vs_reference": 0.10353227771010962, "unparsed_rate": 0.007308160779537149 }, { "baseline": "0.10", "num_samples": 821, "accuracy": 0.5188794153471377, "num_failures": 395, "flip_rate_vs_reference": 0.12545676004872108, "unparsed_rate": 0.001218026796589525 }, { "baseline": "0.15", "num_samples": 821, "accuracy": 0.5408038976857491, "num_failures": 377, "flip_rate_vs_reference": 0.14007308160779536, "unparsed_rate": 0.0048721071863581 }, { "baseline": "0.20", "num_samples": 821, "accuracy": 0.5347137637028014, "num_failures": 382, "flip_rate_vs_reference": 0.15956151035322777, "unparsed_rate": 0.001218026796589525 }, { "baseline": "0.25", "num_samples": 821, "accuracy": 0.5347137637028014, "num_failures": 382, "flip_rate_vs_reference": 0.1656516443361754, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.02192448233861144, "stability": { "always_correct": 343, "always_wrong": 296, "unstable": 182, "always_correct_rate": 0.4177831912302071, "always_wrong_rate": 0.36053593179049936, "unstable_rate": 0.22168087697929353 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 388, "failures_b": 393, "failures_shared": 361, "failure_jaccard": 0.8595238095238096, "failure_recall_a_in_b": 0.9304123711340206, "successes_a": 433, "successes_b": 428, "successes_shared": 401, "success_jaccard": 0.8717391304347826, "answer_agreement": 0.8964677222898904 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 388, "failures_b": 395, "failures_shared": 354, "failure_jaccard": 0.8251748251748252, "failure_recall_a_in_b": 0.9123711340206185, "successes_a": 433, "successes_b": 426, "successes_shared": 392, "success_jaccard": 0.8394004282655246, "answer_agreement": 0.8745432399512789 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 388, "failures_b": 377, "failures_shared": 342, "failure_jaccard": 0.8085106382978723, "failure_recall_a_in_b": 0.8814432989690721, "successes_a": 433, "successes_b": 444, "successes_shared": 398, "success_jaccard": 0.8308977035490606, "answer_agreement": 0.8599269183922046 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 388, "failures_b": 382, "failures_shared": 339, "failure_jaccard": 0.7865429234338747, "failure_recall_a_in_b": 0.8737113402061856, "successes_a": 433, "successes_b": 439, "successes_shared": 390, "success_jaccard": 0.8091286307053942, "answer_agreement": 0.8404384896467723 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 388, "failures_b": 382, "failures_shared": 338, "failure_jaccard": 0.7824074074074074, "failure_recall_a_in_b": 0.8711340206185567, "successes_a": 433, "successes_b": 439, "successes_shared": 389, "success_jaccard": 0.8053830227743272, "answer_agreement": 0.8343483556638246 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 393, "failures_b": 395, "failures_shared": 363, "failure_jaccard": 0.8541176470588235, "failure_recall_a_in_b": 0.9236641221374046, "successes_a": 428, "successes_b": 426, "successes_shared": 396, "success_jaccard": 0.8646288209606987, "answer_agreement": 0.8855054811205847 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 393, "failures_b": 377, "failures_shared": 342, "failure_jaccard": 0.7990654205607477, "failure_recall_a_in_b": 0.8702290076335878, "successes_a": 428, "successes_b": 444, "successes_shared": 393, "success_jaccard": 0.8204592901878914, "answer_agreement": 0.8635809987819733 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 393, "failures_b": 382, "failures_shared": 343, "failure_jaccard": 0.7939814814814815, "failure_recall_a_in_b": 0.8727735368956743, "successes_a": 428, "successes_b": 439, "successes_shared": 389, "success_jaccard": 0.8138075313807531, "answer_agreement": 0.8453105968331304 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 393, "failures_b": 382, "failures_shared": 338, "failure_jaccard": 0.7734553775743707, "failure_recall_a_in_b": 0.8600508905852418, "successes_a": 428, "successes_b": 439, "successes_shared": 384, "success_jaccard": 0.7950310559006211, "answer_agreement": 0.8258221680876979 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 395, "failures_b": 377, "failures_shared": 349, "failure_jaccard": 0.8250591016548463, "failure_recall_a_in_b": 0.8835443037974684, "successes_a": 426, "successes_b": 444, "successes_shared": 398, "success_jaccard": 0.8432203389830508, "answer_agreement": 0.8781973203410475 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 395, "failures_b": 382, "failures_shared": 350, "failure_jaccard": 0.819672131147541, "failure_recall_a_in_b": 0.8860759493670886, "successes_a": 426, "successes_b": 439, "successes_shared": 394, "success_jaccard": 0.8365180467091295, "answer_agreement": 0.8684531059683313 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 395, "failures_b": 382, "failures_shared": 347, "failure_jaccard": 0.8069767441860465, "failure_recall_a_in_b": 0.8784810126582279, "successes_a": 426, "successes_b": 439, "successes_shared": 391, "success_jaccard": 0.8248945147679325, "answer_agreement": 0.8514007308160779 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 377, "failures_b": 382, "failures_shared": 341, "failure_jaccard": 0.8157894736842105, "failure_recall_a_in_b": 0.9045092838196287, "successes_a": 444, "successes_b": 439, "successes_shared": 403, "success_jaccard": 0.8395833333333333, "answer_agreement": 0.8721071863580999 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 377, "failures_b": 382, "failures_shared": 338, "failure_jaccard": 0.8028503562945368, "failure_recall_a_in_b": 0.896551724137931, "successes_a": 444, "successes_b": 439, "successes_shared": 400, "success_jaccard": 0.8281573498964804, "answer_agreement": 0.8611449451887941 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 382, "failures_b": 382, "failures_shared": 346, "failure_jaccard": 0.8277511961722488, "failure_recall_a_in_b": 0.9057591623036649, "successes_a": 439, "successes_b": 439, "successes_shared": 403, "success_jaccard": 0.848421052631579, "answer_agreement": 0.8781973203410475 } ] }, { "model_id": "Qwen/Qwen3.5-4B", "task": "youtube_level2", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 764, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 764, "accuracy": 0.3913612565445026, "num_failures": 465, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 764, "accuracy": 0.3900523560209424, "num_failures": 466, "flip_rate_vs_reference": 0.12172774869109948, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 764, "accuracy": 0.38219895287958117, "num_failures": 472, "flip_rate_vs_reference": 0.14136125654450263, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 764, "accuracy": 0.387434554973822, "num_failures": 468, "flip_rate_vs_reference": 0.16361256544502617, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 764, "accuracy": 0.3795811518324607, "num_failures": 474, "flip_rate_vs_reference": 0.17146596858638743, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 764, "accuracy": 0.38481675392670156, "num_failures": 470, "flip_rate_vs_reference": 0.18717277486910994, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.011780104712041883, "stability": { "always_correct": 230, "always_wrong": 401, "unstable": 133, "always_correct_rate": 0.3010471204188482, "always_wrong_rate": 0.524869109947644, "unstable_rate": 0.17408376963350786 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 465, "failures_b": 466, "failures_shared": 438, "failure_jaccard": 0.8884381338742393, "failure_recall_a_in_b": 0.9419354838709677, "successes_a": 299, "successes_b": 298, "successes_shared": 271, "success_jaccard": 0.8312883435582822, "answer_agreement": 0.8782722513089005 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 465, "failures_b": 472, "failures_shared": 438, "failure_jaccard": 0.8777555110220441, "failure_recall_a_in_b": 0.9419354838709677, "successes_a": 299, "successes_b": 292, "successes_shared": 265, "success_jaccard": 0.8128834355828221, "answer_agreement": 0.8586387434554974 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 465, "failures_b": 468, "failures_shared": 433, "failure_jaccard": 0.866, "failure_recall_a_in_b": 0.9311827956989247, "successes_a": 299, "successes_b": 296, "successes_shared": 264, "success_jaccard": 0.797583081570997, "answer_agreement": 0.8363874345549738 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 465, "failures_b": 474, "failures_shared": 434, "failure_jaccard": 0.8594059405940594, "failure_recall_a_in_b": 0.9333333333333333, "successes_a": 299, "successes_b": 290, "successes_shared": 259, "success_jaccard": 0.7848484848484848, "answer_agreement": 0.8285340314136126 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 465, "failures_b": 470, "failures_shared": 423, "failure_jaccard": 0.826171875, "failure_recall_a_in_b": 0.9096774193548387, "successes_a": 299, "successes_b": 294, "successes_shared": 252, "success_jaccard": 0.7390029325513197, "answer_agreement": 0.81282722513089 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 466, "failures_b": 472, "failures_shared": 441, "failure_jaccard": 0.8873239436619719, "failure_recall_a_in_b": 0.9463519313304721, "successes_a": 298, "successes_b": 292, "successes_shared": 267, "success_jaccard": 0.826625386996904, "answer_agreement": 0.8821989528795812 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 466, "failures_b": 468, "failures_shared": 438, "failure_jaccard": 0.8830645161290323, "failure_recall_a_in_b": 0.9399141630901288, "successes_a": 298, "successes_b": 296, "successes_shared": 268, "success_jaccard": 0.8220858895705522, "answer_agreement": 0.8664921465968587 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 466, "failures_b": 474, "failures_shared": 437, "failure_jaccard": 0.8687872763419483, "failure_recall_a_in_b": 0.9377682403433476, "successes_a": 298, "successes_b": 290, "successes_shared": 261, "success_jaccard": 0.7981651376146789, "answer_agreement": 0.856020942408377 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 466, "failures_b": 470, "failures_shared": 432, "failure_jaccard": 0.8571428571428571, "failure_recall_a_in_b": 0.927038626609442, "successes_a": 298, "successes_b": 294, "successes_shared": 260, "success_jaccard": 0.7831325301204819, "answer_agreement": 0.8599476439790575 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 472, "failures_b": 468, "failures_shared": 445, "failure_jaccard": 0.898989898989899, "failure_recall_a_in_b": 0.9427966101694916, "successes_a": 292, "successes_b": 296, "successes_shared": 269, "success_jaccard": 0.8432601880877743, "answer_agreement": 0.8821989528795812 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 472, "failures_b": 474, "failures_shared": 447, "failure_jaccard": 0.8957915831663327, "failure_recall_a_in_b": 0.9470338983050848, "successes_a": 292, "successes_b": 290, "successes_shared": 265, "success_jaccard": 0.8359621451104101, "answer_agreement": 0.8678010471204188 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 472, "failures_b": 470, "failures_shared": 438, "failure_jaccard": 0.8690476190476191, "failure_recall_a_in_b": 0.9279661016949152, "successes_a": 292, "successes_b": 294, "successes_shared": 260, "success_jaccard": 0.7975460122699386, "answer_agreement": 0.856020942408377 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 468, "failures_b": 474, "failures_shared": 446, "failure_jaccard": 0.8991935483870968, "failure_recall_a_in_b": 0.9529914529914529, "successes_a": 296, "successes_b": 290, "successes_shared": 268, "success_jaccard": 0.8427672955974843, "answer_agreement": 0.8704188481675392 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 468, "failures_b": 470, "failures_shared": 440, "failure_jaccard": 0.8835341365461847, "failure_recall_a_in_b": 0.9401709401709402, "successes_a": 296, "successes_b": 294, "successes_shared": 266, "success_jaccard": 0.8209876543209876, "answer_agreement": 0.8704188481675392 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 474, "failures_b": 470, "failures_shared": 443, "failure_jaccard": 0.8842315369261478, "failure_recall_a_in_b": 0.9345991561181435, "successes_a": 290, "successes_b": 294, "successes_shared": 263, "success_jaccard": 0.8193146417445483, "answer_agreement": 0.8520942408376964 } ] }, { "model_id": "Qwen/Qwen3.5-4B", "task": "youtube_level3", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 836, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 836, "accuracy": 0.2930622009569378, "num_failures": 591, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 836, "accuracy": 0.29904306220095694, "num_failures": 586, "flip_rate_vs_reference": 0.23803827751196172, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 836, "accuracy": 0.2966507177033493, "num_failures": 588, "flip_rate_vs_reference": 0.2595693779904306, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 836, "accuracy": 0.3038277511961722, "num_failures": 582, "flip_rate_vs_reference": 0.2595693779904306, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 836, "accuracy": 0.28827751196172247, "num_failures": 595, "flip_rate_vs_reference": 0.27870813397129185, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 836, "accuracy": 0.2930622009569378, "num_failures": 591, "flip_rate_vs_reference": 0.29545454545454547, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.015550239234449759, "stability": { "always_correct": 144, "always_wrong": 474, "unstable": 218, "always_correct_rate": 0.1722488038277512, "always_wrong_rate": 0.5669856459330144, "unstable_rate": 0.2607655502392344 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 591, "failures_b": 586, "failures_shared": 539, "failure_jaccard": 0.8448275862068966, "failure_recall_a_in_b": 0.9120135363790186, "successes_a": 245, "successes_b": 250, "successes_shared": 198, "success_jaccard": 0.6666666666666666, "answer_agreement": 0.7619617224880383 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 591, "failures_b": 588, "failures_shared": 534, "failure_jaccard": 0.827906976744186, "failure_recall_a_in_b": 0.9035532994923858, "successes_a": 245, "successes_b": 248, "successes_shared": 191, "success_jaccard": 0.6324503311258278, "answer_agreement": 0.7404306220095693 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 591, "failures_b": 582, "failures_shared": 530, "failure_jaccard": 0.8242612752721618, "failure_recall_a_in_b": 0.8967851099830795, "successes_a": 245, "successes_b": 254, "successes_shared": 193, "success_jaccard": 0.630718954248366, "answer_agreement": 0.7404306220095693 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 591, "failures_b": 595, "failures_shared": 531, "failure_jaccard": 0.8106870229007633, "failure_recall_a_in_b": 0.8984771573604061, "successes_a": 245, "successes_b": 241, "successes_shared": 181, "success_jaccard": 0.5934426229508196, "answer_agreement": 0.7212918660287081 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 591, "failures_b": 591, "failures_shared": 525, "failure_jaccard": 0.7990867579908676, "failure_recall_a_in_b": 0.8883248730964467, "successes_a": 245, "successes_b": 245, "successes_shared": 179, "success_jaccard": 0.5755627009646302, "answer_agreement": 0.7045454545454546 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 586, "failures_b": 588, "failures_shared": 546, "failure_jaccard": 0.8694267515923567, "failure_recall_a_in_b": 0.931740614334471, "successes_a": 250, "successes_b": 248, "successes_shared": 208, "success_jaccard": 0.7172413793103448, "answer_agreement": 0.8229665071770335 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 586, "failures_b": 582, "failures_shared": 540, "failure_jaccard": 0.8598726114649682, "failure_recall_a_in_b": 0.9215017064846417, "successes_a": 250, "successes_b": 254, "successes_shared": 208, "success_jaccard": 0.7027027027027027, "answer_agreement": 0.7918660287081339 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 586, "failures_b": 595, "failures_shared": 546, "failure_jaccard": 0.8598425196850393, "failure_recall_a_in_b": 0.931740614334471, "successes_a": 250, "successes_b": 241, "successes_shared": 201, "success_jaccard": 0.6931034482758621, "answer_agreement": 0.7811004784688995 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 586, "failures_b": 591, "failures_shared": 541, "failure_jaccard": 0.85062893081761, "failure_recall_a_in_b": 0.9232081911262798, "successes_a": 250, "successes_b": 245, "successes_shared": 200, "success_jaccard": 0.6779661016949152, "answer_agreement": 0.7822966507177034 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 588, "failures_b": 582, "failures_shared": 544, "failure_jaccard": 0.8690095846645367, "failure_recall_a_in_b": 0.9251700680272109, "successes_a": 248, "successes_b": 254, "successes_shared": 210, "success_jaccard": 0.7191780821917808, "answer_agreement": 0.8145933014354066 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 588, "failures_b": 595, "failures_shared": 555, "failure_jaccard": 0.8837579617834395, "failure_recall_a_in_b": 0.9438775510204082, "successes_a": 248, "successes_b": 241, "successes_shared": 208, "success_jaccard": 0.7402135231316725, "answer_agreement": 0.8050239234449761 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 588, "failures_b": 591, "failures_shared": 538, "failure_jaccard": 0.8393135725429017, "failure_recall_a_in_b": 0.9149659863945578, "successes_a": 248, "successes_b": 245, "successes_shared": 195, "success_jaccard": 0.6543624161073825, "answer_agreement": 0.7870813397129187 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 582, "failures_b": 595, "failures_shared": 551, "failure_jaccard": 0.8801916932907349, "failure_recall_a_in_b": 0.9467353951890034, "successes_a": 254, "successes_b": 241, "successes_shared": 210, "success_jaccard": 0.7368421052631579, "answer_agreement": 0.819377990430622 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 582, "failures_b": 591, "failures_shared": 539, "failure_jaccard": 0.8501577287066246, "failure_recall_a_in_b": 0.9261168384879725, "successes_a": 254, "successes_b": 245, "successes_shared": 202, "success_jaccard": 0.6801346801346801, "answer_agreement": 0.7930622009569378 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 595, "failures_b": 591, "failures_shared": 550, "failure_jaccard": 0.8647798742138365, "failure_recall_a_in_b": 0.9243697478991597, "successes_a": 241, "successes_b": 245, "successes_shared": 200, "success_jaccard": 0.6993006993006993, "answer_agreement": 0.7918660287081339 } ] }, { "model_id": "Qwen/Qwen3.5-4B-Base", "task": "vsr", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 2195, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 2195, "accuracy": 0.8318906605922551, "num_failures": 369, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 2195, "accuracy": 0.8314350797266514, "num_failures": 370, "flip_rate_vs_reference": 0.04601366742596811, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 2195, "accuracy": 0.8323462414578587, "num_failures": 368, "flip_rate_vs_reference": 0.05330296127562642, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 2195, "accuracy": 0.8277904328018223, "num_failures": 378, "flip_rate_vs_reference": 0.05968109339407745, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 2195, "accuracy": 0.8241457858769932, "num_failures": 386, "flip_rate_vs_reference": 0.07881548974943052, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 2195, "accuracy": 0.8164009111617312, "num_failures": 403, "flip_rate_vs_reference": 0.0856492027334852, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.015945330296127547, "stability": { "always_correct": 1653, "always_wrong": 240, "unstable": 302, "always_correct_rate": 0.7530751708428246, "always_wrong_rate": 0.10933940774487472, "unstable_rate": 0.13758542141230068 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 369, "failures_b": 370, "failures_shared": 319, "failure_jaccard": 0.7595238095238095, "failure_recall_a_in_b": 0.8644986449864499, "successes_a": 1826, "successes_b": 1825, "successes_shared": 1775, "success_jaccard": 0.9461620469083155, "answer_agreement": 0.9539863325740319 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 369, "failures_b": 368, "failures_shared": 310, "failure_jaccard": 0.7259953161592506, "failure_recall_a_in_b": 0.8401084010840109, "successes_a": 1826, "successes_b": 1827, "successes_shared": 1768, "success_jaccard": 0.9379310344827586, "answer_agreement": 0.9466970387243736 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 369, "failures_b": 378, "failures_shared": 308, "failure_jaccard": 0.7015945330296127, "failure_recall_a_in_b": 0.8346883468834688, "successes_a": 1826, "successes_b": 1817, "successes_shared": 1756, "success_jaccard": 0.9305776364599894, "answer_agreement": 0.9403189066059225 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 369, "failures_b": 386, "failures_shared": 291, "failure_jaccard": 0.6271551724137931, "failure_recall_a_in_b": 0.7886178861788617, "successes_a": 1826, "successes_b": 1809, "successes_shared": 1731, "success_jaccard": 0.9091386554621849, "answer_agreement": 0.9211845102505695 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 369, "failures_b": 403, "failures_shared": 292, "failure_jaccard": 0.6083333333333333, "failure_recall_a_in_b": 0.7913279132791328, "successes_a": 1826, "successes_b": 1792, "successes_shared": 1715, "success_jaccard": 0.9012086179716238, "answer_agreement": 0.9143507972665148 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 370, "failures_b": 368, "failures_shared": 319, "failure_jaccard": 0.7613365155131265, "failure_recall_a_in_b": 0.8621621621621621, "successes_a": 1825, "successes_b": 1827, "successes_shared": 1776, "success_jaccard": 0.9466950959488273, "answer_agreement": 0.9544419134396356 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 370, "failures_b": 378, "failures_shared": 310, "failure_jaccard": 0.7077625570776256, "failure_recall_a_in_b": 0.8378378378378378, "successes_a": 1825, "successes_b": 1817, "successes_shared": 1757, "success_jaccard": 0.9320954907161804, "answer_agreement": 0.9416856492027335 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 370, "failures_b": 386, "failures_shared": 302, "failure_jaccard": 0.6651982378854625, "failure_recall_a_in_b": 0.8162162162162162, "successes_a": 1825, "successes_b": 1809, "successes_shared": 1741, "success_jaccard": 0.9197041732699419, "answer_agreement": 0.9307517084282461 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 370, "failures_b": 403, "failures_shared": 298, "failure_jaccard": 0.6273684210526316, "failure_recall_a_in_b": 0.8054054054054054, "successes_a": 1825, "successes_b": 1792, "successes_shared": 1720, "success_jaccard": 0.9066947812335266, "answer_agreement": 0.9193621867881548 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 368, "failures_b": 378, "failures_shared": 310, "failure_jaccard": 0.7110091743119266, "failure_recall_a_in_b": 0.842391304347826, "successes_a": 1827, "successes_b": 1817, "successes_shared": 1759, "success_jaccard": 0.9331564986737401, "answer_agreement": 0.9425968109339408 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 368, "failures_b": 386, "failures_shared": 316, "failure_jaccard": 0.7214611872146118, "failure_recall_a_in_b": 0.8586956521739131, "successes_a": 1827, "successes_b": 1809, "successes_shared": 1757, "success_jaccard": 0.9350718467269824, "answer_agreement": 0.9444191343963554 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 368, "failures_b": 403, "failures_shared": 310, "failure_jaccard": 0.6724511930585684, "failure_recall_a_in_b": 0.842391304347826, "successes_a": 1827, "successes_b": 1792, "successes_shared": 1734, "success_jaccard": 0.919893899204244, "answer_agreement": 0.9312072892938497 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 378, "failures_b": 386, "failures_shared": 323, "failure_jaccard": 0.7324263038548753, "failure_recall_a_in_b": 0.8544973544973545, "successes_a": 1817, "successes_b": 1809, "successes_shared": 1754, "success_jaccard": 0.936965811965812, "answer_agreement": 0.9462414578587699 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 378, "failures_b": 403, "failures_shared": 325, "failure_jaccard": 0.7127192982456141, "failure_recall_a_in_b": 0.8597883597883598, "successes_a": 1817, "successes_b": 1792, "successes_shared": 1739, "success_jaccard": 0.9299465240641711, "answer_agreement": 0.9403189066059225 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 386, "failures_b": 403, "failures_shared": 336, "failure_jaccard": 0.7417218543046358, "failure_recall_a_in_b": 0.8704663212435233, "successes_a": 1809, "successes_b": 1792, "successes_shared": 1742, "success_jaccard": 0.9370629370629371, "answer_agreement": 0.9466970387243736 } ] }, { "model_id": "Qwen/Qwen3.5-4B-Base", "task": "youtube_level1", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 821, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 821, "accuracy": 0.5420219244823387, "num_failures": 376, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 821, "accuracy": 0.5456760048721072, "num_failures": 373, "flip_rate_vs_reference": 0.10840438489646773, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 821, "accuracy": 0.5602923264311814, "num_failures": 361, "flip_rate_vs_reference": 0.11693057247259439, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 821, "accuracy": 0.5724725943970768, "num_failures": 351, "flip_rate_vs_reference": 0.13276492082825822, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 821, "accuracy": 0.5663824604141291, "num_failures": 356, "flip_rate_vs_reference": 0.14981729598051158, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 821, "accuracy": 0.5554202192448234, "num_failures": 365, "flip_rate_vs_reference": 0.15712545676004872, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.030450669914738104, "stability": { "always_correct": 364, "always_wrong": 286, "unstable": 171, "always_correct_rate": 0.44336175395858707, "always_wrong_rate": 0.34835566382460414, "unstable_rate": 0.20828258221680876 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 376, "failures_b": 373, "failures_shared": 341, "failure_jaccard": 0.8357843137254902, "failure_recall_a_in_b": 0.9069148936170213, "successes_a": 445, "successes_b": 448, "successes_shared": 413, "success_jaccard": 0.8604166666666667, "answer_agreement": 0.8915956151035322 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 376, "failures_b": 361, "failures_shared": 332, "failure_jaccard": 0.8197530864197531, "failure_recall_a_in_b": 0.8829787234042553, "successes_a": 445, "successes_b": 460, "successes_shared": 416, "success_jaccard": 0.8507157464212679, "answer_agreement": 0.8830694275274056 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 376, "failures_b": 351, "failures_shared": 322, "failure_jaccard": 0.7950617283950617, "failure_recall_a_in_b": 0.8563829787234043, "successes_a": 445, "successes_b": 470, "successes_shared": 416, "success_jaccard": 0.8336673346693386, "answer_agreement": 0.8672350791717418 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 376, "failures_b": 356, "failures_shared": 319, "failure_jaccard": 0.7723970944309927, "failure_recall_a_in_b": 0.848404255319149, "successes_a": 445, "successes_b": 465, "successes_shared": 408, "success_jaccard": 0.8127490039840638, "answer_agreement": 0.8501827040194885 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 376, "failures_b": 365, "failures_shared": 320, "failure_jaccard": 0.7600950118764845, "failure_recall_a_in_b": 0.851063829787234, "successes_a": 445, "successes_b": 456, "successes_shared": 400, "success_jaccard": 0.7984031936127745, "answer_agreement": 0.8428745432399513 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 373, "failures_b": 361, "failures_shared": 338, "failure_jaccard": 0.8535353535353535, "failure_recall_a_in_b": 0.9061662198391421, "successes_a": 448, "successes_b": 460, "successes_shared": 425, "success_jaccard": 0.8799171842650103, "answer_agreement": 0.9037758830694276 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 373, "failures_b": 351, "failures_shared": 321, "failure_jaccard": 0.7965260545905707, "failure_recall_a_in_b": 0.8605898123324397, "successes_a": 448, "successes_b": 470, "successes_shared": 418, "success_jaccard": 0.836, "answer_agreement": 0.8672350791717418 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 373, "failures_b": 356, "failures_shared": 318, "failure_jaccard": 0.7737226277372263, "failure_recall_a_in_b": 0.8525469168900804, "successes_a": 448, "successes_b": 465, "successes_shared": 410, "success_jaccard": 0.8151093439363817, "answer_agreement": 0.8489646772228989 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 373, "failures_b": 365, "failures_shared": 319, "failure_jaccard": 0.7613365155131265, "failure_recall_a_in_b": 0.8552278820375335, "successes_a": 448, "successes_b": 456, "successes_shared": 402, "success_jaccard": 0.8007968127490039, "answer_agreement": 0.8453105968331304 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 361, "failures_b": 351, "failures_shared": 318, "failure_jaccard": 0.8071065989847716, "failure_recall_a_in_b": 0.8808864265927978, "successes_a": 460, "successes_b": 470, "successes_shared": 427, "success_jaccard": 0.8489065606361829, "answer_agreement": 0.876979293544458 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 361, "failures_b": 356, "failures_shared": 325, "failure_jaccard": 0.8290816326530612, "failure_recall_a_in_b": 0.9002770083102493, "successes_a": 460, "successes_b": 465, "successes_shared": 429, "success_jaccard": 0.8649193548387096, "answer_agreement": 0.8855054811205847 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 361, "failures_b": 365, "failures_shared": 318, "failure_jaccard": 0.7794117647058824, "failure_recall_a_in_b": 0.8808864265927978, "successes_a": 460, "successes_b": 456, "successes_shared": 413, "success_jaccard": 0.8210735586481114, "answer_agreement": 0.8587088915956151 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 351, "failures_b": 356, "failures_shared": 320, "failure_jaccard": 0.8268733850129198, "failure_recall_a_in_b": 0.9116809116809117, "successes_a": 470, "successes_b": 465, "successes_shared": 434, "success_jaccard": 0.8662674650698603, "answer_agreement": 0.8842874543239951 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 351, "failures_b": 365, "failures_shared": 325, "failure_jaccard": 0.8312020460358056, "failure_recall_a_in_b": 0.9259259259259259, "successes_a": 470, "successes_b": 456, "successes_shared": 430, "success_jaccard": 0.8669354838709677, "answer_agreement": 0.8964677222898904 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 356, "failures_b": 365, "failures_shared": 324, "failure_jaccard": 0.8161209068010076, "failure_recall_a_in_b": 0.9101123595505618, "successes_a": 465, "successes_b": 456, "successes_shared": 424, "success_jaccard": 0.8531187122736419, "answer_agreement": 0.8891595615103532 } ] }, { "model_id": "Qwen/Qwen3.5-4B-Base", "task": "youtube_level2", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 764, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 764, "accuracy": 0.4424083769633508, "num_failures": 426, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 764, "accuracy": 0.45287958115183247, "num_failures": 418, "flip_rate_vs_reference": 0.13089005235602094, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 764, "accuracy": 0.450261780104712, "num_failures": 420, "flip_rate_vs_reference": 0.13743455497382198, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 764, "accuracy": 0.45418848167539266, "num_failures": 417, "flip_rate_vs_reference": 0.14790575916230367, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 764, "accuracy": 0.450261780104712, "num_failures": 420, "flip_rate_vs_reference": 0.17801047120418848, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 764, "accuracy": 0.4397905759162304, "num_failures": 428, "flip_rate_vs_reference": 0.18848167539267016, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.014397905759162277, "stability": { "always_correct": 264, "always_wrong": 350, "unstable": 150, "always_correct_rate": 0.34554973821989526, "always_wrong_rate": 0.4581151832460733, "unstable_rate": 0.19633507853403143 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 426, "failures_b": 418, "failures_shared": 389, "failure_jaccard": 0.8549450549450549, "failure_recall_a_in_b": 0.9131455399061033, "successes_a": 338, "successes_b": 346, "successes_shared": 309, "success_jaccard": 0.824, "answer_agreement": 0.8691099476439791 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 426, "failures_b": 420, "failures_shared": 388, "failure_jaccard": 0.8471615720524017, "failure_recall_a_in_b": 0.9107981220657277, "successes_a": 338, "successes_b": 344, "successes_shared": 306, "success_jaccard": 0.8138297872340425, "answer_agreement": 0.862565445026178 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 426, "failures_b": 417, "failures_shared": 384, "failure_jaccard": 0.8366013071895425, "failure_recall_a_in_b": 0.9014084507042254, "successes_a": 338, "successes_b": 347, "successes_shared": 305, "success_jaccard": 0.8026315789473685, "answer_agreement": 0.8520942408376964 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 426, "failures_b": 420, "failures_shared": 382, "failure_jaccard": 0.8232758620689655, "failure_recall_a_in_b": 0.8967136150234741, "successes_a": 338, "successes_b": 344, "successes_shared": 300, "success_jaccard": 0.7853403141361257, "answer_agreement": 0.8219895287958116 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 426, "failures_b": 428, "failures_shared": 381, "failure_jaccard": 0.8054968287526427, "failure_recall_a_in_b": 0.8943661971830986, "successes_a": 338, "successes_b": 336, "successes_shared": 291, "success_jaccard": 0.7597911227154047, "answer_agreement": 0.8115183246073299 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 418, "failures_b": 420, "failures_shared": 388, "failure_jaccard": 0.8622222222222222, "failure_recall_a_in_b": 0.9282296650717703, "successes_a": 346, "successes_b": 344, "successes_shared": 314, "success_jaccard": 0.8351063829787234, "answer_agreement": 0.8808900523560209 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 418, "failures_b": 417, "failures_shared": 390, "failure_jaccard": 0.8764044943820225, "failure_recall_a_in_b": 0.9330143540669856, "successes_a": 346, "successes_b": 347, "successes_shared": 319, "success_jaccard": 0.8529411764705882, "answer_agreement": 0.8795811518324608 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 418, "failures_b": 420, "failures_shared": 384, "failure_jaccard": 0.8458149779735683, "failure_recall_a_in_b": 0.9186602870813397, "successes_a": 346, "successes_b": 344, "successes_shared": 310, "success_jaccard": 0.8157894736842105, "answer_agreement": 0.8416230366492147 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 418, "failures_b": 428, "failures_shared": 383, "failure_jaccard": 0.8272138228941684, "failure_recall_a_in_b": 0.916267942583732, "successes_a": 346, "successes_b": 336, "successes_shared": 301, "success_jaccard": 0.7900262467191601, "answer_agreement": 0.8350785340314136 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 420, "failures_b": 417, "failures_shared": 397, "failure_jaccard": 0.9022727272727272, "failure_recall_a_in_b": 0.9452380952380952, "successes_a": 344, "successes_b": 347, "successes_shared": 324, "success_jaccard": 0.8828337874659401, "answer_agreement": 0.9070680628272252 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 420, "failures_b": 420, "failures_shared": 391, "failure_jaccard": 0.8708240534521158, "failure_recall_a_in_b": 0.930952380952381, "successes_a": 344, "successes_b": 344, "successes_shared": 315, "success_jaccard": 0.8445040214477212, "answer_agreement": 0.8638743455497382 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 420, "failures_b": 428, "failures_shared": 386, "failure_jaccard": 0.8354978354978355, "failure_recall_a_in_b": 0.919047619047619, "successes_a": 344, "successes_b": 336, "successes_shared": 302, "success_jaccard": 0.798941798941799, "answer_agreement": 0.8520942408376964 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 417, "failures_b": 420, "failures_shared": 393, "failure_jaccard": 0.8851351351351351, "failure_recall_a_in_b": 0.9424460431654677, "successes_a": 347, "successes_b": 344, "successes_shared": 320, "success_jaccard": 0.862533692722372, "answer_agreement": 0.8808900523560209 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 417, "failures_b": 428, "failures_shared": 391, "failure_jaccard": 0.8612334801762115, "failure_recall_a_in_b": 0.9376498800959233, "successes_a": 347, "successes_b": 336, "successes_shared": 310, "success_jaccard": 0.8310991957104558, "answer_agreement": 0.8730366492146597 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 420, "failures_b": 428, "failures_shared": 395, "failure_jaccard": 0.8719646799116998, "failure_recall_a_in_b": 0.9404761904761905, "successes_a": 344, "successes_b": 336, "successes_shared": 311, "success_jaccard": 0.8428184281842819, "answer_agreement": 0.8848167539267016 } ] }, { "model_id": "Qwen/Qwen3.5-4B-Base", "task": "youtube_level3", "baselines": [ "0.00", "0.05", "0.10", "0.15", "0.20", "0.25" ], "reference_baseline": "0.00", "num_common_samples": 836, "dropped_per_baseline": { "0.00": 0, "0.05": 0, "0.10": 0, "0.15": 0, "0.20": 0, "0.25": 0 }, "per_baseline": [ { "baseline": "0.00", "num_samples": 836, "accuracy": 0.3026315789473684, "num_failures": 583, "flip_rate_vs_reference": 0.0, "unparsed_rate": 0.0 }, { "baseline": "0.05", "num_samples": 836, "accuracy": 0.2715311004784689, "num_failures": 609, "flip_rate_vs_reference": 0.24760765550239233, "unparsed_rate": 0.0 }, { "baseline": "0.10", "num_samples": 836, "accuracy": 0.27751196172248804, "num_failures": 604, "flip_rate_vs_reference": 0.2452153110047847, "unparsed_rate": 0.0 }, { "baseline": "0.15", "num_samples": 836, "accuracy": 0.26674641148325356, "num_failures": 613, "flip_rate_vs_reference": 0.26555023923444976, "unparsed_rate": 0.0 }, { "baseline": "0.20", "num_samples": 836, "accuracy": 0.2727272727272727, "num_failures": 608, "flip_rate_vs_reference": 0.27631578947368424, "unparsed_rate": 0.0 }, { "baseline": "0.25", "num_samples": 836, "accuracy": 0.2822966507177033, "num_failures": 600, "flip_rate_vs_reference": 0.284688995215311, "unparsed_rate": 0.0 } ], "accuracy_spread": 0.03588516746411485, "stability": { "always_correct": 137, "always_wrong": 482, "unstable": 217, "always_correct_rate": 0.1638755980861244, "always_wrong_rate": 0.5765550239234449, "unstable_rate": 0.2595693779904306 }, "pairwise": [ { "baseline_a": "0.00", "baseline_b": "0.05", "failures_a": 583, "failures_b": 609, "failures_shared": 545, "failure_jaccard": 0.8423493044822257, "failure_recall_a_in_b": 0.934819897084048, "successes_a": 253, "successes_b": 227, "successes_shared": 189, "success_jaccard": 0.6494845360824743, "answer_agreement": 0.7523923444976076 }, { "baseline_a": "0.00", "baseline_b": "0.10", "failures_a": 583, "failures_b": 604, "failures_shared": 541, "failure_jaccard": 0.8374613003095975, "failure_recall_a_in_b": 0.9279588336192109, "successes_a": 253, "successes_b": 232, "successes_shared": 190, "success_jaccard": 0.6440677966101694, "answer_agreement": 0.7547846889952153 }, { "baseline_a": "0.00", "baseline_b": "0.15", "failures_a": 583, "failures_b": 613, "failures_shared": 542, "failure_jaccard": 0.8287461773700305, "failure_recall_a_in_b": 0.9296740994854202, "successes_a": 253, "successes_b": 223, "successes_shared": 182, "success_jaccard": 0.6190476190476191, "answer_agreement": 0.7344497607655502 }, { "baseline_a": "0.00", "baseline_b": "0.20", "failures_a": 583, "failures_b": 608, "failures_shared": 534, "failure_jaccard": 0.8127853881278538, "failure_recall_a_in_b": 0.9159519725557461, "successes_a": 253, "successes_b": 228, "successes_shared": 179, "success_jaccard": 0.5927152317880795, "answer_agreement": 0.7236842105263158 }, { "baseline_a": "0.00", "baseline_b": "0.25", "failures_a": 583, "failures_b": 600, "failures_shared": 528, "failure_jaccard": 0.8061068702290076, "failure_recall_a_in_b": 0.9056603773584906, "successes_a": 253, "successes_b": 236, "successes_shared": 181, "success_jaccard": 0.5876623376623377, "answer_agreement": 0.715311004784689 }, { "baseline_a": "0.05", "baseline_b": "0.10", "failures_a": 609, "failures_b": 604, "failures_shared": 563, "failure_jaccard": 0.8661538461538462, "failure_recall_a_in_b": 0.9244663382594417, "successes_a": 227, "successes_b": 232, "successes_shared": 186, "success_jaccard": 0.6813186813186813, "answer_agreement": 0.7906698564593302 }, { "baseline_a": "0.05", "baseline_b": "0.15", "failures_a": 609, "failures_b": 613, "failures_shared": 567, "failure_jaccard": 0.8656488549618321, "failure_recall_a_in_b": 0.9310344827586207, "successes_a": 227, "successes_b": 223, "successes_shared": 181, "success_jaccard": 0.6728624535315985, "answer_agreement": 0.7727272727272727 }, { "baseline_a": "0.05", "baseline_b": "0.20", "failures_a": 609, "failures_b": 608, "failures_shared": 562, "failure_jaccard": 0.8580152671755725, "failure_recall_a_in_b": 0.922824302134647, "successes_a": 227, "successes_b": 228, "successes_shared": 181, "success_jaccard": 0.6605839416058394, "answer_agreement": 0.7691387559808612 }, { "baseline_a": "0.05", "baseline_b": "0.25", "failures_a": 609, "failures_b": 600, "failures_shared": 550, "failure_jaccard": 0.834597875569044, "failure_recall_a_in_b": 0.90311986863711, "successes_a": 227, "successes_b": 236, "successes_shared": 177, "success_jaccard": 0.6188811188811189, "answer_agreement": 0.7559808612440191 }, { "baseline_a": "0.10", "baseline_b": "0.15", "failures_a": 604, "failures_b": 613, "failures_shared": 569, "failure_jaccard": 0.8780864197530864, "failure_recall_a_in_b": 0.9420529801324503, "successes_a": 232, "successes_b": 223, "successes_shared": 188, "success_jaccard": 0.704119850187266, "answer_agreement": 0.819377990430622 }, { "baseline_a": "0.10", "baseline_b": "0.20", "failures_a": 604, "failures_b": 608, "failures_shared": 566, "failure_jaccard": 0.8761609907120743, "failure_recall_a_in_b": 0.9370860927152318, "successes_a": 232, "successes_b": 228, "successes_shared": 190, "success_jaccard": 0.7037037037037037, "answer_agreement": 0.8038277511961722 }, { "baseline_a": "0.10", "baseline_b": "0.25", "failures_a": 604, "failures_b": 600, "failures_shared": 560, "failure_jaccard": 0.8695652173913043, "failure_recall_a_in_b": 0.9271523178807947, "successes_a": 232, "successes_b": 236, "successes_shared": 192, "success_jaccard": 0.6956521739130435, "answer_agreement": 0.7954545454545454 }, { "baseline_a": "0.15", "baseline_b": "0.20", "failures_a": 613, "failures_b": 608, "failures_shared": 570, "failure_jaccard": 0.8755760368663594, "failure_recall_a_in_b": 0.9298531810766721, "successes_a": 223, "successes_b": 228, "successes_shared": 185, "success_jaccard": 0.6954887218045113, "answer_agreement": 0.8098086124401914 }, { "baseline_a": "0.15", "baseline_b": "0.25", "failures_a": 613, "failures_b": 600, "failures_shared": 563, "failure_jaccard": 0.8661538461538462, "failure_recall_a_in_b": 0.9184339314845025, "successes_a": 223, "successes_b": 236, "successes_shared": 186, "success_jaccard": 0.6813186813186813, "answer_agreement": 0.7894736842105263 }, { "baseline_a": "0.20", "baseline_b": "0.25", "failures_a": 608, "failures_b": 600, "failures_shared": 560, "failure_jaccard": 0.8641975308641975, "failure_recall_a_in_b": 0.9210526315789473, "successes_a": 228, "successes_b": 236, "successes_shared": 188, "success_jaccard": 0.6811594202898551, "answer_agreement": 0.7978468899521531 } ] } ] }