[ { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":1, "wins_b":120, "ties":0, "win_rate_a_over_b":0.0082644628, "mean_diff_a_minus_b":-0.2415859185, "median_diff_a_minus_b":-0.2228611507, "W_stat":11.0, "p_two_sided":1.797749072e-21, "z_equiv":-9.5161147732, "effect_size_r":0.865101343, "p_holm_global":1.348311804e-19, "p_holm_within_metric":2.696623608e-20 }, { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":4, "wins_b":117, "ties":0, "win_rate_a_over_b":0.0330578512, "mean_diff_a_minus_b":-0.2070521511, "median_diff_a_minus_b":-0.1616377295, "W_stat":33.0, "p_two_sided":3.102799787e-21, "z_equiv":-9.4592094932, "effect_size_r":0.8599281357, "p_holm_global":2.296071842e-19, "p_holm_within_metric":4.343919702e-20 }, { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":6, "wins_b":115, "ties":0, "win_rate_a_over_b":0.0495867769, "mean_diff_a_minus_b":-0.2224755243, "median_diff_a_minus_b":-0.1828913777, "W_stat":54.0, "p_two_sided":5.208425291e-21, "z_equiv":-9.4048908169, "effect_size_r":0.8549900743, "p_holm_global":3.802150463e-19, "p_holm_within_metric":6.770952879e-20 }, { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":5, "wins_b":116, "ties":0, "win_rate_a_over_b":0.041322314, "mean_diff_a_minus_b":-0.2760772691, "median_diff_a_minus_b":-0.2530477867, "W_stat":63.0, "p_two_sided":6.497205758e-21, "z_equiv":-9.3816113842, "effect_size_r":0.8528737622, "p_holm_global":4.677988146e-19, "p_holm_within_metric":7.79664691e-20 }, { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":117, "wins_a":15, "wins_b":102, "ties":4, "win_rate_a_over_b":0.1404958678, "mean_diff_a_minus_b":-0.1459002825, "median_diff_a_minus_b":-0.0794756578, "W_stat":265.0, "p_two_sided":4.503316115e-18, "z_equiv":-8.6652984778, "effect_size_r":0.8011071277, "p_holm_global":2.656956508e-16, "p_holm_within_metric":4.953647726e-17 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":16, "wins_b":104, "ties":1, "win_rate_a_over_b":0.1363636364, "mean_diff_a_minus_b":-0.1301769866, "median_diff_a_minus_b":-0.0911340416, "W_stat":469.0, "p_two_sided":1.263853299e-16, "z_equiv":-8.2769357893, "effect_size_r":0.7555774065, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_108", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":15, "wins_b":106, "ties":0, "win_rate_a_over_b":0.1239669421, "mean_diff_a_minus_b":-0.0536017448, "median_diff_a_minus_b":-0.0410954431, "W_stat":634.0, "p_two_sided":0.0, "z_equiv":-7.9046607086, "effect_size_r":0.718605519, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_108", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":120, "wins_a":95, "wins_b":25, "ties":1, "win_rate_a_over_b":0.7892561983, "mean_diff_a_minus_b":0.069025118, "median_diff_a_minus_b":0.0391470212, "W_stat":769.0, "p_two_sided":0.0, "z_equiv":7.4912750595, "effect_size_r":0.6838567224, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":28, "wins_b":93, "ties":0, "win_rate_a_over_b":0.2314049587, "mean_diff_a_minus_b":-0.0956856361, "median_diff_a_minus_b":-0.0559161572, "W_stat":854.0, "p_two_sided":0.0, "z_equiv":-7.3356079089, "effect_size_r":0.6668734463, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":39, "wins_b":82, "ties":0, "win_rate_a_over_b":0.3223140496, "mean_diff_a_minus_b":-0.0765752418, "median_diff_a_minus_b":-0.0288892186, "W_stat":1595.0, "p_two_sided":0.00000006, "z_equiv":-5.4189346153, "effect_size_r":0.4926304196, "p_holm_global":0.0000018586, "p_holm_within_metric":0.0000003597 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":119, "wins_a":37, "wins_b":82, "ties":2, "win_rate_a_over_b":0.3140495868, "mean_diff_a_minus_b":-0.0611518687, "median_diff_a_minus_b":-0.023088851, "W_stat":1675.0, "p_two_sided":0.0000005064, "z_equiv":-5.0238559266, "effect_size_r":0.4605361177, "p_holm_global":0.0000141804, "p_holm_within_metric":0.0000025322 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":83, "wins_b":38, "ties":0, "win_rate_a_over_b":0.6859504132, "mean_diff_a_minus_b":0.0345337674, "median_diff_a_minus_b":0.0187233128, "W_stat":2059.0, "p_two_sided":0.000024566, "z_equiv":4.2187505287, "effect_size_r":0.3835227753, "p_holm_global":0.0005158858, "p_holm_within_metric":0.000098264 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":43, "wins_b":78, "ties":0, "win_rate_a_over_b":0.3553719008, "mean_diff_a_minus_b":-0.0344913506, "median_diff_a_minus_b":-0.0212858766, "W_stat":2138.0, "p_two_sided":0.000059595, "z_equiv":-4.0144088415, "effect_size_r":0.3649462583, "p_holm_global":0.001044842, "p_holm_within_metric":0.0001787849 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":77, "wins_b":44, "ties":0, "win_rate_a_over_b":0.6363636364, "mean_diff_a_minus_b":0.0191103942, "median_diff_a_minus_b":0.0181754008, "W_stat":2506.0, "p_two_sided":0.0021946809, "z_equiv":3.0625387038, "effect_size_r":0.2784126094, "p_holm_global":0.0197521283, "p_holm_within_metric":0.0043893618 }, { "metric":"mean_cert_acc_108", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":65, "wins_b":56, "ties":0, "win_rate_a_over_b":0.5371900826, "mean_diff_a_minus_b":0.0154233732, "median_diff_a_minus_b":0.0037754271, "W_stat":3069.0, "p_two_sided":0.1082122144, "z_equiv":1.6062808573, "effect_size_r":0.1460255325, "p_holm_global":0.8656977152, "p_holm_within_metric":0.1082122144 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":112, "wins_a":1, "wins_b":111, "ties":9, "win_rate_a_over_b":0.0454545455, "mean_diff_a_minus_b":-0.1173470639, "median_diff_a_minus_b":-0.0569680196, "W_stat":10.0, "p_two_sided":5.435127869e-20, "z_equiv":-9.1550129731, "effect_size_r":0.8650674134, "p_holm_global":3.587184393e-18, "p_holm_within_metric":8.152691803e-19 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":114, "wins_a":2, "wins_b":112, "ties":7, "win_rate_a_over_b":0.0454545455, "mean_diff_a_minus_b":-0.1497530324, "median_diff_a_minus_b":-0.0803308832, "W_stat":62.0, "p_two_sided":9.896991201e-20, "z_equiv":-9.0900759336, "effect_size_r":0.8513636146, "p_holm_global":6.334074369e-18, "p_holm_within_metric":1.385578768e-18 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":107, "wins_a":3, "wins_b":104, "ties":14, "win_rate_a_over_b":0.0826446281, "mean_diff_a_minus_b":-0.0919241391, "median_diff_a_minus_b":-0.0476176738, "W_stat":23.0, "p_two_sided":5.290354121e-19, "z_equiv":-8.9060071017, "effect_size_r":0.8609762037, "p_holm_global":3.227116014e-17, "p_holm_within_metric":6.877460357e-18 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":110, "wins_a":5, "wins_b":105, "ties":11, "win_rate_a_over_b":0.0867768595, "mean_diff_a_minus_b":-0.0971526016, "median_diff_a_minus_b":-0.036305353, "W_stat":122.0, "p_two_sided":2.369681608e-18, "z_equiv":-8.7381455094, "effect_size_r":0.8331494843, "p_holm_global":1.421808965e-16, "p_holm_within_metric":2.84361793e-17 }, { "metric":"mean_cert_acc_255", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":114, "wins_a":11, "wins_b":103, "ties":7, "win_rate_a_over_b":0.1198347107, "mean_diff_a_minus_b":-0.0526004308, "median_diff_a_minus_b":-0.0312724002, "W_stat":258.0, "p_two_sided":1.390608438e-17, "z_equiv":-8.5359064521, "effect_size_r":0.7994608872, "p_holm_global":8.065528942e-16, "p_holm_within_metric":1.529669282e-16 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":114, "wins_a":12, "wins_b":102, "ties":7, "win_rate_a_over_b":0.1280991736, "mean_diff_a_minus_b":-0.0844428848, "median_diff_a_minus_b":-0.0396959223, "W_stat":342.0, "p_two_sided":1.05518212e-16, "z_equiv":-8.2984052458, "effect_size_r":0.7772168612, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":93, "wins_a":8, "wins_b":85, "ties":28, "win_rate_a_over_b":0.1818181818, "mean_diff_a_minus_b":-0.0653101477, "median_diff_a_minus_b":-0.0160309789, "W_stat":182.0, "p_two_sided":0.0, "z_equiv":-7.6746815803, "effect_size_r":0.7958274071, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_255", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":115, "wins_a":90, "wins_b":25, "ties":6, "win_rate_a_over_b":0.7685950413, "mean_diff_a_minus_b":0.0578288934, "median_diff_a_minus_b":0.0239693746, "W_stat":977.0, "p_two_sided":0.0, "z_equiv":6.5792049823, "effect_size_r":0.613514028, "p_holm_global":0.0000000017, "p_holm_within_metric":0.0000000004 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":112, "wins_a":23, "wins_b":89, "ties":9, "win_rate_a_over_b":0.2272727273, "mean_diff_a_minus_b":-0.0520369162, "median_diff_a_minus_b":-0.0241820474, "W_stat":927.0, "p_two_sided":0.0000000001, "z_equiv":-6.4928449387, "effect_size_r":0.6135161789, "p_holm_global":0.000000003, "p_holm_within_metric":0.0000000006 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":115, "wins_a":80, "wins_b":35, "ties":6, "win_rate_a_over_b":0.6859504132, "mean_diff_a_minus_b":0.0254229248, "median_diff_a_minus_b":0.0174753755, "W_stat":1603.0, "p_two_sided":0.0000013504, "z_equiv":4.8321923338, "effect_size_r":0.4506042586, "p_holm_global":0.0000337594, "p_holm_within_metric":0.0000081023 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":106, "wins_a":27, "wins_b":79, "ties":15, "win_rate_a_over_b":0.2851239669, "mean_diff_a_minus_b":-0.0266139914, "median_diff_a_minus_b":-0.0090909088, "W_stat":1323.0, "p_two_sided":0.0000018824, "z_equiv":-4.76565708, "effect_size_r":0.4628815347, "p_holm_global":0.0000451774, "p_holm_within_metric":0.000009412 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":111, "wins_a":36, "wins_b":75, "ties":10, "win_rate_a_over_b":0.3388429752, "mean_diff_a_minus_b":-0.031842454, "median_diff_a_minus_b":-0.0084677431, "W_stat":1741.0, "p_two_sided":0.0000580468, "z_equiv":-4.0206134822, "effect_size_r":0.3816197434, "p_holm_global":0.001044842, "p_holm_within_metric":0.0002321871 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":117, "wins_a":77, "wins_b":40, "ties":4, "win_rate_a_over_b":0.652892562, "mean_diff_a_minus_b":0.0201944623, "median_diff_a_minus_b":0.0131578948, "W_stat":1982.0, "p_two_sided":0.0000645869, "z_equiv":3.9953934287, "effect_size_r":0.3693742531, "p_holm_global":0.001044842, "p_holm_within_metric":0.0002321871 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":118, "wins_a":45, "wins_b":73, "ties":3, "win_rate_a_over_b":0.3842975207, "mean_diff_a_minus_b":-0.0324059685, "median_diff_a_minus_b":-0.0077698529, "W_stat":2234.0, "p_two_sided":0.0006111136, "z_equiv":-3.426633354, "effect_size_r":0.3154471691, "p_holm_global":0.0072473427, "p_holm_within_metric":0.0012222272 }, { "metric":"mean_cert_acc_255", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":113, "wins_a":55, "wins_b":58, "ties":8, "win_rate_a_over_b":0.4876033058, "mean_diff_a_minus_b":0.0052284626, "median_diff_a_minus_b":0.0, "W_stat":3068.0, "p_two_sided":0.6632312125, "z_equiv":0.4354563548, "effect_size_r":0.040964288, "p_holm_global":1.0, "p_holm_within_metric":0.6632312125 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":6, "wins_b":115, "ties":0, "win_rate_a_over_b":0.0495867769, "mean_diff_a_minus_b":-0.2438353373, "median_diff_a_minus_b":-0.231873706, "W_stat":84.0, "p_two_sided":1.086095964e-20, "z_equiv":-9.3272927078, "effect_size_r":0.8479357007, "p_holm_global":7.711281345e-19, "p_holm_within_metric":1.629143946e-19 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":6, "wins_b":115, "ties":0, "win_rate_a_over_b":0.0495867769, "mean_diff_a_minus_b":-0.2706310854, "median_diff_a_minus_b":-0.2436474561, "W_stat":88.0, "p_two_sided":1.197367966e-20, "z_equiv":-9.3169462933, "effect_size_r":0.8469951176, "p_holm_global":8.381575764e-19, "p_holm_within_metric":1.676315153e-19 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":6, "wins_b":115, "ties":0, "win_rate_a_over_b":0.0495867769, "mean_diff_a_minus_b":-0.2215721657, "median_diff_a_minus_b":-0.1912159696, "W_stat":145.0, "p_two_sided":4.751745038e-20, "z_equiv":-9.1695098861, "effect_size_r":0.8335918078, "p_holm_global":3.183669175e-18, "p_holm_within_metric":6.177268549e-19 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":10, "wins_b":111, "ties":0, "win_rate_a_over_b":0.0826446281, "mean_diff_a_minus_b":-0.2304439908, "median_diff_a_minus_b":-0.218675293, "W_stat":196.0, "p_two_sided":1.60159291e-19, "z_equiv":-9.0375931007, "effect_size_r":0.8215993728, "p_holm_global":1.009003533e-17, "p_holm_within_metric":1.921911492e-18 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":121, "wins_a":16, "wins_b":105, "ties":0, "win_rate_a_over_b":0.132231405, "mean_diff_a_minus_b":-0.1632065656, "median_diff_a_minus_b":-0.1217573732, "W_stat":579.0, "p_two_sided":8.490093064e-16, "z_equiv":-8.0469239085, "effect_size_r":0.7315385371, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":28, "wins_b":93, "ties":0, "win_rate_a_over_b":0.2314049587, "mean_diff_a_minus_b":-0.1074245198, "median_diff_a_minus_b":-0.0510568023, "W_stat":884.0, "p_two_sided":0.0, "z_equiv":-7.2580097998, "effect_size_r":0.6598190727, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_36", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":97, "wins_b":24, "ties":0, "win_rate_a_over_b":0.8016528926, "mean_diff_a_minus_b":0.0490589197, "median_diff_a_minus_b":0.0198676884, "W_stat":1005.0, "p_two_sided":0.0, "z_equiv":6.94503076, "effect_size_r":0.6313664327, "p_holm_global":0.0000000002, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_36", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":24, "wins_b":96, "ties":1, "win_rate_a_over_b":0.2024793388, "mean_diff_a_minus_b":-0.0401870946, "median_diff_a_minus_b":-0.0274580419, "W_stat":1002.0, "p_two_sided":0.0, "z_equiv":-6.8810785592, "effect_size_r":0.6281536578, "p_holm_global":0.0000000002, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":39, "wins_b":82, "ties":0, "win_rate_a_over_b":0.3223140496, "mean_diff_a_minus_b":-0.0806287717, "median_diff_a_minus_b":-0.0372283832, "W_stat":1356.0, "p_two_sided":0.0000000016, "z_equiv":-6.0371328841, "effect_size_r":0.5488302622, "p_holm_global":0.0000000518, "p_holm_within_metric":0.000000011 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":47, "wins_b":74, "ties":0, "win_rate_a_over_b":0.3884297521, "mean_diff_a_minus_b":-0.0672374252, "median_diff_a_minus_b":-0.0189958066, "W_stat":1855.0, "p_two_sided":0.0000020705, "z_equiv":-4.7464176702, "effect_size_r":0.4314925155, "p_holm_global":0.0000476217, "p_holm_within_metric":0.0000124231 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":49, "wins_b":72, "ties":0, "win_rate_a_over_b":0.4049586777, "mean_diff_a_minus_b":-0.0583656001, "median_diff_a_minus_b":-0.0074773338, "W_stat":2119.0, "p_two_sided":0.0000483311, "z_equiv":-4.0635543106, "effect_size_r":0.3694140282, "p_holm_global":0.0009182903, "p_holm_within_metric":0.0002416554 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":41, "wins_b":80, "ties":0, "win_rate_a_over_b":0.3388429752, "mean_diff_a_minus_b":-0.0267957481, "median_diff_a_minus_b":-0.0168480575, "W_stat":2142.0, "p_two_sided":0.0000622639, "z_equiv":-4.004062427, "effect_size_r":0.3640056752, "p_holm_global":0.001044842, "p_holm_within_metric":0.0002490557 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":120, "wins_a":81, "wins_b":39, "ties":1, "win_rate_a_over_b":0.673553719, "mean_diff_a_minus_b":0.0222631716, "median_diff_a_minus_b":0.0118150264, "W_stat":2104.0, "p_two_sided":0.0000646711, "z_equiv":3.9950848115, "effect_size_r":0.3646996784, "p_holm_global":0.001044842, "p_holm_within_metric":0.0002490557 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":78, "wins_b":43, "ties":0, "win_rate_a_over_b":0.6446280992, "mean_diff_a_minus_b":0.0133913465, "median_diff_a_minus_b":0.0125584751, "W_stat":2429.0, "p_two_sided":0.0011074347, "z_equiv":3.2617071837, "effect_size_r":0.2965188349, "p_holm_global":0.0110743473, "p_holm_within_metric":0.0022148695 }, { "metric":"mean_cert_acc_36", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":60, "wins_b":61, "ties":0, "win_rate_a_over_b":0.4958677686, "mean_diff_a_minus_b":0.0088718251, "median_diff_a_minus_b":-0.0004384369, "W_stat":3076.0, "p_two_sided":0.1122468558, "z_equiv":1.5881746319, "effect_size_r":0.144379512, "p_holm_global":0.8656977152, "p_holm_within_metric":0.1122468558 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":7, "wins_b":114, "ties":0, "win_rate_a_over_b":0.0578512397, "mean_diff_a_minus_b":-0.2932351713, "median_diff_a_minus_b":-0.2614153504, "W_stat":107.0, "p_two_sided":1.900233353e-20, "z_equiv":-9.2678008242, "effect_size_r":0.8425273477, "p_holm_global":1.311161014e-18, "p_holm_within_metric":2.85035003e-19 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":5, "wins_b":116, "ties":0, "win_rate_a_over_b":0.041322314, "mean_diff_a_minus_b":-0.2631359926, "median_diff_a_minus_b":-0.2326939851, "W_stat":116.0, "p_two_sided":2.362958291e-20, "z_equiv":-9.2445213915, "effect_size_r":0.8404110356, "p_holm_global":1.606811638e-18, "p_holm_within_metric":3.308141607e-19 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":5, "wins_b":116, "ties":0, "win_rate_a_over_b":0.041322314, "mean_diff_a_minus_b":-0.2314897135, "median_diff_a_minus_b":-0.1927602664, "W_stat":162.0, "p_two_sided":7.138107477e-20, "z_equiv":-9.1255376243, "effect_size_r":0.8295943295, "p_holm_global":4.63976986e-18, "p_holm_within_metric":9.27953972e-19 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":11, "wins_b":110, "ties":0, "win_rate_a_over_b":0.0909090909, "mean_diff_a_minus_b":-0.2412404211, "median_diff_a_minus_b":-0.2063548118, "W_stat":214.0, "p_two_sided":2.449147439e-19, "z_equiv":-8.9910342353, "effect_size_r":0.8173667487, "p_holm_global":1.518471412e-17, "p_holm_within_metric":2.938976926e-18 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":119, "wins_a":19, "wins_b":100, "ties":2, "win_rate_a_over_b":0.1652892562, "mean_diff_a_minus_b":-0.1632772665, "median_diff_a_minus_b":-0.125, "W_stat":557.0, "p_two_sided":0.0, "z_equiv":-7.988580617, "effect_size_r":0.7323119845, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":20, "wins_b":101, "ties":0, "win_rate_a_over_b":0.1652892562, "mean_diff_a_minus_b":-0.1299579048, "median_diff_a_minus_b":-0.0772011476, "W_stat":607.0, "p_two_sided":0.0, "z_equiv":-7.9744990067, "effect_size_r":0.7249544552, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_72", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":19, "wins_b":101, "ties":1, "win_rate_a_over_b":0.1611570248, "mean_diff_a_minus_b":-0.0519947502, "median_diff_a_minus_b":-0.034627907, "W_stat":601.0, "p_two_sided":0.0, "z_equiv":-7.9312450682, "effect_size_r":0.7240203055, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_72", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":120, "wins_a":95, "wins_b":25, "ties":1, "win_rate_a_over_b":0.7892561983, "mean_diff_a_minus_b":0.0617454578, "median_diff_a_minus_b":0.0392772481, "W_stat":801.0, "p_two_sided":0.0, "z_equiv":7.4074712483, "effect_size_r":0.6762065161, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":30, "wins_b":91, "ties":0, "win_rate_a_over_b":0.2479338843, "mean_diff_a_minus_b":-0.0998587261, "median_diff_a_minus_b":-0.062678976, "W_stat":930.0, "p_two_sided":0.0, "z_equiv":-7.1390260326, "effect_size_r":0.6490023666, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":42, "wins_b":79, "ties":0, "win_rate_a_over_b":0.347107438, "mean_diff_a_minus_b":-0.0779631546, "median_diff_a_minus_b":-0.020989418, "W_stat":1696.0, "p_two_sided":0.00000025, "z_equiv":-5.1576876482, "effect_size_r":0.4688806953, "p_holm_global":0.0000075005, "p_holm_within_metric":0.0000015001 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":42, "wins_b":79, "ties":0, "win_rate_a_over_b":0.347107438, "mean_diff_a_minus_b":-0.068212447, "median_diff_a_minus_b":-0.0213127583, "W_stat":1776.0, "p_two_sided":0.0000007392, "z_equiv":-4.9507593574, "effect_size_r":0.4500690325, "p_holm_global":0.0000199596, "p_holm_within_metric":0.0000036962 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":83, "wins_b":38, "ties":0, "win_rate_a_over_b":0.6859504132, "mean_diff_a_minus_b":0.0316462791, "median_diff_a_minus_b":0.0191756338, "W_stat":1938.0, "p_two_sided":0.0000058503, "z_equiv":4.5317295685, "effect_size_r":0.4119754153, "p_holm_global":0.000128706, "p_holm_within_metric":0.0000234011 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":40, "wins_b":81, "ties":0, "win_rate_a_over_b":0.3305785124, "mean_diff_a_minus_b":-0.0300991787, "median_diff_a_minus_b":-0.0196958482, "W_stat":2062.0, "p_two_sided":0.0000254253, "z_equiv":-4.2109907178, "effect_size_r":0.382817338, "p_holm_global":0.0005158858, "p_holm_within_metric":0.000076276 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":78, "wins_b":43, "ties":0, "win_rate_a_over_b":0.6446280992, "mean_diff_a_minus_b":0.0218955715, "median_diff_a_minus_b":0.0177944824, "W_stat":2306.0, "p_two_sided":0.0003437791, "z_equiv":3.5798594309, "effect_size_r":0.3254417664, "p_holm_global":0.0044691282, "p_holm_within_metric":0.0006875582 }, { "metric":"mean_cert_acc_72", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":66, "wins_b":55, "ties":0, "win_rate_a_over_b":0.5454545455, "mean_diff_a_minus_b":0.0097507076, "median_diff_a_minus_b":0.0040983558, "W_stat":3178.0, "p_two_sided":0.1853897994, "z_equiv":1.3243410611, "effect_size_r":0.1203946419, "p_holm_global":0.8656977152, "p_holm_within_metric":0.1853897994 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":121, "wins_a":16, "wins_b":105, "ties":0, "win_rate_a_over_b":0.132231405, "mean_diff_a_minus_b":-0.0928145234, "median_diff_a_minus_b":-0.0567989122, "W_stat":412.0, "p_two_sided":2.273589155e-17, "z_equiv":-8.4788867156, "effect_size_r":0.7708078832, "p_holm_global":0.0, "p_holm_within_metric":3.410383732e-16 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":18, "wins_b":103, "ties":0, "win_rate_a_over_b":0.1487603306, "mean_diff_a_minus_b":-0.0920325933, "median_diff_a_minus_b":-0.0594664378, "W_stat":518.0, "p_two_sided":2.311544437e-16, "z_equiv":-8.2047067302, "effect_size_r":0.74588243, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":22, "wins_b":99, "ties":0, "win_rate_a_over_b":0.1818181818, "mean_diff_a_minus_b":-0.0683567538, "median_diff_a_minus_b":-0.0351344055, "W_stat":806.0, "p_two_sided":0.0, "z_equiv":-7.4597648834, "effect_size_r":0.6781604439, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":27, "wins_b":94, "ties":0, "win_rate_a_over_b":0.2231404959, "mean_diff_a_minus_b":-0.0674542894, "median_diff_a_minus_b":-0.0264984947, "W_stat":971.0, "p_two_sided":0.0, "z_equiv":-7.0329752836, "effect_size_r":0.6393613894, "p_holm_global":0.0000000001, "p_holm_within_metric":0.0 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":25, "wins_b":96, "ties":0, "win_rate_a_over_b":0.2066115702, "mean_diff_a_minus_b":-0.0727332781, "median_diff_a_minus_b":-0.0313389237, "W_stat":1013.0, "p_two_sided":0.0, "z_equiv":-6.9243379309, "effect_size_r":0.6294852664, "p_holm_global":0.0000000002, "p_holm_within_metric":0.0 }, { "metric":"mean_test_acc", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":31, "wins_b":89, "ties":1, "win_rate_a_over_b":0.2603305785, "mean_diff_a_minus_b":-0.0245783039, "median_diff_a_minus_b":-0.0125617273, "W_stat":1269.0, "p_two_sided":0.0000000006, "z_equiv":-6.1818405096, "effect_size_r":0.564322249, "p_holm_global":0.0000000222, "p_holm_within_metric":0.0000000063 }, { "metric":"mean_test_acc", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":94, "wins_b":27, "ties":0, "win_rate_a_over_b":0.7768595041, "mean_diff_a_minus_b":0.025360234, "median_diff_a_minus_b":0.013893891, "W_stat":1309.0, "p_two_sided":0.0000000007, "z_equiv":6.158703255, "effect_size_r":0.5598821141, "p_holm_global":0.0000000249, "p_holm_within_metric":0.0000000066 }, { "metric":"mean_test_acc", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":86, "wins_b":35, "ties":0, "win_rate_a_over_b":0.7107438017, "mean_diff_a_minus_b":0.0244577696, "median_diff_a_minus_b":0.0110695032, "W_stat":1503.0, "p_two_sided":0.0000000154, "z_equiv":5.6569021498, "effect_size_r":0.5142638318, "p_holm_global":0.0000004932, "p_holm_within_metric":0.0000001233 }, { "metric":"mean_test_acc", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":120, "wins_a":87, "wins_b":33, "ties":1, "win_rate_a_over_b":0.7231404959, "mean_diff_a_minus_b":0.0200812453, "median_diff_a_minus_b":0.0108439309, "W_stat":1689.0, "p_two_sided":0.0000003736, "z_equiv":5.0819154878, "effect_size_r":0.4639132913, "p_holm_global":0.0000108358, "p_holm_within_metric":0.0000026155 }, { "metric":"mean_test_acc", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":83, "wins_b":38, "ties":0, "win_rate_a_over_b":0.6859504132, "mean_diff_a_minus_b":0.0236758395, "median_diff_a_minus_b":0.0089801928, "W_stat":1789.0, "p_two_sided":0.0000008782, "z_equiv":4.9171335102, "effect_size_r":0.4470121373, "p_holm_global":0.0000228334, "p_holm_within_metric":0.0000052692 }, { "metric":"mean_test_acc", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":43, "wins_b":78, "ties":0, "win_rate_a_over_b":0.3553719008, "mean_diff_a_minus_b":-0.0192993151, "median_diff_a_minus_b":-0.0067946254, "W_stat":2364.0, "p_two_sided":0.0006039452, "z_equiv":-3.42983642, "effect_size_r":0.3118033109, "p_holm_global":0.0072473427, "p_holm_within_metric":0.0030197261 }, { "metric":"mean_test_acc", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":120, "wins_a":66, "wins_b":54, "ties":1, "win_rate_a_over_b":0.5495867769, "mean_diff_a_minus_b":0.0043765243, "median_diff_a_minus_b":0.0019017287, "W_stat":3070.0, "p_two_sided":0.1428507422, "z_equiv":1.4652572612, "effect_size_r":0.1337590758, "p_holm_global":0.8656977152, "p_holm_within_metric":0.5714029689 }, { "metric":"mean_test_acc", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":71, "wins_b":50, "ties":0, "win_rate_a_over_b":0.5867768595, "mean_diff_a_minus_b":0.0052789887, "median_diff_a_minus_b":0.002452642, "W_stat":3138.0, "p_two_sided":0.1533479338, "z_equiv":1.4278052065, "effect_size_r":0.1298004733, "p_holm_global":0.8656977152, "p_holm_within_metric":0.5714029689 }, { "metric":"mean_test_acc", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":60, "wins_b":61, "ties":0, "win_rate_a_over_b":0.4958677686, "mean_diff_a_minus_b":-0.0009024644, "median_diff_a_minus_b":-0.0003659998, "W_stat":3623.0, "p_two_sided":0.8624137031, "z_equiv":-0.1733024435, "effect_size_r":0.0157547676, "p_holm_global":1.0, "p_holm_within_metric":1.0 }, { "metric":"mean_test_acc", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":59, "wins_b":62, "ties":0, "win_rate_a_over_b":0.4876033058, "mean_diff_a_minus_b":0.0007819301, "median_diff_a_minus_b":-0.0001641521, "W_stat":3624.0, "p_two_sided":0.8644472072, "z_equiv":0.1707158399, "effect_size_r":0.0155196218, "p_holm_global":1.0, "p_holm_within_metric":1.0 } ]