[ { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":1, "wins_b":120, "ties":0, "win_rate_a_over_b":0.0082644628, "mean_diff_a_minus_b":-0.2739230915, "median_diff_a_minus_b":-0.2317528344, "W_stat":45.0, "p_two_sided":4.173048145e-21, "z_equiv":-9.4281702496, "effect_size_r":0.8571063863, "p_holm_global":3.046325146e-19, "p_holm_within_metric":6.259572217e-20 }, { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":120, "wins_a":3, "wins_b":117, "ties":1, "win_rate_a_over_b":0.0289256198, "mean_diff_a_minus_b":-0.2489971982, "median_diff_a_minus_b":-0.2098822109, "W_stat":60.0, "p_two_sided":8.927572293e-21, "z_equiv":-9.3480532511, "effect_size_r":0.8533566057, "p_holm_global":6.249300605e-19, "p_holm_within_metric":1.249860121e-19 }, { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":5, "wins_b":116, "ties":0, "win_rate_a_over_b":0.041322314, "mean_diff_a_minus_b":-0.2999183728, "median_diff_a_minus_b":-0.2735294178, "W_stat":82.0, "p_two_sided":1.034358853e-20, "z_equiv":-9.3324659151, "effect_size_r":0.8484059923, "p_holm_global":7.033640204e-19, "p_holm_within_metric":1.34466651e-19 }, { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":117, "wins_a":1, "wins_b":116, "ties":4, "win_rate_a_over_b":0.0247933884, "mean_diff_a_minus_b":-0.2222316045, "median_diff_a_minus_b":-0.1720663907, "W_stat":34.0, "p_two_sided":1.49193415e-20, "z_equiv":-9.2935734145, "effect_size_r":0.8591911661, "p_holm_global":9.995958805e-19, "p_holm_within_metric":1.79032098e-19 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":14, "wins_b":107, "ties":0, "win_rate_a_over_b":0.1157024793, "mean_diff_a_minus_b":-0.1491411817, "median_diff_a_minus_b":-0.0818716213, "W_stat":450.0, "p_two_sided":5.266046065e-17, "z_equiv":-8.3805957774, "effect_size_r":0.7618723434, "p_holm_global":0.0, "p_holm_within_metric":5.792650671e-16 }, { "metric":"mean_cert_acc_108", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":96, "wins_a":6, "wins_b":90, "ties":25, "win_rate_a_over_b":0.152892562, "mean_diff_a_minus_b":-0.1507771911, "median_diff_a_minus_b":-0.0671557561, "W_stat":113.0, "p_two_sided":5.846618541e-16, "z_equiv":-8.0924712901, "effect_size_r":0.8259343925, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":25, "wins_b":96, "ties":0, "win_rate_a_over_b":0.2066115702, "mean_diff_a_minus_b":-0.1231459003, "median_diff_a_minus_b":-0.0714981307, "W_stat":733.0, "p_two_sided":0.0, "z_equiv":-7.6485869487, "effect_size_r":0.6953260862, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_108", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":18, "wins_b":102, "ties":1, "win_rate_a_over_b":0.152892562, "mean_diff_a_minus_b":-0.0509211747, "median_diff_a_minus_b":-0.0375725031, "W_stat":945.0, "p_two_sided":0.0, "z_equiv":-7.0303540979, "effect_size_r":0.6417805878, "p_holm_global":0.0000000001, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_108", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":93, "wins_b":28, "ties":0, "win_rate_a_over_b":0.7685950413, "mean_diff_a_minus_b":0.0776867683, "median_diff_a_minus_b":0.043483858, "W_stat":995.0, "p_two_sided":0.0, "z_equiv":6.9708967963, "effect_size_r":0.6337178906, "p_holm_global":0.0000000001, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":120, "wins_a":32, "wins_b":88, "ties":1, "win_rate_a_over_b":0.2685950413, "mean_diff_a_minus_b":-0.098220007, "median_diff_a_minus_b":-0.035191671, "W_stat":1203.0, "p_two_sided":0.0000000002, "z_equiv":-6.3546858702, "effect_size_r":0.5801007995, "p_holm_global":0.0000000077, "p_holm_within_metric":0.0000000013 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":117, "wins_a":35, "wins_b":82, "ties":4, "win_rate_a_over_b":0.305785124, "mean_diff_a_minus_b":-0.0714544134, "median_diff_a_minus_b":-0.0293723642, "W_stat":1335.0, "p_two_sided":0.0000000087, "z_equiv":-5.7551072125, "effect_size_r":0.53205985, "p_holm_global":0.0000002598, "p_holm_within_metric":0.0000000433 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":89, "wins_b":32, "ties":0, "win_rate_a_over_b":0.7355371901, "mean_diff_a_minus_b":0.0516914869, "median_diff_a_minus_b":0.0233454705, "W_stat":1603.0, "p_two_sided":0.0000000673, "z_equiv":5.3982417863, "effect_size_r":0.4907492533, "p_holm_global":0.0000018843, "p_holm_within_metric":0.0000002692 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":81, "wins_b":40, "ties":0, "win_rate_a_over_b":0.6694214876, "mean_diff_a_minus_b":0.0249258933, "median_diff_a_minus_b":0.0182105228, "W_stat":2277.0, "p_two_sided":0.0002573114, "z_equiv":3.6548709363, "effect_size_r":0.3322609942, "p_holm_global":0.004631605, "p_holm_within_metric":0.0007719342 }, { "metric":"mean_cert_acc_108", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":54, "wins_b":67, "ties":0, "win_rate_a_over_b":0.4462809917, "mean_diff_a_minus_b":-0.0259952814, "median_diff_a_minus_b":-0.0096504689, "W_stat":2867.0, "p_two_sided":0.0332728975, "z_equiv":-2.1287747916, "effect_size_r":0.1935249811, "p_holm_global":0.3078706868, "p_holm_within_metric":0.066545795 }, { "metric":"mean_cert_acc_108", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":67, "wins_b":54, "ties":0, "win_rate_a_over_b":0.5537190083, "mean_diff_a_minus_b":0.0267655936, "median_diff_a_minus_b":0.0061113834, "W_stat":2930.0, "p_two_sided":0.0493195497, "z_equiv":1.9658187626, "effect_size_r":0.1787107966, "p_holm_global":0.3945563974, "p_holm_within_metric":0.066545795 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":110, "wins_a":0, "wins_b":110, "ties":11, "win_rate_a_over_b":0.0454545455, "mean_diff_a_minus_b":-0.1337634204, "median_diff_a_minus_b":-0.0716657937, "W_stat":0.0, "p_two_sided":8.869456124e-20, "z_equiv":-9.1019863804, "effect_size_r":0.8678403502, "p_holm_global":5.587757358e-18, "p_holm_within_metric":1.330418419e-18 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":114, "wins_a":2, "wins_b":112, "ties":7, "win_rate_a_over_b":0.0454545455, "mean_diff_a_minus_b":-0.1626829197, "median_diff_a_minus_b":-0.0885858461, "W_stat":74.0, "p_two_sided":1.351396746e-19, "z_equiv":-9.0561471899, "effect_size_r":0.8481858966, "p_holm_global":8.378659827e-18, "p_holm_within_metric":1.891955445e-18 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":103, "wins_a":2, "wins_b":101, "ties":18, "win_rate_a_over_b":0.0909090909, "mean_diff_a_minus_b":-0.1051036161, "median_diff_a_minus_b":-0.0429337709, "W_stat":22.0, "p_two_sided":2.407012276e-18, "z_equiv":-8.7363791132, "effect_size_r":0.8608210125, "p_holm_global":1.468277489e-16, "p_holm_within_metric":3.129115959e-17 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":105, "wins_a":2, "wins_b":103, "ties":16, "win_rate_a_over_b":0.0826446281, "mean_diff_a_minus_b":-0.1135240161, "median_diff_a_minus_b":-0.0488128485, "W_stat":71.0, "p_two_sided":4.454047917e-18, "z_equiv":-8.66655163, "effect_size_r":0.8457688368, "p_holm_global":2.67242875e-16, "p_holm_within_metric":5.344857501e-17 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":114, "wins_a":10, "wins_b":104, "ties":7, "win_rate_a_over_b":0.1115702479, "mean_diff_a_minus_b":-0.0879753088, "median_diff_a_minus_b":-0.0350877196, "W_stat":415.0, "p_two_sided":5.869031305e-16, "z_equiv":-8.0920053879, "effect_size_r":0.7578857434, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_255", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":114, "wins_a":17, "wins_b":97, "ties":7, "win_rate_a_over_b":0.1694214876, "mean_diff_a_minus_b":-0.0491589036, "median_diff_a_minus_b":-0.0273425561, "W_stat":465.0, "p_two_sided":0.0, "z_equiv":-7.9506356222, "effect_size_r":0.7446452517, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_255", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":87, "wins_a":3, "wins_b":84, "ties":34, "win_rate_a_over_b":0.1652892562, "mean_diff_a_minus_b":-0.0747076109, "median_diff_a_minus_b":-0.0172084123, "W_stat":48.0, "p_two_sided":0.0, "z_equiv":-7.8955227184, "effect_size_r":0.8464888875, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":110, "wins_a":20, "wins_b":90, "ties":11, "win_rate_a_over_b":0.2107438017, "mean_diff_a_minus_b":-0.0590558095, "median_diff_a_minus_b":-0.0295574367, "W_stat":685.0, "p_two_sided":0.0, "z_equiv":-7.0591093586, "effect_size_r":0.6730596687, "p_holm_global":0.0000000001, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_255", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":114, "wins_a":84, "wins_b":30, "ties":7, "win_rate_a_over_b":0.7231404959, "mean_diff_a_minus_b":0.0575793037, "median_diff_a_minus_b":0.0163958966, "W_stat":1043.0, "p_two_sided":0.0000000003, "z_equiv":6.3164011309, "effect_size_r":0.5915851679, "p_holm_global":0.0000000096, "p_holm_within_metric":0.0000000019 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":111, "wins_a":80, "wins_b":31, "ties":10, "win_rate_a_over_b":0.7024793388, "mean_diff_a_minus_b":0.0286598044, "median_diff_a_minus_b":0.0146750305, "W_stat":1338.0, "p_two_sided":0.0000001926, "z_equiv":5.2063487426, "effect_size_r":0.4941647538, "p_holm_global":0.0000052, "p_holm_within_metric":0.0000011556 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":103, "wins_a":27, "wins_b":76, "ties":18, "win_rate_a_over_b":0.2975206612, "mean_diff_a_minus_b":-0.0303960051, "median_diff_a_minus_b":-0.0085921446, "W_stat":1234.0, "p_two_sided":0.0000020443, "z_equiv":-4.7489976464, "effect_size_r":0.4679326423, "p_holm_global":0.000049814, "p_holm_within_metric":0.0000102214 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":112, "wins_a":79, "wins_b":33, "ties":9, "win_rate_a_over_b":0.6900826446, "mean_diff_a_minus_b":0.0202394043, "median_diff_a_minus_b":0.0148802027, "W_stat":1541.0, "p_two_sided":0.0000024732, "z_equiv":4.7103245755, "effect_size_r":0.4450838365, "p_holm_global":0.0000568842, "p_holm_within_metric":0.0000102214 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":104, "wins_a":30, "wins_b":74, "ties":17, "win_rate_a_over_b":0.3181818182, "mean_diff_a_minus_b":-0.0388164052, "median_diff_a_minus_b":-0.0039353251, "W_stat":1287.0, "p_two_sided":0.0000029004, "z_equiv":-4.6777497092, "effect_size_r":0.4586910971, "p_holm_global":0.0000638089, "p_holm_within_metric":0.0000102214 }, { "metric":"mean_cert_acc_255", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":115, "wins_a":45, "wins_b":70, "ties":6, "win_rate_a_over_b":0.3966942149, "mean_diff_a_minus_b":-0.0289194993, "median_diff_a_minus_b":-0.0067800414, "W_stat":2292.0, "p_two_sided":0.003621671, "z_equiv":-2.9093621184, "effect_size_r":0.2712994164, "p_holm_global":0.050703394, "p_holm_within_metric":0.007243342 }, { "metric":"mean_cert_acc_255", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":107, "wins_a":51, "wins_b":56, "ties":14, "win_rate_a_over_b":0.479338843, "mean_diff_a_minus_b":0.0084204, "median_diff_a_minus_b":0.0, "W_stat":2810.0, "p_two_sided":0.8072471563, "z_equiv":0.2439789068, "effect_size_r":0.0235863312, "p_holm_global":1.0, "p_holm_within_metric":0.8072471563 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":3, "wins_b":118, "ties":0, "win_rate_a_over_b":0.0247933884, "mean_diff_a_minus_b":-0.3159544249, "median_diff_a_minus_b":-0.2903496213, "W_stat":53.0, "p_two_sided":5.081865377e-21, "z_equiv":-9.4074774205, "effect_size_r":0.85522522, "p_holm_global":3.608124418e-19, "p_holm_within_metric":7.622798066e-20 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":4, "wins_b":117, "ties":0, "win_rate_a_over_b":0.0330578512, "mean_diff_a_minus_b":-0.2939407802, "median_diff_a_minus_b":-0.2717649229, "W_stat":115.0, "p_two_sided":2.306486848e-20, "z_equiv":-9.2471079952, "effect_size_r":0.8406461814, "p_holm_global":1.52228132e-18, "p_holm_within_metric":3.229081588e-19 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":9, "wins_b":112, "ties":0, "win_rate_a_over_b":0.0743801653, "mean_diff_a_minus_b":-0.3362883789, "median_diff_a_minus_b":-0.3413665891, "W_stat":135.0, "p_two_sided":3.736856648e-20, "z_equiv":-9.1953759225, "effect_size_r":0.8359432657, "p_holm_global":2.428956821e-18, "p_holm_within_metric":4.857913642e-19 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":8, "wins_b":113, "ties":0, "win_rate_a_over_b":0.0661157025, "mean_diff_a_minus_b":-0.3145207473, "median_diff_a_minus_b":-0.298160661, "W_stat":140.0, "p_two_sided":4.214205424e-20, "z_equiv":-9.1824429043, "effect_size_r":0.8347675368, "p_holm_global":2.697091471e-18, "p_holm_within_metric":5.057046509e-19 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":22, "wins_b":99, "ties":0, "win_rate_a_over_b":0.1818181818, "mean_diff_a_minus_b":-0.1593331515, "median_diff_a_minus_b":-0.0761357992, "W_stat":770.0, "p_two_sided":0.0, "z_equiv":-7.5528826142, "effect_size_r":0.6866256922, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_36", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":117, "wins_a":22, "wins_b":95, "ties":4, "win_rate_a_over_b":0.1983471074, "mean_diff_a_minus_b":-0.1769552274, "median_diff_a_minus_b":-0.1111815423, "W_stat":872.0, "p_two_sided":0.0, "z_equiv":-7.0143768908, "effect_size_r":0.648479373, "p_holm_global":0.0000000001, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":34, "wins_b":87, "ties":0, "win_rate_a_over_b":0.2809917355, "mean_diff_a_minus_b":-0.1389991975, "median_diff_a_minus_b":-0.0613140762, "W_stat":1073.0, "p_two_sided":0.0, "z_equiv":-6.7691417128, "effect_size_r":0.6153765193, "p_holm_global":0.0000000005, "p_holm_within_metric":0.0000000001 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":38, "wins_b":83, "ties":0, "win_rate_a_over_b":0.3140495868, "mean_diff_a_minus_b":-0.1169855528, "median_diff_a_minus_b":-0.0544650611, "W_stat":1275.0, "p_two_sided":0.0000000004, "z_equiv":-6.2466477785, "effect_size_r":0.5678770708, "p_holm_global":0.0000000143, "p_holm_within_metric":0.0000000034 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":40, "wins_b":81, "ties":0, "win_rate_a_over_b":0.3305785124, "mean_diff_a_minus_b":-0.1375655199, "median_diff_a_minus_b":-0.058108862, "W_stat":1346.0, "p_two_sided":0.0000000013, "z_equiv":-6.0629989205, "effect_size_r":0.55118172, "p_holm_global":0.0000000428, "p_holm_within_metric":0.0000000094 }, { "metric":"mean_cert_acc_36", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":89, "wins_b":32, "ties":0, "win_rate_a_over_b":0.7355371901, "mean_diff_a_minus_b":0.0423475987, "median_diff_a_minus_b":0.0236002803, "W_stat":1459.0, "p_two_sided":0.0000000079, "z_equiv":5.7707127097, "effect_size_r":0.5246102463, "p_holm_global":0.0000002447, "p_holm_within_metric":0.0000000474 }, { "metric":"mean_cert_acc_36", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":33, "wins_b":87, "ties":1, "win_rate_a_over_b":0.2768595041, "mean_diff_a_minus_b":-0.0217676316, "median_diff_a_minus_b":-0.0164444894, "W_stat":1846.0, "p_two_sided":0.000003001, "z_equiv":-4.6707530392, "effect_size_r":0.4263794667, "p_holm_global":0.0000638089, "p_holm_within_metric":0.0000150049 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":81, "wins_b":40, "ties":0, "win_rate_a_over_b":0.6694214876, "mean_diff_a_minus_b":0.0220136447, "median_diff_a_minus_b":0.0100490861, "W_stat":2305.0, "p_two_sided":0.000340392, "z_equiv":3.5824460345, "effect_size_r":0.3256769122, "p_holm_global":0.0054462712, "p_holm_within_metric":0.0013615678 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":50, "wins_b":71, "ties":0, "win_rate_a_over_b":0.4132231405, "mean_diff_a_minus_b":-0.020333954, "median_diff_a_minus_b":-0.0093774051, "W_stat":2753.0, "p_two_sided":0.0153655075, "z_equiv":-2.423647606, "effect_size_r":0.2203316005, "p_holm_global":0.1843860902, "p_holm_within_metric":0.0460965226 }, { "metric":"mean_cert_acc_36", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":65, "wins_b":56, "ties":0, "win_rate_a_over_b":0.5371900826, "mean_diff_a_minus_b":0.0205799671, "median_diff_a_minus_b":0.0014110952, "W_stat":2942.0, "p_two_sided":0.0530173574, "z_equiv":1.934779519, "effect_size_r":0.1758890472, "p_holm_global":0.3945563974, "p_holm_within_metric":0.1060347149 }, { "metric":"mean_cert_acc_36", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":75, "wins_b":46, "ties":0, "win_rate_a_over_b":0.6198347107, "mean_diff_a_minus_b":0.0014336775, "median_diff_a_minus_b":0.0064880252, "W_stat":3172.0, "p_two_sided":0.1802906425, "z_equiv":1.3398606829, "effect_size_r":0.1218055166, "p_holm_global":1.0, "p_holm_within_metric":0.1802906425 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":0, "wins_b":121, "ties":0, "win_rate_a_over_b":0.0, "mean_diff_a_minus_b":-0.2656215986, "median_diff_a_minus_b":-0.2418231331, "W_stat":0.0, "p_two_sided":1.36676811e-21, "z_equiv":-9.5445674132, "effect_size_r":0.8676879467, "p_holm_global":1.025076083e-19, "p_holm_within_metric":2.050152165e-20 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":1, "wins_b":120, "ties":0, "win_rate_a_over_b":0.0082644628, "mean_diff_a_minus_b":-0.3045117887, "median_diff_a_minus_b":-0.2825206725, "W_stat":25.0, "p_two_sided":2.545212099e-21, "z_equiv":-9.4799023223, "effect_size_r":0.861809302, "p_holm_global":1.883456953e-19, "p_holm_within_metric":3.563296938e-20 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":4, "wins_b":117, "ties":0, "win_rate_a_over_b":0.0330578512, "mean_diff_a_minus_b":-0.2930986073, "median_diff_a_minus_b":-0.2681985311, "W_stat":45.0, "p_two_sided":4.173048145e-21, "z_equiv":-9.4281702496, "effect_size_r":0.8571063863, "p_holm_global":3.046325146e-19, "p_holm_within_metric":5.424962588e-20 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":6, "wins_b":115, "ties":0, "win_rate_a_over_b":0.0495867769, "mean_diff_a_minus_b":-0.332593737, "median_diff_a_minus_b":-0.3329101392, "W_stat":80.0, "p_two_sided":9.850602055e-21, "z_equiv":-9.3376391224, "effect_size_r":0.8488762839, "p_holm_global":6.796915418e-19, "p_holm_within_metric":1.182072247e-19 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":16, "wins_b":105, "ties":0, "win_rate_a_over_b":0.132231405, "mean_diff_a_minus_b":-0.1617923106, "median_diff_a_minus_b":-0.0864403248, "W_stat":509.0, "p_two_sided":1.903870686e-16, "z_equiv":-8.227986163, "effect_size_r":0.7479987421, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_72", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":103, "wins_a":10, "wins_b":93, "ties":18, "win_rate_a_over_b":0.1570247934, "mean_diff_a_minus_b":-0.1708014264, "median_diff_a_minus_b":-0.0849279724, "W_stat":304.0, "p_two_sided":0.0, "z_equiv":-7.8086220392, "effect_size_r":0.7694063917, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":30, "wins_b":91, "ties":0, "win_rate_a_over_b":0.2479338843, "mean_diff_a_minus_b":-0.1337103623, "median_diff_a_minus_b":-0.0764530534, "W_stat":873.0, "p_two_sided":0.0, "z_equiv":-7.2864624398, "effect_size_r":0.6624056763, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_cert_acc_72", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":95, "wins_b":26, "ties":0, "win_rate_a_over_b":0.7851239669, "mean_diff_a_minus_b":0.0669721384, "median_diff_a_minus_b":0.039064005, "W_stat":1169.0, "p_two_sided":0.0000000001, "z_equiv":6.5208277639, "effect_size_r":0.592802524, "p_holm_global":0.0000000028, "p_holm_within_metric":0.0000000006 }, { "metric":"mean_cert_acc_72", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":22, "wins_b":98, "ties":1, "win_rate_a_over_b":0.1859504132, "mean_diff_a_minus_b":-0.0394951297, "median_diff_a_minus_b":-0.0318346173, "W_stat":1154.0, "p_two_sided":0.0000000001, "z_equiv":-6.4830104561, "effect_size_r":0.5918151779, "p_holm_global":0.0000000035, "p_holm_within_metric":0.0000000006 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":34, "wins_b":87, "ties":0, "win_rate_a_over_b":0.2809917355, "mean_diff_a_minus_b":-0.0948201722, "median_diff_a_minus_b":-0.04369919, "W_stat":1219.0, "p_two_sided":0.0000000002, "z_equiv":-6.3914975821, "effect_size_r":0.5810452347, "p_holm_global":0.0000000062, "p_holm_within_metric":0.000000001 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":36, "wins_b":85, "ties":0, "win_rate_a_over_b":0.2975206612, "mean_diff_a_minus_b":-0.1222971809, "median_diff_a_minus_b":-0.0452102409, "W_stat":1256.0, "p_two_sided":0.0000000003, "z_equiv":-6.2957932476, "effect_size_r":0.5723448407, "p_holm_global":0.0000000107, "p_holm_within_metric":0.0000000015 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":89, "wins_b":32, "ties":0, "win_rate_a_over_b":0.7355371901, "mean_diff_a_minus_b":0.03889019, "median_diff_a_minus_b":0.0201615543, "W_stat":1852.0, "p_two_sided":0.0000019926, "z_equiv":4.7541774811, "effect_size_r":0.4321979528, "p_holm_global":0.000049814, "p_holm_within_metric":0.0000079702 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":49, "wins_b":72, "ties":0, "win_rate_a_over_b":0.4049586777, "mean_diff_a_minus_b":-0.0280819483, "median_diff_a_minus_b":-0.0133254528, "W_stat":2656.0, "p_two_sided":0.0074830028, "z_equiv":-2.6745481586, "effect_size_r":0.2431407417, "p_holm_global":0.0972790362, "p_holm_within_metric":0.0224490084 }, { "metric":"mean_cert_acc_72", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":67, "wins_b":54, "ties":0, "win_rate_a_over_b":0.5537190083, "mean_diff_a_minus_b":0.0274770087, "median_diff_a_minus_b":0.0045558959, "W_stat":2784.0, "p_two_sided":0.0191056626, "z_equiv":2.3434628933, "effect_size_r":0.2130420812, "p_holm_global":0.2101622889, "p_holm_within_metric":0.0382113252 }, { "metric":"mean_cert_acc_72", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":74, "wins_b":47, "ties":0, "win_rate_a_over_b":0.6115702479, "mean_diff_a_minus_b":0.0114131814, "median_diff_a_minus_b":0.0088901073, "W_stat":2855.0, "p_two_sided":0.0307870687, "z_equiv":2.1598140352, "effect_size_r":0.1963467305, "p_holm_global":0.3078706868, "p_holm_within_metric":0.0382113252 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"ldlt", "n_common":121, "n_nonzero":121, "wins_a":16, "wins_b":105, "ties":0, "win_rate_a_over_b":0.132231405, "mean_diff_a_minus_b":-0.1195835257, "median_diff_a_minus_b":-0.0631501061, "W_stat":409.0, "p_two_sided":2.126854418e-17, "z_equiv":-8.4866465265, "effect_size_r":0.7715133206, "p_holm_global":0.0, "p_holm_within_metric":3.190281628e-16 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":23, "wins_b":98, "ties":0, "win_rate_a_over_b":0.1900826446, "mean_diff_a_minus_b":-0.1113283867, "median_diff_a_minus_b":-0.0656148695, "W_stat":698.0, "p_two_sided":0.0, "z_equiv":-7.7391180759, "effect_size_r":0.7035561887, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":25, "wins_b":96, "ties":0, "win_rate_a_over_b":0.2066115702, "mean_diff_a_minus_b":-0.0986620727, "median_diff_a_minus_b":-0.0525094743, "W_stat":908.0, "p_two_sided":0.0, "z_equiv":-7.1959313126, "effect_size_r":0.6541755739, "p_holm_global":0.0, "p_holm_within_metric":0.0 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":24, "wins_b":97, "ties":0, "win_rate_a_over_b":0.1983471074, "mean_diff_a_minus_b":-0.0961238247, "median_diff_a_minus_b":-0.0445100629, "W_stat":1047.0, "p_two_sided":0.0, "z_equiv":-6.8363934073, "effect_size_r":0.6214903098, "p_holm_global":0.0000000003, "p_holm_within_metric":0.0000000001 }, { "metric":"mean_test_acc", "alg_a":"aol", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":27, "wins_b":94, "ties":0, "win_rate_a_over_b":0.2231404959, "mean_diff_a_minus_b":-0.0920606953, "median_diff_a_minus_b":-0.0356908378, "W_stat":1307.0, "p_two_sided":0.0000000007, "z_equiv":-6.1638764622, "effect_size_r":0.5603524057, "p_holm_global":0.0000000234, "p_holm_within_metric":0.0000000078 }, { "metric":"mean_test_acc", "alg_a":"ldlt", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":86, "wins_b":35, "ties":0, "win_rate_a_over_b":0.7107438017, "mean_diff_a_minus_b":0.023459701, "median_diff_a_minus_b":0.0102784065, "W_stat":1511.0, "p_two_sided":0.0000000174, "z_equiv":5.6362093207, "effect_size_r":0.5123826655, "p_holm_global":0.0000005041, "p_holm_within_metric":0.0000001738 }, { "metric":"mean_test_acc", "alg_a":"ldlt", "alg_b":"ldlt-resnet", "n_common":121, "n_nonzero":121, "wins_a":87, "wins_b":34, "ties":0, "win_rate_a_over_b":0.7190082645, "mean_diff_a_minus_b":0.0275228304, "median_diff_a_minus_b":0.0124811598, "W_stat":1779.0, "p_two_sided":0.0000007693, "z_equiv":4.9429995465, "effect_size_r":0.4493635951, "p_holm_global":0.0000200017, "p_holm_within_metric":0.0000069237 }, { "metric":"mean_test_acc", "alg_a":"ldlt", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":78, "wins_b":43, "ties":0, "win_rate_a_over_b":0.6446280992, "mean_diff_a_minus_b":0.020921453, "median_diff_a_minus_b":0.010356299, "W_stat":2002.0, "p_two_sided":0.0000126434, "z_equiv":4.3661869359, "effect_size_r":0.3969260851, "p_holm_global":0.0002528686, "p_holm_within_metric":0.0001011474 }, { "metric":"mean_test_acc", "alg_a":"ortho", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":39, "wins_b":81, "ties":1, "win_rate_a_over_b":0.326446281, "mean_diff_a_minus_b":-0.012666314, "median_diff_a_minus_b":-0.0078946893, "W_stat":2171.0, "p_two_sided":0.0001336571, "z_equiv":-3.8196205818, "effect_size_r":0.348682059, "p_holm_global":0.0025394851, "p_holm_within_metric":0.0009355998 }, { "metric":"mean_test_acc", "alg_a":"sandwich", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":80, "wins_b":41, "ties":0, "win_rate_a_over_b":0.6611570248, "mean_diff_a_minus_b":0.015204562, "median_diff_a_minus_b":0.006305895, "W_stat":2279.0, "p_two_sided":0.0002625491, "z_equiv":3.649697729, "effect_size_r":0.3317907026, "p_holm_global":0.004631605, "p_holm_within_metric":0.0015752945 }, { "metric":"mean_test_acc", "alg_a":"ldlt-resnet", "alg_b":"sandwich", "n_common":121, "n_nonzero":121, "wins_a":47, "wins_b":74, "ties":0, "win_rate_a_over_b":0.3884297521, "mean_diff_a_minus_b":-0.0192676914, "median_diff_a_minus_b":-0.01009705, "W_stat":2411.0, "p_two_sided":0.0009387558, "z_equiv":-3.3082660492, "effect_size_r":0.300751459, "p_holm_global":0.0140813373, "p_holm_within_metric":0.0046937791 }, { "metric":"mean_test_acc", "alg_a":"ldlt", "alg_b":"sandwich", "n_common":121, "n_nonzero":120, "wins_a":64, "wins_b":56, "ties":1, "win_rate_a_over_b":0.5330578512, "mean_diff_a_minus_b":0.008255139, "median_diff_a_minus_b":0.0008221223, "W_stat":3145.0, "p_two_sided":0.2044973939, "z_equiv":1.2688420788, "effect_size_r":0.1158289047, "p_holm_global":1.0, "p_holm_within_metric":0.8179895757 }, { "metric":"mean_test_acc", "alg_a":"ldlt-resnet", "alg_b":"ortho", "n_common":121, "n_nonzero":121, "wins_a":54, "wins_b":67, "ties":0, "win_rate_a_over_b":0.4462809917, "mean_diff_a_minus_b":-0.0066013774, "median_diff_a_minus_b":-0.0023532391, "W_stat":3206.0, "p_two_sided":0.2106004159, "z_equiv":-1.2519161593, "effect_size_r":0.1138105599, "p_holm_global":1.0, "p_holm_within_metric":0.8179895757 }, { "metric":"mean_test_acc", "alg_a":"ortho", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":60, "wins_b":61, "ties":0, "win_rate_a_over_b":0.4958677686, "mean_diff_a_minus_b":0.0025382481, "median_diff_a_minus_b":-0.0000065726, "W_stat":3382.0, "p_two_sided":0.4256404345, "z_equiv":0.7966739196, "effect_size_r":0.0724249018, "p_holm_global":1.0, "p_holm_within_metric":0.8512808691 }, { "metric":"mean_test_acc", "alg_a":"ldlt-resnet", "alg_b":"sdp", "n_common":121, "n_nonzero":121, "wins_a":58, "wins_b":63, "ties":0, "win_rate_a_over_b":0.479338843, "mean_diff_a_minus_b":-0.0040631294, "median_diff_a_minus_b":-0.0009775593, "W_stat":3627.0, "p_two_sided":0.8705530556, "z_equiv":-0.162956029, "effect_size_r":0.0148141845, "p_holm_global":1.0, "p_holm_within_metric":0.8705530556 } ]