{ "dataset": "humaneval", "mean_accept_allmask": 3.948973607038123, "iters_allmask": 1705, "mean_accept_soft": 4.188315724052206, "iters_soft": 1609, "per_slot_accept": { "allmask": { "p": [ 0.7853372434017595, 0.7356235997012696, 0.7441624365482233, 0.732605729877217, 0.776536312849162, 0.7458033573141487, 0.7041800643086816, 0.7351598173515982, 0.7080745341614907, 0.6929824561403509, 0.7088607594936709, 0.6785714285714286, 0.47368421052631576, 0.6666666666666666, 0.75 ], "n": [ 1705, 1339, 985, 733, 537, 417, 311, 219, 161, 114, 79, 56, 38, 18, 12 ] }, "soft_all": { "p": [ 0.7942821628340584, 0.7449139280125195, 0.75, 0.7703081232492998, 0.7454545454545455, 0.7829268292682927, 0.7570093457943925, 0.7407407407407407, 0.7722222222222223, 0.7553956834532374, 0.7428571428571429, 0.7435897435897436, 0.8103448275862069, 0.6595744680851063, 0.7741935483870968 ], "n": [ 1609, 1278, 952, 714, 550, 410, 321, 243, 180, 139, 105, 78, 58, 47, 31 ] }, "soft_injected": { "p": [ 0.7880921895006402, 0.7389162561576355, 0.7528089887640449, 0.7659574468085106, 0.7387755102040816, 0.7867435158501441, 0.7431906614785992, 0.7325581395348837, 0.7433628318584071, 0.75, 0.6739130434782609, 0.7307692307692307, 0.8, 0.8333333333333334, NaN ], "n": [ 1562, 1218, 890, 658, 490, 347, 257, 172, 113, 72, 46, 26, 15, 6, 0 ] }, "soft_plain": { "p": [ 1.0, 0.8666666666666667, 0.7096774193548387, 0.8214285714285714, 0.8, 0.7619047619047619, 0.8125, 0.7605633802816901, 0.8208955223880597, 0.7611940298507462, 0.7966101694915254, 0.75, 0.813953488372093, 0.6341463414634146, 0.7741935483870968 ], "n": [ 47, 60, 62, 56, 60, 63, 64, 71, 67, 67, 59, 52, 43, 41, 31 ] } }, "stale": { "n": 5843, "p_stale_correct": 0.289748416909122, "p_draft_eq_stale": 0.2750299503679617, "p_draft_correct_given_stale_correct": 0.8688718251624336, "p_draft_correct_given_stale_wrong": 0.7168674698795181, "n_stale_correct": 1693, "n_stale_wrong": 4150 }, "draft_true_acc": { "allmask": [ 0.7487670004483634, 6691 ], "soft_injected": [ 0.7609104911860346, 5843 ], "soft_plain": [ 0.7933491686460807, 842 ] }, "confidence": { "allmask": { "draft_entropy": [ 0.4822262190999417, 0.6194321467266822, 0.6793310558552478, 0.6951289749774279, 0.7095502910723125, 0.7399152832886364, 0.7771486154409833, 0.6533597869547272, 0.7813663820822925, 0.940931878819683, 0.9723743908295166, 0.867812811854362, 0.9898493239460023, 0.8508801045568867, 1.164206553871433 ], "draft_margin": [ 5.468511730205279, 4.690020537714712, 4.683819796954315, 4.545446793997272, 4.5856610800744875, 4.2651378896882495, 4.143287781350482, 4.2117579908675795, 3.6952639751552794, 3.419407894736842, 2.7484177215189876, 3.0111607142857144, 2.4703947368421053, 2.4791666666666665, 1.8072916666666667 ], "draft_post_prob": [ 0.7525642334602753, 0.6954548170649785, 0.6997965261496853, 0.6781012368041063, 0.7096506602097706, 0.6873723726877926, 0.6469180324712746, 0.678749335313006, 0.6453455868062328, 0.6160269136982901, 0.6329416942965428, 0.5705339292053395, 0.478534352867619, 0.5680604233833341, 0.637175852432847 ], "kl": [ 2.635718817033946, 4.834327692850442, 5.598808940099261, 7.196740593228585, 6.566966955378826, 8.10424973854649, 9.414402867214653, 9.195208116797332, 10.81202663909116, 11.349533233687145, 11.22842841142644, 12.921131833057318, 18.919279036651318, 14.600540276383981, 13.099440122644106 ] }, "soft_injected": { "draft_entropy": [ 0.47434058730222434, 0.5976078964026429, 0.6541828303927636, 0.6483106892582394, 0.6967833905792554, 0.7131875165372659, 0.7081649251949111, 0.7298880264506594, 0.6840055754767016, 0.7725380352594584, 0.6942947078243141, 0.761553277915272, 0.5613124983385205, 0.6289710681885481, NaN ], "draft_margin": [ 5.487996158770807, 4.832409688013136, 4.649719101123596, 4.576747720364741, 4.507525510204082, 4.495677233429395, 4.290612840466926, 4.031613372093023, 3.91537610619469, 3.4097222222222223, 3.7880434782608696, 3.9326923076923075, 3.529166666666667, 3.3333333333333335, NaN ], "draft_post_prob": [ 0.7567602613848452, 0.7002803349261569, 0.7088243131601883, 0.7115280219297397, 0.696094905271069, 0.720909270695346, 0.685116020765508, 0.6801537426454478, 0.6789160468052832, 0.684002336745228, 0.6509463993883816, 0.6517678998195782, 0.7327756682410836, 0.78127870708704, NaN ], "kl": [ 2.453038260544986, 4.73336213359238, 5.513123838561617, 6.0578736871354675, 6.580009719487561, 6.7250926806411435, 8.198265500825059, 8.607726810001665, 9.023444945351438, 7.700491157008906, 10.707569581861405, 9.40604196316473, 6.346413653592268, 5.048184854288896, NaN ] }, "soft_plain": { "draft_entropy": [ 0.040405993775713814, 0.27616663798263896, 0.42970561586158845, 0.4241469690191317, 0.5458362212698072, 0.4874553030188378, 0.4994212769747861, 0.4692442683110873, 0.5282427542925457, 0.6807720537847795, 0.613856958338825, 0.714026040254304, 0.9425503540686656, 1.1083221515845052, 0.9593555971138901 ], "draft_margin": [ 8.944148936170214, 6.361458333333333, 5.487903225806452, 5.924107142857143, 5.865625, 5.396825396825397, 5.365234375, 5.640845070422535, 4.450559701492537, 3.7453358208955225, 3.680084745762712, 3.1923076923076925, 3.1482558139534884, 2.6966463414634148, 2.314516129032258 ], "draft_post_prob": [ 0.9916122390868816, 0.8269412138111268, 0.7114935512466717, 0.7927069298457354, 0.7616262567336283, 0.7271546019735055, 0.7808423230530934, 0.7364120416238275, 0.7670932991543973, 0.706616341604012, 0.7224049867778006, 0.6781610697955609, 0.6953246948460854, 0.51585699517348, 0.6654305310258942 ], "kl": [ 0.06694674533132262, 1.7980495042359357, 3.0492179376994843, 4.306691158987178, 5.311214989855671, 6.763043598033619, 5.468546570148263, 7.569672409044575, 6.989644391011315, 8.733376897949684, 7.799720932311236, 9.114654757749612, 9.055333296096949, 15.086656015139164, 10.431995185632859 ] } }, "num_reuse_mean": 10.720323182100683, "num_reuse_hist": [ 47, 16, 11, 20, 27, 34, 41, 62, 78, 88, 136, 159, 233, 326, 331, 0 ], "accept_vs_num_reuse": { "0": [ 5.851063829787234, 47 ], "1": [ 4.3125, 16 ], "2": [ 7.2727272727272725, 11 ], "3": [ 5.85, 20 ], "4": [ 5.555555555555555, 27 ], "5": [ 5.882352941176471, 34 ], "6": [ 5.682926829268292, 41 ], "7": [ 5.774193548387097, 62 ], "8": [ 5.089743589743589, 78 ], "9": [ 5.511363636363637, 88 ], "10": [ 4.213235294117647, 136 ], "11": [ 3.9622641509433962, 159 ], "12": [ 3.6051502145922747, 233 ], "13": [ 3.745398773006135, 326 ], "14": [ 3.3564954682779455, 331 ] } }