daim / exp /analysis /humaneval_behavior.json
yaro1214's picture
Upload dflash source snapshot
09af6cc verified
Raw History Blame Contribute Delete
9.63 kB
Invalid JSON:Unexpected token 'N', ...", NaN "... is not valid JSON
{
"dataset": "humaneval",
"mean_accept_allmask": 3.948973607038123,
"iters_allmask": 1705,
"mean_accept_soft": 4.188315724052206,
"iters_soft": 1609,
"per_slot_accept": {
"allmask": {
"p": [
0.7853372434017595,
0.7356235997012696,
0.7441624365482233,
0.732605729877217,
0.776536312849162,
0.7458033573141487,
0.7041800643086816,
0.7351598173515982,
0.7080745341614907,
0.6929824561403509,
0.7088607594936709,
0.6785714285714286,
0.47368421052631576,
0.6666666666666666,
0.75
],
"n": [
1705,
1339,
985,
733,
537,
417,
311,
219,
161,
114,
79,
56,
38,
18,
12
]
},
"soft_all": {
"p": [
0.7942821628340584,
0.7449139280125195,
0.75,
0.7703081232492998,
0.7454545454545455,
0.7829268292682927,
0.7570093457943925,
0.7407407407407407,
0.7722222222222223,
0.7553956834532374,
0.7428571428571429,
0.7435897435897436,
0.8103448275862069,
0.6595744680851063,
0.7741935483870968
],
"n": [
1609,
1278,
952,
714,
550,
410,
321,
243,
180,
139,
105,
78,
58,
47,
31
]
},
"soft_injected": {
"p": [
0.7880921895006402,
0.7389162561576355,
0.7528089887640449,
0.7659574468085106,
0.7387755102040816,
0.7867435158501441,
0.7431906614785992,
0.7325581395348837,
0.7433628318584071,
0.75,
0.6739130434782609,
0.7307692307692307,
0.8,
0.8333333333333334,
NaN
],
"n": [
1562,
1218,
890,
658,
490,
347,
257,
172,
113,
72,
46,
26,
15,
6,
0
]
},
"soft_plain": {
"p": [
1.0,
0.8666666666666667,
0.7096774193548387,
0.8214285714285714,
0.8,
0.7619047619047619,
0.8125,
0.7605633802816901,
0.8208955223880597,
0.7611940298507462,
0.7966101694915254,
0.75,
0.813953488372093,
0.6341463414634146,
0.7741935483870968
],
"n": [
47,
60,
62,
56,
60,
63,
64,
71,
67,
67,
59,
52,
43,
41,
31
]
}
},
"stale": {
"n": 5843,
"p_stale_correct": 0.289748416909122,
"p_draft_eq_stale": 0.2750299503679617,
"p_draft_correct_given_stale_correct": 0.8688718251624336,
"p_draft_correct_given_stale_wrong": 0.7168674698795181,
"n_stale_correct": 1693,
"n_stale_wrong": 4150
},
"draft_true_acc": {
"allmask": [
0.7487670004483634,
6691
],
"soft_injected": [
0.7609104911860346,
5843
],
"soft_plain": [
0.7933491686460807,
842
]
},
"confidence": {
"allmask": {
"draft_entropy": [
0.4822262190999417,
0.6194321467266822,
0.6793310558552478,
0.6951289749774279,
0.7095502910723125,
0.7399152832886364,
0.7771486154409833,
0.6533597869547272,
0.7813663820822925,
0.940931878819683,
0.9723743908295166,
0.867812811854362,
0.9898493239460023,
0.8508801045568867,
1.164206553871433
],
"draft_margin": [
5.468511730205279,
4.690020537714712,
4.683819796954315,
4.545446793997272,
4.5856610800744875,
4.2651378896882495,
4.143287781350482,
4.2117579908675795,
3.6952639751552794,
3.419407894736842,
2.7484177215189876,
3.0111607142857144,
2.4703947368421053,
2.4791666666666665,
1.8072916666666667
],
"draft_post_prob": [
0.7525642334602753,
0.6954548170649785,
0.6997965261496853,
0.6781012368041063,
0.7096506602097706,
0.6873723726877926,
0.6469180324712746,
0.678749335313006,
0.6453455868062328,
0.6160269136982901,
0.6329416942965428,
0.5705339292053395,
0.478534352867619,
0.5680604233833341,
0.637175852432847
],
"kl": [
2.635718817033946,
4.834327692850442,
5.598808940099261,
7.196740593228585,
6.566966955378826,
8.10424973854649,
9.414402867214653,
9.195208116797332,
10.81202663909116,
11.349533233687145,
11.22842841142644,
12.921131833057318,
18.919279036651318,
14.600540276383981,
13.099440122644106
]
},
"soft_injected": {
"draft_entropy": [
0.47434058730222434,
0.5976078964026429,
0.6541828303927636,
0.6483106892582394,
0.6967833905792554,
0.7131875165372659,
0.7081649251949111,
0.7298880264506594,
0.6840055754767016,
0.7725380352594584,
0.6942947078243141,
0.761553277915272,
0.5613124983385205,
0.6289710681885481,
NaN
],
"draft_margin": [
5.487996158770807,
4.832409688013136,
4.649719101123596,
4.576747720364741,
4.507525510204082,
4.495677233429395,
4.290612840466926,
4.031613372093023,
3.91537610619469,
3.4097222222222223,
3.7880434782608696,
3.9326923076923075,
3.529166666666667,
3.3333333333333335,
NaN
],
"draft_post_prob": [
0.7567602613848452,
0.7002803349261569,
0.7088243131601883,
0.7115280219297397,
0.696094905271069,
0.720909270695346,
0.685116020765508,
0.6801537426454478,
0.6789160468052832,
0.684002336745228,
0.6509463993883816,
0.6517678998195782,
0.7327756682410836,
0.78127870708704,
NaN
],
"kl": [
2.453038260544986,
4.73336213359238,
5.513123838561617,
6.0578736871354675,
6.580009719487561,
6.7250926806411435,
8.198265500825059,
8.607726810001665,
9.023444945351438,
7.700491157008906,
10.707569581861405,
9.40604196316473,
6.346413653592268,
5.048184854288896,
NaN
]
},
"soft_plain": {
"draft_entropy": [
0.040405993775713814,
0.27616663798263896,
0.42970561586158845,
0.4241469690191317,
0.5458362212698072,
0.4874553030188378,
0.4994212769747861,
0.4692442683110873,
0.5282427542925457,
0.6807720537847795,
0.613856958338825,
0.714026040254304,
0.9425503540686656,
1.1083221515845052,
0.9593555971138901
],
"draft_margin": [
8.944148936170214,
6.361458333333333,
5.487903225806452,
5.924107142857143,
5.865625,
5.396825396825397,
5.365234375,
5.640845070422535,
4.450559701492537,
3.7453358208955225,
3.680084745762712,
3.1923076923076925,
3.1482558139534884,
2.6966463414634148,
2.314516129032258
],
"draft_post_prob": [
0.9916122390868816,
0.8269412138111268,
0.7114935512466717,
0.7927069298457354,
0.7616262567336283,
0.7271546019735055,
0.7808423230530934,
0.7364120416238275,
0.7670932991543973,
0.706616341604012,
0.7224049867778006,
0.6781610697955609,
0.6953246948460854,
0.51585699517348,
0.6654305310258942
],
"kl": [
0.06694674533132262,
1.7980495042359357,
3.0492179376994843,
4.306691158987178,
5.311214989855671,
6.763043598033619,
5.468546570148263,
7.569672409044575,
6.989644391011315,
8.733376897949684,
7.799720932311236,
9.114654757749612,
9.055333296096949,
15.086656015139164,
10.431995185632859
]
}
},
"num_reuse_mean": 10.720323182100683,
"num_reuse_hist": [
47,
16,
11,
20,
27,
34,
41,
62,
78,
88,
136,
159,
233,
326,
331,
0
],
"accept_vs_num_reuse": {
"0": [
5.851063829787234,
47
],
"1": [
4.3125,
16
],
"2": [
7.2727272727272725,
11
],
"3": [
5.85,
20
],
"4": [
5.555555555555555,
27
],
"5": [
5.882352941176471,
34
],
"6": [
5.682926829268292,
41
],
"7": [
5.774193548387097,
62
],
"8": [
5.089743589743589,
78
],
"9": [
5.511363636363637,
88
],
"10": [
4.213235294117647,
136
],
"11": [
3.9622641509433962,
159
],
"12": [
3.6051502145922747,
233
],
"13": [
3.745398773006135,
326
],
"14": [
3.3564954682779455,
331
]
}
}