Add ctx32k/eval from udgamlabs/udgam3-ctx32k
Browse files- ctx32k/eval/eval_base2k.json +241 -0
- ctx32k/eval/eval_base2k.log +20 -0
- ctx32k/eval/eval_ctx32k.json +241 -0
- ctx32k/eval/eval_ctx32k.log +20 -0
ctx32k/eval/eval_base2k.json
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"ckpt": "/content/init/udgam3-base-v1/model_ema_076294.pt",
|
| 3 |
+
"config": {
|
| 4 |
+
"sequence_len": 8193,
|
| 5 |
+
"vocab_size": 32768,
|
| 6 |
+
"n_layer": 28,
|
| 7 |
+
"n_head": 10,
|
| 8 |
+
"n_kv_head": 2,
|
| 9 |
+
"n_embd": 640,
|
| 10 |
+
"head_dim": 64,
|
| 11 |
+
"global_head_dim": 128,
|
| 12 |
+
"intermediate_size": 1792,
|
| 13 |
+
"ple_dim": 0,
|
| 14 |
+
"window_pattern": "SSSL",
|
| 15 |
+
"sliding_window": 512,
|
| 16 |
+
"rope_theta_sliding": 10000.0,
|
| 17 |
+
"rope_theta_full": 1000000.0,
|
| 18 |
+
"final_logit_softcap": 15.0,
|
| 19 |
+
"rms_norm_eps": 1e-06,
|
| 20 |
+
"loop_start": -1,
|
| 21 |
+
"loop_end": -1,
|
| 22 |
+
"n_loops": 1,
|
| 23 |
+
"mtp": false,
|
| 24 |
+
"loop_loss": false,
|
| 25 |
+
"loop_loss_stride": 4,
|
| 26 |
+
"loop_loss_weight": -1.0,
|
| 27 |
+
"loop_reinject": false,
|
| 28 |
+
"abacus": 0,
|
| 29 |
+
"abacus_offset": 0,
|
| 30 |
+
"digit_ids": [
|
| 31 |
+
291,
|
| 32 |
+
292,
|
| 33 |
+
293,
|
| 34 |
+
294,
|
| 35 |
+
295,
|
| 36 |
+
296,
|
| 37 |
+
297,
|
| 38 |
+
298,
|
| 39 |
+
299,
|
| 40 |
+
300
|
| 41 |
+
],
|
| 42 |
+
"digit_space_ids": [
|
| 43 |
+
1150,
|
| 44 |
+
958,
|
| 45 |
+
994,
|
| 46 |
+
1106,
|
| 47 |
+
1189,
|
| 48 |
+
1182,
|
| 49 |
+
1348,
|
| 50 |
+
1506,
|
| 51 |
+
1500,
|
| 52 |
+
1564
|
| 53 |
+
],
|
| 54 |
+
"tie_embeddings": true
|
| 55 |
+
},
|
| 56 |
+
"bpb2048_web": 0.8501416386046416,
|
| 57 |
+
"bpb2048_task": 0.634110372765497,
|
| 58 |
+
"bpb2048_reason": 0.6844764738368659,
|
| 59 |
+
"bpb2048_fresh": 0.9209135476121134,
|
| 60 |
+
"long_bpb_L8192": {
|
| 61 |
+
"0-512": 1.173462,
|
| 62 |
+
"512-1024": 1.035295,
|
| 63 |
+
"1024-2048": 1.012183,
|
| 64 |
+
"2048-3072": 1.082558,
|
| 65 |
+
"3072-4096": 1.27177,
|
| 66 |
+
"4096-6144": 1.366089,
|
| 67 |
+
"6144-8192": 1.347903,
|
| 68 |
+
"n_docs": 400,
|
| 69 |
+
"per_src_first_half_second_half": {
|
| 70 |
+
"pre1929": [
|
| 71 |
+
1.20924,
|
| 72 |
+
1.47077,
|
| 73 |
+
60
|
| 74 |
+
],
|
| 75 |
+
"gutenberg": [
|
| 76 |
+
1.25172,
|
| 77 |
+
1.53167,
|
| 78 |
+
112
|
| 79 |
+
],
|
| 80 |
+
"arxiv": [
|
| 81 |
+
0.91582,
|
| 82 |
+
1.12526,
|
| 83 |
+
35
|
| 84 |
+
],
|
| 85 |
+
"loc": [
|
| 86 |
+
1.41106,
|
| 87 |
+
1.63034,
|
| 88 |
+
59
|
| 89 |
+
],
|
| 90 |
+
"doab": [
|
| 91 |
+
0.88866,
|
| 92 |
+
1.10496,
|
| 93 |
+
54
|
| 94 |
+
],
|
| 95 |
+
"hansard_commons": [
|
| 96 |
+
0.93938,
|
| 97 |
+
1.13722,
|
| 98 |
+
80
|
| 99 |
+
]
|
| 100 |
+
}
|
| 101 |
+
},
|
| 102 |
+
"long_bpb_L16384": {
|
| 103 |
+
"0-512": 1.173462,
|
| 104 |
+
"512-1024": 1.035295,
|
| 105 |
+
"1024-2048": 1.012183,
|
| 106 |
+
"2048-3072": 1.082558,
|
| 107 |
+
"3072-4096": 1.27177,
|
| 108 |
+
"4096-6144": 1.366089,
|
| 109 |
+
"6144-8192": 1.347903,
|
| 110 |
+
"8192-12288": 1.368215,
|
| 111 |
+
"12288-16384": 1.388605,
|
| 112 |
+
"n_docs": 400,
|
| 113 |
+
"per_src_first_half_second_half": {
|
| 114 |
+
"pre1929": [
|
| 115 |
+
1.33858,
|
| 116 |
+
1.49199,
|
| 117 |
+
60
|
| 118 |
+
],
|
| 119 |
+
"gutenberg": [
|
| 120 |
+
1.3932,
|
| 121 |
+
1.5649,
|
| 122 |
+
112
|
| 123 |
+
],
|
| 124 |
+
"arxiv": [
|
| 125 |
+
1.0123,
|
| 126 |
+
1.13282,
|
| 127 |
+
35
|
| 128 |
+
],
|
| 129 |
+
"loc": [
|
| 130 |
+
1.52222,
|
| 131 |
+
1.59875,
|
| 132 |
+
59
|
| 133 |
+
],
|
| 134 |
+
"doab": [
|
| 135 |
+
0.99651,
|
| 136 |
+
1.11498,
|
| 137 |
+
54
|
| 138 |
+
],
|
| 139 |
+
"hansard_commons": [
|
| 140 |
+
1.0388,
|
| 141 |
+
1.17896,
|
| 142 |
+
80
|
| 143 |
+
]
|
| 144 |
+
}
|
| 145 |
+
},
|
| 146 |
+
"long_bpb_L32768": {
|
| 147 |
+
"0-512": 1.173462,
|
| 148 |
+
"512-1024": 1.035295,
|
| 149 |
+
"1024-2048": 1.012183,
|
| 150 |
+
"2048-3072": 1.082558,
|
| 151 |
+
"3072-4096": 1.27177,
|
| 152 |
+
"4096-6144": 1.366089,
|
| 153 |
+
"6144-8192": 1.347903,
|
| 154 |
+
"8192-12288": 1.368215,
|
| 155 |
+
"12288-16384": 1.388605,
|
| 156 |
+
"16384-24576": 1.403997,
|
| 157 |
+
"24576-32768": 1.436084,
|
| 158 |
+
"n_docs": 400,
|
| 159 |
+
"per_src_first_half_second_half": {
|
| 160 |
+
"pre1929": [
|
| 161 |
+
1.41471,
|
| 162 |
+
1.50851,
|
| 163 |
+
60
|
| 164 |
+
],
|
| 165 |
+
"gutenberg": [
|
| 166 |
+
1.47915,
|
| 167 |
+
1.60659,
|
| 168 |
+
112
|
| 169 |
+
],
|
| 170 |
+
"arxiv": [
|
| 171 |
+
1.06798,
|
| 172 |
+
1.26401,
|
| 173 |
+
35
|
| 174 |
+
],
|
| 175 |
+
"loc": [
|
| 176 |
+
1.56117,
|
| 177 |
+
1.61969,
|
| 178 |
+
59
|
| 179 |
+
],
|
| 180 |
+
"doab": [
|
| 181 |
+
1.05531,
|
| 182 |
+
1.19035,
|
| 183 |
+
54
|
| 184 |
+
],
|
| 185 |
+
"hansard_commons": [
|
| 186 |
+
1.10888,
|
| 187 |
+
1.20284,
|
| 188 |
+
80
|
| 189 |
+
]
|
| 190 |
+
}
|
| 191 |
+
},
|
| 192 |
+
"needle": {
|
| 193 |
+
"1024": {
|
| 194 |
+
"exact": 1.0,
|
| 195 |
+
"mean_logprob_key_token": -0.0224,
|
| 196 |
+
"distance_tokens_approx": 920
|
| 197 |
+
},
|
| 198 |
+
"2048": {
|
| 199 |
+
"exact": 1.0,
|
| 200 |
+
"mean_logprob_key_token": -0.0422,
|
| 201 |
+
"distance_tokens_approx": 1944
|
| 202 |
+
},
|
| 203 |
+
"4096": {
|
| 204 |
+
"exact": 0.0,
|
| 205 |
+
"mean_logprob_key_token": -2.6239,
|
| 206 |
+
"distance_tokens_approx": 3992
|
| 207 |
+
},
|
| 208 |
+
"8192": {
|
| 209 |
+
"exact": 0.0,
|
| 210 |
+
"mean_logprob_key_token": -3.6672,
|
| 211 |
+
"distance_tokens_approx": 8088
|
| 212 |
+
},
|
| 213 |
+
"12288": {
|
| 214 |
+
"exact": 0.0,
|
| 215 |
+
"mean_logprob_key_token": -3.7545,
|
| 216 |
+
"distance_tokens_approx": 12184
|
| 217 |
+
},
|
| 218 |
+
"16384": {
|
| 219 |
+
"exact": 0.0,
|
| 220 |
+
"mean_logprob_key_token": -3.9069,
|
| 221 |
+
"distance_tokens_approx": 16280
|
| 222 |
+
},
|
| 223 |
+
"24576": {
|
| 224 |
+
"exact": 0.0,
|
| 225 |
+
"mean_logprob_key_token": -4.1492,
|
| 226 |
+
"distance_tokens_approx": 24472
|
| 227 |
+
},
|
| 228 |
+
"32768": {
|
| 229 |
+
"exact": 0.0,
|
| 230 |
+
"mean_logprob_key_token": -4.1836,
|
| 231 |
+
"distance_tokens_approx": 32664
|
| 232 |
+
}
|
| 233 |
+
},
|
| 234 |
+
"copy": {
|
| 235 |
+
"2048": 0.9475,
|
| 236 |
+
"8192": 0.043,
|
| 237 |
+
"16384": 0.0225,
|
| 238 |
+
"32768": 0.023
|
| 239 |
+
},
|
| 240 |
+
"seconds": 597.9
|
| 241 |
+
}
|
ctx32k/eval/eval_base2k.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
bpb@2048 web: 0.850142
|
| 2 |
+
bpb@2048 task: 0.634110
|
| 3 |
+
bpb@2048 reason: 0.684476
|
| 4 |
+
bpb@2048 fresh: 0.920914
|
| 5 |
+
long docs L=8192: {'0-512': 1.173462, '512-1024': 1.035295, '1024-2048': 1.012183, '2048-3072': 1.082558, '3072-4096': 1.27177, '4096-6144': 1.366089, '6144-8192': 1.347903, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.20924, 1.47077, 60], 'gutenberg': [1.25172, 1.53167, 112], 'arxiv': [0.91582, 1.12526, 35], 'loc': [1.41106, 1.63034, 59], 'doab': [0.88866, 1.10496, 54], 'hansard_commons': [0.93938, 1.13722, 80]}}
|
| 6 |
+
long docs L=16384: {'0-512': 1.173462, '512-1024': 1.035295, '1024-2048': 1.012183, '2048-3072': 1.082558, '3072-4096': 1.27177, '4096-6144': 1.366089, '6144-8192': 1.347903, '8192-12288': 1.368215, '12288-16384': 1.388605, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.33858, 1.49199, 60], 'gutenberg': [1.3932, 1.5649, 112], 'arxiv': [1.0123, 1.13282, 35], 'loc': [1.52222, 1.59875, 59], 'doab': [0.99651, 1.11498, 54], 'hansard_commons': [1.0388, 1.17896, 80]}}
|
| 7 |
+
long docs L=32768: {'0-512': 1.173462, '512-1024': 1.035295, '1024-2048': 1.012183, '2048-3072': 1.082558, '3072-4096': 1.27177, '4096-6144': 1.366089, '6144-8192': 1.347903, '8192-12288': 1.368215, '12288-16384': 1.388605, '16384-24576': 1.403997, '24576-32768': 1.436084, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.41471, 1.50851, 60], 'gutenberg': [1.47915, 1.60659, 112], 'arxiv': [1.06798, 1.26401, 35], 'loc': [1.56117, 1.61969, 59], 'doab': [1.05531, 1.19035, 54], 'hansard_commons': [1.10888, 1.20284, 80]}}
|
| 8 |
+
needle ctx 1024: {'exact': 1.0, 'mean_logprob_key_token': -0.0224, 'distance_tokens_approx': 920}
|
| 9 |
+
needle ctx 2048: {'exact': 1.0, 'mean_logprob_key_token': -0.0422, 'distance_tokens_approx': 1944}
|
| 10 |
+
needle ctx 4096: {'exact': 0.0, 'mean_logprob_key_token': -2.6239, 'distance_tokens_approx': 3992}
|
| 11 |
+
needle ctx 8192: {'exact': 0.0, 'mean_logprob_key_token': -3.6672, 'distance_tokens_approx': 8088}
|
| 12 |
+
needle ctx 12288: {'exact': 0.0, 'mean_logprob_key_token': -3.7545, 'distance_tokens_approx': 12184}
|
| 13 |
+
needle ctx 16384: {'exact': 0.0, 'mean_logprob_key_token': -3.9069, 'distance_tokens_approx': 16280}
|
| 14 |
+
needle ctx 24576: {'exact': 0.0, 'mean_logprob_key_token': -4.1492, 'distance_tokens_approx': 24472}
|
| 15 |
+
needle ctx 32768: {'exact': 0.0, 'mean_logprob_key_token': -4.1836, 'distance_tokens_approx': 32664}
|
| 16 |
+
copy ctx 2048: next-token acc 0.9475
|
| 17 |
+
copy ctx 8192: next-token acc 0.043
|
| 18 |
+
copy ctx 16384: next-token acc 0.0225
|
| 19 |
+
copy ctx 32768: next-token acc 0.023
|
| 20 |
+
wrote /content/eval_base2k.json
|
ctx32k/eval/eval_ctx32k.json
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"ckpt": "/content/cache/base_checkpoints/udgam3-ctx32k/model_001900.pt",
|
| 3 |
+
"config": {
|
| 4 |
+
"sequence_len": 32768,
|
| 5 |
+
"vocab_size": 32768,
|
| 6 |
+
"n_layer": 28,
|
| 7 |
+
"n_head": 10,
|
| 8 |
+
"n_kv_head": 2,
|
| 9 |
+
"n_embd": 640,
|
| 10 |
+
"head_dim": 64,
|
| 11 |
+
"global_head_dim": 128,
|
| 12 |
+
"intermediate_size": 1792,
|
| 13 |
+
"ple_dim": 0,
|
| 14 |
+
"window_pattern": "SSSL",
|
| 15 |
+
"sliding_window": 512,
|
| 16 |
+
"rope_theta_sliding": 10000.0,
|
| 17 |
+
"rope_theta_full": 1000000.0,
|
| 18 |
+
"final_logit_softcap": 15.0,
|
| 19 |
+
"rms_norm_eps": 1e-06,
|
| 20 |
+
"loop_start": -1,
|
| 21 |
+
"loop_end": -1,
|
| 22 |
+
"n_loops": 1,
|
| 23 |
+
"mtp": false,
|
| 24 |
+
"loop_loss": false,
|
| 25 |
+
"loop_loss_stride": 4,
|
| 26 |
+
"loop_loss_weight": -1.0,
|
| 27 |
+
"loop_reinject": false,
|
| 28 |
+
"abacus": 0,
|
| 29 |
+
"abacus_offset": 0,
|
| 30 |
+
"digit_ids": [
|
| 31 |
+
291,
|
| 32 |
+
292,
|
| 33 |
+
293,
|
| 34 |
+
294,
|
| 35 |
+
295,
|
| 36 |
+
296,
|
| 37 |
+
297,
|
| 38 |
+
298,
|
| 39 |
+
299,
|
| 40 |
+
300
|
| 41 |
+
],
|
| 42 |
+
"digit_space_ids": [
|
| 43 |
+
1150,
|
| 44 |
+
958,
|
| 45 |
+
994,
|
| 46 |
+
1106,
|
| 47 |
+
1189,
|
| 48 |
+
1182,
|
| 49 |
+
1348,
|
| 50 |
+
1506,
|
| 51 |
+
1500,
|
| 52 |
+
1564
|
| 53 |
+
],
|
| 54 |
+
"tie_embeddings": true
|
| 55 |
+
},
|
| 56 |
+
"bpb2048_web": 0.8535457788260455,
|
| 57 |
+
"bpb2048_task": 0.6393049352887717,
|
| 58 |
+
"bpb2048_reason": 0.6919760354978337,
|
| 59 |
+
"bpb2048_fresh": 0.9205743961059951,
|
| 60 |
+
"long_bpb_L8192": {
|
| 61 |
+
"0-512": 1.026994,
|
| 62 |
+
"512-1024": 0.935882,
|
| 63 |
+
"1024-2048": 0.944786,
|
| 64 |
+
"2048-3072": 0.930071,
|
| 65 |
+
"3072-4096": 0.921109,
|
| 66 |
+
"4096-6144": 0.928579,
|
| 67 |
+
"6144-8192": 0.925182,
|
| 68 |
+
"n_docs": 400,
|
| 69 |
+
"per_src_first_half_second_half": {
|
| 70 |
+
"pre1929": [
|
| 71 |
+
1.06866,
|
| 72 |
+
1.07843,
|
| 73 |
+
60
|
| 74 |
+
],
|
| 75 |
+
"gutenberg": [
|
| 76 |
+
1.0833,
|
| 77 |
+
1.06462,
|
| 78 |
+
112
|
| 79 |
+
],
|
| 80 |
+
"arxiv": [
|
| 81 |
+
0.76825,
|
| 82 |
+
0.6845,
|
| 83 |
+
35
|
| 84 |
+
],
|
| 85 |
+
"loc": [
|
| 86 |
+
1.21512,
|
| 87 |
+
1.12338,
|
| 88 |
+
59
|
| 89 |
+
],
|
| 90 |
+
"doab": [
|
| 91 |
+
0.68966,
|
| 92 |
+
0.69926,
|
| 93 |
+
54
|
| 94 |
+
],
|
| 95 |
+
"hansard_commons": [
|
| 96 |
+
0.75608,
|
| 97 |
+
0.75785,
|
| 98 |
+
80
|
| 99 |
+
]
|
| 100 |
+
}
|
| 101 |
+
},
|
| 102 |
+
"long_bpb_L16384": {
|
| 103 |
+
"0-512": 1.026994,
|
| 104 |
+
"512-1024": 0.935882,
|
| 105 |
+
"1024-2048": 0.944786,
|
| 106 |
+
"2048-3072": 0.930071,
|
| 107 |
+
"3072-4096": 0.921109,
|
| 108 |
+
"4096-6144": 0.928579,
|
| 109 |
+
"6144-8192": 0.925182,
|
| 110 |
+
"8192-12288": 0.928408,
|
| 111 |
+
"12288-16384": 0.938268,
|
| 112 |
+
"n_docs": 400,
|
| 113 |
+
"per_src_first_half_second_half": {
|
| 114 |
+
"pre1929": [
|
| 115 |
+
1.07349,
|
| 116 |
+
1.08106,
|
| 117 |
+
60
|
| 118 |
+
],
|
| 119 |
+
"gutenberg": [
|
| 120 |
+
1.07386,
|
| 121 |
+
1.08415,
|
| 122 |
+
112
|
| 123 |
+
],
|
| 124 |
+
"arxiv": [
|
| 125 |
+
0.72967,
|
| 126 |
+
0.6699,
|
| 127 |
+
35
|
| 128 |
+
],
|
| 129 |
+
"loc": [
|
| 130 |
+
1.16862,
|
| 131 |
+
1.104,
|
| 132 |
+
59
|
| 133 |
+
],
|
| 134 |
+
"doab": [
|
| 135 |
+
0.69445,
|
| 136 |
+
0.68398,
|
| 137 |
+
54
|
| 138 |
+
],
|
| 139 |
+
"hansard_commons": [
|
| 140 |
+
0.75697,
|
| 141 |
+
0.77872,
|
| 142 |
+
80
|
| 143 |
+
]
|
| 144 |
+
}
|
| 145 |
+
},
|
| 146 |
+
"long_bpb_L32768": {
|
| 147 |
+
"0-512": 1.026994,
|
| 148 |
+
"512-1024": 0.935882,
|
| 149 |
+
"1024-2048": 0.944786,
|
| 150 |
+
"2048-3072": 0.930071,
|
| 151 |
+
"3072-4096": 0.921109,
|
| 152 |
+
"4096-6144": 0.928579,
|
| 153 |
+
"6144-8192": 0.925182,
|
| 154 |
+
"8192-12288": 0.928408,
|
| 155 |
+
"12288-16384": 0.938268,
|
| 156 |
+
"16384-24576": 0.943074,
|
| 157 |
+
"24576-32768": 0.940369,
|
| 158 |
+
"n_docs": 400,
|
| 159 |
+
"per_src_first_half_second_half": {
|
| 160 |
+
"pre1929": [
|
| 161 |
+
1.07725,
|
| 162 |
+
1.07459,
|
| 163 |
+
60
|
| 164 |
+
],
|
| 165 |
+
"gutenberg": [
|
| 166 |
+
1.07901,
|
| 167 |
+
1.09185,
|
| 168 |
+
112
|
| 169 |
+
],
|
| 170 |
+
"arxiv": [
|
| 171 |
+
0.70206,
|
| 172 |
+
0.68107,
|
| 173 |
+
35
|
| 174 |
+
],
|
| 175 |
+
"loc": [
|
| 176 |
+
1.13573,
|
| 177 |
+
1.09291,
|
| 178 |
+
59
|
| 179 |
+
],
|
| 180 |
+
"doab": [
|
| 181 |
+
0.68925,
|
| 182 |
+
0.69537,
|
| 183 |
+
54
|
| 184 |
+
],
|
| 185 |
+
"hansard_commons": [
|
| 186 |
+
0.76784,
|
| 187 |
+
0.80331,
|
| 188 |
+
80
|
| 189 |
+
]
|
| 190 |
+
}
|
| 191 |
+
},
|
| 192 |
+
"needle": {
|
| 193 |
+
"1024": {
|
| 194 |
+
"exact": 1.0,
|
| 195 |
+
"mean_logprob_key_token": -0.0238,
|
| 196 |
+
"distance_tokens_approx": 920
|
| 197 |
+
},
|
| 198 |
+
"2048": {
|
| 199 |
+
"exact": 1.0,
|
| 200 |
+
"mean_logprob_key_token": -0.0283,
|
| 201 |
+
"distance_tokens_approx": 1944
|
| 202 |
+
},
|
| 203 |
+
"4096": {
|
| 204 |
+
"exact": 1.0,
|
| 205 |
+
"mean_logprob_key_token": -0.0521,
|
| 206 |
+
"distance_tokens_approx": 3992
|
| 207 |
+
},
|
| 208 |
+
"8192": {
|
| 209 |
+
"exact": 1.0,
|
| 210 |
+
"mean_logprob_key_token": -0.0644,
|
| 211 |
+
"distance_tokens_approx": 8088
|
| 212 |
+
},
|
| 213 |
+
"12288": {
|
| 214 |
+
"exact": 0.99,
|
| 215 |
+
"mean_logprob_key_token": -0.1275,
|
| 216 |
+
"distance_tokens_approx": 12184
|
| 217 |
+
},
|
| 218 |
+
"16384": {
|
| 219 |
+
"exact": 0.96,
|
| 220 |
+
"mean_logprob_key_token": -0.1908,
|
| 221 |
+
"distance_tokens_approx": 16280
|
| 222 |
+
},
|
| 223 |
+
"24576": {
|
| 224 |
+
"exact": 0.0,
|
| 225 |
+
"mean_logprob_key_token": -2.4617,
|
| 226 |
+
"distance_tokens_approx": 24472
|
| 227 |
+
},
|
| 228 |
+
"32768": {
|
| 229 |
+
"exact": 0.0,
|
| 230 |
+
"mean_logprob_key_token": -2.9218,
|
| 231 |
+
"distance_tokens_approx": 32664
|
| 232 |
+
}
|
| 233 |
+
},
|
| 234 |
+
"copy": {
|
| 235 |
+
"2048": 0.947,
|
| 236 |
+
"8192": 0.8895,
|
| 237 |
+
"16384": 0.749,
|
| 238 |
+
"32768": 0.192
|
| 239 |
+
},
|
| 240 |
+
"seconds": 598.7
|
| 241 |
+
}
|
ctx32k/eval/eval_ctx32k.log
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
bpb@2048 web: 0.853546
|
| 2 |
+
bpb@2048 task: 0.639305
|
| 3 |
+
bpb@2048 reason: 0.691976
|
| 4 |
+
bpb@2048 fresh: 0.920574
|
| 5 |
+
long docs L=8192: {'0-512': 1.026994, '512-1024': 0.935882, '1024-2048': 0.944786, '2048-3072': 0.930071, '3072-4096': 0.921109, '4096-6144': 0.928579, '6144-8192': 0.925182, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.06866, 1.07843, 60], 'gutenberg': [1.0833, 1.06462, 112], 'arxiv': [0.76825, 0.6845, 35], 'loc': [1.21512, 1.12338, 59], 'doab': [0.68966, 0.69926, 54], 'hansard_commons': [0.75608, 0.75785, 80]}}
|
| 6 |
+
long docs L=16384: {'0-512': 1.026994, '512-1024': 0.935882, '1024-2048': 0.944786, '2048-3072': 0.930071, '3072-4096': 0.921109, '4096-6144': 0.928579, '6144-8192': 0.925182, '8192-12288': 0.928408, '12288-16384': 0.938268, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.07349, 1.08106, 60], 'gutenberg': [1.07386, 1.08415, 112], 'arxiv': [0.72967, 0.6699, 35], 'loc': [1.16862, 1.104, 59], 'doab': [0.69445, 0.68398, 54], 'hansard_commons': [0.75697, 0.77872, 80]}}
|
| 7 |
+
long docs L=32768: {'0-512': 1.026994, '512-1024': 0.935882, '1024-2048': 0.944786, '2048-3072': 0.930071, '3072-4096': 0.921109, '4096-6144': 0.928579, '6144-8192': 0.925182, '8192-12288': 0.928408, '12288-16384': 0.938268, '16384-24576': 0.943074, '24576-32768': 0.940369, 'n_docs': 400, 'per_src_first_half_second_half': {'pre1929': [1.07725, 1.07459, 60], 'gutenberg': [1.07901, 1.09185, 112], 'arxiv': [0.70206, 0.68107, 35], 'loc': [1.13573, 1.09291, 59], 'doab': [0.68925, 0.69537, 54], 'hansard_commons': [0.76784, 0.80331, 80]}}
|
| 8 |
+
needle ctx 1024: {'exact': 1.0, 'mean_logprob_key_token': -0.0238, 'distance_tokens_approx': 920}
|
| 9 |
+
needle ctx 2048: {'exact': 1.0, 'mean_logprob_key_token': -0.0283, 'distance_tokens_approx': 1944}
|
| 10 |
+
needle ctx 4096: {'exact': 1.0, 'mean_logprob_key_token': -0.0521, 'distance_tokens_approx': 3992}
|
| 11 |
+
needle ctx 8192: {'exact': 1.0, 'mean_logprob_key_token': -0.0644, 'distance_tokens_approx': 8088}
|
| 12 |
+
needle ctx 12288: {'exact': 0.99, 'mean_logprob_key_token': -0.1275, 'distance_tokens_approx': 12184}
|
| 13 |
+
needle ctx 16384: {'exact': 0.96, 'mean_logprob_key_token': -0.1908, 'distance_tokens_approx': 16280}
|
| 14 |
+
needle ctx 24576: {'exact': 0.0, 'mean_logprob_key_token': -2.4617, 'distance_tokens_approx': 24472}
|
| 15 |
+
needle ctx 32768: {'exact': 0.0, 'mean_logprob_key_token': -2.9218, 'distance_tokens_approx': 32664}
|
| 16 |
+
copy ctx 2048: next-token acc 0.947
|
| 17 |
+
copy ctx 8192: next-token acc 0.8895
|
| 18 |
+
copy ctx 16384: next-token acc 0.749
|
| 19 |
+
copy ctx 32768: next-token acc 0.192
|
| 20 |
+
wrote /content/eval_ctx32k.json
|