Hugging Face's logo

vuhaian
/
results_two4

Model card Files
xet
 Community
vuhaian commited on
Commit
ff9f028
·
verified ·
1 Parent(s): ba4b4ce

Delete full_4323_5e_4.json

Browse files
Files changed (1) hide show
  1. full_4323_5e_4.json +0 -1999
full_4323_5e_4.json DELETED
@@ -1,1999 +0,0 @@
1
- {
2
- "teacher": "Qwen/Qwen3.5-4B",
3
- "max_new_tokens": 512,
4
- "block_seed": 987654321,
5
- "tensor_parallel_size": 1,
6
- "n_prompts": 10,
7
- "n_prompts_filtered": 0,
8
- "min_completion_tokens": 10,
9
- "students": {
10
- "vuhaian/full-4323-5e-4": {
11
- "chat_probe": {
12
- "pass": true,
13
- "reason": "",
14
- "prompts_tested": 4,
15
- "prompts_terminated": 4,
16
- "prompts_non_empty": 4,
17
- "mean_gen_tokens": 54.5,
18
- "mean_content_chars": 194.0,
19
- "mean_reasoning_fraction": 0.0,
20
- "samples": [
21
- {
22
- "prompt": "hi",
23
- "gen_tokens": 6,
24
- "terminated": true,
25
- "content_chars": 10,
26
- "reasoning_frac": 0.0,
27
- "content_preview": "Roarbox #5",
28
- "raw_prompt": "<|im_start|>user\nhi<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n",
29
- "raw_output": "Roarbox #5"
30
- },
31
- {
32
- "prompt": "What is 2+2?",
33
- "gen_tokens": 3,
34
- "terminated": true,
35
- "content_chars": 2,
36
- "reasoning_frac": 0.0,
37
- "content_preview": "22",
38
- "raw_prompt": "<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n",
39
- "raw_output": "22"
40
- },
41
- {
42
- "prompt": "Say hello in one word.",
43
- "gen_tokens": 206,
44
- "terminated": true,
45
- "content_chars": 762,
46
- "reasoning_frac": 0.0,
47
- "content_preview": "Here is a unique, classic combinatorics, often referred to as \"handshake problem",
48
- "raw_prompt": "<|im_start|>user\nSay hello in one word.<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n",
49
- "raw_output": "Here is a unique, classic combinatorics, often referred to as \"handshake problems\" or finding a unique sense of smell, like sets a favorite scrot.\n\nHere is a **indirect cause:**\n* ** Sometimes lists include a **indirect cause** (shakes, no \"feast\" or a direct action).\n* **Validate:** Sometimes lists include a **indirect cause** (shakes, no \"feast\" or a direct action).\n* **Validate:** Sometimes lists include a **indirect cause** (shakes, no \"feast\" or a direct action).\n* **Validate:** Sometimes lists include a **indirect cause** (shakes, no \"feast\" or a direct action).\n* **Validate:** Sometimes lists include a **indirect cause** (shakes, no \"feast\" or a direct action).\n* **Success Metric:** Sometimes lists include:\n\n### 1. Does this directly"
50
- },
51
- {
52
- "prompt": "Reply with just the word: yes",
53
- "gen_tokens": 3,
54
- "terminated": true,
55
- "content_chars": 2,
56
- "reasoning_frac": 0.0,
57
- "content_preview": "13",
58
- "raw_prompt": "<|im_start|>user\nReply with just the word: yes<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n",
59
- "raw_output": "13"
60
- }
61
- ]
62
- },
63
- "capability": {
64
- "n": 36,
65
- "correct": 5,
66
- "pass_frac": 0.139,
67
- "teacher_pass_frac": 0.889,
68
- "items": [
69
- {
70
- "q": "What is the capital of China? One word.",
71
- "expected": "beijing",
72
- "kind": "word_alt",
73
- "pred": "the",
74
- "ok": false,
75
- "tail": "The capital of China is Tokyo."
76
- },
77
- {
78
- "q": "Who wrote 'Hamlet'? Two words.",
79
- "expected": "william shakespeare",
80
- "kind": "phrase",
81
- "pred": "Hamlet** is a very common ancestor of life, and the author of **Hamlet** is a rhythmic dance between human and anagram of expanse. Both novels fundamentally explore the underlying physics of modern life, yet they often imagine",
82
- "ok": false,
83
- "tail": " and anagram of expanse. Both novels fundamentally explore the underlying physics of modern life, yet they often imagine"
84
- },
85
- {
86
- "q": "How many seconds in one hour? Answer with only the number.",
87
- "expected": "3600",
88
- "kind": "int",
89
- "pred": "13",
90
- "ok": false,
91
- "tail": "13"
92
- },
93
- {
94
- "q": "What is the smallest country in the world? Two words or one hyphenated.",
95
- "expected": "vatican city",
96
- "kind": "phrase",
97
- "pred": "The smallest country in the world is the sum of the entire familytree, specifically every continent in the central Mexico. It is a deciduous tree, meaning no single butterfly completes the entire round trip, but its broad leaves exactly once existed.",
98
- "ok": false,
99
- "tail": " deciduous tree, meaning no single butterfly completes the entire round trip, but its broad leaves exactly once existed."
100
- },
101
- {
102
- "q": "Is the sun considered a star? Answer yes or no.",
103
- "expected": "yes",
104
- "kind": "yesno",
105
- "pred": "",
106
- "ok": false,
107
- "tail": " life of a star is primarily caused by the sun's rotational, electrons, and the Earth's axial tilt of approximately 23.5"
108
- },
109
- {
110
- "q": "How many millimeters in a kilometer? Answer with only the number.",
111
- "expected": "1000000",
112
- "kind": "int",
113
- "pred": "13",
114
- "ok": false,
115
- "tail": "13"
116
- },
117
- {
118
- "q": "What does 'len([1, 2, 3, 4])' evaluate to in Python? Answer with only the number.",
119
- "expected": "4",
120
- "kind": "int",
121
- "pred": "2",
122
- "ok": false,
123
- "tail": "2"
124
- },
125
- {
126
- "q": "Is 0 an even number? Answer yes or no.",
127
- "expected": "yes",
128
- "kind": "yesno",
129
- "pred": "",
130
- "ok": false,
131
- "tail": "0"
132
- },
133
- {
134
- "q": "What is the powerhouse of the cell? One word.",
135
- "expected": "mitochondria",
136
- "kind": "word_alt",
137
- "pred": "the",
138
- "ok": false,
139
- "tail": "The user is the of the cell."
140
- },
141
- {
142
- "q": "What color do you get by mixing red and blue? One word.",
143
- "expected": "purple",
144
- "kind": "word_alt",
145
- "pred": "here",
146
- "ok": false,
147
- "tail": " This recipe uses standard pantry ingredients and yields about 24 cookies.\n\n### \ud83c\udf6a Classic Chocolate Chip Cookies\n\n**Prep"
148
- },
149
- {
150
- "q": "What is the capital of Thailand? One word.",
151
- "expected": "bangkok",
152
- "kind": "word",
153
- "pred": "the",
154
- "ok": false,
155
- "tail": "s natural migration is not completed by a single individual but a single person, and by which the same isotope notation."
156
- },
157
- {
158
- "q": "What is the capital of Egypt? One word.",
159
- "expected": "cairo",
160
- "kind": "word",
161
- "pred": "the",
162
- "ok": false,
163
- "tail": "ek \"Alpine Railway\" of Egypt).\n\nThe capital of Egypt is defined by the steep, dense forests of central Mexico. While the"
164
- },
165
- {
166
- "q": "How many characters are in the word 'lemon'? Answer with only the number.",
167
- "expected": "5",
168
- "kind": "int",
169
- "pred": "3",
170
- "ok": false,
171
- "tail": "3"
172
- },
173
- {
174
- "q": "What is 4 * 8? Answer with only the number.",
175
- "expected": "32",
176
- "kind": "int",
177
- "pred": "42",
178
- "ok": false,
179
- "tail": "42"
180
- },
181
- {
182
- "q": "Is 83 divisible by 9? Answer yes or no.",
183
- "expected": "no",
184
- "kind": "yesno",
185
- "pred": "no",
186
- "ok": true,
187
- "tail": "no"
188
- },
189
- {
190
- "q": "What is 81 / 9? Answer with only the number.",
191
- "expected": "9",
192
- "kind": "int",
193
- "pred": "10",
194
- "ok": false,
195
- "tail": "10"
196
- },
197
- {
198
- "q": "Is 90244 even or odd? Answer with one word: 'even' or 'odd'.",
199
- "expected": "even",
200
- "kind": "word",
201
- "pred": "even",
202
- "ok": true,
203
- "tail": "even"
204
- },
205
- {
206
- "q": "What is 759 - 11? Answer with only the number.",
207
- "expected": "748",
208
- "kind": "int",
209
- "pred": "759",
210
- "ok": false,
211
- "tail": "759"
212
- },
213
- {
214
- "q": "What is 3 to the power of 3? Answer with only the number.",
215
- "expected": "27",
216
- "kind": "int",
217
- "pred": "30",
218
- "ok": false,
219
- "tail": "30"
220
- },
221
- {
222
- "q": "How many vowels are in the word 'delta'? Answer with only the number.",
223
- "expected": "2",
224
- "kind": "int",
225
- "pred": "2",
226
- "ok": true,
227
- "tail": "2"
228
- },
229
- {
230
- "q": "What is 6 * 7? Answer with only the number.",
231
- "expected": "42",
232
- "kind": "int",
233
- "pred": "67",
234
- "ok": false,
235
- "tail": "67"
236
- },
237
- {
238
- "q": "What is 26 + 274? Answer with only the number.",
239
- "expected": "300",
240
- "kind": "int",
241
- "pred": "274",
242
- "ok": false,
243
- "tail": "274"
244
- },
245
- {
246
- "q": "Is 770 divisible by 5? Answer yes or no.",
247
- "expected": "yes",
248
- "kind": "yesno",
249
- "pred": "no",
250
- "ok": false,
251
- "tail": "no"
252
- },
253
- {
254
- "q": "What is the minimum of: 29, 82, 58, 27, 79? Answer with only the number.",
255
- "expected": "27",
256
- "kind": "int",
257
- "pred": "22",
258
- "ok": false,
259
- "tail": "22"
260
- },
261
- {
262
- "q": "Is 612 divisible by 5? Answer yes or no.",
263
- "expected": "no",
264
- "kind": "yesno",
265
- "pred": "no",
266
- "ok": true,
267
- "tail": "no"
268
- },
269
- {
270
- "q": "What is 262 + 68? Answer with only the number.",
271
- "expected": "330",
272
- "kind": "int",
273
- "pred": "623",
274
- "ok": false,
275
- "tail": "623"
276
- },
277
- {
278
- "q": "What is 61 mod 5? Answer with only the number.",
279
- "expected": "1",
280
- "kind": "int",
281
- "pred": "2",
282
- "ok": false,
283
- "tail": "2"
284
- },
285
- {
286
- "q": "How many even numbers are in this list: 65, 33, 23, 93, 54, 14? Answer with only the number.",
287
- "expected": "2",
288
- "kind": "int",
289
- "pred": "3",
290
- "ok": false,
291
- "tail": "3"
292
- },
293
- {
294
- "q": "Is 246 divisible by 8? Answer yes or no.",
295
- "expected": "no",
296
- "kind": "yesno",
297
- "pred": "no",
298
- "ok": true,
299
- "tail": "no"
300
- },
301
- {
302
- "q": "What is 287 - 78? Answer with only the number.",
303
- "expected": "209",
304
- "kind": "int",
305
- "pred": "278",
306
- "ok": false,
307
- "tail": "278"
308
- },
309
- {
310
- "q": "What is 33 + 178? Answer with only the number.",
311
- "expected": "211",
312
- "kind": "int",
313
- "pred": "333",
314
- "ok": false,
315
- "tail": "333"
316
- },
317
- {
318
- "q": "What is 368 + 389? Answer with only the number.",
319
- "expected": "757",
320
- "kind": "int",
321
- "pred": "383",
322
- "ok": false,
323
- "tail": "383"
324
- },
325
- {
326
- "q": "What is the minimum of: 61, 97, 28, 18, 59, 36? Answer with only the number.",
327
- "expected": "18",
328
- "kind": "int",
329
- "pred": "11",
330
- "ok": false,
331
- "tail": "11"
332
- },
333
- {
334
- "q": "What is 4 to the power of 2? Answer with only the number.",
335
- "expected": "16",
336
- "kind": "int",
337
- "pred": "23",
338
- "ok": false,
339
- "tail": "23"
340
- },
341
- {
342
- "q": "What is 793 - 91? Answer with only the number.",
343
- "expected": "702",
344
- "kind": "int",
345
- "pred": "739",
346
- "ok": false,
347
- "tail": "739"
348
- },
349
- {
350
- "q": "What is 54 / 3? Answer with only the number.",
351
- "expected": "18",
352
- "kind": "int",
353
- "pred": "5",
354
- "ok": false,
355
- "tail": "5"
356
- }
357
- ]
358
- },
359
- "judge_probe_meta": {
360
- "n_prompts": 16,
361
- "mean_gen_tokens": 104.1,
362
- "collected_at": 140.0
363
- },
364
- "chat_turns_probe_meta": {
365
- "n_convos": 6,
366
- "n_turns": 3,
367
- "mean_gen_tokens": 196.6,
368
- "collected_at": 240.6
369
- },
370
- "activation_fingerprint": {
371
- "layer_fingerprints": {
372
- "0": [
373
- -0.02358,
374
- 0.023047,
375
- -0.018652,
376
- -0.06203,
377
- 0.065354,
378
- 0.105742,
379
- 0.131179,
380
- 0.005227,
381
- 0.120648,
382
- -0.158617,
383
- -0.058113,
384
- 0.084517,
385
- -0.026138,
386
- -0.063506,
387
- -0.077578,
388
- -0.034811,
389
- -0.074237,
390
- -0.006084,
391
- -0.05654,
392
- -0.030059,
393
- -0.103639,
394
- -0.117317,
395
- 0.124655,
396
- -0.088633,
397
- 0.096639,
398
- 0.116544,
399
- -0.076829,
400
- 0.117099,
401
- 0.032611,
402
- -0.216403,
403
- -0.114123,
404
- -0.039659,
405
- 0.141624,
406
- -0.079809,
407
- 0.003829,
408
- 0.14665,
409
- 0.150806,
410
- -0.108851,
411
- -0.042686,
412
- -0.060563,
413
- 0.179756,
414
- 0.107547,
415
- -0.001415,
416
- -0.101834,
417
- 0.048482,
418
- -0.06239,
419
- 0.094574,
420
- -0.187087,
421
- -0.05525,
422
- 0.071986,
423
- 0.030619,
424
- -0.038882,
425
- -0.111968,
426
- -0.207109,
427
- 0.024401,
428
- -0.118496,
429
- 0.092319,
430
- 0.051842,
431
- 0.100322,
432
- -0.167014,
433
- 0.087614,
434
- -0.000598,
435
- 0.051001,
436
- 0.045138,
437
- 0.013818,
438
- -0.144281,
439
- -0.073048,
440
- -0.046344,
441
- 0.016473,
442
- 0.068888,
443
- 0.011685,
444
- -0.004957,
445
- -0.097593,
446
- 0.08579,
447
- 0.073644,
448
- 0.191274,
449
- 0.158977,
450
- -0.074912,
451
- 0.005852,
452
- 0.156189,
453
- -0.101118,
454
- 0.003868,
455
- -0.031783,
456
- -0.213898,
457
- -0.023444,
458
- -0.117981,
459
- -0.171241,
460
- 0.123527,
461
- 0.023992,
462
- -0.109726,
463
- 0.096026,
464
- -0.021349,
465
- -0.153388,
466
- -0.091153,
467
- 0.069493,
468
- 0.02735,
469
- 0.032176,
470
- 0.026836,
471
- -0.198828,
472
- 0.019355,
473
- 0.021652,
474
- 0.050302,
475
- -0.029157,
476
- -0.012547,
477
- -0.017701,
478
- 0.00725,
479
- 0.01213,
480
- -0.083942,
481
- -0.0002,
482
- 0.170217,
483
- 0.006994,
484
- -0.067303,
485
- 0.046642,
486
- 0.010163,
487
- -0.041537,
488
- 0.048544,
489
- 0.081023,
490
- 0.020891,
491
- 0.054447,
492
- 0.089388,
493
- -0.263624,
494
- -0.114355,
495
- -0.021575,
496
- 0.192974,
497
- -0.089267,
498
- 0.049052,
499
- -0.032412,
500
- -0.111175
501
- ],
502
- "8": [
503
- -0.356313,
504
- 0.542448,
505
- -0.026062,
506
- -0.247746,
507
- 0.047553,
508
- 0.173991,
509
- 0.250737,
510
- 0.043759,
511
- 0.869366,
512
- -0.387574,
513
- 0.275757,
514
- 0.337193,
515
- -0.166669,
516
- -0.013,
517
- -0.360152,
518
- -0.22412,
519
- -0.063324,
520
- -0.424334,
521
- -0.182127,
522
- -0.071875,
523
- -0.269472,
524
- 0.023085,
525
- 0.218655,
526
- -0.508551,
527
- 0.317973,
528
- 0.412306,
529
- 0.077114,
530
- 0.314127,
531
- 0.944412,
532
- -0.25864,
533
- -0.213614,
534
- -0.207512,
535
- -0.071897,
536
- -0.547037,
537
- -0.045923,
538
- -0.048846,
539
- 0.67462,
540
- -0.14531,
541
- 0.09615,
542
- 0.158321,
543
- 0.431241,
544
- 0.245608,
545
- 0.08166,
546
- -0.157362,
547
- -0.560843,
548
- -0.188654,
549
- 0.022864,
550
- -0.571392,
551
- 0.312103,
552
- 0.116028,
553
- -0.16853,
554
- -0.077114,
555
- 0.015948,
556
- -0.850711,
557
- -0.053794,
558
- -0.56074,
559
- 0.35169,
560
- 0.127887,
561
- 0.140828,
562
- -0.331677,
563
- 0.655454,
564
- -0.252755,
565
- 0.190753,
566
- 0.268501,
567
- 0.160194,
568
- -0.264688,
569
- -0.30078,
570
- -0.127256,
571
- 0.611192,
572
- 0.049363,
573
- -0.307033,
574
- -0.000383,
575
- 0.053589,
576
- -0.251357,
577
- 0.460809,
578
- 0.401471,
579
- 0.342244,
580
- -0.28945,
581
- -0.332274,
582
- 0.470964,
583
- -0.339503,
584
- -0.334885,
585
- 0.184045,
586
- -0.635249,
587
- -0.127951,
588
- -0.268283,
589
- -0.202101,
590
- 0.175005,
591
- -0.069824,
592
- -0.513116,
593
- -0.112091,
594
- -0.270226,
595
- -0.496494,
596
- -0.23215,
597
- -0.168239,
598
- 0.132642,
599
- -0.103555,
600
- 0.013657,
601
- -0.044037,
602
- -0.103405,
603
- 0.04829,
604
- 0.050528,
605
- -0.409297,
606
- -0.080933,
607
- -0.062962,
608
- -0.570068,
609
- 0.111535,
610
- 0.604951,
611
- -0.358704,
612
- 0.226629,
613
- 0.023521,
614
- 0.10636,
615
- 0.308478,
616
- 0.149058,
617
- -0.132217,
618
- -0.072786,
619
- 0.368518,
620
- -0.137814,
621
- 0.223361,
622
- -0.176459,
623
- -0.841371,
624
- 0.076617,
625
- -0.363967,
626
- 0.333049,
627
- -0.697313,
628
- 0.267288,
629
- -0.497775,
630
- -0.773132
631
- ],
632
- "16": [
633
- -0.553848,
634
- 0.663364,
635
- 0.247362,
636
- -0.031048,
637
- 0.029107,
638
- -0.069239,
639
- 0.319555,
640
- -0.204498,
641
- 0.696275,
642
- -0.55421,
643
- 0.147498,
644
- 0.171642,
645
- -0.402901,
646
- -0.658237,
647
- -0.056252,
648
- -1.144753,
649
- 1.411752,
650
- -0.208825,
651
- 0.150669,
652
- 0.098111,
653
- 0.262871,
654
- 0.740419,
655
- 0.555316,
656
- -0.53737,
657
- 0.757213,
658
- 0.237095,
659
- -0.1508,
660
- 0.682908,
661
- -2.698931,
662
- -0.537254,
663
- -0.370926,
664
- -0.072592,
665
- -0.187218,
666
- 0.316548,
667
- 0.259634,
668
- -0.045607,
669
- 0.489293,
670
- -0.25361,
671
- -0.156073,
672
- 0.021973,
673
- 0.020636,
674
- 0.516373,
675
- -0.730363,
676
- 0.158852,
677
- -0.244986,
678
- -0.852696,
679
- -0.341619,
680
- -0.524388,
681
- 0.493624,
682
- 0.19849,
683
- -0.069968,
684
- -0.447185,
685
- -0.384867,
686
- -0.200382,
687
- -0.327482,
688
- -0.420265,
689
- 0.238882,
690
- 0.211189,
691
- 0.245478,
692
- -0.46331,
693
- 0.610028,
694
- -0.157139,
695
- 0.04278,
696
- -0.028567,
697
- 0.059746,
698
- -0.875951,
699
- 0.288316,
700
- -2.8e-05,
701
- 0.246938,
702
- -0.671251,
703
- 0.089834,
704
- 0.046384,
705
- 0.200573,
706
- -0.568695,
707
- -0.035238,
708
- 0.44219,
709
- 0.340631,
710
- 0.345957,
711
- -0.575619,
712
- 0.332026,
713
- -0.02053,
714
- -0.681751,
715
- 0.585333,
716
- -0.359494,
717
- 0.2522,
718
- -0.196143,
719
- 0.346302,
720
- 0.047533,
721
- -0.387082,
722
- 0.414155,
723
- 0.080459,
724
- -0.197517,
725
- 0.037546,
726
- 0.206393,
727
- -0.200278,
728
- 0.622899,
729
- -0.400144,
730
- 0.019957,
731
- -0.016437,
732
- -0.063687,
733
- 0.22555,
734
- -0.29304,
735
- -0.390129,
736
- 0.104182,
737
- -0.004765,
738
- -0.249057,
739
- -0.042419,
740
- 0.310793,
741
- -0.222476,
742
- 0.093852,
743
- 0.041026,
744
- -0.53854,
745
- 0.090736,
746
- 0.257589,
747
- 0.431975,
748
- -0.162952,
749
- 0.106623,
750
- -0.306464,
751
- 0.294026,
752
- -0.285186,
753
- 0.303658,
754
- 0.617457,
755
- -0.199019,
756
- 0.578837,
757
- -0.193669,
758
- -0.127395,
759
- -0.586367,
760
- -0.779146
761
- ],
762
- "23": [
763
- -0.258527,
764
- 0.896605,
765
- 0.77685,
766
- 0.033634,
767
- 0.711418,
768
- 0.002111,
769
- -0.400077,
770
- -0.4999,
771
- 0.499398,
772
- -0.568787,
773
- 0.429323,
774
- -0.28107,
775
- -0.041556,
776
- -0.483025,
777
- -0.571643,
778
- -0.72305,
779
- 4.596298,
780
- -0.205575,
781
- 0.185375,
782
- 0.362442,
783
- 0.329307,
784
- 0.077613,
785
- 0.757329,
786
- 0.431228,
787
- 0.732175,
788
- -0.380133,
789
- -0.174373,
790
- 1.007456,
791
- -7.00988,
792
- 0.242204,
793
- -0.680006,
794
- -0.472774,
795
- 0.054452,
796
- 0.244063,
797
- 0.278304,
798
- 0.031637,
799
- -0.005795,
800
- 0.403786,
801
- 0.224216,
802
- -0.417353,
803
- -0.284475,
804
- 0.733055,
805
- -0.56216,
806
- 0.477628,
807
- -0.349799,
808
- -2.79653,
809
- -1.267487,
810
- -0.661888,
811
- 0.497994,
812
- 0.297001,
813
- -0.50287,
814
- -0.271973,
815
- -0.160612,
816
- 0.996306,
817
- 0.116682,
818
- -0.478855,
819
- 0.437648,
820
- 0.414956,
821
- 0.439904,
822
- -0.160444,
823
- 0.970315,
824
- -0.280247,
825
- 0.378928,
826
- -0.902866,
827
- 0.202645,
828
- -1.120171,
829
- 0.548113,
830
- 0.225239,
831
- 0.155128,
832
- -0.719004,
833
- -0.088457,
834
- -0.047934,
835
- -0.290179,
836
- -0.713527,
837
- 0.109485,
838
- -0.058881,
839
- 0.197997,
840
- 0.782527,
841
- -0.340713,
842
- 0.402086,
843
- 0.617989,
844
- -0.644194,
845
- 0.212122,
846
- -0.178868,
847
- 0.450748,
848
- -0.34007,
849
- 0.700888,
850
- -0.11438,
851
- -0.437129,
852
- 0.876918,
853
- 0.115418,
854
- 0.400021,
855
- 0.38345,
856
- 0.228783,
857
- -0.712939,
858
- 1.033931,
859
- 0.019141,
860
- 0.234295,
861
- -0.641412,
862
- 0.347562,
863
- 0.281232,
864
- -1.025538,
865
- -0.565528,
866
- 0.199486,
867
- -0.523468,
868
- -0.048982,
869
- 0.090189,
870
- 0.471387,
871
- -0.122239,
872
- -0.342236,
873
- -0.716396,
874
- -1.235716,
875
- -0.16003,
876
- 0.113693,
877
- 0.671339,
878
- -0.205079,
879
- 0.019947,
880
- -0.585409,
881
- -0.246022,
882
- -1.003072,
883
- 1.76363,
884
- 0.567343,
885
- -0.369613,
886
- 0.613968,
887
- -0.657497,
888
- -0.006477,
889
- -1.256819,
890
- -0.555521
891
- ]
892
- },
893
- "n_layers": 24,
894
- "hidden_size": 1536
895
- },
896
- "status": "scored",
897
- "kl_global_avg": 1.87522,
898
- "kl_per_prompt": [
899
- 1.244923,
900
- 1.428292,
901
- 1.822757,
902
- 1.357092,
903
- 1.802379,
904
- 2.000687,
905
- 2.725963,
906
- 1.818884,
907
- 2.591961,
908
- 1.959264
909
- ],
910
- "prompts_scored": 10,
911
- "effective_total": 10,
912
- "prompts_total": 10,
913
- "scoring_time": 1.6,
914
- "pre_scoring_time": 437.7,
915
- "load_time": 3.6,
916
- "early_stopped": false,
917
- "early_stop_reason": null,
918
- "shadow_topk": {
919
- "kl_conditional_top128": 1.598752,
920
- "student_top128_mass": 0.800853,
921
- "tail_mass_gap_top128": 0.188981,
922
- "teacher_top128_mass": 0.987175
923
- },
924
- "length_axis": {
925
- "student_mean_gen": 54.5,
926
- "teacher_mean_gen": 4.2,
927
- "ratio": 12.824,
928
- "penalty": 0.156,
929
- "too_short_penalty": 1.0,
930
- "too_long_penalty": 0.156,
931
- "min_ratio": 0.08,
932
- "too_short": false,
933
- "source": "chat_probe"
934
- },
935
- "on_policy_rollouts_meta": {
936
- "n_rollouts": 12,
937
- "mean_gen_len": 124.8,
938
- "collected_at": 160.3
939
- },
940
- "math_bench": {
941
- "n": 16,
942
- "correct": 0,
943
- "pass_frac": 0.0,
944
- "wall_s": 9.2,
945
- "items": [
946
- {
947
- "src": "procedural_math/factorial_mod",
948
- "pred": "1",
949
- "gold": "6",
950
- "ok": false,
951
- "gen_tokens": 4,
952
- "tail": "#### 1"
953
- },
954
- {
955
- "src": "procedural_math/polynomial_eval",
956
- "pred": "11",
957
- "gold": "-50",
958
- "ok": false,
959
- "gen_tokens": 5,
960
- "tail": "#### 11"
961
- },
962
- {
963
- "src": "procedural_math/rate_distance",
964
- "pred": "130",
965
- "gold": "101",
966
- "ok": false,
967
- "gen_tokens": 6,
968
- "tail": "#### 130"
969
- },
970
- {
971
- "src": "procedural_math/arithmetic_series",
972
- "pred": "106",
973
- "gold": "510",
974
- "ok": false,
975
- "gen_tokens": 6,
976
- "tail": "#### 106"
977
- },
978
- {
979
- "src": "procedural_math/gcd_lcm",
980
- "pred": "1",
981
- "gold": "2",
982
- "ok": false,
983
- "gen_tokens": 4,
984
- "tail": "#### 1"
985
- },
986
- {
987
- "src": "procedural_math/triangle_area",
988
- "pred": "13",
989
- "gold": "52",
990
- "ok": false,
991
- "gen_tokens": 5,
992
- "tail": "#### 13"
993
- },
994
- {
995
- "src": "procedural_math/proportion",
996
- "pred": "1223",
997
- "gold": "1224",
998
- "ok": false,
999
- "gen_tokens": 7,
1000
- "tail": "#### 1223"
1001
- },
1002
- {
1003
- "src": "procedural_math/percentage",
1004
- "pred": "113",
1005
- "gold": "417",
1006
- "ok": false,
1007
- "gen_tokens": 6,
1008
- "tail": "#### 113"
1009
- },
1010
- {
1011
- "src": "procedural_math/simultaneous",
1012
- "pred": "10",
1013
- "gold": "8",
1014
- "ok": false,
1015
- "gen_tokens": 5,
1016
- "tail": "#### 10"
1017
- },
1018
- {
1019
- "src": "procedural_math/modular_linear",
1020
- "pred": "10",
1021
- "gold": "9",
1022
- "ok": false,
1023
- "gen_tokens": 5,
1024
- "tail": "#### 10"
1025
- },
1026
- {
1027
- "src": "procedural_math/mixture",
1028
- "pred": "4200",
1029
- "gold": "3830",
1030
- "ok": false,
1031
- "gen_tokens": 7,
1032
- "tail": "#### 4200"
1033
- },
1034
- {
1035
- "src": "procedural_math/unit_conversion",
1036
- "pred": "100",
1037
- "gold": "310",
1038
- "ok": false,
1039
- "gen_tokens": 6,
1040
- "tail": "#### 100"
1041
- },
1042
- {
1043
- "src": "procedural_math/geometric_series",
1044
- "pred": "1008",
1045
- "gold": "2912",
1046
- "ok": false,
1047
- "gen_tokens": 7,
1048
- "tail": "#### 1008"
1049
- },
1050
- {
1051
- "src": "procedural_math/probability_count",
1052
- "pred": "10",
1053
- "gold": "15",
1054
- "ok": false,
1055
- "gen_tokens": 5,
1056
- "tail": "#### 10"
1057
- },
1058
- {
1059
- "src": "procedural_math/set_intersect",
1060
- "pred": "107",
1061
- "gold": "87",
1062
- "ok": false,
1063
- "gen_tokens": 6,
1064
- "tail": "#### 107"
1065
- },
1066
- {
1067
- "src": "procedural_math/digit_sum",
1068
- "pred": "122",
1069
- "gold": "126",
1070
- "ok": false,
1071
- "gen_tokens": 6,
1072
- "tail": "#### 122"
1073
- }
1074
- ],
1075
- "error": null,
1076
- "mean_gen_tokens": 5.6,
1077
- "mean_gen_tokens_correct": 0.0
1078
- },
1079
- "code_bench": {
1080
- "n": 10,
1081
- "correct": 4,
1082
- "pass_frac": 0.4,
1083
- "wall_s": 70.9,
1084
- "items": [
1085
- {
1086
- "task_id": "v27/run_length/00",
1087
- "entry_point": "rle",
1088
- "ok": false,
1089
- "gen_tokens": 94,
1090
- "reason": "fail:IndentationError: unindent does not match any outer indentation level",
1091
- "tail": " result.append((current_char, count))\n current_char = char\n count = 1\n result.append((current_char, count))\n return result"
1092
- },
1093
- {
1094
- "task_id": "v27/string_predicate/01",
1095
- "entry_point": "is_balanced",
1096
- "ok": false,
1097
- "gen_tokens": 56,
1098
- "reason": "fail:IndentationError: unexpected indent",
1099
- "tail": " stack.append(char)\n elif char == ')':\n if not stack:\n return False\n stack.pop()\n return len(stack) == 0"
1100
- },
1101
- {
1102
- "task_id": "v27/pair_count/02",
1103
- "entry_point": "pair_count",
1104
- "ok": false,
1105
- "gen_tokens": 53,
1106
- "reason": "fail:IndentationError: unexpected indent",
1107
- "tail": " count = 0\n for i in range(len(xs)):\n for j in range(i + 1, len(xs)):\n if xs[i] + xs[j] == 6:\n count += 1\n return count"
1108
- },
1109
- {
1110
- "task_id": "v27/string_transform/03",
1111
- "entry_point": "swap_pairs",
1112
- "ok": true,
1113
- "gen_tokens": 106,
1114
- "reason": "pass",
1115
- "tail": "n place.\n \"\"\"\n chars = list(s)\n for i in range(0, len(chars) - 1, 2):\n chars[i], chars[i + 1] = chars[i + 1], chars[i]\n return ''.join(chars)"
1116
- },
1117
- {
1118
- "task_id": "v27/aggregate_list/04",
1119
- "entry_point": "aggregate",
1120
- "ok": false,
1121
- "gen_tokens": 18,
1122
- "reason": "fail:SyntaxError: 'return' outside function",
1123
- "tail": " return sum(x for x in xs if x % 2 == 0)"
1124
- },
1125
- {
1126
- "task_id": "v27/digit_sum/05",
1127
- "entry_point": "digit_sum",
1128
- "ok": true,
1129
- "gen_tokens": 53,
1130
- "reason": "pass",
1131
- "tail": "n):\n \"\"\"Return the sum of the decimal digits of the non-negative integer ``n``. ``digit_sum(0)`` is 0.\n \"\"\"\n return sum(int(digit) for digit in str(n))"
1132
- },
1133
- {
1134
- "task_id": "v27/window_sum/06",
1135
- "entry_point": "window_max_sum",
1136
- "ok": true,
1137
- "gen_tokens": 146,
1138
- "reason": "pass",
1139
- "tail": "for i in range(k, len(xs)):\n current_sum += xs[i] - xs[i - k]\n if current_sum > max_sum:\n max_sum = current_sum\n \n return max_sum"
1140
- },
1141
- {
1142
- "task_id": "v27/transform_list/07",
1143
- "entry_point": "transform",
1144
- "ok": false,
1145
- "gen_tokens": 12,
1146
- "reason": "fail:SyntaxError: 'return' outside function",
1147
- "tail": " return [abs(x) for x in xs]"
1148
- },
1149
- {
1150
- "task_id": "v27/run_length/08",
1151
- "entry_point": "rle",
1152
- "ok": false,
1153
- "gen_tokens": 94,
1154
- "reason": "fail:IndentationError: unindent does not match any outer indentation level",
1155
- "tail": " result.append((current_char, count))\n current_char = char\n count = 1\n result.append((current_char, count))\n return result"
1156
- },
1157
- {
1158
- "task_id": "v27/string_predicate/09",
1159
- "entry_point": "is_anagram",
1160
- "ok": true,
1161
- "gen_tokens": 64,
1162
- "reason": "pass",
1163
- "tail": "nagram of string ``b`` (case-sensitive). Different lengths return False.\n \"\"\"\n if len(a) != len(b):\n return False\n return sorted(a) == sorted(b)"
1164
- }
1165
- ],
1166
- "error": null,
1167
- "mean_gen_tokens": 69.6,
1168
- "mean_gen_tokens_correct": 92.2
1169
- },
1170
- "reasoning_bench": {
1171
- "n": 12,
1172
- "correct": 6,
1173
- "pass_frac": 0.5,
1174
- "wall_s": 3.5,
1175
- "items": [
1176
- {
1177
- "src": "procedural_reasoning/analogy_letter",
1178
- "pred": "(A)",
1179
- "gold": "(A)",
1180
- "ok": true,
1181
- "gen_tokens": 3,
1182
- "tail": "(A)"
1183
- },
1184
- {
1185
- "src": "procedural_reasoning/ordering",
1186
- "pred": "(A)",
1187
- "gold": "(A)",
1188
- "ok": true,
1189
- "gen_tokens": 3,
1190
- "tail": "(A)"
1191
- },
1192
- {
1193
- "src": "procedural_reasoning/odd_one_out",
1194
- "pred": "(A)",
1195
- "gold": "(C)",
1196
- "ok": false,
1197
- "gen_tokens": 3,
1198
- "tail": "(A)"
1199
- },
1200
- {
1201
- "src": "procedural_reasoning/deduction",
1202
- "pred": "(B)",
1203
- "gold": "(C)",
1204
- "ok": false,
1205
- "gen_tokens": 3,
1206
- "tail": "(B)"
1207
- },
1208
- {
1209
- "src": "procedural_reasoning/boolean_eval",
1210
- "pred": "(A)",
1211
- "gold": "(B)",
1212
- "ok": false,
1213
- "gen_tokens": 3,
1214
- "tail": "(A)"
1215
- },
1216
- {
1217
- "src": "procedural_reasoning/sequence_next",
1218
- "pred": "(B)",
1219
- "gold": "(B)",
1220
- "ok": true,
1221
- "gen_tokens": 3,
1222
- "tail": "(B)"
1223
- },
1224
- {
1225
- "src": "procedural_reasoning/analogy_letter",
1226
- "pred": "(B)",
1227
- "gold": "(A)",
1228
- "ok": false,
1229
- "gen_tokens": 3,
1230
- "tail": "(B)"
1231
- },
1232
- {
1233
- "src": "procedural_reasoning/ordering",
1234
- "pred": "(D)",
1235
- "gold": "(B)",
1236
- "ok": false,
1237
- "gen_tokens": 3,
1238
- "tail": "(D)"
1239
- },
1240
- {
1241
- "src": "procedural_reasoning/odd_one_out",
1242
- "pred": "(B)",
1243
- "gold": "(B)",
1244
- "ok": true,
1245
- "gen_tokens": 3,
1246
- "tail": "(B)"
1247
- },
1248
- {
1249
- "src": "procedural_reasoning/deduction",
1250
- "pred": "(B)",
1251
- "gold": "(B)",
1252
- "ok": true,
1253
- "gen_tokens": 3,
1254
- "tail": "(B)"
1255
- },
1256
- {
1257
- "src": "procedural_reasoning/boolean_eval",
1258
- "pred": "(A)",
1259
- "gold": "(B)",
1260
- "ok": false,
1261
- "gen_tokens": 3,
1262
- "tail": "(A)"
1263
- },
1264
- {
1265
- "src": "procedural_reasoning/sequence_next",
1266
- "pred": "(B)",
1267
- "gold": "(B)",
1268
- "ok": true,
1269
- "gen_tokens": 3,
1270
- "tail": "(B)"
1271
- }
1272
- ],
1273
- "error": null,
1274
- "mean_gen_tokens": 3.0,
1275
- "mean_gen_tokens_correct": 3.0
1276
- },
1277
- "knowledge_bench": {
1278
- "n": 0,
1279
- "correct": 0,
1280
- "pass_frac": 0.0,
1281
- "wall_s": 0.0,
1282
- "items": [],
1283
- "error": null,
1284
- "mean_gen_tokens": 0.0,
1285
- "mean_gen_tokens_correct": 0.0
1286
- },
1287
- "ifeval_bench": {
1288
- "n": 10,
1289
- "correct": 5,
1290
- "pass_frac": 0.5,
1291
- "wall_s": 103.7,
1292
- "items": [
1293
- {
1294
- "src": "procedural_ifeval/forbid_keyword",
1295
- "instruction_ids": [
1296
- "keywords:forbidden_words"
1297
- ],
1298
- "per_instruction": [
1299
- true
1300
- ],
1301
- "ok": true,
1302
- "gen_tokens": 178,
1303
- "tail": "ommute feels less like a chore and more like a necessary ritual of movement, grounding me before I step into the office."
1304
- },
1305
- {
1306
- "src": "procedural_ifeval/no_comma",
1307
- "instruction_ids": [
1308
- "punctuation:no_comma"
1309
- ],
1310
- "per_instruction": [
1311
- true
1312
- ],
1313
- "ok": true,
1314
- "gen_tokens": 47,
1315
- "tail": "es mental health. Communities bond over shared green spaces. This hobby connects people with nature in concrete jungles."
1316
- },
1317
- {
1318
- "src": "procedural_ifeval/exact_words",
1319
- "instruction_ids": [
1320
- "length_constraints:number_words"
1321
- ],
1322
- "per_instruction": [
1323
- false
1324
- ],
1325
- "ok": false,
1326
- "gen_tokens": 56,
1327
- "tail": "eprint** of modern infrastructure piercing through ancient navigational aids to ensure the safety of preserved soldiers."
1328
- },
1329
- {
1330
- "src": "procedural_ifeval/ends_with_phrase",
1331
- "instruction_ids": [
1332
- "startend:end_checker"
1333
- ],
1334
- "per_instruction": [
1335
- false
1336
- ],
1337
- "ok": false,
1338
- "gen_tokens": 98,
1339
- "tail": "art ensures that fresh, crusty goods are ready to greet customers as the world begins to wake up. Thank you for reading."
1340
- },
1341
- {
1342
- "src": "procedural_ifeval/all_lowercase",
1343
- "instruction_ids": [
1344
- "change_case:english_lowercase"
1345
- ],
1346
- "per_instruction": [
1347
- false
1348
- ],
1349
- "ok": false,
1350
- "gen_tokens": 47,
1351
- "tail": "es mental health. Communities bond over shared green spaces. This hobby connects people with nature in concrete jungles."
1352
- },
1353
- {
1354
- "src": "procedural_ifeval/exact_sentences",
1355
- "instruction_ids": [
1356
- "length_constraints:number_sentences"
1357
- ],
1358
- "per_instruction": [
1359
- false
1360
- ],
1361
- "ok": false,
1362
- "gen_tokens": 68,
1363
- "tail": "ucture projects demonstrate the pinnacle of civil engineering capabilities in overcoming severe geographical challenges."
1364
- },
1365
- {
1366
- "src": "procedural_ifeval/include_keyword",
1367
- "instruction_ids": [
1368
- "keywords:frequency"
1369
- ],
1370
- "per_instruction": [
1371
- true
1372
- ],
1373
- "ok": true,
1374
- "gen_tokens": 256,
1375
- "tail": "y the time I arrived at work, the final **harbor** of the day\u2019s dedication saved countless hours, reminding me that this"
1376
- },
1377
- {
1378
- "src": "procedural_ifeval/title_format",
1379
- "instruction_ids": [
1380
- "detectable_format:title"
1381
- ],
1382
- "per_instruction": [
1383
- true
1384
- ],
1385
- "ok": true,
1386
- "gen_tokens": 176,
1387
- "tail": "alogue with the elements, requiring humility, vigilance, and an intimate understanding of nature\u2019s unpredictable rhythm."
1388
- },
1389
- {
1390
- "src": "procedural_ifeval/json_format",
1391
- "instruction_ids": [
1392
- "detectable_format:json_format"
1393
- ],
1394
- "per_instruction": [
1395
- false
1396
- ],
1397
- "ok": false,
1398
- "gen_tokens": 53,
1399
- "tail": "oys of Men's Volcano Eruption\" (often shortened to *15 minutes' 16\u201315 minutes of running, but only 2:00 AM to work\n\n{16}"
1400
- },
1401
- {
1402
- "src": "procedural_ifeval/all_uppercase",
1403
- "instruction_ids": [
1404
- "change_case:english_capital"
1405
- ],
1406
- "per_instruction": [
1407
- true
1408
- ],
1409
- "ok": true,
1410
- "gen_tokens": 49,
1411
- "tail": "LVES AND WIND PATTERNS. THIS INFORMATION CRITICALLY GUIDES SHIP NAVIGATION TO ENSURE SAFETY AND EFFICIENT ROUTES ACROSS."
1412
- }
1413
- ],
1414
- "error": null,
1415
- "mean_gen_tokens": 102.8,
1416
- "mean_gen_tokens_correct": 141.2
1417
- },
1418
- "aime_bench": {
1419
- "n": 10,
1420
- "correct": 0,
1421
- "pass_frac": 0.0,
1422
- "wall_s": 5.6,
1423
- "items": [
1424
- {
1425
- "src": "procedural_aime/lcm_residue",
1426
- "pred": "10",
1427
- "gold": "57",
1428
- "ok": false,
1429
- "gen_tokens": 7,
1430
- "tail": "\\boxed{10}"
1431
- },
1432
- {
1433
- "src": "procedural_aime/diophantine",
1434
- "pred": "104",
1435
- "gold": "290",
1436
- "ok": false,
1437
- "gen_tokens": 8,
1438
- "tail": "\\boxed{104}"
1439
- },
1440
- {
1441
- "src": "procedural_aime/factor_chain",
1442
- "pred": "44",
1443
- "gold": "52",
1444
- "ok": false,
1445
- "gen_tokens": 7,
1446
- "tail": "\\boxed{44}"
1447
- },
1448
- {
1449
- "src": "procedural_aime/iterated_digit",
1450
- "pred": "108",
1451
- "gold": "66",
1452
- "ok": false,
1453
- "gen_tokens": 8,
1454
- "tail": "\\boxed{108}"
1455
- },
1456
- {
1457
- "src": "procedural_aime/chained_modular",
1458
- "pred": "10",
1459
- "gold": "81",
1460
- "ok": false,
1461
- "gen_tokens": 7,
1462
- "tail": "\\boxed{10}"
1463
- },
1464
- {
1465
- "src": "procedural_aime/lcm_residue",
1466
- "pred": "10",
1467
- "gold": "43",
1468
- "ok": false,
1469
- "gen_tokens": 7,
1470
- "tail": "\\boxed{10}"
1471
- },
1472
- {
1473
- "src": "procedural_aime/diophantine",
1474
- "pred": "10",
1475
- "gold": "169",
1476
- "ok": false,
1477
- "gen_tokens": 7,
1478
- "tail": "\\boxed{10}"
1479
- },
1480
- {
1481
- "src": "procedural_aime/factor_chain",
1482
- "pred": "44",
1483
- "gold": "62",
1484
- "ok": false,
1485
- "gen_tokens": 7,
1486
- "tail": "\\boxed{44}"
1487
- },
1488
- {
1489
- "src": "procedural_aime/iterated_digit",
1490
- "pred": "108",
1491
- "gold": "66",
1492
- "ok": false,
1493
- "gen_tokens": 8,
1494
- "tail": "\\boxed{108}"
1495
- },
1496
- {
1497
- "src": "procedural_aime/chained_modular",
1498
- "pred": "10",
1499
- "gold": "17",
1500
- "ok": false,
1501
- "gen_tokens": 7,
1502
- "tail": "\\boxed{10}"
1503
- }
1504
- ],
1505
- "error": null,
1506
- "mean_gen_tokens": 7.3,
1507
- "mean_gen_tokens_correct": 0.0
1508
- },
1509
- "mbpp_bench": {
1510
- "n": 10,
1511
- "correct": 10,
1512
- "pass_frac": 1.0,
1513
- "wall_s": 98.5,
1514
- "items": [
1515
- {
1516
- "task_id": "v27/aggregate_list/00",
1517
- "entry_point": "aggregate",
1518
- "ok": true,
1519
- "gen_tokens": 54,
1520
- "reason": "pass",
1521
- "tail": "ate(xs):\n \"\"\"Return the sum of the even integers in ``xs``.\n\n If there are no even integers, return 0.\n \"\"\"\n return sum(x for x in xs if x % 2 == 0)"
1522
- },
1523
- {
1524
- "task_id": "v27/window_sum/01",
1525
- "entry_point": "window_max_sum",
1526
- "ok": true,
1527
- "gen_tokens": 191,
1528
- "reason": "pass",
1529
- "tail": "):\n current_sum = current_sum - xs[i - 1] + xs[i + k - 1]\n if current_sum > max_sum:\n max_sum = current_sum\n \n return max_sum\n```"
1530
- },
1531
- {
1532
- "task_id": "v27/string_predicate/02",
1533
- "entry_point": "is_anagram",
1534
- "ok": true,
1535
- "gen_tokens": 64,
1536
- "reason": "pass",
1537
- "tail": "nagram of string ``b`` (case-sensitive). Different lengths return False.\n \"\"\"\n if len(a) != len(b):\n return False\n return sorted(a) == sorted(b)"
1538
- },
1539
- {
1540
- "task_id": "v27/string_transform/03",
1541
- "entry_point": "alternate_case",
1542
- "ok": true,
1543
- "gen_tokens": 131,
1544
- "reason": "pass",
1545
- "tail": "sult.append(char.upper())\n else:\n result.append(char.lower())\n else:\n result.append(char)\n return ''.join(result)"
1546
- },
1547
- {
1548
- "task_id": "v27/transform_list/04",
1549
- "entry_point": "transform",
1550
- "ok": true,
1551
- "gen_tokens": 61,
1552
- "reason": "pass",
1553
- "tail": "transform(xs):\n \"\"\"Return a list where every element of ``xs`` is doubled.\n\n >>> transform([1, 2, 3])\n [2, 4, 6]\n \"\"\"\n return [x * 2 for x in xs]"
1554
- },
1555
- {
1556
- "task_id": "v27/digit_sum/05",
1557
- "entry_point": "digit_sum",
1558
- "ok": true,
1559
- "gen_tokens": 52,
1560
- "reason": "pass",
1561
- "tail": "git_sum(n):\n \"\"\"Return the sum of the decimal digits of the non-negative integer ``n``. ``digit_sum(0)`` is 0.\n \"\"\"\n return sum(int(d) for d in str(n))"
1562
- },
1563
- {
1564
- "task_id": "v27/pair_count/06",
1565
- "entry_point": "pair_count",
1566
- "ok": true,
1567
- "gen_tokens": 100,
1568
- "reason": "pass",
1569
- "tail": "ount = 0\n n = len(xs)\n for i in range(n):\n for j in range(i + 1, n):\n if xs[i] + xs[j] == 5:\n count += 1\n return count"
1570
- },
1571
- {
1572
- "task_id": "v27/run_length/07",
1573
- "entry_point": "rle",
1574
- "ok": true,
1575
- "gen_tokens": 100,
1576
- "reason": "pass",
1577
- "tail": " result.append((current_char, count))\n current_char = char\n count = 1\n result.append((current_char, count))\n return result"
1578
- },
1579
- {
1580
- "task_id": "v27/aggregate_list/08",
1581
- "entry_point": "aggregate",
1582
- "ok": true,
1583
- "gen_tokens": 45,
1584
- "reason": "pass",
1585
- "tail": " \"\"\"Return the difference between the largest and the smallest value in ``xs``. The list always has at least one element.\n \"\"\"\n return max(xs) - min(xs)"
1586
- },
1587
- {
1588
- "task_id": "v27/window_sum/09",
1589
- "entry_point": "window_max_sum",
1590
- "ok": true,
1591
- "gen_tokens": 191,
1592
- "reason": "pass",
1593
- "tail": "):\n current_sum = current_sum - xs[i - 1] + xs[i + k - 1]\n if current_sum > max_sum:\n max_sum = current_sum\n \n return max_sum\n```"
1594
- }
1595
- ],
1596
- "error": null,
1597
- "mean_gen_tokens": 98.9,
1598
- "mean_gen_tokens_correct": 98.9
1599
- },
1600
- "tool_use_bench": {
1601
- "n": 8,
1602
- "correct": 1,
1603
- "pass_frac": 0.125,
1604
- "wall_s": 6.0,
1605
- "items": [
1606
- {
1607
- "src": "procedural_math/geometric_series",
1608
- "tool_used": false,
1609
- "tool_result": null,
1610
- "pred": "105",
1611
- "gold": "248",
1612
- "ok": false,
1613
- "gen_tokens": 8,
1614
- "tail": "\\boxed{105}"
1615
- },
1616
- {
1617
- "src": "procedural_math/set_intersect",
1618
- "tool_used": false,
1619
- "tool_result": null,
1620
- "pred": "102",
1621
- "gold": "93",
1622
- "ok": false,
1623
- "gen_tokens": 8,
1624
- "tail": "\\boxed{102}"
1625
- },
1626
- {
1627
- "src": "procedural_math/simultaneous",
1628
- "tool_used": false,
1629
- "tool_result": null,
1630
- "pred": "10",
1631
- "gold": "24",
1632
- "ok": false,
1633
- "gen_tokens": 7,
1634
- "tail": "\\boxed{10}"
1635
- },
1636
- {
1637
- "src": "procedural_math/probability_count",
1638
- "tool_used": false,
1639
- "tool_result": null,
1640
- "pred": "10",
1641
- "gold": "56",
1642
- "ok": false,
1643
- "gen_tokens": 7,
1644
- "tail": "\\boxed{10}"
1645
- },
1646
- {
1647
- "src": "procedural_math/triangle_area",
1648
- "tool_used": false,
1649
- "tool_result": null,
1650
- "pred": "10",
1651
- "gold": "35",
1652
- "ok": false,
1653
- "gen_tokens": 7,
1654
- "tail": "\\boxed{10}"
1655
- },
1656
- {
1657
- "src": "procedural_math/gcd_lcm",
1658
- "tool_used": false,
1659
- "tool_result": null,
1660
- "pred": "1",
1661
- "gold": "1",
1662
- "ok": true,
1663
- "gen_tokens": 6,
1664
- "tail": "\\boxed{1}"
1665
- },
1666
- {
1667
- "src": "procedural_math/digit_sum",
1668
- "tool_used": false,
1669
- "tool_result": null,
1670
- "pred": "108",
1671
- "gold": "205",
1672
- "ok": false,
1673
- "gen_tokens": 8,
1674
- "tail": "\\boxed{108}"
1675
- },
1676
- {
1677
- "src": "procedural_math/proportion",
1678
- "tool_used": false,
1679
- "tool_result": null,
1680
- "pred": "1008",
1681
- "gold": "1064",
1682
- "ok": false,
1683
- "gen_tokens": 9,
1684
- "tail": "\\boxed{1008}"
1685
- }
1686
- ],
1687
- "error": null,
1688
- "mean_gen_tokens": 7.5,
1689
- "mean_gen_tokens_correct": 6.0
1690
- },
1691
- "self_consistency_bench": {
1692
- "n": 0,
1693
- "correct": 0,
1694
- "pass_frac": 0.0,
1695
- "wall_s": 0.0,
1696
- "items": [],
1697
- "error": null,
1698
- "mean_gen_tokens": 0.0,
1699
- "mean_gen_tokens_correct": 0.0
1700
- },
1701
- "arc_bench": {
1702
- "n": 0,
1703
- "correct": 0,
1704
- "pass_frac": 0.0,
1705
- "wall_s": 0.0,
1706
- "items": [],
1707
- "error": null,
1708
- "mean_gen_tokens": 0.0,
1709
- "mean_gen_tokens_correct": 0.0
1710
- },
1711
- "truthful_bench": {
1712
- "n": 0,
1713
- "correct": 0,
1714
- "pass_frac": 0.0,
1715
- "wall_s": 0.0,
1716
- "items": [],
1717
- "error": null,
1718
- "mean_gen_tokens": 0.0,
1719
- "mean_gen_tokens_correct": 0.0
1720
- },
1721
- "long_context_bench": {
1722
- "n": 6,
1723
- "correct": 0,
1724
- "pass_frac": 0.0,
1725
- "wall_s": 4.1,
1726
- "items": [
1727
- {
1728
- "src": "long_context",
1729
- "gold": "A93ZP3S",
1730
- "pred_tail": "64P3D9",
1731
- "ok": false,
1732
- "gen_tokens": 7,
1733
- "needle_position": 12,
1734
- "confuser_positions": [
1735
- 9,
1736
- 27,
1737
- 22
1738
- ],
1739
- "confuser_hit": false,
1740
- "gold_in_pred": false
1741
- },
1742
- {
1743
- "src": "long_context",
1744
- "gold": "UM2PTED",
1745
- "pred_tail": "9RZ2U",
1746
- "ok": false,
1747
- "gen_tokens": 6,
1748
- "needle_position": 21,
1749
- "confuser_positions": [
1750
- 0,
1751
- 28,
1752
- 16
1753
- ],
1754
- "confuser_hit": false,
1755
- "gold_in_pred": false
1756
- },
1757
- {
1758
- "src": "long_context",
1759
- "gold": "GX4RZQU",
1760
- "pred_tail": "64V8ZC",
1761
- "ok": false,
1762
- "gen_tokens": 7,
1763
- "needle_position": 24,
1764
- "confuser_positions": [
1765
- 12,
1766
- 19,
1767
- 8
1768
- ],
1769
- "confuser_hit": false,
1770
- "gold_in_pred": false
1771
- },
1772
- {
1773
- "src": "long_context",
1774
- "gold": "DSEBMP8",
1775
- "pred_tail": "3CQMB8",
1776
- "ok": false,
1777
- "gen_tokens": 6,
1778
- "needle_position": 14,
1779
- "confuser_positions": [
1780
- 37,
1781
- 40,
1782
- 8
1783
- ],
1784
- "confuser_hit": false,
1785
- "gold_in_pred": false
1786
- },
1787
- {
1788
- "src": "long_context",
1789
- "gold": "Z6BHR2C",
1790
- "pred_tail": "93C6C2",
1791
- "ok": false,
1792
- "gen_tokens": 7,
1793
- "needle_position": 12,
1794
- "confuser_positions": [
1795
- 8,
1796
- 22,
1797
- 29
1798
- ],
1799
- "confuser_hit": false,
1800
- "gold_in_pred": false
1801
- },
1802
- {
1803
- "src": "long_context",
1804
- "gold": "JDYQJMZ",
1805
- "pred_tail": "XQ24GY",
1806
- "ok": false,
1807
- "gen_tokens": 6,
1808
- "needle_position": 33,
1809
- "confuser_positions": [
1810
- 43,
1811
- 20,
1812
- 38
1813
- ],
1814
- "confuser_hit": false,
1815
- "gold_in_pred": false
1816
- }
1817
- ],
1818
- "error": null,
1819
- "mean_gen_tokens": 6.5,
1820
- "mean_gen_tokens_correct": 0.0
1821
- },
1822
- "procedural_bench": {
1823
- "n": 0,
1824
- "correct": 0,
1825
- "pass_frac": 0.0,
1826
- "wall_s": 0.0,
1827
- "items": [],
1828
- "error": null,
1829
- "mean_gen_tokens": 0.0,
1830
- "mean_gen_tokens_correct": 0.0
1831
- },
1832
- "robustness_bench": {
1833
- "n": 16,
1834
- "correct": 0,
1835
- "pass_frac": 0.0,
1836
- "wall_s": 9.7,
1837
- "items": [
1838
- {
1839
- "src": "procedural_math/geometric_series",
1840
- "perturbation": "imperative_to_question",
1841
- "pred": "100",
1842
- "gold": "189",
1843
- "ok": false,
1844
- "gen_tokens": 6
1845
- },
1846
- {
1847
- "src": "procedural_math/geometric_series",
1848
- "perturbation": "context_prefix",
1849
- "pred": "100",
1850
- "gold": "189",
1851
- "ok": false,
1852
- "gen_tokens": 6
1853
- },
1854
- {
1855
- "src": "procedural_math/arithmetic_series",
1856
- "perturbation": "imperative_to_question",
1857
- "pred": "108",
1858
- "gold": "1122",
1859
- "ok": false,
1860
- "gen_tokens": 6
1861
- },
1862
- {
1863
- "src": "procedural_math/arithmetic_series",
1864
- "perturbation": "context_prefix",
1865
- "pred": "108",
1866
- "gold": "1122",
1867
- "ok": false,
1868
- "gen_tokens": 6
1869
- },
1870
- {
1871
- "src": "procedural_math/proportion",
1872
- "perturbation": "imperative_to_question",
1873
- "pred": "1225",
1874
- "gold": "345",
1875
- "ok": false,
1876
- "gen_tokens": 7
1877
- },
1878
- {
1879
- "src": "procedural_math/proportion",
1880
- "perturbation": "context_prefix",
1881
- "pred": "1021",
1882
- "gold": "345",
1883
- "ok": false,
1884
- "gen_tokens": 7
1885
- },
1886
- {
1887
- "src": "procedural_math/set_intersect",
1888
- "perturbation": "imperative_to_question",
1889
- "pred": "108",
1890
- "gold": "117",
1891
- "ok": false,
1892
- "gen_tokens": 6
1893
- },
1894
- {
1895
- "src": "procedural_math/set_intersect",
1896
- "perturbation": "context_prefix",
1897
- "pred": "108",
1898
- "gold": "117",
1899
- "ok": false,
1900
- "gen_tokens": 6
1901
- },
1902
- {
1903
- "src": "procedural_math/percentage",
1904
- "perturbation": "imperative_to_question",
1905
- "pred": "104",
1906
- "gold": "121",
1907
- "ok": false,
1908
- "gen_tokens": 6
1909
- },
1910
- {
1911
- "src": "procedural_math/percentage",
1912
- "perturbation": "context_prefix",
1913
- "pred": "104",
1914
- "gold": "121",
1915
- "ok": false,
1916
- "gen_tokens": 6
1917
- },
1918
- {
1919
- "src": "procedural_math/probability_count",
1920
- "perturbation": "imperative_to_question",
1921
- "pred": "14",
1922
- "gold": "77",
1923
- "ok": false,
1924
- "gen_tokens": 5
1925
- },
1926
- {
1927
- "src": "procedural_math/probability_count",
1928
- "perturbation": "context_prefix",
1929
- "pred": "10",
1930
- "gold": "77",
1931
- "ok": false,
1932
- "gen_tokens": 5
1933
- },
1934
- {
1935
- "src": "procedural_math/mixture",
1936
- "perturbation": "imperative_to_question",
1937
- "pred": "4200",
1938
- "gold": "6340",
1939
- "ok": false,
1940
- "gen_tokens": 7
1941
- },
1942
- {
1943
- "src": "procedural_math/mixture",
1944
- "perturbation": "context_prefix",
1945
- "pred": "4200",
1946
- "gold": "6340",
1947
- "ok": false,
1948
- "gen_tokens": 7
1949
- },
1950
- {
1951
- "src": "procedural_math/modular_linear",
1952
- "perturbation": "imperative_to_question",
1953
- "pred": "10",
1954
- "gold": "2",
1955
- "ok": false,
1956
- "gen_tokens": 5
1957
- },
1958
- {
1959
- "src": "procedural_math/modular_linear",
1960
- "perturbation": "context_prefix",
1961
- "pred": "10",
1962
- "gold": "2",
1963
- "ok": false,
1964
- "gen_tokens": 5
1965
- }
1966
- ],
1967
- "error": null,
1968
- "mean_gen_tokens": 6.0,
1969
- "mean_gen_tokens_correct": 0.0
1970
- },
1971
- "noise_resistance_bench": {
1972
- "n": 0,
1973
- "correct": 0,
1974
- "pass_frac": 0.0,
1975
- "wall_s": 0.0,
1976
- "items": [],
1977
- "error": null,
1978
- "mean_gen_tokens": 0.0,
1979
- "mean_gen_tokens_correct": 0.0
1980
- },
1981
- "bench_gate": {
1982
- "run": true,
1983
- "reason": "no_king_kl",
1984
- "only_kl_candidates": true,
1985
- "kl_candidate_slack": 0.0,
1986
- "student_kl": 1.87522,
1987
- "king_kl": null,
1988
- "wall_s": 311.3
1989
- }
1990
- }
1991
- },
1992
- "timings": {
1993
- "vllm_startup": 76.7,
1994
- "vllm_generation": 6.7,
1995
- "teacher_probe_refs": 12.3,
1996
- "teacher_logits_pass": 0.0,
1997
- "teacher_hf_load": 0.0
1998
- }
1999
- }