File size: 10,001 Bytes
fd74a4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
{
  "fixture": {
    "purpose": "End-to-end replay of the existing Figure 1 family, selected by its role in the paper; uses both models and arms at base five, without choosing outputs by success.",
    "sample_id": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e",
    "base_count": 5,
    "reference_rule_pairs": [
      [
        "ex",
        "azw"
      ],
      [
        "wa",
        "y"
      ]
    ],
    "task_pair": {
      "CONTROL": {
        "inputs": [
          "gud",
          "czua",
          "bwcied",
          "dzexa",
          "yf",
          "iiiexa"
        ],
        "outputs": [
          "gud",
          "czua",
          "bwcied",
          "dzazy",
          "yf",
          "iiiazy"
        ]
      },
      "SEPARATION": {
        "inputs": [
          "gud",
          "czua",
          "bwcied",
          "dzexa",
          "yf",
          "wbawex"
        ],
        "outputs": [
          "gud",
          "czua",
          "bwcied",
          "dzazy",
          "yf",
          "wbayzw"
        ]
      }
    },
    "request_ids": [
      "M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL",
      "M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION",
      "M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL",
      "M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION"
    ],
    "expected": {
      "status": "PASS",
      "family": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e",
      "base_count": 5,
      "certificates": {
        "CONTROL": {
          "complete_one_rule_solutions": [
            [
              "exa",
              "azy"
            ]
          ],
          "minimum_length": 1,
          "two_rule_reference_passes": true
        },
        "SEPARATION": {
          "complete_one_rule_solutions": [],
          "minimum_length": 2,
          "two_rule_reference_passes": true
        }
      },
      "saved_outputs": {
        "M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": {
          "selected_rules": null,
          "legal_selected_program": false,
          "targets": {
            "displayed": {
              "hits": 0,
              "inputs": 6,
              "all_pass": false
            },
            "common_nine": {
              "hits": 0,
              "inputs": 9,
              "all_pass": false
            },
            "wholly_unshown_pair": {
              "hits": 0,
              "inputs": 2,
              "all_pass": false
            },
            "uniform": {
              "hits": 0,
              "inputs": 32,
              "all_pass": false
            },
            "active": {
              "hits": 0,
              "inputs": 32,
              "all_pass": false
            }
          }
        },
        "M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": {
          "selected_rules": [
            [
              "a",
              "y"
            ],
            [
              "e",
              "a"
            ],
            [
              "x",
              "z"
            ]
          ],
          "legal_selected_program": true,
          "targets": {
            "displayed": {
              "hits": 4,
              "inputs": 6,
              "all_pass": false
            },
            "common_nine": {
              "hits": 5,
              "inputs": 9,
              "all_pass": false
            },
            "wholly_unshown_pair": {
              "hits": 1,
              "inputs": 2,
              "all_pass": false
            },
            "uniform": {
              "hits": 11,
              "inputs": 32,
              "all_pass": false
            },
            "active": {
              "hits": 5,
              "inputs": 32,
              "all_pass": false
            }
          }
        },
        "M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": {
          "selected_rules": [
            [
              "exa",
              "azy"
            ],
            [
              "wex",
              "yzw"
            ]
          ],
          "legal_selected_program": true,
          "targets": {
            "displayed": {
              "hits": 6,
              "inputs": 6,
              "all_pass": true
            },
            "common_nine": {
              "hits": 9,
              "inputs": 9,
              "all_pass": true
            },
            "wholly_unshown_pair": {
              "hits": 2,
              "inputs": 2,
              "all_pass": true
            },
            "uniform": {
              "hits": 32,
              "inputs": 32,
              "all_pass": true
            },
            "active": {
              "hits": 0,
              "inputs": 32,
              "all_pass": false
            }
          }
        },
        "M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": {
          "selected_rules": null,
          "legal_selected_program": false,
          "targets": {
            "displayed": {
              "hits": 0,
              "inputs": 6,
              "all_pass": false
            },
            "common_nine": {
              "hits": 0,
              "inputs": 9,
              "all_pass": false
            },
            "wholly_unshown_pair": {
              "hits": 0,
              "inputs": 2,
              "all_pass": false
            },
            "uniform": {
              "hits": 0,
              "inputs": 32,
              "all_pass": false
            },
            "active": {
              "hits": 0,
              "inputs": 32,
              "all_pass": false
            }
          }
        }
      },
      "model_inference_calls": 0
    }
  },
  "panel": [
    {
      "model": "Qwen3-8B",
      "base_count": "3",
      "displayed_examples": "4",
      "primary_family_N": "119",
      "CONTROL_displayed_successes": "59",
      "SEPARATION_displayed_successes": "15",
      "displayed_delta_pp_and_95CI": "-36.97 [-46.22,-27.73]",
      "unshown_pair_delta_pp_and_95CI": "+13.45 [+7.56,+20.17]",
      "all_legal_N": "120",
      "all_legal_displayed_delta_pp_and_95CI": "-36.67 [-45.83,-27.50]",
      "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
    },
    {
      "model": "Qwen3-8B",
      "base_count": "4",
      "displayed_examples": "5",
      "primary_family_N": "119",
      "CONTROL_displayed_successes": "61",
      "SEPARATION_displayed_successes": "15",
      "displayed_delta_pp_and_95CI": "-38.66 [-47.90,-28.57]",
      "unshown_pair_delta_pp_and_95CI": "+10.92 [+5.88,+16.81]",
      "all_legal_N": "120",
      "all_legal_displayed_delta_pp_and_95CI": "-39.17 [-48.33,-29.17]",
      "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
    },
    {
      "model": "Qwen3-8B",
      "base_count": "5",
      "displayed_examples": "6",
      "primary_family_N": "120",
      "CONTROL_displayed_successes": "67",
      "SEPARATION_displayed_successes": "12",
      "displayed_delta_pp_and_95CI": "-45.83 [-55.00,-36.67]",
      "unshown_pair_delta_pp_and_95CI": "+9.17 [+4.17,+15.00]",
      "all_legal_N": "120",
      "all_legal_displayed_delta_pp_and_95CI": "-45.83 [-55.00,-36.67]",
      "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
    },
    {
      "model": "Qwen3-8B",
      "base_count": "6",
      "displayed_examples": "7",
      "primary_family_N": "120",
      "CONTROL_displayed_successes": "68",
      "SEPARATION_displayed_successes": "19",
      "displayed_delta_pp_and_95CI": "-40.83 [-50.00,-31.67]",
      "unshown_pair_delta_pp_and_95CI": "+12.50 [+6.67,+18.33]",
      "all_legal_N": "120",
      "all_legal_displayed_delta_pp_and_95CI": "-40.83 [-50.00,-31.67]",
      "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
    },
    {
      "model": "Olmo-3-7B-Think",
      "base_count": "3",
      "displayed_examples": "4",
      "primary_family_N": "119",
      "CONTROL_displayed_successes": "53",
      "SEPARATION_displayed_successes": "14",
      "displayed_delta_pp_and_95CI": "-32.77 [-42.86,-22.69]",
      "unshown_pair_delta_pp_and_95CI": "+8.40 [+3.36,+13.45]",
      "all_legal_N": "120",
      "all_legal_displayed_delta_pp_and_95CI": "-33.33 [-43.33,-23.33]",
      "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
    },
    {
      "model": "Olmo-3-7B-Think",
      "base_count": "4",
      "displayed_examples": "5",
      "primary_family_N": "119",
      "CONTROL_displayed_successes": "47",
      "SEPARATION_displayed_successes": "19",
      "displayed_delta_pp_and_95CI": "-23.53 [-34.45,-12.61]",
      "unshown_pair_delta_pp_and_95CI": "+11.76 [+5.88,+17.65]",
      "all_legal_N": "120",
      "all_legal_displayed_delta_pp_and_95CI": "-23.33 [-34.17,-12.50]",
      "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
    },
    {
      "model": "Olmo-3-7B-Think",
      "base_count": "5",
      "displayed_examples": "6",
      "primary_family_N": "120",
      "CONTROL_displayed_successes": "52",
      "SEPARATION_displayed_successes": "15",
      "displayed_delta_pp_and_95CI": "-30.83 [-40.83,-20.83]",
      "unshown_pair_delta_pp_and_95CI": "+11.67 [+6.67,+17.50]",
      "all_legal_N": "120",
      "all_legal_displayed_delta_pp_and_95CI": "-30.83 [-40.83,-20.83]",
      "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
    },
    {
      "model": "Olmo-3-7B-Think",
      "base_count": "6",
      "displayed_examples": "7",
      "primary_family_N": "120",
      "CONTROL_displayed_successes": "54",
      "SEPARATION_displayed_successes": "13",
      "displayed_delta_pp_and_95CI": "-34.17 [-43.33,-25.00]",
      "unshown_pair_delta_pp_and_95CI": "+9.17 [+4.17,+15.00]",
      "all_legal_N": "120",
      "all_legal_displayed_delta_pp_and_95CI": "-34.17 [-43.33,-25.00]",
      "interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
    }
  ]
}