Hugging Face's logo

saketh-chervu
/
rvr-grid-intermediate-correct

Safetensors
qwen3
Model card Files
xet
 Community
saketh-chervu commited on
Commit
a6c489e
·
verified ·
1 Parent(s): 7758b8e

Upload eval_data/meta.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. eval_data/meta.json +278 -0
eval_data/meta.json ADDED
@@ -0,0 +1,278 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "exp": "grid",
3
+ "domain": "grid",
4
+ "n_cells": 9,
5
+ "variant": "intermediate",
6
+ "arm": "correct",
7
+ "seed": 0,
8
+ "composable": [
9
+ "anti_transpose",
10
+ "flip_h",
11
+ "flip_v",
12
+ "rot90",
13
+ "shift_rows"
14
+ ],
15
+ "perms": {
16
+ "rot90": [
17
+ 6,
18
+ 3,
19
+ 0,
20
+ 7,
21
+ 4,
22
+ 1,
23
+ 8,
24
+ 5,
25
+ 2
26
+ ],
27
+ "rot180": [
28
+ 8,
29
+ 7,
30
+ 6,
31
+ 5,
32
+ 4,
33
+ 3,
34
+ 2,
35
+ 1,
36
+ 0
37
+ ],
38
+ "rot270": [
39
+ 2,
40
+ 5,
41
+ 8,
42
+ 1,
43
+ 4,
44
+ 7,
45
+ 0,
46
+ 3,
47
+ 6
48
+ ],
49
+ "transpose": [
50
+ 0,
51
+ 3,
52
+ 6,
53
+ 1,
54
+ 4,
55
+ 7,
56
+ 2,
57
+ 5,
58
+ 8
59
+ ],
60
+ "anti_transpose": [
61
+ 8,
62
+ 5,
63
+ 2,
64
+ 7,
65
+ 4,
66
+ 1,
67
+ 6,
68
+ 3,
69
+ 0
70
+ ],
71
+ "flip_h": [
72
+ 2,
73
+ 1,
74
+ 0,
75
+ 5,
76
+ 4,
77
+ 3,
78
+ 8,
79
+ 7,
80
+ 6
81
+ ],
82
+ "flip_v": [
83
+ 6,
84
+ 7,
85
+ 8,
86
+ 3,
87
+ 4,
88
+ 5,
89
+ 0,
90
+ 1,
91
+ 2
92
+ ],
93
+ "shift_rows": [
94
+ 6,
95
+ 7,
96
+ 8,
97
+ 0,
98
+ 1,
99
+ 2,
100
+ 3,
101
+ 4,
102
+ 5
103
+ ]
104
+ },
105
+ "train_comps": [
106
+ "anti_transpose|rot90",
107
+ "anti_transpose|shift_rows",
108
+ "flip_h|rot90",
109
+ "flip_v|rot90",
110
+ "flip_v|shift_rows",
111
+ "rot90|anti_transpose",
112
+ "rot90|flip_h",
113
+ "rot90|flip_v",
114
+ "rot90|shift_rows",
115
+ "shift_rows|anti_transpose",
116
+ "shift_rows|flip_v",
117
+ "shift_rows|rot90"
118
+ ],
119
+ "heldout_comps": [
120
+ "anti_transpose|flip_h",
121
+ "flip_h|anti_transpose",
122
+ "anti_transpose|flip_v",
123
+ "flip_v|anti_transpose"
124
+ ],
125
+ "heldout_labels": {
126
+ "anti_transpose|flip_h": "never_paired",
127
+ "flip_h|anti_transpose": "never_paired",
128
+ "anti_transpose|flip_v": "never_paired",
129
+ "flip_v|anti_transpose": "never_paired"
130
+ },
131
+ "n_comp_train_per_pairing": 200,
132
+ "total_comp_train": 2400,
133
+ "compositions": {
134
+ "anti_transpose|rot90": {
135
+ "role": "train",
136
+ "n_train": 200,
137
+ "n_seen_eval": 200
138
+ },
139
+ "anti_transpose|shift_rows": {
140
+ "role": "train",
141
+ "n_train": 200,
142
+ "n_seen_eval": 200
143
+ },
144
+ "flip_h|rot90": {
145
+ "role": "train",
146
+ "n_train": 200,
147
+ "n_seen_eval": 200
148
+ },
149
+ "flip_v|rot90": {
150
+ "role": "train",
151
+ "n_train": 200,
152
+ "n_seen_eval": 200
153
+ },
154
+ "flip_v|shift_rows": {
155
+ "role": "train",
156
+ "n_train": 200,
157
+ "n_seen_eval": 200
158
+ },
159
+ "rot90|anti_transpose": {
160
+ "role": "train",
161
+ "n_train": 200,
162
+ "n_seen_eval": 200
163
+ },
164
+ "rot90|flip_h": {
165
+ "role": "train",
166
+ "n_train": 200,
167
+ "n_seen_eval": 200
168
+ },
169
+ "rot90|flip_v": {
170
+ "role": "train",
171
+ "n_train": 200,
172
+ "n_seen_eval": 200
173
+ },
174
+ "rot90|shift_rows": {
175
+ "role": "train",
176
+ "n_train": 200,
177
+ "n_seen_eval": 200
178
+ },
179
+ "shift_rows|anti_transpose": {
180
+ "role": "train",
181
+ "n_train": 200,
182
+ "n_seen_eval": 200
183
+ },
184
+ "shift_rows|flip_v": {
185
+ "role": "train",
186
+ "n_train": 200,
187
+ "n_seen_eval": 200
188
+ },
189
+ "shift_rows|rot90": {
190
+ "role": "train",
191
+ "n_train": 200,
192
+ "n_seen_eval": 200
193
+ },
194
+ "anti_transpose|flip_h": {
195
+ "role": "heldout",
196
+ "label": "never_paired",
197
+ "n_eval": 200,
198
+ "distinct_answers": 200,
199
+ "top_answer_share": 0.005
200
+ },
201
+ "flip_h|anti_transpose": {
202
+ "role": "heldout",
203
+ "label": "never_paired",
204
+ "n_eval": 200,
205
+ "distinct_answers": 200,
206
+ "top_answer_share": 0.005
207
+ },
208
+ "anti_transpose|flip_v": {
209
+ "role": "heldout",
210
+ "label": "never_paired",
211
+ "n_eval": 200,
212
+ "distinct_answers": 200,
213
+ "top_answer_share": 0.005
214
+ },
215
+ "flip_v|anti_transpose": {
216
+ "role": "heldout",
217
+ "label": "never_paired",
218
+ "n_eval": 200,
219
+ "distinct_answers": 200,
220
+ "top_answer_share": 0.005
221
+ }
222
+ },
223
+ "train_heldout_chain_overlap": [],
224
+ "heldout_instance_collisions": 0,
225
+ "coverage": {
226
+ "train_inner_atoms": [
227
+ "anti_transpose",
228
+ "flip_h",
229
+ "flip_v",
230
+ "rot90",
231
+ "shift_rows"
232
+ ],
233
+ "train_outer_atoms": [
234
+ "anti_transpose",
235
+ "flip_h",
236
+ "flip_v",
237
+ "rot90",
238
+ "shift_rows"
239
+ ],
240
+ "heldout_inner_atoms": [
241
+ "anti_transpose",
242
+ "flip_h",
243
+ "flip_v"
244
+ ],
245
+ "heldout_outer_atoms": [
246
+ "anti_transpose",
247
+ "flip_h",
248
+ "flip_v"
249
+ ],
250
+ "heldout_inner_count": 3,
251
+ "heldout_outer_count": 3,
252
+ "per_slot_coverage": {
253
+ "anti_transpose": {
254
+ "train_as_inner": 2,
255
+ "train_as_outer": 2
256
+ },
257
+ "flip_h": {
258
+ "train_as_inner": 1,
259
+ "train_as_outer": 1
260
+ },
261
+ "flip_v": {
262
+ "train_as_inner": 2,
263
+ "train_as_outer": 2
264
+ },
265
+ "rot90": {
266
+ "train_as_inner": 4,
267
+ "train_as_outer": 4
268
+ },
269
+ "shift_rows": {
270
+ "train_as_inner": 3,
271
+ "train_as_outer": 3
272
+ }
273
+ },
274
+ "train_covers_both_slots": true,
275
+ "pairing_collisions": []
276
+ },
277
+ "collisions_total": 0
278
+ }