Batnini commited on
Commit
403f4b4
·
verified ·
1 Parent(s): 2dd609a

Upload write_back_cycle_result.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. write_back_cycle_result.json +532 -52
write_back_cycle_result.json CHANGED
@@ -1,26 +1,40 @@
1
  {
2
  "status": "Auftrag/unverifiziert-lokal",
3
  "source": "AGENTS.md §10.3 · arXiv:2608.22215 §3.4",
4
- "n": 20,
5
  "seed": 42,
6
- "base_model": "Qwen/Qwen2.5-0.5B-Instruct",
7
  "gpu": "Tesla T4",
8
  "llm_base_url": "http://127.0.0.1:11434/v1",
9
  "llm_model": "qwen2.5:1.5b",
10
  "workdir": "/kaggle/working",
11
  "self_update": true,
12
  "approvers": [
13
- "Batnini",
14
- "Taoufik"
15
  ],
16
  "writes": {
17
- "write-new": 7,
18
- "write-update": 6,
19
- "non-write": 3,
20
- "blocked": 4,
21
  "error": 0
22
  },
23
  "write_details": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  {
25
  "question": "What was the title that Traidenis held?",
26
  "ok": true,
@@ -29,91 +43,238 @@
29
  "reason": null
30
  },
31
  {
32
- "question": "What award was An Officer and a Gentleman nominated for?",
33
  "ok": true,
34
  "action": "write-update",
35
  "gate": "allow",
36
  "reason": null
37
  },
38
  {
39
- "question": "What company makes Armoured Multirole Carrier?",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  "ok": false,
41
  "action": "uncertain",
42
  "gate": "ask",
43
  "reason": "Router uncertain -> Human-Gate (§10.4)"
44
  },
45
  {
46
- "question": "What river does Styr River form a part of?",
47
  "ok": true,
48
- "action": "write-update",
49
  "gate": "allow",
50
  "reason": null
51
  },
52
  {
53
- "question": "In what city was Norbert Hummelt born?",
 
 
 
 
 
 
 
54
  "ok": false,
55
  "action": "uncertain",
56
  "gate": "ask",
57
  "reason": "Router uncertain -> Human-Gate (§10.4)"
58
  },
59
  {
60
- "question": "Which director worked on the film A Dog's Life?",
61
  "ok": false,
62
  "action": "uncertain",
63
  "gate": "ask",
64
  "reason": "Router uncertain -> Human-Gate (§10.4)"
65
  },
66
  {
67
- "question": "What city was Christians & Lions formed in?",
 
 
 
 
 
 
 
68
  "ok": true,
69
  "action": "write-new",
70
  "gate": "allow",
71
  "reason": null
72
  },
73
  {
74
- "question": "What was the name of the city or state where KOAN Sound was formed?",
75
  "ok": true,
76
  "action": "write-new",
77
  "gate": "allow",
78
  "reason": null
79
  },
80
  {
81
- "question": "Is the taxon rank of Allium tardiflorum species or genus?",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  "ok": true,
83
  "action": "none",
84
  "gate": "allow",
85
  "reason": null
86
  },
87
  {
88
- "question": "The 9998 ISO was named for whom?",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  "ok": true,
90
  "action": "write-update",
91
  "gate": "allow",
92
  "reason": null
93
  },
94
  {
95
- "question": "What is the name of the founder of Sharecare?",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  "ok": false,
97
  "action": "uncertain",
98
  "gate": "ask",
99
  "reason": "Router uncertain -> Human-Gate (§10.4)"
100
  },
101
  {
102
- "question": "What team is Beau Allred affiliated with?",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  "ok": true,
104
  "action": "write-new",
105
  "gate": "allow",
106
  "reason": null
107
  },
108
  {
109
- "question": "What award was Tweetie Pie nominated for?",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110
  "ok": true,
111
  "action": "write-update",
112
  "gate": "allow",
113
  "reason": null
114
  },
115
  {
116
- "question": "What was the noble family that Tuor was from?",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
  "ok": true,
118
  "action": "write-update",
119
  "gate": "allow",
@@ -127,19 +288,40 @@
127
  "reason": null
128
  },
129
  {
130
- "question": "What programming language was used to write RealVNC?",
 
 
 
 
 
 
 
131
  "ok": true,
132
- "action": "none",
133
  "gate": "allow",
134
  "reason": null
135
  },
136
  {
137
- "question": "What year was the end of Comdial?",
 
 
 
 
 
 
 
138
  "ok": true,
139
  "action": "write-new",
140
  "gate": "allow",
141
  "reason": null
142
  },
 
 
 
 
 
 
 
143
  {
144
  "question": "What noble family was Kunigunde of Austria part of?",
145
  "ok": true,
@@ -148,62 +330,330 @@
148
  "reason": null
149
  },
150
  {
151
- "question": "What was Isaac Nicola's city of birth?",
 
 
 
 
 
 
 
152
  "ok": true,
153
  "action": "write-new",
154
  "gate": "allow",
155
  "reason": null
156
  },
157
  {
158
- "question": "Who is the illustrator of Cycle of the Werewolf?",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  "ok": true,
160
  "action": "write-new",
161
  "gate": "allow",
162
  "reason": null
 
 
 
 
 
 
 
163
  }
164
  ],
165
  "provenance_log": "/kaggle/working/tier3/provenance.jsonl",
166
- "pending": 7,
167
  "pending_keys": [
168
  "what-was-the-title-that-traidenis-held",
169
- "what-city-was-christians-lions-formed-in",
170
- "what-was-the-name-of-the-city-or-state-where-koan-sound-was-",
 
171
  "what-team-is-beau-allred-affiliated-with",
172
- "what-year-was-the-end-of-comdial",
173
  "what-was-isaac-nicola-s-city-of-birth",
174
- "who-is-the-illustrator-of-cycle-of-the-werewolf"
 
 
 
 
 
 
 
 
 
 
 
 
175
  ],
176
- "flushed": [],
177
  "kept": [
178
  "what-was-the-title-that-traidenis-held",
179
- "what-city-was-christians-lions-formed-in",
180
- "what-was-the-name-of-the-city-or-state-where-koan-sound-was-",
 
181
  "what-team-is-beau-allred-affiliated-with",
182
- "what-year-was-the-end-of-comdial",
183
  "what-was-isaac-nicola-s-city-of-birth",
184
- "who-is-the-illustrator-of-cycle-of-the-werewolf"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
185
  ],
186
- "loss": null,
187
- "adapter_path": null,
188
  "write_back": {
189
- "ok": false,
190
- "decision": "abstain",
191
- "flushed": [],
 
 
192
  "kept": [
193
  "what-was-the-title-that-traidenis-held",
194
- "what-city-was-christians-lions-formed-in",
195
- "what-was-the-name-of-the-city-or-state-where-koan-sound-was-",
 
196
  "what-team-is-beau-allred-affiliated-with",
197
- "what-year-was-the-end-of-comdial",
198
  "what-was-isaac-nicola-s-city-of-birth",
199
- "who-is-the-illustrator-of-cycle-of-the-werewolf"
 
 
 
 
 
 
200
  ],
201
- "reason": "base_rate zu niedrig (0/7 < 1) → abstain"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
202
  },
203
  "zsre": {
204
  "ok": true,
205
  "path": "/kaggle/working/zsre_validation.jsonl",
206
- "count": 20,
207
  "hashes": [
208
  "9a2e90866036a10bc18334eeb314469127a67de0b69669a770b742114e91255f",
209
  "baf562e40d9e8ff16f215ebed6571dc6efa5766c2ce0ad510920f104a3bbafbc",
@@ -224,13 +674,43 @@
224
  "4b74b9a72bf6a77efa9c7e4fba3e713016b63cbd67ac1796914f56666dcd943e",
225
  "85b1755b7fd804ee8f7ce3b9e9841ded8a6280fb3616110a2aa47975ae834a10",
226
  "a32e2d23f3c1c149181aa6b84d37e5c1a292f24eefb10f51cecd65c7bd659689",
227
- "fff721e2040788e3c560ebf0ede6be113bea02316caea2860d7ac53efb9609ee"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228
  ],
229
- "requested": 20,
230
  "split": "train",
231
  "dataset": "community-datasets/qa_zre",
232
- "rows_scanned": 49,
233
- "skipped_empty": 29,
234
  "note": "volle Menge aus Quelle verfuegbar; hier gesampeltes n"
235
  },
236
  "dataset": "/kaggle/working/consolidation_sft.jsonl"
 
1
  {
2
  "status": "Auftrag/unverifiziert-lokal",
3
  "source": "AGENTS.md §10.3 · arXiv:2608.22215 §3.4",
4
+ "n": 50,
5
  "seed": 42,
6
+ "base_model": "Qwen/Qwen2.5-3B-Instruct",
7
  "gpu": "Tesla T4",
8
  "llm_base_url": "http://127.0.0.1:11434/v1",
9
  "llm_model": "qwen2.5:1.5b",
10
  "workdir": "/kaggle/working",
11
  "self_update": true,
12
  "approvers": [
13
+ "batnini",
14
+ "taoufik"
15
  ],
16
  "writes": {
17
+ "write-new": 15,
18
+ "write-update": 16,
19
+ "non-write": 6,
20
+ "blocked": 13,
21
  "error": 0
22
  },
23
  "write_details": [
24
+ {
25
+ "question": "On what date did Schermerhorn-Drees cabinet end?",
26
+ "ok": false,
27
+ "action": "uncertain",
28
+ "gate": "ask",
29
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
30
+ },
31
+ {
32
+ "question": "The Du glucose pour Noémie has what character?",
33
+ "ok": false,
34
+ "action": "uncertain",
35
+ "gate": "ask",
36
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
37
+ },
38
  {
39
  "question": "What was the title that Traidenis held?",
40
  "ok": true,
 
43
  "reason": null
44
  },
45
  {
46
+ "question": "What was the noble family that Tuor was from?",
47
  "ok": true,
48
  "action": "write-update",
49
  "gate": "allow",
50
  "reason": null
51
  },
52
  {
53
+ "question": "What river does Styr River form a part of?",
54
+ "ok": true,
55
+ "action": "write-update",
56
+ "gate": "allow",
57
+ "reason": null
58
+ },
59
+ {
60
+ "question": "Who gives out the Frank P. Brown Medal award?",
61
+ "ok": true,
62
+ "action": "write-update",
63
+ "gate": "allow",
64
+ "reason": null
65
+ },
66
+ {
67
+ "question": "Is the taxon rank of Senecio littoralis species or genus?",
68
+ "ok": true,
69
+ "action": "none",
70
+ "gate": "allow",
71
+ "reason": null
72
+ },
73
+ {
74
+ "question": "In what city was Norbert Hummelt born?",
75
  "ok": false,
76
  "action": "uncertain",
77
  "gate": "ask",
78
  "reason": "Router uncertain -> Human-Gate (§10.4)"
79
  },
80
  {
81
+ "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?",
82
  "ok": true,
83
+ "action": "write-new",
84
  "gate": "allow",
85
  "reason": null
86
  },
87
  {
88
+ "question": "What programming language was used to write RealVNC?",
89
+ "ok": true,
90
+ "action": "none",
91
+ "gate": "allow",
92
+ "reason": null
93
+ },
94
+ {
95
+ "question": "In what city did José María Arizmendiarrieta die?",
96
  "ok": false,
97
  "action": "uncertain",
98
  "gate": "ask",
99
  "reason": "Router uncertain -> Human-Gate (§10.4)"
100
  },
101
  {
102
+ "question": "What state is Newport-on-Tay located?",
103
  "ok": false,
104
  "action": "uncertain",
105
  "gate": "ask",
106
  "reason": "Router uncertain -> Human-Gate (§10.4)"
107
  },
108
  {
109
+ "question": "The 9998 ISO was named for whom?",
110
+ "ok": true,
111
+ "action": "write-update",
112
+ "gate": "allow",
113
+ "reason": null
114
+ },
115
+ {
116
+ "question": "Who is the illustrator of Cycle of the Werewolf?",
117
  "ok": true,
118
  "action": "write-new",
119
  "gate": "allow",
120
  "reason": null
121
  },
122
  {
123
+ "question": "Which was the record label for Renincarnated?",
124
  "ok": true,
125
  "action": "write-new",
126
  "gate": "allow",
127
  "reason": null
128
  },
129
  {
130
+ "question": "What is the musical instrument Gymnopédies was intended for?",
131
+ "ok": true,
132
+ "action": "write-update",
133
+ "gate": "allow",
134
+ "reason": null
135
+ },
136
+ {
137
+ "question": "What team is Beau Allred affiliated with?",
138
+ "ok": true,
139
+ "action": "write-new",
140
+ "gate": "allow",
141
+ "reason": null
142
+ },
143
+ {
144
+ "question": "What was the noble family that Princess Margareta of Romania was from?",
145
  "ok": true,
146
  "action": "none",
147
  "gate": "allow",
148
  "reason": null
149
  },
150
  {
151
+ "question": "What is the name of the person who designed ParaFAL?",
152
+ "ok": false,
153
+ "action": "uncertain",
154
+ "gate": "ask",
155
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
156
+ },
157
+ {
158
+ "question": "What was Isaac Nicola's city of birth?",
159
+ "ok": true,
160
+ "action": "write-new",
161
+ "gate": "allow",
162
+ "reason": null
163
+ },
164
+ {
165
+ "question": "Which was the voice type that Lawrence Zazzo had?",
166
  "ok": true,
167
  "action": "write-update",
168
  "gate": "allow",
169
  "reason": null
170
  },
171
  {
172
+ "question": "What was the name of the city or state where KOAN Sound was formed?",
173
+ "ok": true,
174
+ "action": "write-new",
175
+ "gate": "allow",
176
+ "reason": null
177
+ },
178
+ {
179
+ "question": "Which chromosone can you find Endothelial NOS?",
180
+ "ok": true,
181
+ "action": "write-update",
182
+ "gate": "allow",
183
+ "reason": null
184
+ },
185
+ {
186
+ "question": "What person illustrated The Faraway Lurs?",
187
+ "ok": true,
188
+ "action": "write-new",
189
+ "gate": "allow",
190
+ "reason": null
191
+ },
192
+ {
193
+ "question": "What is the name of the editors of The Best American Poetry 1996?",
194
  "ok": false,
195
  "action": "uncertain",
196
  "gate": "ask",
197
  "reason": "Router uncertain -> Human-Gate (§10.4)"
198
  },
199
  {
200
+ "question": "Which was the family of Paragagrellina?",
201
+ "ok": true,
202
+ "action": "write-update",
203
+ "gate": "allow",
204
+ "reason": null
205
+ },
206
+ {
207
+ "question": "What along with privateer was Ferdinando Acton's military rank?",
208
+ "ok": true,
209
+ "action": "write-update",
210
+ "gate": "allow",
211
+ "reason": null
212
+ },
213
+ {
214
+ "question": "What title is held by Adomnán?",
215
+ "ok": true,
216
+ "action": "write-update",
217
+ "gate": "allow",
218
+ "reason": null
219
+ },
220
+ {
221
+ "question": "What is the name of the singer or writer of Odorono?",
222
  "ok": true,
223
  "action": "write-new",
224
  "gate": "allow",
225
  "reason": null
226
  },
227
  {
228
+ "question": "By which body of water is Płock located?",
229
+ "ok": true,
230
+ "action": "none",
231
+ "gate": "allow",
232
+ "reason": null
233
+ },
234
+ {
235
+ "question": "What was the political party of P. S. Sreenivasan?",
236
+ "ok": false,
237
+ "action": "uncertain",
238
+ "gate": "ask",
239
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
240
+ },
241
+ {
242
+ "question": "What town or city does KKBB serve?",
243
+ "ok": true,
244
+ "action": "write-new",
245
+ "gate": "allow",
246
+ "reason": null
247
+ },
248
+ {
249
+ "question": "Which airport does PNG Air operate in?",
250
  "ok": true,
251
  "action": "write-update",
252
  "gate": "allow",
253
  "reason": null
254
  },
255
  {
256
+ "question": "Who served as director for I Hate Valentine's Day?",
257
+ "ok": false,
258
+ "action": "uncertain",
259
+ "gate": "ask",
260
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
261
+ },
262
+ {
263
+ "question": "Which director worked on the film A Dog's Life?",
264
+ "ok": false,
265
+ "action": "uncertain",
266
+ "gate": "ask",
267
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
268
+ },
269
+ {
270
+ "question": "At what airport can you find UTair Aviation?",
271
+ "ok": true,
272
+ "action": "write-new",
273
+ "gate": "allow",
274
+ "reason": null
275
+ },
276
+ {
277
+ "question": "What is the publication year of Porca vacca?",
278
  "ok": true,
279
  "action": "write-update",
280
  "gate": "allow",
 
288
  "reason": null
289
  },
290
  {
291
+ "question": "Which position was held by Landelin?",
292
+ "ok": false,
293
+ "action": "uncertain",
294
+ "gate": "ask",
295
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
296
+ },
297
+ {
298
+ "question": "On what date did 40th government of Turkey end?",
299
  "ok": true,
300
+ "action": "write-new",
301
  "gate": "allow",
302
  "reason": null
303
  },
304
  {
305
+ "question": "What award was An Officer and a Gentleman nominated for?",
306
+ "ok": true,
307
+ "action": "write-update",
308
+ "gate": "allow",
309
+ "reason": null
310
+ },
311
+ {
312
+ "question": "What was the production company for Getting Acquainted?",
313
  "ok": true,
314
  "action": "write-new",
315
  "gate": "allow",
316
  "reason": null
317
  },
318
+ {
319
+ "question": "Is the taxon rank of Allium tardiflorum species or genus?",
320
+ "ok": true,
321
+ "action": "none",
322
+ "gate": "allow",
323
+ "reason": null
324
+ },
325
  {
326
  "question": "What noble family was Kunigunde of Austria part of?",
327
  "ok": true,
 
330
  "reason": null
331
  },
332
  {
333
+ "question": "What company makes Armoured Multirole Carrier?",
334
+ "ok": false,
335
+ "action": "uncertain",
336
+ "gate": "ask",
337
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
338
+ },
339
+ {
340
+ "question": "What city was Christians & Lions formed in?",
341
  "ok": true,
342
  "action": "write-new",
343
  "gate": "allow",
344
  "reason": null
345
  },
346
  {
347
+ "question": "What is the name of the founder of Sharecare?",
348
+ "ok": false,
349
+ "action": "uncertain",
350
+ "gate": "ask",
351
+ "reason": "Router uncertain -> Human-Gate (§10.4)"
352
+ },
353
+ {
354
+ "question": "What award was Tweetie Pie nominated for?",
355
+ "ok": true,
356
+ "action": "write-update",
357
+ "gate": "allow",
358
+ "reason": null
359
+ },
360
+ {
361
+ "question": "What year was the end of Comdial?",
362
  "ok": true,
363
  "action": "write-new",
364
  "gate": "allow",
365
  "reason": null
366
+ },
367
+ {
368
+ "question": "Who was the brother of Casey Neistat?",
369
+ "ok": true,
370
+ "action": "write-update",
371
+ "gate": "allow",
372
+ "reason": null
373
  }
374
  ],
375
  "provenance_log": "/kaggle/working/tier3/provenance.jsonl",
376
+ "pending": 15,
377
  "pending_keys": [
378
  "what-was-the-title-that-traidenis-held",
379
+ "what-year-did-jyllands-posten-muhammad-cartoons-controversy-",
380
+ "who-is-the-illustrator-of-cycle-of-the-werewolf",
381
+ "which-was-the-record-label-for-renincarnated",
382
  "what-team-is-beau-allred-affiliated-with",
 
383
  "what-was-isaac-nicola-s-city-of-birth",
384
+ "what-was-the-name-of-the-city-or-state-where-koan-sound-was-",
385
+ "what-person-illustrated-the-faraway-lurs",
386
+ "what-is-the-name-of-the-singer-or-writer-of-odorono",
387
+ "what-town-or-city-does-kkbb-serve",
388
+ "at-what-airport-can-you-find-utair-aviation",
389
+ "on-what-date-did-40th-government-of-turkey-end",
390
+ "what-was-the-production-company-for-getting-acquainted",
391
+ "what-city-was-christians-lions-formed-in",
392
+ "what-year-was-the-end-of-comdial"
393
+ ],
394
+ "flushed": [
395
+ "what-town-or-city-does-kkbb-serve",
396
+ "on-what-date-did-40th-government-of-turkey-end"
397
  ],
 
398
  "kept": [
399
  "what-was-the-title-that-traidenis-held",
400
+ "what-year-did-jyllands-posten-muhammad-cartoons-controversy-",
401
+ "who-is-the-illustrator-of-cycle-of-the-werewolf",
402
+ "which-was-the-record-label-for-renincarnated",
403
  "what-team-is-beau-allred-affiliated-with",
 
404
  "what-was-isaac-nicola-s-city-of-birth",
405
+ "what-was-the-name-of-the-city-or-state-where-koan-sound-was-",
406
+ "what-person-illustrated-the-faraway-lurs",
407
+ "what-is-the-name-of-the-singer-or-writer-of-odorono",
408
+ "at-what-airport-can-you-find-utair-aviation",
409
+ "what-was-the-production-company-for-getting-acquainted",
410
+ "what-city-was-christians-lions-formed-in",
411
+ "what-year-was-the-end-of-comdial"
412
+ ],
413
+ "loss": 2.295806121826172,
414
+ "adapter_path": "/kaggle/working/slow_consolidation_train/adapter",
415
+ "probe_results": [
416
+ {
417
+ "passed": false,
418
+ "answer": "Duke of Normandy",
419
+ "base_answer": "Duke of Normandy",
420
+ "question": "What was the title that Traidenis held?",
421
+ "gold": "Grand Duke of Lithuania"
422
+ },
423
+ {
424
+ "passed": false,
425
+ "answer": "2005",
426
+ "base_answer": "2005",
427
+ "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?",
428
+ "gold": "30 September 2005"
429
+ },
430
+ {
431
+ "passed": false,
432
+ "answer": "J. C. Vaughn",
433
+ "base_answer": "J. C. Vaughn",
434
+ "question": "Who is the illustrator of Cycle of the Werewolf?",
435
+ "gold": "Bernie Wrightson"
436
+ },
437
+ {
438
+ "passed": false,
439
+ "answer": "Capitol Records",
440
+ "base_answer": "Capitol Records",
441
+ "question": "Which was the record label for Renincarnated?",
442
+ "gold": "Villain Entertainment"
443
+ },
444
+ {
445
+ "passed": false,
446
+ "answer": "Los Angeles Chargers",
447
+ "base_answer": "Los Angeles Chargers",
448
+ "question": "What team is Beau Allred affiliated with?",
449
+ "gold": "Cleveland Indians"
450
+ },
451
+ {
452
+ "passed": false,
453
+ "answer": "New York City",
454
+ "base_answer": "New York City",
455
+ "question": "What was Isaac Nicola's city of birth?",
456
+ "gold": "Havana"
457
+ },
458
+ {
459
+ "passed": false,
460
+ "answer": "Manchester",
461
+ "base_answer": "Manchester",
462
+ "question": "What was the name of the city or state where KOAN Sound was formed?",
463
+ "gold": "Bristol"
464
+ },
465
+ {
466
+ "passed": false,
467
+ "answer": "John Bauer",
468
+ "base_answer": "John Bauer",
469
+ "question": "What person illustrated The Faraway Lurs?",
470
+ "gold": "Harry Behn"
471
+ },
472
+ {
473
+ "passed": false,
474
+ "answer": "Theodoros Kordjis",
475
+ "base_answer": "Theodoros Kordjis",
476
+ "question": "What is the name of the singer or writer of Odorono?",
477
+ "gold": "Pete Townshend"
478
+ },
479
+ {
480
+ "passed": true,
481
+ "answer": "Bakersfield",
482
+ "base_answer": "Bakersfield",
483
+ "question": "What town or city does KKBB serve?",
484
+ "gold": "Bakersfield"
485
+ },
486
+ {
487
+ "passed": false,
488
+ "answer": "Vnukovo International Airport",
489
+ "base_answer": "Vnukovo International Airport",
490
+ "question": "At what airport can you find UTair Aviation?",
491
+ "gold": "Khanty-Mansiysk Airport"
492
+ },
493
+ {
494
+ "passed": true,
495
+ "answer": "1977",
496
+ "base_answer": "1977",
497
+ "question": "On what date did 40th government of Turkey end?",
498
+ "gold": "1977"
499
+ },
500
+ {
501
+ "passed": false,
502
+ "answer": "The Weinstein Company",
503
+ "base_answer": "The Weinstein Company",
504
+ "question": "What was the production company for Getting Acquainted?",
505
+ "gold": "Keystone Studios"
506
+ },
507
+ {
508
+ "passed": false,
509
+ "answer": "Denver",
510
+ "base_answer": "Denver",
511
+ "question": "What city was Christians & Lions formed in?",
512
+ "gold": "Jamaica Plain"
513
+ },
514
+ {
515
+ "passed": false,
516
+ "answer": "2013",
517
+ "base_answer": "2013",
518
+ "question": "What year was the end of Comdial?",
519
+ "gold": "2005"
520
+ }
521
  ],
 
 
522
  "write_back": {
523
+ "ok": true,
524
+ "flushed": [
525
+ "what-town-or-city-does-kkbb-serve",
526
+ "on-what-date-did-40th-government-of-turkey-end"
527
+ ],
528
  "kept": [
529
  "what-was-the-title-that-traidenis-held",
530
+ "what-year-did-jyllands-posten-muhammad-cartoons-controversy-",
531
+ "who-is-the-illustrator-of-cycle-of-the-werewolf",
532
+ "which-was-the-record-label-for-renincarnated",
533
  "what-team-is-beau-allred-affiliated-with",
 
534
  "what-was-isaac-nicola-s-city-of-birth",
535
+ "what-was-the-name-of-the-city-or-state-where-koan-sound-was-",
536
+ "what-person-illustrated-the-faraway-lurs",
537
+ "what-is-the-name-of-the-singer-or-writer-of-odorono",
538
+ "at-what-airport-can-you-find-utair-aviation",
539
+ "what-was-the-production-company-for-getting-acquainted",
540
+ "what-city-was-christians-lions-formed-in",
541
+ "what-year-was-the-end-of-comdial"
542
  ],
543
+ "loss": 2.295806121826172,
544
+ "adapter_path": "/kaggle/working/slow_consolidation_train/adapter",
545
+ "probe_results": [
546
+ {
547
+ "passed": false,
548
+ "answer": "Duke of Normandy",
549
+ "base_answer": "Duke of Normandy",
550
+ "question": "What was the title that Traidenis held?",
551
+ "gold": "Grand Duke of Lithuania"
552
+ },
553
+ {
554
+ "passed": false,
555
+ "answer": "2005",
556
+ "base_answer": "2005",
557
+ "question": "What year did Jyllands-Posten Muhammad cartoons controversy start?",
558
+ "gold": "30 September 2005"
559
+ },
560
+ {
561
+ "passed": false,
562
+ "answer": "J. C. Vaughn",
563
+ "base_answer": "J. C. Vaughn",
564
+ "question": "Who is the illustrator of Cycle of the Werewolf?",
565
+ "gold": "Bernie Wrightson"
566
+ },
567
+ {
568
+ "passed": false,
569
+ "answer": "Capitol Records",
570
+ "base_answer": "Capitol Records",
571
+ "question": "Which was the record label for Renincarnated?",
572
+ "gold": "Villain Entertainment"
573
+ },
574
+ {
575
+ "passed": false,
576
+ "answer": "Los Angeles Chargers",
577
+ "base_answer": "Los Angeles Chargers",
578
+ "question": "What team is Beau Allred affiliated with?",
579
+ "gold": "Cleveland Indians"
580
+ },
581
+ {
582
+ "passed": false,
583
+ "answer": "New York City",
584
+ "base_answer": "New York City",
585
+ "question": "What was Isaac Nicola's city of birth?",
586
+ "gold": "Havana"
587
+ },
588
+ {
589
+ "passed": false,
590
+ "answer": "Manchester",
591
+ "base_answer": "Manchester",
592
+ "question": "What was the name of the city or state where KOAN Sound was formed?",
593
+ "gold": "Bristol"
594
+ },
595
+ {
596
+ "passed": false,
597
+ "answer": "John Bauer",
598
+ "base_answer": "John Bauer",
599
+ "question": "What person illustrated The Faraway Lurs?",
600
+ "gold": "Harry Behn"
601
+ },
602
+ {
603
+ "passed": false,
604
+ "answer": "Theodoros Kordjis",
605
+ "base_answer": "Theodoros Kordjis",
606
+ "question": "What is the name of the singer or writer of Odorono?",
607
+ "gold": "Pete Townshend"
608
+ },
609
+ {
610
+ "passed": true,
611
+ "answer": "Bakersfield",
612
+ "base_answer": "Bakersfield",
613
+ "question": "What town or city does KKBB serve?",
614
+ "gold": "Bakersfield"
615
+ },
616
+ {
617
+ "passed": false,
618
+ "answer": "Vnukovo International Airport",
619
+ "base_answer": "Vnukovo International Airport",
620
+ "question": "At what airport can you find UTair Aviation?",
621
+ "gold": "Khanty-Mansiysk Airport"
622
+ },
623
+ {
624
+ "passed": true,
625
+ "answer": "1977",
626
+ "base_answer": "1977",
627
+ "question": "On what date did 40th government of Turkey end?",
628
+ "gold": "1977"
629
+ },
630
+ {
631
+ "passed": false,
632
+ "answer": "The Weinstein Company",
633
+ "base_answer": "The Weinstein Company",
634
+ "question": "What was the production company for Getting Acquainted?",
635
+ "gold": "Keystone Studios"
636
+ },
637
+ {
638
+ "passed": false,
639
+ "answer": "Denver",
640
+ "base_answer": "Denver",
641
+ "question": "What city was Christians & Lions formed in?",
642
+ "gold": "Jamaica Plain"
643
+ },
644
+ {
645
+ "passed": false,
646
+ "answer": "2013",
647
+ "base_answer": "2013",
648
+ "question": "What year was the end of Comdial?",
649
+ "gold": "2005"
650
+ }
651
+ ]
652
  },
653
  "zsre": {
654
  "ok": true,
655
  "path": "/kaggle/working/zsre_validation.jsonl",
656
+ "count": 50,
657
  "hashes": [
658
  "9a2e90866036a10bc18334eeb314469127a67de0b69669a770b742114e91255f",
659
  "baf562e40d9e8ff16f215ebed6571dc6efa5766c2ce0ad510920f104a3bbafbc",
 
674
  "4b74b9a72bf6a77efa9c7e4fba3e713016b63cbd67ac1796914f56666dcd943e",
675
  "85b1755b7fd804ee8f7ce3b9e9841ded8a6280fb3616110a2aa47975ae834a10",
676
  "a32e2d23f3c1c149181aa6b84d37e5c1a292f24eefb10f51cecd65c7bd659689",
677
+ "fff721e2040788e3c560ebf0ede6be113bea02316caea2860d7ac53efb9609ee",
678
+ "5b29a1f4117213c73324249e15456c3443238cfac2dd7b985a15f7673a948eae",
679
+ "2a02404f674f2a6ce3ef6b977c024733812426a5f758afabfa02227c6f61222a",
680
+ "0df4258fa40979ab1028bb46a22e5ad484f68e76ba1ad15d73b8d9cae30f3ea5",
681
+ "b2000d4eed8bc686409cbbbbd1001b45b6342a693d32b4232b26e3dd81e9cb0e",
682
+ "d1fa0ee81a2811cf562daf2da3696ad23ce85f968e06ada7eecacdcf34633806",
683
+ "7b2feb2452644070c22df31327a17a46f6b83ce3cb42f434caced2c339e48648",
684
+ "3c8019d7dc2ef0a86bd3e3aff7d0909fd9b1d034c2a11e664aa5f481f9d887c0",
685
+ "48f4d5e63a28c54e6214c39bb9c0bcde5f33330d8b76fd65538ce4573aafdd4a",
686
+ "fd142b2cdb5f8cbdfdd5e406f9eb3b68f03d6cd9b4631f6199a70f28ca096943",
687
+ "7ed26005d6eb17392c6b5ad516557e0f6ef381fec3c62cd8950c563a7013ed35",
688
+ "8be6020b10a31b56b3250f43a3b02bf5b3489ec9a3550aa3f2a95939c1f721dd",
689
+ "76cd29eca3268b9016ff4aa811a7899eeefdf85a176c010a4a507f4b84dcffd5",
690
+ "7d1e1972bdc53090072397ff02a4438d5efea8537617cb941c756e24d3e67fdb",
691
+ "93726ed587e0df201ec0f62d9dc9142a321815ff0da21881fa5d9d2e584d4ed7",
692
+ "76a28fe90bb268f32456d29eaf86a6d66ea500fb60b46e163289e451f18f3219",
693
+ "25bf369ba023cad955e1d17f57673b467bfae2528317988440d654ed9acad5f5",
694
+ "a2ebfef45fefa1e547748a682b22bbc108f4770766cb4fbf43281c5e6c3268db",
695
+ "bead11a2eeed353c9817b771a0c5d219299a74bd98185af351dcdb70b148e24c",
696
+ "f88bdb4abbfaa3cdcd843df3dd0695def7d297946dd428584964f5ef1d29bcf6",
697
+ "d3ead4f1ce02140312d4db00a9907d1f2e0c52831ddfb24e290797f726155681",
698
+ "94864688ac110ff7470dfc0e5bba1f5cca3516c576ee61dadb82e54a5a6a5fcd",
699
+ "de8232289f42171210cbafa88f5ada2cb8d4ed8b84da5311b8dde5be909293e3",
700
+ "3e943dc112961053e6171ae71ff1fdb200f8229db1967df230b6183291690e9e",
701
+ "9329aa19ec57aa8a4beec95c461649708a70fcde832b10e613583b30b0592c2c",
702
+ "4ff52295ad070e17f97fd38cea3344613d276c25bdeb969c152334121b2ad732",
703
+ "90f02a377a82697755975ef8202235665d21200ae529debe4a061063f88f5c1b",
704
+ "4a367091736597fcd4d682aaf233d3c699e77b20abe51afe43eb1407759b1dbc",
705
+ "7f4bf8be42bb56a22b43e26c5daa15df273aadeb52e460bcdd8c15a798ad1072",
706
+ "98185e874520dd154c307faa44c9423255047ca06391c1b50eac2d40d82443d2",
707
+ "c9a99c58024bc5fd3ba38660bfd18005e71fb47fc9c0f265423a288c8c1f2dc9"
708
  ],
709
+ "requested": 50,
710
  "split": "train",
711
  "dataset": "community-datasets/qa_zre",
712
+ "rows_scanned": 105,
713
+ "skipped_empty": 55,
714
  "note": "volle Menge aus Quelle verfuegbar; hier gesampeltes n"
715
  },
716
  "dataset": "/kaggle/working/consolidation_sft.jsonl"