tns Claude Opus 4.6 commited on
Commit
4f3e2c1
·
1 Parent(s): fb86de2

Add [START]/[STEP]/[END] structured output to inference.py

Browse files

Validator requires structured markers in stdout to parse results.
Added flush=True on all markers across every code path.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. inference.py +27 -7
inference.py CHANGED
@@ -167,14 +167,18 @@ def main() -> None:
167
  print(f"\n{'=' * 60}")
168
  print(f"Task: {task_id}")
169
  print(f"{'=' * 60}")
 
170
 
171
  obs = env_reset(task_id)
172
  if "observation" in obs:
173
  obs = obs["observation"]
174
 
 
175
  done = obs.get("done", False)
176
  if done:
177
- results[task_id] = obs.get("current_score", 0.0)
 
 
178
  continue
179
 
180
  total_issues = obs.get("total_issues", 0)
@@ -190,7 +194,6 @@ def main() -> None:
190
  columns.append(col_name)
191
 
192
  inspection_results = {}
193
- step_count = 0
194
  for col in columns:
195
  if done:
196
  break
@@ -201,12 +204,16 @@ def main() -> None:
201
  if "observation" in obs:
202
  obs = obs["observation"]
203
  done = obs.get("done", False)
 
 
204
  feedback = obs.get("feedback", "")
205
  inspection_results[col] = feedback
206
 
207
  if done:
208
- results[task_id] = obs.get("current_score", 0.0)
209
- print(f" Done during inspection. Score: {results[task_id]:.4f}")
 
 
210
  continue
211
 
212
  # --- Phase 1.5: Filter to only columns WITH issues ---
@@ -261,10 +268,14 @@ def main() -> None:
261
  plan_text = completion.choices[0].message.content or ""
262
  except Exception as exc:
263
  print(f" LLM error: {exc}. Submitting.")
 
264
  obs = env_step("submit()")
265
  if "observation" in obs:
266
  obs = obs["observation"]
267
- results[task_id] = obs.get("current_score", 0.0)
 
 
 
268
  continue
269
 
270
  plan = extract_json_plan(plan_text)
@@ -306,14 +317,18 @@ def main() -> None:
306
  if "observation" in obs:
307
  obs = obs["observation"]
308
  done = obs.get("done", False)
 
 
309
  remaining = obs.get("actions_remaining", 0)
310
  if not done:
311
  fb = obs.get("feedback", "")
312
  fallback_messages.append({"role": "user", "content": f"Result: {fb}\nFixed: {obs.get('issues_fixed',0)}/{obs.get('total_issues',0)}. Remaining steps: {remaining}."})
313
  if len(fallback_messages) > 30:
314
  fallback_messages = [fallback_messages[0]] + fallback_messages[-28:]
315
- results[task_id] = obs.get("current_score", 0.0)
316
- print(f" Final score for {task_id}: {results[task_id]:.4f}")
 
 
317
  continue
318
 
319
  print(f" Plan has {len(plan)} actions (expected ~{total_issues}).")
@@ -334,6 +349,8 @@ def main() -> None:
334
  if "observation" in obs:
335
  obs = obs["observation"]
336
  done = obs.get("done", False)
 
 
337
  remaining = obs.get("actions_remaining", 0)
338
 
339
  feedback = obs.get("feedback", "")
@@ -347,10 +364,13 @@ def main() -> None:
347
  obs = env_step("submit()")
348
  if "observation" in obs:
349
  obs = obs["observation"]
 
 
350
 
351
  score = obs.get("current_score", 0.0)
352
  results[task_id] = score
353
  print(f" Final score for {task_id}: {score:.4f}")
 
354
 
355
  # --- Results Summary ---
356
  print(f"\n{'=' * 60}")
 
167
  print(f"\n{'=' * 60}")
168
  print(f"Task: {task_id}")
169
  print(f"{'=' * 60}")
170
+ print(f"[START] task={task_id}", flush=True)
171
 
172
  obs = env_reset(task_id)
173
  if "observation" in obs:
174
  obs = obs["observation"]
175
 
176
+ step_count = 0
177
  done = obs.get("done", False)
178
  if done:
179
+ score = obs.get("current_score", 0.0)
180
+ results[task_id] = score
181
+ print(f"[END] task={task_id} score={score} steps=0", flush=True)
182
  continue
183
 
184
  total_issues = obs.get("total_issues", 0)
 
194
  columns.append(col_name)
195
 
196
  inspection_results = {}
 
197
  for col in columns:
198
  if done:
199
  break
 
204
  if "observation" in obs:
205
  obs = obs["observation"]
206
  done = obs.get("done", False)
207
+ reward = obs.get("current_score", 0.0)
208
+ print(f"[STEP] step={step_count} reward={reward}", flush=True)
209
  feedback = obs.get("feedback", "")
210
  inspection_results[col] = feedback
211
 
212
  if done:
213
+ score = obs.get("current_score", 0.0)
214
+ results[task_id] = score
215
+ print(f" Done during inspection. Score: {score:.4f}")
216
+ print(f"[END] task={task_id} score={score} steps={step_count}", flush=True)
217
  continue
218
 
219
  # --- Phase 1.5: Filter to only columns WITH issues ---
 
268
  plan_text = completion.choices[0].message.content or ""
269
  except Exception as exc:
270
  print(f" LLM error: {exc}. Submitting.")
271
+ step_count += 1
272
  obs = env_step("submit()")
273
  if "observation" in obs:
274
  obs = obs["observation"]
275
+ score = obs.get("current_score", 0.0)
276
+ results[task_id] = score
277
+ print(f"[STEP] step={step_count} reward={score}", flush=True)
278
+ print(f"[END] task={task_id} score={score} steps={step_count}", flush=True)
279
  continue
280
 
281
  plan = extract_json_plan(plan_text)
 
317
  if "observation" in obs:
318
  obs = obs["observation"]
319
  done = obs.get("done", False)
320
+ reward = obs.get("current_score", 0.0)
321
+ print(f"[STEP] step={step_count} reward={reward}", flush=True)
322
  remaining = obs.get("actions_remaining", 0)
323
  if not done:
324
  fb = obs.get("feedback", "")
325
  fallback_messages.append({"role": "user", "content": f"Result: {fb}\nFixed: {obs.get('issues_fixed',0)}/{obs.get('total_issues',0)}. Remaining steps: {remaining}."})
326
  if len(fallback_messages) > 30:
327
  fallback_messages = [fallback_messages[0]] + fallback_messages[-28:]
328
+ score = obs.get("current_score", 0.0)
329
+ results[task_id] = score
330
+ print(f" Final score for {task_id}: {score:.4f}")
331
+ print(f"[END] task={task_id} score={score} steps={step_count}", flush=True)
332
  continue
333
 
334
  print(f" Plan has {len(plan)} actions (expected ~{total_issues}).")
 
349
  if "observation" in obs:
350
  obs = obs["observation"]
351
  done = obs.get("done", False)
352
+ reward = obs.get("current_score", 0.0)
353
+ print(f"[STEP] step={step_count} reward={reward}", flush=True)
354
  remaining = obs.get("actions_remaining", 0)
355
 
356
  feedback = obs.get("feedback", "")
 
364
  obs = env_step("submit()")
365
  if "observation" in obs:
366
  obs = obs["observation"]
367
+ reward = obs.get("current_score", 0.0)
368
+ print(f"[STEP] step={step_count} reward={reward}", flush=True)
369
 
370
  score = obs.get("current_score", 0.0)
371
  results[task_id] = score
372
  print(f" Final score for {task_id}: {score:.4f}")
373
+ print(f"[END] task={task_id} score={score} steps={step_count}", flush=True)
374
 
375
  # --- Results Summary ---
376
  print(f"\n{'=' * 60}")