immortalindeed commited on
Commit
d270d2a
Β·
1 Parent(s): b4f20cf

Revert incorrect log parsing changes and fix reward summation logic

Browse files
Files changed (1) hide show
  1. inference.py +14 -14
inference.py CHANGED
@@ -214,19 +214,17 @@ def run_task(client: OpenAI, task_id: str) -> float:
214
  resp = requests.post(f"{ENV_URL}/reset", json={"task_id": task_id}, timeout=30)
215
  data = resp.json()
216
 
217
- import uuid
218
  if "error" in data and not data.get("episode_id"):
219
  # ── MANDATORY: [START] line even on error ──
220
- episode_id = f"ep-{uuid.uuid4().hex[:8]}"
221
- print(f"[START] task_id={task_id} episode_id={episode_id}", flush=True)
222
- print(f"[END] task_id={task_id} episode_id={episode_id} total_reward=0.0000 steps=0", flush=True)
223
  return 0.0
224
 
225
- episode_id = data.get("episode_id", f"ep-{uuid.uuid4().hex[:8]}")
226
  obs = data.get("observation", data)
227
 
228
  # ── MANDATORY [START] β€” exact spec format ──
229
- print(f"[START] task_id={task_id} episode_id={episode_id}", flush=True)
230
 
231
  rewards = []
232
  history = []
@@ -262,7 +260,7 @@ def run_task(client: OpenAI, task_id: str) -> float:
262
  except Exception as e:
263
  error_msg = str(e)
264
  # ── MANDATORY [STEP] line on connection error ──
265
- print(f"[STEP] task_id={task_id} step={step_num} action={action_type} reward=0.0000 done=True", flush=True)
266
  rewards.append(0.0)
267
  break
268
 
@@ -281,18 +279,22 @@ def run_task(client: OpenAI, task_id: str) -> float:
281
  display_action = "invalid"
282
 
283
  # ── MANDATORY [STEP] β€” exact spec format ──
284
- print(f"[STEP] task_id={task_id} step={step_num} action={display_action} reward={reward:.4f} done={done}", flush=True)
 
285
 
286
  if done:
287
  break
288
 
289
  # Sum the rewards for multi-turn accumulation
290
  total_reward = sum(rewards) if rewards else 0.0
 
 
 
291
 
292
  # ── MANDATORY [END] β€” exact spec format ──
293
- print(f"[END] task_id={task_id} episode_id={episode_id} total_reward={total_reward:.4f} steps={step_num}", flush=True)
294
 
295
- return total_reward
296
 
297
 
298
  def main() -> None:
@@ -317,10 +319,8 @@ def main() -> None:
317
  try:
318
  scores[task_id] = run_task(client, task_id)
319
  except Exception as e:
320
- import uuid
321
- episode_id = f"ep-{uuid.uuid4().hex[:8]}"
322
- print(f"[START] task_id={task_id} episode_id={episode_id}", flush=True)
323
- print(f"[END] task_id={task_id} episode_id={episode_id} total_reward=0.0000 steps=0", flush=True)
324
  scores[task_id] = 0.0
325
 
326
  avg = round(sum(scores.values()) / max(len(scores), 1), 2)
 
214
  resp = requests.post(f"{ENV_URL}/reset", json={"task_id": task_id}, timeout=30)
215
  data = resp.json()
216
 
 
217
  if "error" in data and not data.get("episode_id"):
218
  # ── MANDATORY: [START] line even on error ──
219
+ print(f"[START] task={task_id} env={BENCHMARK} model={MODEL_NAME}", flush=True)
220
+ print(f"[END] success=false steps=0 score=0.00 rewards=", flush=True)
 
221
  return 0.0
222
 
223
+ episode_id = data.get("episode_id", "unknown")
224
  obs = data.get("observation", data)
225
 
226
  # ── MANDATORY [START] β€” exact spec format ──
227
+ print(f"[START] task={task_id} env={BENCHMARK} model={MODEL_NAME}", flush=True)
228
 
229
  rewards = []
230
  history = []
 
260
  except Exception as e:
261
  error_msg = str(e)
262
  # ── MANDATORY [STEP] line on connection error ──
263
+ print(f"[STEP] step={step_num} action={action_type} reward=0.00 done=true error={error_msg}", flush=True)
264
  rewards.append(0.0)
265
  break
266
 
 
279
  display_action = "invalid"
280
 
281
  # ── MANDATORY [STEP] β€” exact spec format ──
282
+ error_val = step_error if step_error else "null"
283
+ print(f"[STEP] step={step_num} action={display_action} reward={reward:.2f} done={str(done).lower()} error={error_val}", flush=True)
284
 
285
  if done:
286
  break
287
 
288
  # Sum the rewards for multi-turn accumulation
289
  total_reward = sum(rewards) if rewards else 0.0
290
+ score = round(min(max(total_reward, 0.0), 1.0), 2)
291
+ success = score > 0.0
292
+ rewards_str = ",".join(f"{r:.2f}" for r in rewards)
293
 
294
  # ── MANDATORY [END] β€” exact spec format ──
295
+ print(f"[END] success={str(success).lower()} steps={step_num} score={score:.2f} rewards={rewards_str}", flush=True)
296
 
297
+ return score
298
 
299
 
300
  def main() -> None:
 
319
  try:
320
  scores[task_id] = run_task(client, task_id)
321
  except Exception as e:
322
+ print(f"[START] task={task_id} env={BENCHMARK} model={MODEL_NAME}", flush=True)
323
+ print(f"[END] success=false steps=0 score=0.00 rewards=", flush=True)
 
 
324
  scores[task_id] = 0.0
325
 
326
  avg = round(sum(scores.values()) / max(len(scores), 1), 2)