Harden score outputs to strict open interval

inference.py

@@ -74,6 +74,25 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
+def _strict_score(value: Any) -> float:
+    """Normalize any numeric-like score to strict open interval (0, 1)."""
+    try:
+        numeric = float(value)
+    except (TypeError, ValueError):
+        numeric = 0.01
+    return max(0.01, min(0.99, numeric))
+
+
+def _sanitize_task_result(task_result: Dict[str, Any]) -> Dict[str, Any]:
+    """Ensure task result contains evaluator-safe score fields."""
+    safe = dict(task_result)
+    safe["steps"] = int(safe.get("steps", 0) or 0)
+    safe["total_reward"] = _strict_score(safe.get("total_reward", 0.01))
+    safe["avg_reward"] = _strict_score(safe.get("avg_reward", 0.01))
+    safe["elapsed"] = float(safe.get("elapsed", 0.0) or 0.0)
+    return safe
+
+
 # ──────────────────────────────────────────────────────────────────
 # LLM Client  (uses OpenAI SDK — required by checklist item 4)
 # ──────────────────────────────────────────────────────────────────
@@ -306,7 +325,8 @@ def run_task(env_client: EnvClient, task_id: str) -> Dict[str, Any]:
         )
 
         step_count += 1
-        step_reward = result.get("reward", 0.0)
+        # Guard against endpoint-side boundary values (0.0 or 1.0)
+        step_reward = _strict_score(result.get("reward", 0.01))
         total_reward += step_reward
         done = result.get("done", False)
         obs = result.get("observation", {})
@@ -324,8 +344,7 @@ def run_task(env_client: EnvClient, task_id: str) -> Dict[str, Any]:
         )
 
     # Compute average reward for this task — clamped to strict (0, 1)
-    avg_reward = total_reward / max(step_count, 1)
-    avg_reward = max(0.01, min(0.99, avg_reward))
+    avg_reward = _strict_score(total_reward / max(step_count, 1))
     elapsed = time.time() - start_time
 
     logger.info(
@@ -339,7 +358,7 @@ def run_task(env_client: EnvClient, task_id: str) -> Dict[str, Any]:
     return {
         "task_id": task_id,
         "steps": step_count,
-        "total_reward": total_reward,
+        "total_reward": _strict_score(total_reward),
         "avg_reward": avg_reward,
         "elapsed": elapsed,
     }
@@ -361,6 +380,33 @@ def main():
     logger.info("=" * 60)
 
     env_client = EnvClient(base_url=ENV_BASE_URL)
+    task_ids = ["easy_faq", "medium_refund", "hard_escalation"]
+
+    def _write_results(results: List[Dict[str, Any]]) -> float:
+        """Write sanitized results and return sanitized final score."""
+        sanitized_results = [_sanitize_task_result(r) for r in results]
+        total_avg = sum(r["avg_reward"] for r in sanitized_results)
+        final = _strict_score(total_avg / len(sanitized_results)) if sanitized_results else 0.01
+
+        output = {
+            "final_score": final,
+            "task_results": sanitized_results,
+            "config": {
+                "api_base_url": API_BASE_URL,
+                "model_name": MODEL_NAME,
+                "env_base_url": ENV_BASE_URL,
+            },
+        }
+
+        try:
+            os.makedirs("outputs", exist_ok=True)
+            with open("outputs/inference_results.json", "w") as f:
+                json.dump(output, f, indent=2)
+            logger.info("\nResults saved to outputs/inference_results.json")
+        except Exception as e:
+            logger.error(f"[ERROR] Failed to save results: {e}")
+
+        return final
 
     # Wait for environment to be ready
     logger.info("[START] Waiting for environment server...")
@@ -371,11 +417,20 @@ def main():
         time.sleep(2)
     else:
         logger.error("[ERROR] Environment server not available after 60 seconds.")
-        # Return 0.0 score instead of sys.exit(1) to avoid non-zero exit code
-        return 0.0
+        # Emit safe fallback scores so evaluator never sees 0.0/1.0 task values.
+        fallback_results = [
+            {
+                "task_id": tid,
+                "steps": 0,
+                "total_reward": 0.01,
+                "avg_reward": 0.01,
+                "elapsed": 0.0,
+                "error": "environment_unavailable",
+            }
+            for tid in task_ids
+        ]
+        return _write_results(fallback_results)
 
-    # Task order: easy -> medium -> hard
-    task_ids = ["easy_faq", "medium_refund", "hard_escalation"]
     results = []
 
     for task_id in task_ids:
@@ -383,7 +438,7 @@ def main():
         logger.info("-" * 40)
         try:
             result = run_task(env_client, task_id)
-            results.append(result)
+            results.append(_sanitize_task_result(result))
         except Exception as e:
             logger.error(f"[ERROR] Task {task_id} failed: {e}")
             results.append({
@@ -412,32 +467,12 @@ def main():
         )
         total_avg += r.get("avg_reward", 0)
 
-    final_score = total_avg / len(results) if results else 0.01
-    final_score = max(0.01, min(0.99, final_score))  # strict (0, 1)
+    final_score = _strict_score(total_avg / len(results)) if results else 0.01
     logger.info("-" * 60)
     logger.info(f"  FINAL SCORE: {final_score:.4f} (0.0 -- 1.0)")
     logger.info("=" * 60)
 
-    # Save results to file
-    output = {
-        "final_score": final_score,
-        "task_results": results,
-        "config": {
-            "api_base_url": API_BASE_URL,
-            "model_name": MODEL_NAME,
-            "env_base_url": ENV_BASE_URL,
-        },
-    }
-
-    try:
-        os.makedirs("outputs", exist_ok=True)
-        with open("outputs/inference_results.json", "w") as f:
-            json.dump(output, f, indent=2)
-        logger.info(f"\nResults saved to outputs/inference_results.json")
-    except Exception as e:
-        logger.error(f"[ERROR] Failed to save results: {e}")
-
-    return final_score
+    return _write_results(results)
 
 
 if __name__ == "__main__":