fix(evals): harden suite model selection and response extraction

Handle reasoning-only OpenAI-compatible responses correctly, avoid false PASS results on identical error strings, and let suite benchmarks honor the CLI-selected model instead of defaulting to gpt-4o-mini.

Generated with [Claude Code](https://claude.ai/code)
via [Happy](https://happy.engineering)

Co-Authored-By: Claude <noreply@anthropic.com>
Co-Authored-By: Happy <yesreply@happy.engineering>

headroom/evals/runners/before_after.py

@@ -377,6 +377,10 @@ Answer:"""
             response_compressed = f"ERROR: {e}"
         latency_compressed = (time.time() - start) * 1000
 
+        # Detect error responses — two identical errors must not count as a pass
+        original_errored = response_original.startswith("ERROR:")
+        compressed_errored = response_compressed.startswith("ERROR:")
+
         # Compute metrics
         exact_match = compute_exact_match(response_original, response_compressed)
         f1_score = compute_f1(response_original, response_compressed)
@@ -400,9 +404,13 @@ Answer:"""
 
         # Determine accuracy preservation
         accuracy_preserved = (
-            f1_score > 0.7
-            or (semantic_sim is not None and semantic_sim > 0.85)
-            or contains_ground_truth is True
+            not original_errored
+            and not compressed_errored
+            and (
+                f1_score > 0.7
+                or (semantic_sim is not None and semantic_sim > 0.85)
+                or contains_ground_truth is True
+            )
         )
 
         return EvalResult(

headroom/evals/runners/utils.py

@@ -9,11 +9,15 @@ from typing import Any
 def extract_openai_text(response: Any) -> str:
     """Extract text content from an OpenAI-compatible chat completion response.
 
-    Handles three shapes returned by different providers / OpenRouter models:
+    Handles shapes returned by different providers / models:
     1. Standard: ``message.content`` is a non-empty string.
-    2. Content-part list: ``message.content`` is a list of typed blocks (vision models).
-    3. Reasoning-only: ``message.content`` is null but ``message.reasoning`` has the
-       answer embedded in a code block (e.g. z-ai/glm-4.5-air via OpenRouter).
+    2. Content-part list: ``message.content`` is a list of typed blocks (vision).
+    3. Reasoning models (Step 3.7 Flash, GLM via OpenRouter):
+       ``message.content`` is null but ``message.reasoning_content``
+       or ``message.reasoning`` holds the actual answer — extract it directly,
+       with code-block extraction as a fallback.
+    4. Reasoning models that embed the final answer in a code block within
+       ``message.reasoning``.
 
     Raises:
         ValueError: When the response carries no usable text in any of the above forms.
@@ -21,9 +25,13 @@ def extract_openai_text(response: Any) -> str:
     if not response.choices:
         raise ValueError("model returned no choices")
     message = response.choices[0].message
+
+    # Shape 1: standard string content
     content = getattr(message, "content", None)
     if isinstance(content, str) and content.strip():
         return content
+
+    # Shape 2: content-part list (vision models)
     if isinstance(content, list):
         text_parts = [
             part.text
@@ -32,9 +40,25 @@ def extract_openai_text(response: Any) -> str:
         ]
         if text_parts:
             return "\n".join(text_parts)
+
+    # Shape 3: reasoning_content (Step 3.7 Flash / some NVIDIA NIM models)
+    # Use as direct answer — may be the full response with no code blocks.
+    reasoning_content = getattr(message, "reasoning_content", None)
+    if isinstance(reasoning_content, str) and reasoning_content.strip():
+        # Try code-block extraction first
+        match = re.search(r"```(?:python)?\s*(.*?)```", reasoning_content, re.DOTALL)
+        if match:
+            return match.group(1).strip()
+        # No code block — use reasoning_content as-is (full answer)
+        return reasoning_content.strip()
+
+    # Shape 4: reasoning (OpenRouter reasoning models)
     reasoning = getattr(message, "reasoning", None)
     if isinstance(reasoning, str) and reasoning.strip():
         match = re.search(r"```(?:python)?\s*(.*?)```", reasoning, re.DOTALL)
         if match:
-            return match.group(0)
+            return match.group(1).strip()
+        # Fall back to reasoning text itself as answer
+        return reasoning.strip()
+
     raise ValueError("model returned no content")

headroom/evals/suite_runner.py

@@ -38,7 +38,7 @@ class BenchmarkSpec:
     tier: int
     runner_type: Literal["lm_eval", "before_after", "compression_only", "livecodebench"]
     sample_size: int
-    model: str = "gpt-4o-mini"
+    model: str | None = None
     dataset_name: str | None = None  # For before_after runner
     lm_eval_tasks: list[str] | None = None  # For lm_eval runner
     primary_metric: str = "accuracy"
@@ -372,6 +372,12 @@ class SuiteRunner:
             return self._proxy_proc is not None
         return False
 
+    def _get_proxy_url(self) -> str | None:
+        """Return proxy URL if a Headroom proxy is reachable, else None."""
+        if _check_proxy(self.headroom_port):
+            return f"http://localhost:{self.headroom_port}"
+        return None
+
     def _cleanup_proxy(self) -> None:
         """Stop proxy if we started it."""
         if self._proxy_proc:
@@ -456,9 +462,7 @@ class SuiteRunner:
             suite = load_dataset_by_name(spec.dataset_name or spec.name.lower(), n=spec.sample_size)
 
         # Configure runner — use proxy for full-stack eval (compression + CCR)
-        proxy_url = (
-            f"http://localhost:{self.headroom_port}" if _check_proxy(self.headroom_port) else None
-        )
+        proxy_url = self._get_proxy_url()
         runner = BeforeAfterRunner(
             llm_config=LLMConfig(
                 provider=spec.provider,
@@ -506,14 +510,14 @@ class SuiteRunner:
         from headroom.evals.runners.livecodebench_runner import LiveCodeBenchRunner
 
         suite = load_dataset_by_name(spec.dataset_name or "livecodebench", n=spec.sample_size)
+        proxy_url = self._get_proxy_url()
+
         runner = LiveCodeBenchRunner(
             llm_config=LLMConfig(
                 provider=spec.provider,
                 model=spec.model or self.model,
                 temperature=0.0,
-                headroom_proxy_url=(
-                    f"http://localhost:{self.headroom_port}" if _check_proxy(self.headroom_port) else None
-                ),
+                headroom_proxy_url=proxy_url,
             )
         )
         result = runner.run(suite)

tests/test_evals/test_eval_runners.py

@@ -148,11 +148,18 @@ def test_livecodebench_extract_openai_text_from_reasoning_code_block() -> None:
     message = SimpleNamespace(content=None, reasoning="```python\nprint(123)\n```")
     response = SimpleNamespace(choices=[SimpleNamespace(message=message)])
 
-    assert extract_openai_text(response) == "```python\nprint(123)\n```"
+    assert extract_openai_text(response) == "print(123)"
+
+
+def test_livecodebench_extract_openai_text_from_reasoning_content() -> None:
+    message = SimpleNamespace(content=None, reasoning=None, reasoning_content="print(123)")
+    response = SimpleNamespace(choices=[SimpleNamespace(message=message)])
+
+    assert extract_openai_text(response) == "print(123)"
 
 
 def test_livecodebench_extract_openai_text_raises_on_empty_response() -> None:
-    message = SimpleNamespace(content=None, reasoning=None)
+    message = SimpleNamespace(content=None, reasoning=None, reasoning_content=None)
     response = SimpleNamespace(choices=[SimpleNamespace(message=message)])
 
     with pytest.raises(ValueError, match="model returned no content"):