updated to print

.gitignore

@@ -12,3 +12,4 @@ anatomy_full.pdf
 *.csv
 .pycache/*
 .worktrees/
+__pycache__/*

app.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import traceback
 from pathlib import Path
 
 from dotenv import load_dotenv
@@ -26,7 +27,19 @@ class LilithAgent:
         self.cfg = cfg or Config.from_env()
         self.client = client or ScoringApiClient()
         self.graph = build_react_agent(self.cfg)
-        print(f"LilithAgent initialized (caveman={self.cfg.caveman}/{self.cfg.caveman_mode}).")
+        print(f"LilithAgent initialized (caveman={self.cfg.caveman}/{self.cfg.caveman_mode}).", flush=True)
+        print(
+            "[config] "
+            f"cheap={self.cfg.cheap_provider}/{self.cfg.cheap_model} "
+            f"strong={self.cfg.strong_provider}/{self.cfg.strong_model} "
+            f"extra={self.cfg.extra_strong_provider}/{self.cfg.extra_strong_model} "
+            f"vision={self.cfg.vision_provider}/{self.cfg.vision_model} "
+            f"recursion_limit={self.cfg.recursion_limit} "
+            f"budget_warn_at={self.cfg.budget_warn_at} "
+            f"budget_hard_cap={self.cfg.budget_hard_cap} "
+            f"checkpoint_dir={self.cfg.checkpoint_dir}",
+            flush=True,
+        )
 
 
 def run_and_submit_all(profile: gr.OAuthProfile | None):
@@ -34,9 +47,9 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
 
     if profile:
         username = profile.username
-        print(f"User logged in: {username}")
+        print(f"User logged in: {username}", flush=True)
     else:
-        print("User not logged in.")
+        print("User not logged in.", flush=True)
         return "Please Login to Hugging Face with the button.", None
 
     submit_url = f"{DEFAULT_API_URL}/submit"
@@ -44,31 +57,40 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     try:
         agent = LilithAgent()
     except Exception as e:
-        print(f"Error instantiating agent: {e}")
+        print(f"Error instantiating agent: {e}", flush=True)
+        traceback.print_exc()
         return f"Error initializing agent: {e}", None
 
     agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
-    print(agent_code)
+    print(agent_code, flush=True)
 
-    print("Fetching questions from scoring API...")
+    print("Fetching questions from scoring API...", flush=True)
     try:
         questions_data = agent.client.get_questions()
     except requests.exceptions.RequestException as e:
+        print(f"Error fetching questions: {e}", flush=True)
+        traceback.print_exc()
         return f"Error fetching questions: {e}", None
     if agent.client.last_warning:
-        print(agent.client.last_warning)
+        print(agent.client.last_warning, flush=True)
 
     if not questions_data:
         return "Fetched questions list is empty or invalid format.", None
-    print(f"Fetched {len(questions_data)} questions.")
-
-    print(f"Running agent on {len(questions_data)} questions...")
-    answers_payload = run_agent_on_questions(
-        agent.graph,
-        questions_data,
-        agent.cfg.checkpoint_dir,
-        client=agent.client,
-    )
+    print(f"Fetched {len(questions_data)} questions.", flush=True)
+
+    print(f"Running agent on {len(questions_data)} questions...", flush=True)
+    try:
+        answers_payload = run_agent_on_questions(
+            agent.graph,
+            questions_data,
+            agent.cfg.checkpoint_dir,
+            client=agent.client,
+        )
+    except Exception as e:
+        print(f"[app] runner failed type={type(e).__name__} error={e}", flush=True)
+        traceback.print_exc()
+        return f"Agent runner failed: {type(e).__name__}: {e}", None
+    print(f"[app] runner produced {len(answers_payload)} answers", flush=True)
     answers_by_id = {a["task_id"]: a["submitted_answer"] for a in answers_payload}
     results_log = [
         {
@@ -88,12 +110,20 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         "agent_code": agent_code,
         "answers": answers_payload,
     }
-    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
+    print(f"Submitting {len(answers_payload)} answers to: {submit_url}", flush=True)
 
     try:
         response = requests.post(submit_url, json=submission_data, timeout=60)
+        print(f"[submit] status_code={response.status_code}", flush=True)
         response.raise_for_status()
         result_data = response.json()
+        print(
+            "[submit] success "
+            f"score={result_data.get('score', 'N/A')} "
+            f"correct={result_data.get('correct_count', '?')} "
+            f"attempted={result_data.get('total_attempted', '?')}",
+            flush=True,
+        )
         final_status = (
             f"Submission Successful!\n"
             f"User: {result_data.get('username')}\n"
@@ -109,12 +139,18 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
             error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
         except requests.exceptions.JSONDecodeError:
             error_detail += f" Response: {e.response.text[:500]}"
+        print(f"[submit] http_error {error_detail}", flush=True)
         return f"Submission Failed: {error_detail}", pd.DataFrame(results_log)
     except requests.exceptions.Timeout:
+        print("[submit] timeout", flush=True)
         return "Submission Failed: The request timed out.", pd.DataFrame(results_log)
     except requests.exceptions.RequestException as e:
+        print(f"[submit] network_error {e}", flush=True)
+        traceback.print_exc()
         return f"Submission Failed: Network error - {e}", pd.DataFrame(results_log)
     except Exception as e:
+        print(f"[submit] unexpected_error type={type(e).__name__} error={e}", flush=True)
+        traceback.print_exc()
         return f"An unexpected error occurred during submission: {e}", pd.DataFrame(results_log)
 
 
@@ -144,24 +180,24 @@ with gr.Blocks() as demo:
 
 
 if __name__ == "__main__":
-    print("\n" + "-" * 30 + " App Starting " + "-" * 30)
+    print("\n" + "-" * 30 + " App Starting " + "-" * 30, flush=True)
     space_host_startup = os.getenv("SPACE_HOST")
     space_id_startup = os.getenv("SPACE_ID")
 
     if space_host_startup:
-        print(f"✅ SPACE_HOST found: {space_host_startup}")
-        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
+        print(f"✅ SPACE_HOST found: {space_host_startup}", flush=True)
+        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space", flush=True)
     else:
-        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")
+        print("ℹ️  SPACE_HOST environment variable not found (running locally?).", flush=True)
 
     if space_id_startup:
-        print(f"✅ SPACE_ID found: {space_id_startup}")
-        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
-        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
+        print(f"✅ SPACE_ID found: {space_id_startup}", flush=True)
+        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}", flush=True)
+        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main", flush=True)
     else:
-        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
+        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.", flush=True)
 
-    print("-" * (60 + len(" App Starting ")) + "\n")
+    print("-" * (60 + len(" App Starting ")) + "\n", flush=True)
 
-    print("Launching Gradio Interface for Lilith Agent Evaluation...")
+    print("Launching Gradio Interface for Lilith Agent Evaluation...", flush=True)
     demo.launch(debug=True, share=False)

src/lilith_agent/app.py

@@ -307,8 +307,16 @@ def _route_after_model(
     AIMessage has tool_calls; otherwise END.
     """
     if state.get("iterations", 0) >= recursion_limit - 2:
+        print(
+            f"[route] recursion threshold reached iter={state.get('iterations', 0)} limit={recursion_limit}",
+            flush=True,
+        )
         return "fail_safe"
     if _count_tool_calls_since_last_human(state["messages"]) >= budget_hard_cap:
+        print(
+            f"[route] hard cap reached tool_calls={_count_tool_calls_since_last_human(state['messages'])} cap={budget_hard_cap}",
+            flush=True,
+        )
         log.info("[hard_cap] per-question tool-call cap hit; forcing fail_safe")
         return "fail_safe"
     last = state["messages"][-1]
@@ -335,6 +343,10 @@ def _build_tool_node(
         tool_calls = getattr(last, "tool_calls", None) or []
         todo_state_update = None
         if tool_calls:
+            print(
+                f"[tools] dispatching count={len(tool_calls)} names={[tc.get('name') for tc in tool_calls]}",
+                flush=True,
+            )
             log_tools.info(
                 "[tools] dispatching %d call(s): %s",
                 len(tool_calls),
@@ -368,6 +380,7 @@ def _build_tool_node(
 
             key = _call_key(name, args)
             if key in seen:
+                print(f"[tools] dedup tool={name}", flush=True)
                 log.info("[dedup] short-circuited repeat tool call: %s %s", name, args)
                 results.append(ToolMessage(
                     tool_call_id=tc_id,
@@ -392,6 +405,7 @@ def _build_tool_node(
                         if score > best_score:
                             best_prior, best_score = prior_q, score
                     if best_score >= semantic_dedup_threshold:
+                        print(f"[tools] semantic_dedup score={best_score:.2f} tool={name}", flush=True)
                         log.info("[semantic_dedup] %.2f match vs prior: %r ~ %r", best_score, q, best_prior)
                         results.append(ToolMessage(
                             tool_call_id=tc_id,
@@ -409,6 +423,7 @@ def _build_tool_node(
 
             cooldown_limit = _cooldown_limit_for(name)
             if count_recent_errors(name) >= cooldown_limit:
+                print(f"[tools] cooldown tool={name} limit={cooldown_limit}", flush=True)
                 log.info("[loop_breaker] force-cooldown %s (limit=%d)", name, cooldown_limit)
                 results.append(ToolMessage(
                     tool_call_id=tc_id,
@@ -425,6 +440,7 @@ def _build_tool_node(
 
             tool = tools_by_name.get(name)
             if tool is None:
+                print(f"[tools] unknown tool={name}", flush=True)
                 results.append(ToolMessage(
                     tool_call_id=tc_id,
                     name=name or "unknown",
@@ -440,10 +456,12 @@ def _build_tool_node(
             if len(args_preview) > _TOOL_ARG_PREVIEW_CHARS:
                 args_preview = args_preview[:_TOOL_ARG_PREVIEW_CHARS] + "…"
             log_tools.info("[tools] calling tool=%s args=%s", name, args_preview)
+            print(f"[tools] calling tool={name} args={args_preview}", flush=True)
 
             try:
                 out = tool.invoke(args)
             except Exception as e:
+                print(f"[tools] error tool={name} type={type(e).__name__} msg={e}", flush=True)
                 log_tools.warning("[tools] %s raised: %s", name, e)
                 out = f"ERROR: {type(e).__name__}: {e}"
                 if len(out) > 1000:
@@ -471,6 +489,7 @@ def _build_tool_node(
             if len(preview) > _TOOL_RESULT_PREVIEW_CHARS:
                 preview = preview[:_TOOL_RESULT_PREVIEW_CHARS] + "…"
             log_tools.info("[tools] tool result (%d chars): %s", len(out_str), preview)
+            print(f"[tools] result tool={name} chars={len(out_str)} preview={preview}", flush=True)
             results.append(ToolMessage(tool_call_id=tc_id, name=name, content=out_str))
 
         update = {"messages": results}
@@ -522,6 +541,15 @@ def build_react_agent(cfg: Config):
         supervisor_model = None
     summarize_fn = _make_tool_result_summarizer(cfg) if cfg.compact_summarize else None
 
+    def _initial_question_from_state(state) -> str:
+        for m in state["messages"]:
+            if isinstance(m, HumanMessage):
+                raw = str(m.content).split("--- BENCHMARK SCORING RULES ---")[0].strip()
+                if raw.startswith("<gaia_question>") and raw.endswith("</gaia_question>"):
+                    raw = raw[len("<gaia_question>"):-len("</gaia_question>")].strip()
+                return raw
+        return ""
+
     def model_node(state):
         from langchain_core.messages import SystemMessage
         from lilith_agent.memory import retrieve_relevant_context
@@ -604,6 +632,10 @@ def build_react_agent(cfg: Config):
             tool_calls_this_turn,
             len(compacted),
         )
+        print(
+            f"[model] invoking iter={iteration} tool_calls_so_far={tool_calls_this_turn} msgs={len(compacted)}",
+            flush=True,
+        )
         response = model.invoke(prompt_msgs)
 
         # Clean up Gemini signatures and unhelpful metadata to reduce log noise and context bloat
@@ -618,6 +650,7 @@ def build_react_agent(cfg: Config):
 
         # Fallback for empty responses
         if not response.content and not getattr(response, "tool_calls", None):
+            print("[model] blank response detected", flush=True)
             log_model.warning("[model] blank response detected; injecting system nudge")
             response = AIMessage(content=(
                 "SYSTEM: Your previous response was empty. If you have enough information, "
@@ -627,6 +660,7 @@ def build_react_agent(cfg: Config):
         else:
             requested = [tc.get("name") for tc in (getattr(response, "tool_calls", None) or [])]
             if requested:
+                print(f"[model] requested tool_calls={requested}", flush=True)
                 log_model.info("[model] requested tool_calls=%s", requested)
             else:
                 content_text = response.content
@@ -638,22 +672,27 @@ def build_react_agent(cfg: Config):
                 content_text = str(content_text or "").strip().replace("\n", " ")
                 if len(content_text) > _TOOL_RESULT_PREVIEW_CHARS:
                     content_text = content_text[:_TOOL_RESULT_PREVIEW_CHARS] + "…"
+                print(f"[model] finished content={content_text!r}", flush=True)
                 log_model.info("[model] finished content=%r", content_text)
 
         return {"messages": [response], "iterations": iteration + 1}
 
     def fail_safe_node(state):
+        print(f"[fail_safe] emergency override: iter={state.get('iterations', 0)}", flush=True)
         log_fail_safe.warning(
             "[fail_safe] emergency override: iter=%d",
             state.get("iterations", 0),
         )
+        original_question = _initial_question_from_state(state)
         sys_prompt = (
             "SYSTEM EMERGENCY OVERRIDE: You have hit the absolute maximum iteration limit for this task. "
-            "You are FORCED to stop. Provide a brief 'Final Answer:' summarizing what you have tried, "
-            "why it failed, and what the best conclusion is so far."
+            "You are FORCED to stop tool use. Answer the original question, not an intermediate hop. "
+            "Provide a bare final answer in 'Final Answer: ...' form using the best conclusion supported so far. "
+            f"Original question: {original_question}"
         )
         compacted = _compact_old_tool_messages(state["messages"], summarize_fn=summarize_fn)
         response = base_model.invoke([SystemMessage(sys_prompt)] + compacted)
+        print(f"[fail_safe] produced content={_message_text(getattr(response, 'content', ''))[:240]!r}", flush=True)
         return {"messages": [response]}
 
     def supervisor_node(state):
@@ -694,6 +733,10 @@ def build_react_agent(cfg: Config):
             best_answer[:80],
             guidance[:160],
         )
+        print(
+            f"[supervisor] status={status} best={best_answer[:80]!r} guidance={guidance[:160]!r}",
+            flush=True,
+        )
         update = {
             "supervisor_decision": status,
             "supervisor_best_answer": best_answer,
@@ -712,16 +755,21 @@ def build_react_agent(cfg: Config):
     def supervisor_finalizer_node(state):
         best_answer = str(state.get("supervisor_best_answer", "") or "").strip()
         if best_answer:
+            print(f"[supervisor_finalizer] finalizing best={best_answer[:160]!r}", flush=True)
             return {"messages": [AIMessage(content=f"Final Answer: {best_answer}")]}
         guidance = str(state.get("supervisor_guidance", "") or "").strip()
+        original_question = _initial_question_from_state(state)
         compacted = _compact_old_tool_messages(state["messages"], summarize_fn=summarize_fn)
         response = base_model.invoke([
             SystemMessage(content=(
-                "SUPERVISOR FINALIZER: Stop tool use. Produce the best possible Final Answer "
-                f"using the existing evidence. Supervisor guidance: {guidance}"
+                "SUPERVISOR FINALIZER: Stop tool use. Answer the original question, not an intermediate hop. "
+                "Produce a bare final answer in 'Final Answer: ...' form using the existing evidence. "
+                f"Original question: {original_question}\n"
+                f"Supervisor guidance: {guidance}"
             )),
             *compacted,
         ])
+        print(f"[supervisor_finalizer] produced content={_message_text(getattr(response, 'content', ''))[:240]!r}", flush=True)
         return {"messages": [response]}
 
     def extract_memory_node(state):

src/lilith_agent/config.py

@@ -49,6 +49,8 @@ class Config:
     semantic_dedup_threshold: float = 0.5
     compact_summarize: bool = True
     llm_formatter_enabled: bool = True
+    answer_contract_enabled: bool = True
+    give_up_recovery_enabled: bool = True
 
     @classmethod
     def from_env(cls) -> "Config":
@@ -79,4 +81,6 @@ class Config:
             semantic_dedup_threshold=_get_float_env("GAIA_SEMANTIC_DEDUP_THRESHOLD", "0.5"),
             compact_summarize=os.getenv("GAIA_COMPACT_SUMMARIZE", "true").lower() == "true",
             llm_formatter_enabled=os.getenv("GAIA_LLM_FORMATTER_ENABLED", "true").lower() == "true",
+            answer_contract_enabled=os.getenv("GAIA_ANSWER_CONTRACT_ENABLED", "true").lower() == "true",
+            give_up_recovery_enabled=os.getenv("GAIA_GIVE_UP_RECOVERY_ENABLED", "true").lower() == "true",
         )

src/lilith_agent/runner.py

@@ -56,6 +56,51 @@ _LLM_FORMATTER_LEN_GATE = 40
 _ASSIGNMENT_PREFIX = re.compile(r"^\s*(?:x|y|answer|result)\s*[:=]\s*(.+?)\s*$", re.IGNORECASE)
 _COMMA_GROUPED_INTEGER = re.compile(r"^[+-]?\d{1,3}(?:,\d{3})+$")
 _SCALAR_NUMBER = re.compile(r"^[+-]?(?:\d+(?:\.\d+)?|\.\d+)$")
+# Matches "to N decimal places" or "to the nearest tenth/hundredth/thousandth"
+_DECIMAL_PLACES_RE = re.compile(
+    r"(?:to|rounded?\s+to|nearest)\s+"
+    r"(?:(\d+)\s+decimal\s+place[s]?|the\s+nearest\s+(tenth|hundredth|thousandth|ten[-\s]?thousandth))",
+    re.IGNORECASE,
+)
+_PRECISION_WORDS = {
+    "tenth": 1, "hundredth": 2, "thousandth": 3,
+    "ten-thousandth": 4, "ten thousandth": 4,
+}
+def _required_decimal_places(question: str) -> int | None:
+    """Return the number of decimal places the question demands, or None."""
+    m = _DECIMAL_PLACES_RE.search(question)
+    if not m:
+        return None
+    if m.group(1):
+        return int(m.group(1))
+    word = m.group(2).lower().replace(" ", "-")
+    return _PRECISION_WORDS.get(word)
+
+
+def _apply_decimal_precision(s: str, places: int) -> str:
+    """Reformat a numeric string to exactly `places` decimal places."""
+    try:
+        value = float(s)
+        return f"{value:.{places}f}"
+    except (ValueError, OverflowError):
+        return s
+
+
+_ANSWER_CONTRACT_QUESTION_MARKERS = (
+    "country", "countries", "capital", "arrival", "time", "meter", "metre",
+    "label", "score", "passenger", "title", "author", "date", "year",
+    "how many",
+)
+_GIVE_UP_PHRASES = (
+    "unknown",
+    "i don't know",
+    "cannot determine",
+    "could not determine",
+    "unable to determine",
+    "not enough information",
+    "could not complete",
+    "why it failed",
+)
 
 
 def _wrap_user_question(text: str) -> str:
@@ -167,9 +212,201 @@ def _normalize_gaia_submission(question: str, answer: str) -> str:
     if ";" in s:
         s = re.sub(r"\s*;\s*", "; ", s).strip()
 
+    required_places = _required_decimal_places(question)
+    if required_places is not None and _SCALAR_NUMBER.fullmatch(s):
+        s = _apply_decimal_precision(s, required_places)
+
     return s
 
 
+def _is_give_up_answer(answer: str) -> bool:
+    s = answer.strip().lower()
+    if not s:
+        return True
+    if s.startswith("agent error"):
+        return False
+    if s in {"unknown", "n/a", "not found"}:
+        return True
+    if len(s) <= 240 and any(phrase in s for phrase in _GIVE_UP_PHRASES):
+        return True
+    return False
+
+
+def _question_has_answer_contract_marker(question: str) -> bool:
+    q = question.lower()
+    return any(re.search(rf"(?<!\w){re.escape(marker)}(?!\w)", q) for marker in _ANSWER_CONTRACT_QUESTION_MARKERS)
+
+
+def _needs_answer_contract_check(question: str, answer: str) -> bool:
+    if _is_give_up_answer(answer):
+        return False
+    q = question.lower()
+    if not _question_has_answer_contract_marker(question):
+        return False
+    if _SCALAR_NUMBER.fullmatch(answer.strip()) and not any(marker in q for marker in ("time", "arrival", "date", "year")):
+        return False
+    return True
+
+
+def _parse_contract_response(content: Any) -> dict[str, str]:
+    if isinstance(content, list):
+        text = "".join(
+            part.get("text", "") if isinstance(part, dict) else str(part)
+            for part in content
+        )
+    else:
+        text = str(content or "")
+    text = text.strip()
+    try:
+        parsed = json.loads(text)
+    except Exception:
+        match = re.search(r"\{.*\}", text, flags=re.S)
+        if not match:
+            return {"status": "ok", "submitted_answer": "", "reason": ""}
+        try:
+            parsed = json.loads(match.group(0))
+        except Exception:
+            return {"status": "ok", "submitted_answer": "", "reason": ""}
+    if not isinstance(parsed, dict):
+        return {"status": "ok", "submitted_answer": "", "reason": ""}
+    status = str(parsed.get("status", "ok") or "ok").strip().lower()
+    if status not in {"ok", "repair"}:
+        status = "ok"
+    return {
+        "status": status,
+        "submitted_answer": str(parsed.get("submitted_answer", "") or "").strip(),
+        "reason": str(parsed.get("reason", "") or "").strip(),
+    }
+
+
+def _repair_supported_by_context(question: str, reasoning_trace: str, repaired: str) -> bool:
+    context = f"{question}\n{reasoning_trace}".lower()
+    pieces = [
+        piece.strip(" \t\r\n.,;:()[]{}\"'`")
+        for piece in re.split(r"\s*(?:,|;|\band\b)\s*", repaired)
+    ]
+    pieces = [piece for piece in pieces if piece]
+    if not pieces:
+        return False
+    return all(piece.lower() in context for piece in pieces)
+
+
+def _apply_answer_contract(
+    model: Any,
+    question: str,
+    answer: str,
+    reasoning_trace: str,
+    *,
+    enabled: bool = True,
+) -> str:
+    if not enabled or not _needs_answer_contract_check(question, answer):
+        return answer
+    from langchain_core.messages import SystemMessage, HumanMessage
+
+    prompt = (
+        "You are a GAIA benchmark answer contract verifier. Check whether the submitted "
+        "answer answers the ORIGINAL question, not an intermediate hop. If the answer type "
+        "matches the question, return JSON {\"status\":\"ok\"}. If the answer is clearly "
+        "the wrong type and the evidence trace contains the correct final answer, return "
+        "JSON {\"status\":\"repair\",\"submitted_answer\":\"...\",\"reason\":\"...\"}. "
+        "Do not guess. Do not repair unless the replacement appears in the evidence trace."
+    )
+    user = (
+        f"ORIGINAL QUESTION:\n{question}\n\n"
+        f"SUBMITTED ANSWER:\n{answer}\n\n"
+        f"EVIDENCE TRACE:\n{reasoning_trace[-4000:]}"
+    )
+    try:
+        response = model.invoke([SystemMessage(content=prompt), HumanMessage(content=user)])
+    except Exception as exc:
+        log.warning("answer_contract: verifier failed (%s), keeping original answer", exc)
+        return answer
+    decision = _parse_contract_response(getattr(response, "content", ""))
+    if decision["status"] != "repair":
+        return answer
+    repaired = _normalize_gaia_submission(question, decision["submitted_answer"])
+    if not repaired:
+        return answer
+    if not _repair_supported_by_context(question, reasoning_trace, repaired):
+        log.warning("answer_contract: rejected unsupported repair %r", repaired)
+        return answer
+    log.info("answer_contract: repaired answer %r -> %r", answer, repaired)
+    return repaired
+
+
+def _parse_recovery_response(content: Any) -> dict[str, str]:
+    if isinstance(content, list):
+        text = "".join(
+            part.get("text", "") if isinstance(part, dict) else str(part)
+            for part in content
+        )
+    else:
+        text = str(content or "")
+    text = text.strip()
+    try:
+        parsed = json.loads(text)
+    except Exception:
+        match = re.search(r"\{.*\}", text, flags=re.S)
+        if not match:
+            return {"status": "keep", "submitted_answer": "", "reason": ""}
+        try:
+            parsed = json.loads(match.group(0))
+        except Exception:
+            return {"status": "keep", "submitted_answer": "", "reason": ""}
+    if not isinstance(parsed, dict):
+        return {"status": "keep", "submitted_answer": "", "reason": ""}
+    status = str(parsed.get("status", "keep") or "keep").strip().lower()
+    if status not in {"keep", "answer"}:
+        status = "keep"
+    return {
+        "status": status,
+        "submitted_answer": str(parsed.get("submitted_answer", "") or "").strip(),
+        "reason": str(parsed.get("reason", "") or "").strip(),
+    }
+
+
+def _apply_give_up_recovery(
+    model: Any,
+    question: str,
+    answer: str,
+    reasoning_trace: str,
+    *,
+    enabled: bool = True,
+) -> str:
+    if not enabled or not _is_give_up_answer(answer):
+        return answer
+    from langchain_core.messages import SystemMessage, HumanMessage
+
+    prompt = (
+        "You are a GAIA benchmark give-up recovery verifier. The submitted answer is "
+        "empty, unknown, or a failure summary. If the evidence trace contains a concrete "
+        "answer to the original question, return JSON {\"status\":\"answer\","
+        "\"submitted_answer\":\"...\",\"reason\":\"...\"}. Otherwise return JSON "
+        "{\"status\":\"keep\"}. Do not guess. The submitted_answer must appear in the trace."
+    )
+    user = (
+        f"ORIGINAL QUESTION:\n{question}\n\n"
+        f"CURRENT SUBMITTED ANSWER:\n{answer}\n\n"
+        f"EVIDENCE TRACE:\n{reasoning_trace[-4000:]}"
+    )
+    try:
+        response = model.invoke([SystemMessage(content=prompt), HumanMessage(content=user)])
+    except Exception as exc:
+        log.warning("give_up_recovery: verifier failed (%s), keeping original answer", exc)
+        return answer
+    decision = _parse_recovery_response(getattr(response, "content", ""))
+    if decision["status"] != "answer":
+        return answer
+    recovered = _normalize_gaia_submission(question, decision["submitted_answer"])
+    if not recovered:
+        return answer
+    if not _repair_supported_by_context(question, reasoning_trace, recovered):
+        log.warning("give_up_recovery: rejected unsupported answer %r", recovered)
+        return answer
+    log.info("give_up_recovery: recovered answer %r -> %r", answer, recovered)
+    return recovered
+
+
 def _write_checkpoint_atomic(path: Path, data: dict) -> None:
     """Serialize first, then rename. A crash mid-serialize leaves the prior file intact.
 
@@ -239,6 +476,7 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
     cheap_model = get_cheap_model(cfg)
 
     total = len(questions)
+    print(f"[runner] starting batch total={total} checkpoint_dir={checkpoint_root}", flush=True)
 
     def _invoke_task_once(task_state: dict, task_id: str):
         from lilith_agent.memory import ephemeral_memory
@@ -251,6 +489,7 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
         pause_seconds = batch_rate_limit_pause_seconds()
         if pause_seconds is None:
             return
+        print(f"[runner] pausing batch seconds={pause_seconds} reason=rate_limit_window", flush=True)
         log_runner.warning("[runner] pausing batch for %ss due to rate limit window", pause_seconds)
         time.sleep(pause_seconds)
         clear_batch_rate_limit_window()
@@ -260,13 +499,18 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
         task_id = question.get("task_id")
         prompt = question.get("question")
         if not task_id or not prompt:
+            print(f"[runner] skipping invalid question idx={idx} task_id={task_id!r}", flush=True)
             continue
 
         file_name = question.get("file_name")
         if file_name and client:
+            print(f"[runner] task={task_id} downloading file={file_name}", flush=True)
             file_path = client.download_file(task_id, dest_dir=checkpoint_root / "files")
             if file_path:
+                print(f"[runner] task={task_id} file_path={file_path.absolute()}", flush=True)
                 prompt += f"\n\n[Attached File Path: {file_path.absolute()}]"
+            else:
+                print(f"[runner] task={task_id} file_download_missing file={file_name}", flush=True)
 
         # Scoring rules removed from here to reduce per-turn context bloat.
         # They are now applied in a final post-processing step.
@@ -276,6 +520,7 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
             try:
                 checkpoint = json.loads(checkpoint_path.read_text())
                 log_runner.info("[runner] task=%s (%d/%d) skipped (checkpoint exists)", task_id, idx, total)
+                print(f"[runner] task={task_id} ({idx}/{total}) skipped checkpoint={checkpoint_path}", flush=True)
                 answers.append(
                     {
                         "task_id": task_id,
@@ -284,12 +529,14 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
                 )
                 continue
             except Exception:
+                print(f"[runner] task={task_id} checkpoint unreadable path={checkpoint_path}", flush=True)
                 pass
 
         log_runner.info(
             "[runner] task=%s (%d/%d) starting q=%r",
             task_id, idx, total, (prompt[:160] + "…") if len(prompt) > 160 else prompt,
         )
+        print(f"[runner] task={task_id} ({idx}/{total}) starting", flush=True)
 
         state = {
             "messages": [HumanMessage(content=_wrap_user_question(prompt))],
@@ -300,6 +547,10 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
             try:
                 result = _invoke_task_once(state, task_id)
             except RateLimitCooldownError as exc:
+                print(
+                    f"[runner] task={task_id} rate_limited provider={exc.provider} model={exc.model} cooldown={exc.cooldown_seconds}",
+                    flush=True,
+                )
                 log_runner.warning(
                     "[runner] task=%s rate limited provider=%s model=%s cooldown=%s",
                     task_id,
@@ -308,18 +559,22 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
                     exc.cooldown_seconds,
                 )
                 time.sleep(exc.cooldown_seconds)
+                print(f"[runner] task={task_id} retrying after cooldown", flush=True)
                 result = _invoke_task_once(state, task_id)
         except RateLimitCooldownError as exc:
+            print(f"[runner] task={task_id} rate_limited_after_retry error={exc}", flush=True)
             log_runner.warning("[runner] task=%s rate limited after retry: %s", task_id, exc)
             answers.append({"task_id": task_id, "submitted_answer": "AGENT ERROR: RATE LIMITED"})
             _maybe_pause_for_batch_rate_limit()
             continue
         except QuestionRateLimitStreakError as exc:
+            print(f"[runner] task={task_id} rate_limit_streak error={exc}", flush=True)
             log_runner.warning("[runner] task=%s rate limit streak: %s", task_id, exc)
             answers.append({"task_id": task_id, "submitted_answer": "AGENT ERROR: RATE LIMITED"})
             _maybe_pause_for_batch_rate_limit()
             continue
         except BatchAbortRateLimitError as exc:
+            print(f"[runner] task={task_id} batch_abort_rate_limit reason={exc.reason}", flush=True)
             log_runner.warning("[runner] task=%s batch abort rate limit: %s", task_id, exc)
             answers.append({"task_id": task_id, "submitted_answer": "AGENT ERROR: RATE LIMITED"})
             _write_checkpoint_atomic(
@@ -333,6 +588,7 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
             _maybe_pause_for_batch_rate_limit()
             continue
         except Exception as exc:
+            print(f"[runner] task={task_id} agent_error type={type(exc).__name__} error={exc}", flush=True)
             log_runner.warning("[runner] task=%s agent error: %s", task_id, exc)
             answers.append(
                 {
@@ -359,8 +615,22 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
             llm_formatter_enabled=cfg.llm_formatter_enabled,
         )
         submitted_answer = _normalize_gaia_submission(prompt, submitted_answer)
-        
+
         reasoning_trace = _render_reasoning_trace(result["messages"])
+        submitted_answer = _apply_answer_contract(
+            cheap_model,
+            prompt,
+            submitted_answer,
+            reasoning_trace,
+            enabled=cfg.answer_contract_enabled,
+        )
+        submitted_answer = _apply_give_up_recovery(
+            cheap_model,
+            prompt,
+            submitted_answer,
+            reasoning_trace,
+            enabled=cfg.give_up_recovery_enabled,
+        )
 
         checkpoint = {
             "task_id": task_id,
@@ -371,15 +641,19 @@ def run_agent_on_questions(graph: Any, questions: list[dict], checkpoint_dir: st
         
         if submitted_answer and not submitted_answer.startswith("AGENT ERROR"):
             _write_checkpoint_atomic(checkpoint_path, checkpoint)
+            print(f"[runner] task={task_id} checkpoint_written path={checkpoint_path}", flush=True)
 
         log_runner.info(
             "[runner] task=%s (%d/%d) answer=%r",
             task_id, idx, total,
             (submitted_answer[:160] + "…") if len(submitted_answer) > 160 else submitted_answer,
         )
+        answer_preview = (submitted_answer[:160] + "…") if len(submitted_answer) > 160 else submitted_answer
+        print(f"[runner] task={task_id} ({idx}/{total}) answer={answer_preview!r}", flush=True)
         answers.append({"task_id": task_id, "submitted_answer": submitted_answer.strip()})
         _maybe_pause_for_batch_rate_limit()
 
+    print(f"[runner] finished batch produced={len(answers)}", flush=True)
     return answers
 
 

src/lilith_agent/scoring_client.py

@@ -81,6 +81,7 @@ class ScoringApiClient:
             f"Scoring API unavailable while trying to {action} ({detail}); "
             f"falling back to GAIA dataset."
         )
+        print(self.last_warning, flush=True)
         log.warning(self.last_warning)
         return dataset_client
 

tests/test_config.py

@@ -40,3 +40,19 @@ def test_compact_summarize_defaults_on_and_is_env_overridable(monkeypatch):
 
     monkeypatch.setenv("GAIA_COMPACT_SUMMARIZE", "false")
     assert Config.from_env().compact_summarize is False
+
+
+def test_answer_contract_defaults_on_and_is_env_overridable(monkeypatch):
+    monkeypatch.delenv("GAIA_ANSWER_CONTRACT_ENABLED", raising=False)
+    assert Config.from_env().answer_contract_enabled is True
+
+    monkeypatch.setenv("GAIA_ANSWER_CONTRACT_ENABLED", "false")
+    assert Config.from_env().answer_contract_enabled is False
+
+
+def test_give_up_recovery_defaults_on_and_is_env_overridable(monkeypatch):
+    monkeypatch.delenv("GAIA_GIVE_UP_RECOVERY_ENABLED", raising=False)
+    assert Config.from_env().give_up_recovery_enabled is True
+
+    monkeypatch.setenv("GAIA_GIVE_UP_RECOVERY_ENABLED", "false")
+    assert Config.from_env().give_up_recovery_enabled is False

tests/test_formatter.py

@@ -230,6 +230,44 @@ def test_gaia_submission_normalizer_leaves_risky_answers_unchanged(answer: str):
     assert _normalize_gaia_submission("Q", answer) == answer
 
 
+@pytest.mark.parametrize(
+    ("question", "answer", "expected"),
+    [
+        # rounds excess decimal places down
+        ("Give answer to 3 decimal places.", "1.4560", "1.456"),
+        # pads too-short answer to required precision
+        ("Round to 3 decimal places.", "17.06", "17.060"),
+        # nearest-tenth keyword
+        ("Express to the nearest tenth.", "1.46", "1.5"),
+        # nearest-hundredth keyword
+        ("Round to the nearest hundredth.", "0.2690", "0.27"),
+        # nearest-thousandth keyword
+        ("Round to the nearest thousandth.", "0.2690", "0.269"),
+        # no precision requirement → untouched
+        ("What is the answer?", "1.456", "1.456"),
+    ],
+)
+def test_decimal_precision_normalization(question: str, answer: str, expected: str):
+    from lilith_agent.runner import _normalize_gaia_submission
+
+    assert _normalize_gaia_submission(question, answer) == expected
+
+
+@pytest.mark.parametrize(
+    ("question", "answer"),
+    [
+        # non-numeric answer — precision check skipped
+        ("Give answer to 3 decimal places.", "Paris"),
+        # dotted identifier — not a bare scalar, precision check skipped
+        ("Round to 3 decimal places.", "3.1.3.1"),
+    ],
+)
+def test_decimal_precision_skipped_for_non_numeric(question: str, answer: str):
+    from lilith_agent.runner import _normalize_gaia_submission
+
+    assert _normalize_gaia_submission(question, answer) == answer
+
+
 def test_config_llm_formatter_enabled_defaults_on_and_is_env_overridable(monkeypatch):
     from lilith_agent.config import Config
 

tests/test_graph.py

@@ -27,7 +27,7 @@ def test_router_ends_when_no_tool_calls():
     assert _route_after_model(state) == "extract_memory"
 
 
-def test_graph_returns_fail_safe_answer_when_hard_cap_hits_near_recursion_limit(monkeypatch, tmp_path):
+def test_graph_returns_fail_safe_answer_when_hard_cap_hits_near_recursion_limit(monkeypatch, tmp_path, capsys):
     class FakeModel:
         def __init__(self):
             self.calls = 0
@@ -65,6 +65,9 @@ def test_graph_returns_fail_safe_answer_when_hard_cap_hits_near_recursion_limit(
         {"configurable": {"thread_id": "hard-cap-test"}},
     )
 
+    captured = capsys.readouterr().out
+    assert "[route] recursion threshold reached" in captured
+    assert "[fail_safe] emergency override" in captured
     assert result["messages"][-1].content == "Final Answer: best effort answer"
 
 
@@ -115,6 +118,53 @@ def test_fail_safe_uses_unbound_model_to_prevent_more_tool_calls(monkeypatch, tm
     assert not getattr(result["messages"][-1], "tool_calls", None)
 
 
+def test_fail_safe_prompt_reinforces_original_question_contract(monkeypatch, tmp_path):
+    class FakeBoundModel:
+        def invoke(self, messages):
+            return _ai_with_calls([
+                {
+                    "id": "bound-call",
+                    "name": "echo_tool",
+                    "args": {"text": "intermediate"},
+                }
+            ])
+
+    class FakeModel:
+        def __init__(self):
+            self.bound = FakeBoundModel()
+            self.fail_safe_prompt = ""
+
+        def bind_tools(self, tools):
+            return self.bound
+
+        def invoke(self, messages):
+            self.fail_safe_prompt = str(messages[0].content)
+            return AIMessage(content="Final Answer: best effort")
+
+    fake_model = FakeModel()
+    cfg = Config.from_env()
+    cfg.recursion_limit = 4
+    cfg.budget_hard_cap = 1
+    cfg.budget_warn_at = 99
+    cfg.compact_summarize = False
+    monkeypatch.setenv("LILITH_HOME", str(tmp_path / ".lilith"))
+    monkeypatch.setattr("lilith_agent.app.get_extra_strong_model", lambda cfg: fake_model)
+    monkeypatch.setattr("lilith_agent.app.get_cheap_model", lambda cfg: fake_model)
+    monkeypatch.setattr("lilith_agent.tools.build_tools", lambda cfg: [echo_tool])
+    monkeypatch.setattr("lilith_agent.memory.extract_and_compress_facts", lambda messages, model: None)
+
+    graph = build_react_agent(cfg)
+    graph.invoke(
+        {"messages": [HumanMessage(content="What country corresponds to this capital?")], "iterations": 0, "todos": []},
+        {"configurable": {"thread_id": "fail-safe-contract-prompt-test"}},
+    )
+
+    prompt = fake_model.fail_safe_prompt.lower()
+    assert "original question" in prompt
+    assert "not an intermediate" in prompt
+    assert "bare final answer" in prompt
+
+
 def test_supervisor_nudges_agent_to_answer_when_evidence_is_enough(monkeypatch, tmp_path):
     class FakeBoundModel:
         def __init__(self):
@@ -175,6 +225,58 @@ def test_supervisor_nudges_agent_to_answer_when_evidence_is_enough(monkeypatch,
     assert strong.bound.calls == 2
 
 
+def test_supervisor_finalizer_prompt_reinforces_original_question_contract(monkeypatch, tmp_path):
+    class FakeBoundModel:
+        def invoke(self, messages):
+            return _ai_with_calls([
+                {
+                    "id": "call",
+                    "name": "echo_tool",
+                    "args": {"text": "evidence"},
+                }
+            ])
+
+    class FakeStrongModel:
+        def __init__(self):
+            self.bound = FakeBoundModel()
+            self.finalizer_prompt = ""
+
+        def bind_tools(self, tools):
+            return self.bound
+
+        def invoke(self, messages):
+            self.finalizer_prompt = str(messages[0].content)
+            return AIMessage(content="Final Answer: final")
+
+    class FakeSupervisorModel:
+        def invoke(self, messages):
+            return AIMessage(content='{"status":"finalize","best_answer":"","guidance":"Existing evidence is enough."}')
+
+    strong = FakeStrongModel()
+    cfg = Config.from_env()
+    cfg.recursion_limit = 10
+    cfg.budget_hard_cap = 99
+    cfg.budget_warn_at = 99
+    cfg.compact_summarize = False
+    monkeypatch.setenv("LILITH_HOME", str(tmp_path / ".lilith"))
+    monkeypatch.setattr("lilith_agent.app._SUPERVISOR_MIN_TOOL_CALLS", 1, raising=False)
+    monkeypatch.setattr("lilith_agent.app.get_extra_strong_model", lambda cfg: strong)
+    monkeypatch.setattr("lilith_agent.app.get_cheap_model", lambda cfg: FakeSupervisorModel())
+    monkeypatch.setattr("lilith_agent.tools.build_tools", lambda cfg: [echo_tool])
+    monkeypatch.setattr("lilith_agent.memory.extract_and_compress_facts", lambda messages, model: None)
+
+    graph = build_react_agent(cfg)
+    graph.invoke(
+        {"messages": [HumanMessage(content="What final entity answers the original question?")], "iterations": 0, "todos": []},
+        {"configurable": {"thread_id": "supervisor-finalizer-contract-prompt-test"}},
+    )
+
+    prompt = strong.finalizer_prompt.lower()
+    assert "original question" in prompt
+    assert "not an intermediate" in prompt
+    assert "bare final answer" in prompt
+
+
 def test_supervisor_finalizes_when_agent_ignores_prior_nudge(monkeypatch, tmp_path):
     class FakeBoundModel:
         def __init__(self):

tests/test_runner.py

@@ -131,6 +131,23 @@ def test_runner_retries_same_question_once_after_cooldown(monkeypatch, tmp_path:
     assert (tmp_path / "task-1.json").exists()
 
 
+def test_runner_prints_hf_visible_progress_and_success(monkeypatch, tmp_path: Path, capsys):
+    monkeypatch.setattr("lilith_agent.runner._final_formatting_cleanup", lambda model, question, raw, llm_formatter_enabled=True: raw)
+
+    answers = run_agent_on_questions(
+        _GraphAlwaysSucceeds(),
+        [{"task_id": "task-print", "question": "What is visible?"}],
+        tmp_path,
+    )
+
+    captured = capsys.readouterr().out
+    assert "[runner] starting batch total=1" in captured
+    assert "[runner] task=task-print (1/1) starting" in captured
+    assert "[runner] task=task-print (1/1) answer='answer-1'" in captured
+    assert "[runner] finished batch produced=1" in captured
+    assert answers == [{"task_id": "task-print", "submitted_answer": "answer-1"}]
+
+
 class _GraphAlwaysCooldown:
     def __init__(self):
         self.calls = 0
@@ -256,6 +273,169 @@ class _GraphReturnsAssignmentAnswer:
         return {"messages": [AIMessage(content="x = 563.9")]}
 
 
+class _GraphReturnsWrongTypeWithEvidence:
+    def invoke(self, state, config):
+        return {
+            "messages": [
+                AIMessage(content="Evidence: Dili is the capital of Timor-Leste. Naypyidaw is the capital of Myanmar."),
+                AIMessage(content="Final Answer: Dili, Naypyidaw"),
+            ]
+        }
+
+
+class _GraphReturnsUnknownWithEvidence:
+    def invoke(self, state, config):
+        return {
+            "messages": [
+                AIMessage(content="Evidence gathered from the page: the exact UI label is Citations."),
+                AIMessage(content="Final Answer: unknown"),
+            ]
+        }
+
+
+class _FakeContractModel:
+    def __init__(self, response: str):
+        self.response = response
+        self.called = False
+
+    def invoke(self, _messages):
+        self.called = True
+
+        class _Resp:
+            pass
+
+        r = _Resp()
+        r.content = self.response
+        return r
+
+
+class _RaiseIfContractCalled:
+    def invoke(self, _messages):
+        raise AssertionError("contract verifier should not have been called")
+
+
+def test_answer_contract_repairs_wrong_type_when_repair_is_supported_by_trace():
+    from lilith_agent.runner import _apply_answer_contract
+
+    model = _FakeContractModel('{"status":"repair","submitted_answer":"Timor-Leste, Myanmar"}')
+
+    out = _apply_answer_contract(
+        model,
+        "What countries have the capitals Dili and Naypyidaw?",
+        "Dili, Naypyidaw",
+        "Dili is the capital of Timor-Leste. Naypyidaw is the capital of Myanmar.",
+    )
+
+    assert model.called is True
+    assert out == "Timor-Leste, Myanmar"
+
+
+def test_answer_contract_rejects_unsupported_repair():
+    from lilith_agent.runner import _apply_answer_contract
+
+    model = _FakeContractModel('{"status":"repair","submitted_answer":"Indonesia, Myanmar"}')
+
+    out = _apply_answer_contract(
+        model,
+        "What countries have the capitals Dili and Naypyidaw?",
+        "Dili, Naypyidaw",
+        "Dili is a capital city. Naypyidaw is the capital of Myanmar.",
+    )
+
+    assert model.called is True
+    assert out == "Dili, Naypyidaw"
+
+
+def test_answer_contract_skips_unambiguous_scalar_answer():
+    from lilith_agent.runner import _apply_answer_contract
+
+    out = _apply_answer_contract(
+        _RaiseIfContractCalled(),
+        "What is 6*7?",
+        "42",
+        "",
+    )
+
+    assert out == "42"
+
+
+def test_answer_contract_skips_generic_which_question_without_type_marker():
+    from lilith_agent.runner import _apply_answer_contract
+
+    model = _FakeContractModel('{"status":"ok"}')
+
+    out = _apply_answer_contract(
+        model,
+        "Which mountain is the tallest?",
+        "Mount Everest",
+        "Mount Everest is the tallest mountain.",
+    )
+
+    assert model.called is False
+    assert out == "Mount Everest"
+
+
+def test_answer_contract_marker_matching_avoids_word_internal_false_positive():
+    from lilith_agent.runner import _apply_answer_contract
+
+    model = _FakeContractModel('{"status":"ok"}')
+
+    out = _apply_answer_contract(
+        model,
+        "Which candidate won the race?",
+        "Alice",
+        "Alice won the race.",
+    )
+
+    assert model.called is False
+    assert out == "Alice"
+
+
+def test_give_up_recovery_uses_supported_trace_answer():
+    from lilith_agent.runner import _apply_give_up_recovery
+
+    model = _FakeContractModel('{"status":"answer","submitted_answer":"Citations"}')
+
+    out = _apply_give_up_recovery(
+        model,
+        "What is the exact UI label?",
+        "unknown",
+        "Evidence gathered from the page: the exact UI label is Citations.",
+    )
+
+    assert model.called is True
+    assert out == "Citations"
+
+
+def test_give_up_recovery_rejects_unsupported_answer():
+    from lilith_agent.runner import _apply_give_up_recovery
+
+    model = _FakeContractModel('{"status":"answer","submitted_answer":"Downloads"}')
+
+    out = _apply_give_up_recovery(
+        model,
+        "What is the exact UI label?",
+        "unknown",
+        "Evidence gathered from the page: the exact UI label is Citations.",
+    )
+
+    assert model.called is True
+    assert out == "unknown"
+
+
+def test_give_up_recovery_skips_confident_answer():
+    from lilith_agent.runner import _apply_give_up_recovery
+
+    out = _apply_give_up_recovery(
+        _RaiseIfContractCalled(),
+        "What is the exact UI label?",
+        "Citations",
+        "Evidence gathered from the page: the exact UI label is Citations.",
+    )
+
+    assert out == "Citations"
+
+
 def test_runner_applies_gaia_submission_normalizer(tmp_path: Path):
     answers = run_agent_on_questions(
         _GraphReturnsAssignmentAnswer(),
@@ -268,6 +448,38 @@ def test_runner_applies_gaia_submission_normalizer(tmp_path: Path):
     assert checkpoint["submitted_answer"] == "563.9"
 
 
+def test_runner_applies_answer_contract_repair(monkeypatch, tmp_path: Path):
+    model = _FakeContractModel('{"status":"repair","submitted_answer":"Timor-Leste, Myanmar"}')
+    monkeypatch.setattr("lilith_agent.models.get_cheap_model", lambda cfg: model)
+
+    answers = run_agent_on_questions(
+        _GraphReturnsWrongTypeWithEvidence(),
+        [{"task_id": "task-contract", "question": "What countries have the capitals Dili and Naypyidaw?"}],
+        tmp_path,
+    )
+
+    assert model.called is True
+    assert answers == [{"task_id": "task-contract", "submitted_answer": "Timor-Leste, Myanmar"}]
+    checkpoint = json.loads((tmp_path / "task-contract.json").read_text())
+    assert checkpoint["submitted_answer"] == "Timor-Leste, Myanmar"
+
+
+def test_runner_applies_give_up_recovery(monkeypatch, tmp_path: Path):
+    model = _FakeContractModel('{"status":"answer","submitted_answer":"Citations"}')
+    monkeypatch.setattr("lilith_agent.models.get_cheap_model", lambda cfg: model)
+
+    answers = run_agent_on_questions(
+        _GraphReturnsUnknownWithEvidence(),
+        [{"task_id": "task-recovery", "question": "What is the exact UI label?"}],
+        tmp_path,
+    )
+
+    assert model.called is True
+    assert answers == [{"task_id": "task-recovery", "submitted_answer": "Citations"}]
+    checkpoint = json.loads((tmp_path / "task-recovery.json").read_text())
+    assert checkpoint["submitted_answer"] == "Citations"
+
+
 def test_runner_pauses_batch_when_window_trips(monkeypatch, tmp_path: Path):
     pauses = [300, None]
     sleeps = []

tests/test_scoring_client.py

@@ -49,6 +49,30 @@ class ScoringApiClientTests(unittest.TestCase):
         self.assertIn("429", client.last_warning or "")
         dataset_client.get_questions.assert_called_once_with()
 
+    def test_fallback_prints_hf_visible_warning(self) -> None:
+        response = Mock(status_code=503, headers={})
+        error = requests.HTTPError("Service Unavailable", response=response)
+        response.raise_for_status.side_effect = error
+
+        session = Mock()
+        session.get.return_value = response
+        dataset_client = Mock()
+        dataset_client.get_questions.return_value = []
+
+        client = ScoringApiClient(
+            api_url="https://example.com",
+            session=session,
+            dataset_client=dataset_client,
+        )
+
+        with patch("builtins.print") as printed:
+            client.get_questions()
+
+        printed.assert_any_call(
+            "Scoring API unavailable while trying to fetch questions (status=503); falling back to GAIA dataset.",
+            flush=True,
+        )
+
     def test_download_file_falls_back_to_dataset_on_429(self) -> None:
         response = Mock(status_code=429, headers={})
         error = requests.HTTPError("Too Many Requests", response=response)