Upload 2 files

app/graph/nodes.py

@@ -25,20 +25,66 @@ Text:
 
 AGREEMENT_THRESHOLD = 0.5  # min confidence gap tolerated before escalation
 
-ADJUDICATOR_PROMPT = """Two independent verification methods disagree on whether \
-the evidence supports a scientific claim. Review both assessments and the evidence, \
-then make the final call yourself. Respond with strict JSON only.
+MAX_AGENT_ITERATIONS = 3  # ReAct-loop safety cap — avoids runaway free-tier usage
+
+AGENT_SYSTEM_PROMPT = """You are resolving a disagreement between two independent \
+methods that assessed whether a piece of evidence supports a scientific claim.
 
 Claim: {claim}
-Evidence: {evidence}
+Evidence currently available: {evidence}
 
-Assessment A (fine-tuned NLI model): {label_a}, confidence {confidence_a}
-Assessment B (zero-shot LLM verifier): {label_b}, confidence {confidence_b}
+Assessment A (fine-tuned NLI model): {label_a} (confidence {confidence_a})
+Assessment B (zero-shot LLM verifier): {label_b} (confidence {confidence_b})
 Assessment B's reasoning: {reasoning_b}
 
-Classify the relationship as exactly one of: SUPPORT, NOT_ENOUGH_INFO, CONTRADICT.
-Respond as JSON: {{"label": "...", "confidence": 0.0-1.0, "reasoning": "one sentence \
-explaining why you sided the way you did"}}"""
+You have tools available to investigate further before concluding. Use them if \
+they would genuinely help; call `conclude` as soon as you have a well-supported \
+answer. Do not call more tools than necessary. If you cannot resolve the \
+disagreement with more evidence, concluding NOT_ENOUGH_INFO is a legitimate, \
+complete answer — it is not a failure, and is preferable to guessing."""
+
+AGENT_TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "retry_with_query",
+            "description": ("Search Semantic Scholar again with a different query, "
+                             "in case the currently available evidence missed the "
+                             "relevant passage. Write your own search query."),
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string", "description": "New search query"}},
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "request_second_opinion",
+            "description": ("Get an independent, blind re-verification of the claim "
+                             "against the current evidence (no prior verdicts shown "
+                             "to it, to avoid anchoring bias)."),
+            "parameters": {"type": "object", "properties": {}},
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "conclude",
+            "description": "Give your final, complete answer. This ends the investigation.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "label": {"type": "string", "enum": ["SUPPORT", "NOT_ENOUGH_INFO", "CONTRADICT"]},
+                    "confidence": {"type": "number"},
+                    "reasoning": {"type": "string"},
+                },
+                "required": ["label", "confidence", "reasoning"],
+            },
+        },
+    },
+]
 
 
 def _llm():
@@ -46,12 +92,14 @@ def _llm():
                      api_key=os.environ.get("GROQ_API_KEY"))
 
 
-def _adjudicator_llm():
-    # Deliberately a different model from the verifier (llm_verifier.py uses
-    # gpt-oss-20b) — the arbiter shouldn't be the same model marking its own
-    # disagreement.
-    return ChatGroq(model="llama-3.3-70b-versatile", temperature=0.0,
-                     api_key=os.environ.get("GROQ_API_KEY"))
+def _planner_llm():
+    # Deliberately a different, actively-supported model from the verifier
+    # (gpt-oss-20b) — a same-model critic tends to rubber-stamp its own kind
+    # of reasoning (Panickssery et al., 2024). llama-3.3-70b-versatile was
+    # deprecated by Groq (announced 2026-06-17); gpt-oss-120b is Groq's own
+    # recommended replacement and remains actively supported.
+    return ChatGroq(model="openai/gpt-oss-120b", temperature=0.0,
+                     api_key=os.environ.get("GROQ_API_KEY")).bind_tools(AGENT_TOOLS)
 
 
 async def extract_claims(state: GraphState) -> GraphState:
@@ -112,8 +160,7 @@ async def verify_dual(state: GraphState) -> GraphState:
                          "source": "llm"},
         "agreement": agree,
         "escalated": False,
-        "escalation_retry_succeeded": None,
-        "adjudicator_reasoning": None,
+        "escalation_trace": None,
         "final_verdict": deberta_verdict if agree else None,
         "attribution": None,
         "resolution_note": None,
@@ -143,8 +190,7 @@ async def handle_unresolved_citation(state: GraphState) -> GraphState:
         "llm_verdict": None,
         "agreement": None,
         "escalated": False,
-        "escalation_retry_succeeded": None,
-        "adjudicator_reasoning": None,
+        "escalation_trace": None,
         "final_verdict": None,
         "attribution": None,
         "resolution_note": ("Citation could not be resolved to usable evidence — "
@@ -161,12 +207,13 @@ def route_after_verify(state: GraphState) -> str:
 
 
 async def escalate(state: GraphState) -> GraphState:
-    """On disagreement: retry with a reformulated query (claim's own keywords
-    instead of the reference string, in case the first-resolved paper's
-    abstract missed the relevant passage). If the verifiers now agree, done.
-    If they still disagree, an adjudicator model (different from the verifier
-    model) makes the final call — the agent always resolves autonomously,
-    no human-in-the-loop required.
+    """On disagreement: a planner model (gpt-oss-120b, distinct from the
+    gpt-oss-20b verifier) autonomously decides how to resolve it -- it can
+    retry retrieval with its own reformulated query, request a blind second
+    opinion, or conclude directly, in whatever order and however many times
+    (up to MAX_AGENT_ITERATIONS) it judges necessary. This is a ReAct-style
+    reason-act-observe loop: the control flow is decided by the model at
+    each step, not by fixed code. See docs/dev_log.md for design rationale.
     """
     idx = state["current_index"]
     cc = state["claims"][idx]
@@ -174,55 +221,106 @@ async def escalate(state: GraphState) -> GraphState:
     last = audits[-1]
     last["escalated"] = True
 
-    # ── Retry: reformulated query on the claim itself, not the reference string ──
-    s2_key = os.environ.get("S2_API_KEY")
-    retry_query = semantic_scholar.extract_query(cc["claim"])
-    retry_paper = await semantic_scholar.search_paper(retry_query, api_key=s2_key)
-
-    llm_res = None
-    if retry_paper and retry_paper.get("abstract"):
-        retry_evidence = retry_paper["abstract"]
-        deberta_res = await claim_client.analyze(cc["claim"], retry_evidence)
-        winner = deberta_res["winner"]
-        llm_res = await llm_verifier.verify(cc["claim"], retry_evidence)
-
-        if winner["label"] == llm_res.get("label"):
-            # Retry resolved the disagreement — adopt the new verdict values,
-            # but do NOT overwrite `agreement` — it must stay False, since it
-            # records the INITIAL disagreement that triggered escalation.
-            # build_report's initial_agreement_rate depends on this staying accurate.
-            last["winner_sentence"] = winner["sentence"]
-            last["attribution_available"] = deberta_res.get("attribution_available", False)
-            last["deberta_verdict"] = {"label": winner["label"],
-                                        "confidence": winner["confidence"], "source": "deberta"}
-            last["llm_verdict"] = {"label": llm_res.get("label"),
-                                    "confidence": llm_res.get("confidence", 0.0), "source": "llm"}
-            last["escalation_retry_succeeded"] = True
-            last["final_verdict"] = last["deberta_verdict"]
-            return {**state, "audits": audits}
+    from langchain_core.messages import SystemMessage, ToolMessage
 
-    # ── Retry didn't resolve it (or found nothing usable) — adjudicate ──
-    last["escalation_retry_succeeded"] = False
-    prompt = ADJUDICATOR_PROMPT.format(
+    current_evidence = cc["evidence_text"] or ""
+    system_prompt = AGENT_SYSTEM_PROMPT.format(
         claim=cc["claim"],
-        evidence=cc["evidence_text"] or "",
+        evidence=current_evidence,
         label_a=last["deberta_verdict"]["label"],
         confidence_a=last["deberta_verdict"]["confidence"],
         label_b=last["llm_verdict"]["label"],
         confidence_b=last["llm_verdict"]["confidence"],
-        reasoning_b=llm_res.get("reasoning", "") if llm_res else "",
+        reasoning_b="",  # llm_verifier.verify doesn't currently surface reasoning to verify_dual
     )
-    response = await _adjudicator_llm().ainvoke(prompt)
-    try:
-        verdict = json.loads(response.content)
-    except json.JSONDecodeError:
-        verdict = {"label": last["deberta_verdict"]["label"], "confidence": 0.0,
-                   "reasoning": "adjudicator_parse_error"}
+    messages = [SystemMessage(content=system_prompt)]
+    trace = []
+    planner = _planner_llm()
+    s2_key = os.environ.get("S2_API_KEY")
+
+    for _ in range(MAX_AGENT_ITERATIONS):
+        try:
+            response = await planner.ainvoke(messages)
+        except Exception as exc:
+            # Groq's tool-call parser can hard-fail (400) if the model blends
+            # free-text reasoning with a malformed tool call, rather than
+            # returning a plain response we could just re-prompt. Observed in
+            # practice after repeated tool failures (e.g. two 429s in a row)
+            # pushed the model toward verbose reasoning instead of a clean
+            # tool call. Treat this the same as "no usable answer yet" rather
+            # than crashing the whole graph.
+            trace.append({"action": "planner_error", "input": None, "observation": str(exc)})
+            break
+
+        messages.append(response)
+
+        if not response.tool_calls:
+            messages.append(SystemMessage(content="Please call the `conclude` tool with your final answer."))
+            continue
+
+        tool_call = response.tool_calls[0]  # one action at a time, per ReAct
+        name, args = tool_call["name"], tool_call["args"]
+
+        if name == "conclude":
+            last["final_verdict"] = {"label": args["label"], "confidence": args.get("confidence", 0.0),
+                                      "source": "agent"}
+            trace.append({"action": "conclude", "input": args, "observation": None})
+            last["escalation_trace"] = trace
+            return {**state, "audits": audits}
 
-    last["adjudicator_reasoning"] = verdict.get("reasoning", "")
-    last["final_verdict"] = {"label": verdict.get("label"),
-                              "confidence": verdict.get("confidence", 0.0),
-                              "source": "adjudicator"}
+        elif name == "retry_with_query":
+            paper = await semantic_scholar.search_paper(args["query"], api_key=s2_key)
+            if paper and paper.get("abstract"):
+                current_evidence = paper["abstract"]
+                deberta_res = await claim_client.analyze(cc["claim"], current_evidence)
+                winner = deberta_res["winner"]
+                new_llm_res = await llm_verifier.verify(cc["claim"], current_evidence)
+                observation = (f"New evidence found: \"{current_evidence[:300]}\". "
+                                f"Re-verified: NLI model says {winner['label']} "
+                                f"({winner['confidence']:.2f}), LLM verifier says "
+                                f"{new_llm_res.get('label')} ({new_llm_res.get('confidence', 0):.2f}).")
+                last["winner_sentence"] = winner["sentence"]
+                last["attribution_available"] = deberta_res.get("attribution_available", False)
+                last["deberta_verdict"] = {"label": winner["label"], "confidence": winner["confidence"], "source": "deberta"}
+                last["llm_verdict"] = {"label": new_llm_res.get("label"), "confidence": new_llm_res.get("confidence", 0.0), "source": "llm"}
+            else:
+                observation = "No usable evidence found for that query (no match or no abstract indexed)."
+            trace.append({"action": "retry_with_query", "input": args, "observation": observation})
+            messages.append(ToolMessage(content=observation, tool_call_id=tool_call["id"]))
+
+        elif name == "request_second_opinion":
+            second = await llm_verifier.verify(cc["claim"], current_evidence)  # blind -- no prior verdicts in this call
+            observation = (f"Second opinion (blind, independent): {second.get('label')} "
+                            f"(confidence {second.get('confidence', 0):.2f}). "
+                            f"Reasoning: {second.get('reasoning', '')}")
+            trace.append({"action": "request_second_opinion", "input": {}, "observation": observation})
+            messages.append(ToolMessage(content=observation, tool_call_id=tool_call["id"]))
+
+    # Iteration cap hit (or planner errored above) without a `conclude` call --
+    # force a final answer rather than leaving the claim unresolved. Per
+    # ReAct/Reflexion literature, a calibrated NOT_ENOUGH_INFO is a legitimate
+    # complete answer, not a failure.
+    messages.append(SystemMessage(content="You must conclude now with your best answer, "
+                                           "using only the `conclude` tool."))
+    try:
+        final_response = await planner.ainvoke(messages)
+    except Exception as exc:
+        trace.append({"action": "planner_error_on_forced_conclude", "input": None, "observation": str(exc)})
+        final_response = None
+
+    if final_response and final_response.tool_calls and final_response.tool_calls[0]["name"] == "conclude":
+        args = final_response.tool_calls[0]["args"]
+        last["final_verdict"] = {"label": args["label"], "confidence": args.get("confidence", 0.0), "source": "agent"}
+        trace.append({"action": "conclude (forced at iteration cap)", "input": args, "observation": None})
+    else:
+        # Model never produced a valid conclude call, either it declined to
+        # or Groq's parser failed on it. Fall back to NOT_ENOUGH_INFO rather
+        # than guess, consistent with the "refusal over fabrication" principle,
+        # and rather than crash the request.
+        last["final_verdict"] = {"label": "NOT_ENOUGH_INFO", "confidence": 0.0, "source": "agent_fallback"}
+        trace.append({"action": "forced_fallback", "input": None, "observation": "Model did not call conclude within iteration cap."})
+
+    last["escalation_trace"] = trace
     return {**state, "audits": audits}
 
 
@@ -232,8 +330,8 @@ async def explain(state: GraphState) -> GraphState:
     audits = state["audits"]
     last = audits[-1]
     if last["final_verdict"] is None:
-        # Should not normally happen — verify_dual sets it on agreement,
-        # escalate sets it via retry-success or adjudicator. Fallback just in case.
+        # Should not normally happen -- verify_dual sets it on agreement,
+        # escalate always sets it via conclude/forced-conclude/fallback.
         last["final_verdict"] = last["deberta_verdict"]
 
     if last["attribution_available"]:
@@ -262,9 +360,10 @@ async def build_report(state: GraphState) -> GraphState:
     verified = [a for a in audits if a["agreement"] is not None]  # excludes unresolved citations
     agreement_rate = (sum(a["agreement"] for a in verified) / len(verified)) if verified else 0.0
     n_escalated = sum(a["escalated"] for a in audits)
-    n_retry_resolved = sum(bool(a.get("escalation_retry_succeeded")) for a in audits)
-    n_adjudicated = sum(
-        1 for a in audits if a["escalated"] and not a.get("escalation_retry_succeeded")
+    escalated_audits = [a for a in audits if a["escalated"]]
+    avg_agent_iterations = (
+        sum(len(a.get("escalation_trace") or []) for a in escalated_audits) / len(escalated_audits)
+        if escalated_audits else 0.0
     )
     n_unresolved = sum(1 for a in audits if a.get("resolution_note"))
     report = {
@@ -272,8 +371,7 @@ async def build_report(state: GraphState) -> GraphState:
         "n_unresolved_citations": n_unresolved,
         "initial_agreement_rate": agreement_rate,  # computed over resolved claims only
         "n_escalated": n_escalated,
-        "n_resolved_by_retry": n_retry_resolved,
-        "n_resolved_by_adjudicator": n_adjudicated,
+        "avg_agent_iterations_when_escalated": avg_agent_iterations,
         "claims": audits,
     }
     return {**state, "report": report}

app/graph/state.py

@@ -25,8 +25,7 @@ class ClaimAudit(TypedDict):
     llm_verdict: Optional[Verdict]
     agreement: Optional[bool]
     escalated: bool
-    escalation_retry_succeeded: Optional[bool]  # True if re-retrieval resolved disagreement
-    adjudicator_reasoning: Optional[str]        # set only if adjudicator step ran
+    escalation_trace: Optional[list]      # [{"action": ..., "input": ..., "observation": ...}, ...]
     final_verdict: Optional[Verdict]
     attribution: Optional[list]           # [{"token": ..., "score": ...}, ...] or None
     resolution_note: Optional[str]        # set when citation could not be resolved to evidence