""" backend/agents/critic.py The Critic Agent — quality assurance through reflection. Responsibilities: 1. Evaluate the final output quality (0–100 score) 2. Check if the task was actually completed 3. Identify gaps, errors, or hallucinations 4. Decide: approve OR request replanning 5. Generate improvement suggestions This implements the Reflection pattern from AI agent research. Key insight: having a separate evaluator LLM dramatically reduces errors compared to self-evaluation by the same model. """ from __future__ import annotations import json import re from datetime import datetime, timezone from langchain_core.messages import SystemMessage, HumanMessage from ..core.llm import get_llm from ..state.graph_state import ( WorkflowState, TaskStatus, AgentRole, make_agent_event ) from ..core.config import get_settings from ..core.logger import get_logger log = get_logger(__name__) CRITIC_SYSTEM = """Evaluate task completion. Score 0-100. Approve if score>=60, else request_replan. You MUST respond with valid JSON only - no extra text, no markdown fences: {"thinking":"one line","score":75,"decision":"approve","critique":"none","suggestions":[]}""" def critic_node(state: WorkflowState) -> WorkflowState: """LangGraph node - runs the Critic agent for reflection.""" settings = get_settings() llm = get_llm("critic", temperature=0.1) log.info("Critic running", task_id=state["task_id"]) # Don't replan indefinitely if state["iteration"] >= settings.max_iterations - 1: log.warning("Max iterations reached, force-approving") return { **state, "status": TaskStatus.COMPLETED, "quality_score": 60.0, "critique": "Approved by iteration limit", "updated_at": datetime.now(timezone.utc).isoformat(), "events": state["events"] + [ make_agent_event(AgentRole.CRITIC, "force_approved", "Approved due to max iteration limit") ], } # Build evaluation context completed_steps = [s for s in state["plan"] if s["status"] == "done"] failed_steps = [s for s in state["plan"] if s["status"] == "failed"] results_summary = {} for step_id, result in state["step_results"].items(): result_str = json.dumps(result, default=str) results_summary[step_id] = result_str[:400] steps_summary = ", ".join(f"[{s['status']}]{s['title']}" for s in state["plan"]) eval_context = ( f"TASK: {state['task']}\n" f"STEPS: {steps_summary}\n" f"FAILED: {len(failed_steps)}\n" f"OUTPUT: {str(state.get('final_output', 'none'))[:600]}" ) try: # No tool calling — plain JSON text is more reliable across providers response = llm.invoke( [SystemMessage(content=CRITIC_SYSTEM), HumanMessage(content=eval_context)], ) tokens = response.usage_metadata.get("total_tokens", 0) if response.usage_metadata else 0 text = response.content or "" # Strip markdown fences and extract JSON stripped = text.strip() for fence in ("```json", "```"): stripped = stripped.removeprefix(fence) stripped = stripped.removesuffix("```").strip() # Find first {...} block in case model adds prose match = re.search(r'\{.*\}', stripped, re.DOTALL) if match: stripped = match.group(0) try: c = json.loads(stripped) except Exception: raise ValueError(f"Critic JSON parse failed. Response: {text[:200]}") score = int(c.get("score", 70)) decision = c.get("decision", "approve") critique = c.get("critique", "") thinking = c.get("thinking", "") log.info("Critic decision", score=score, decision=decision, critique=critique[:100]) needs_replan = (decision == "request_replan") and (state["iteration"] < settings.max_iterations - 2) new_status = TaskStatus.COMPLETED if needs_replan: new_status = TaskStatus.PLANNING return { **state, "status": new_status, "critique": critique, "quality_score": float(score), "needs_replanning": needs_replan, "total_tokens": state["total_tokens"] + tokens, "updated_at": datetime.now(timezone.utc).isoformat(), "new_memories": state["new_memories"] + [ { "content": f"Task '{state['task'][:80]}' completed with score {score}. " f"Key approach: {'; '.join(s['title'] for s in completed_steps[:3])}", "memory_type": "episodic", "importance": min(score / 100, 0.9), "tags": ["task_completion", f"score_{score}"], } ], "events": state["events"] + [ make_agent_event( AgentRole.CRITIC, "critique_complete", f"Score: {score}/100 — Decision: {decision.upper()}. {critique[:150]}", { "score": score, "decision": decision, "completeness": c.get("completeness_score"), "accuracy": c.get("accuracy_score"), "usefulness": c.get("usefulness_score"), "quality": c.get("quality_score"), "suggestions": c.get("suggestions", []), }, ) ], } except Exception as e: log.error("Critic failed", error=str(e)) return { **state, "status": TaskStatus.COMPLETED, "quality_score": 65.0, "critique": f"Critic error — auto-approved: {e}", "updated_at": datetime.now(timezone.utc).isoformat(), "events": state["events"] + [ make_agent_event(AgentRole.CRITIC, "error", f"Critic failed: {e}") ], }