feat: auto-resume orphaned workflows on server restart + cancel button

Durable restart (requirement #2):
- On startup, queries all RUNNING workflows from DB
- For each, creates WorkflowExecutor and calls resume()
- LangGraph loads checkpoint from PostgreSQL, continues from last node
- If resume fails, marks workflow as FAILED
- Waits 5s after startup for DB pool initialization

Auto-detect stuck workflows:
- Any workflow RUNNING for >5 min with no progress checkpoint
auto-marks as FAILED when polled by frontend

Cancel button:
- POST /workflows/{id}/cancel endpoint marks as FAILED
- Red Cancel button on WorkflowDetail for RUNNING/PENDING workflows
- Immediate escape hatch without waiting for timeout

Fix workspace delete:
- Navigate first, refresh after (prevents double-click issue)

Token capture logging:
- Added explicit INFO/WARNING logs to diagnose token tracking

backend/app/api/workflows.py

@@ -134,6 +134,27 @@ def get_workflow_by_document_version(
     if workspace is None:
         raise HTTPException(status_code=403, detail="You do not have access to this workflow.")
 
+    # Auto-detect stuck workflows: if RUNNING for >5 min with no progress, mark as FAILED
+    if workflow.status == WorkflowStatus.RUNNING:
+        from datetime import datetime, timezone, timedelta
+        now = datetime.now(timezone.utc)
+        started = workflow.started_at.replace(tzinfo=timezone.utc) if workflow.started_at.tzinfo is None else workflow.started_at
+        if now - started > timedelta(minutes=5):
+            # Check if there's been recent progress
+            latest_checkpoint = None
+            for cp in reversed(workflow.checkpoints or []):
+                if cp.agent_name == "PROGRESS":
+                    latest_checkpoint = cp
+                    break
+            stale = True
+            if latest_checkpoint and latest_checkpoint.created_at:
+                cp_time = latest_checkpoint.created_at.replace(tzinfo=timezone.utc) if latest_checkpoint.created_at.tzinfo is None else latest_checkpoint.created_at
+                if now - cp_time < timedelta(minutes=5):
+                    stale = False
+            if stale:
+                workflow.status = WorkflowStatus.FAILED
+                db.commit()
+
     return _workflow_response(workflow)
 
 
@@ -170,4 +191,58 @@ def get_workflow(
             detail="You do not have access to this workflow.",
         )
 
+    # Auto-detect stuck workflows: if RUNNING for >5 min with no progress, mark as FAILED
+    if workflow.status == WorkflowStatus.RUNNING:
+        from datetime import datetime, timezone, timedelta
+        now = datetime.now(timezone.utc)
+        started = workflow.started_at.replace(tzinfo=timezone.utc) if workflow.started_at.tzinfo is None else workflow.started_at
+        if now - started > timedelta(minutes=5):
+            # Check if there's been recent progress
+            latest_checkpoint = None
+            for cp in reversed(workflow.checkpoints or []):
+                if cp.agent_name == "PROGRESS":
+                    latest_checkpoint = cp
+                    break
+            stale = True
+            if latest_checkpoint and latest_checkpoint.created_at:
+                cp_time = latest_checkpoint.created_at.replace(tzinfo=timezone.utc) if latest_checkpoint.created_at.tzinfo is None else latest_checkpoint.created_at
+                if now - cp_time < timedelta(minutes=5):
+                    stale = False
+            if stale:
+                workflow.status = WorkflowStatus.FAILED
+                db.commit()
+
     return _workflow_response(workflow)
+
+
+@router.post("/{workflow_id}/cancel", status_code=200)
+def cancel_workflow(
+    workflow_id: UUID,
+    current_user: User = Depends(get_current_user),
+    db: Session = Depends(get_db),
+):
+    """Mark a stuck/running workflow as FAILED so it stops polling."""
+    workflow = (
+        db.query(WorkflowRun)
+        .filter(WorkflowRun.id == workflow_id)
+        .first()
+    )
+
+    if workflow is None:
+        raise HTTPException(status_code=404, detail="Workflow not found.")
+
+    workspace = (
+        db.query(Workspace)
+        .filter(Workspace.id == workflow.workspace_id, Workspace.created_by == current_user.id)
+        .first()
+    )
+    if workspace is None:
+        raise HTTPException(status_code=403, detail="You do not have access to this workflow.")
+
+    if workflow.status in (WorkflowStatus.COMPLETED, WorkflowStatus.CANCELLED):
+        return {"id": str(workflow.id), "status": workflow.status.value, "message": "Already terminal."}
+
+    workflow.status = WorkflowStatus.FAILED
+    db.commit()
+
+    return {"id": str(workflow.id), "status": "FAILED", "message": "Workflow marked as failed."}

backend/app/main.py

@@ -26,6 +26,7 @@ def preload_models():
     """
     Pre-load heavy ML models at startup so the first document
     upload doesn't pay the cold-start penalty.
+    Also resumes any orphaned RUNNING workflows from before a crash.
     """
     import threading
 
@@ -43,8 +44,54 @@ def preload_models():
         except Exception as e:
             logger.warning("Model pre-load failed (non-fatal): %s", e)
 
-    # Load in background thread so server starts accepting requests immediately
+    def _resume_orphaned_workflows():
+        """
+        Resume workflows that were RUNNING when the server died.
+        LangGraph checkpoints after every node, so we can resume
+        from the last completed stage.
+        """
+        import time
+        time.sleep(5)  # Wait for DB pool to be ready
+
+        try:
+            from app.database.database import SessionLocal
+            from app.models.workflow_run import WorkflowRun, WorkflowStatus
+            from app.workflow import WorkflowExecutor
+
+            db = SessionLocal()
+            try:
+                orphaned = (
+                    db.query(WorkflowRun)
+                    .filter(WorkflowRun.status == WorkflowStatus.RUNNING)
+                    .all()
+                )
+
+                if not orphaned:
+                    return
+
+                logger.info("Found %d orphaned RUNNING workflow(s). Resuming...", len(orphaned))
+
+                for workflow in orphaned:
+                    try:
+                        logger.info("  Resuming workflow %s...", workflow.id)
+                        executor = WorkflowExecutor(db, interrupt_before=["human_review"])
+                        executor.resume(workflow)
+                        executor.close()
+                        logger.info("  Workflow %s resumed successfully (status: %s)", workflow.id, workflow.status.value)
+                    except Exception as e:
+                        logger.warning("  Could not resume workflow %s: %s", workflow.id, str(e)[:100])
+                        # Mark as failed if we can't resume
+                        workflow.status = WorkflowStatus.FAILED
+                        db.commit()
+            finally:
+                db.close()
+        except Exception as e:
+            logger.warning("Orphaned workflow resume failed: %s", e)
+
+    # Load model in background
     threading.Thread(target=_load, daemon=True).start()
+    # Resume orphaned workflows in background
+    threading.Thread(target=_resume_orphaned_workflows, daemon=True).start()
 
 
 app.add_middleware(

backend/app/workflow/nodes/knowledge.py

@@ -165,6 +165,8 @@ def knowledge(
             return state
 
     # Extract token usage from raw LLM response
+    import logging
+    _log = logging.getLogger(__name__)
     try:
         if raw_response and hasattr(raw_response, "response_metadata"):
             meta = raw_response.response_metadata or {}
@@ -177,23 +179,20 @@ def knowledge(
                     input_tokens=input_tokens,
                     output_tokens=output_tokens,
                 )
+                _log.info("Tokens captured: in=%d out=%d", input_tokens, output_tokens)
             else:
-                import logging
-                logging.getLogger(__name__).warning("Token usage was 0. Meta: %s", meta)
+                _log.warning("Token usage was 0. Meta keys: %s, usage: %s", list(meta.keys()), usage)
         elif raw_response and hasattr(raw_response, "usage_metadata"):
             um = raw_response.usage_metadata
             if um:
-                tracker.record_llm_usage(
-                    "knowledge_extraction",
-                    input_tokens=getattr(um, "input_tokens", 0) or 0,
-                    output_tokens=getattr(um, "output_tokens", 0) or 0,
-                )
+                in_t = getattr(um, "input_tokens", 0) or 0
+                out_t = getattr(um, "output_tokens", 0) or 0
+                tracker.record_llm_usage("knowledge_extraction", input_tokens=in_t, output_tokens=out_t)
+                _log.info("Tokens captured via usage_metadata: in=%d out=%d", in_t, out_t)
         else:
-            import logging
-            logging.getLogger(__name__).warning("No raw_response or no response_metadata. raw_response type: %s", type(raw_response))
+            _log.warning("No raw_response or no response_metadata. raw_response=%s type=%s", raw_response is not None, type(raw_response))
     except Exception as e:
-        import logging
-        logging.getLogger(__name__).warning("Token capture exception: %s", e)
+        _log.warning("Token capture exception: %s", e)
 
     tracker.end_stage("knowledge_extraction")
 

frontend/src/pages/Settings.jsx

@@ -426,8 +426,8 @@ export default function Settings() {
                                 onClick={async () => {
                                     if (!confirm(`Delete workspace "${workspace.name}"?\n\nThis action cannot be undone. All documents, knowledge, and workflows will become inaccessible.`)) return;
                                     await deleteWorkspace(workspace.id);
-                                    await refresh();
                                     navigate("/");
+                                    refresh();
                                 }}
                             >
                                 Delete workspace

frontend/src/pages/WorkflowDetail.jsx

@@ -3,11 +3,12 @@ import { useParams, useNavigate } from "react-router-dom";
 import { getWorkflow } from "../api/workflows";
 import { getWorkflowMetrics } from "../api/metrics";
 import { listPendingProposals } from "../api/proposals";
+import { apiRequest } from "../api/client";
 import { WorkflowStatusBadge } from "../components/workflow/WorkflowStatusBadge";
 import { WorkflowPipeline } from "../components/workflow/WorkflowPipeline";
 import { WorkflowTimeline } from "../components/workflow/WorkflowTimeline";
 import { ProposalReviewList } from "../components/proposals/ProposalReviewList";
-import { Card, CardHeader, CardBody, LoadingState, EmptyState, Glossary } from "../components/ui";
+import { Card, CardHeader, CardBody, Button, LoadingState, EmptyState, Glossary } from "../components/ui";
 import { GLOSSARY } from "../utils/labels";
 import "./WorkflowDetail.css";
 
@@ -88,7 +89,19 @@ export default function WorkflowDetail() {
             <Card>
                 <CardHeader
                     title="Workflow status"
-                    action={<WorkflowStatusBadge status={workflow.status} />}
+                    action={
+                        <span style={{ display: "flex", alignItems: "center", gap: "var(--space-3)" }}>
+                            <WorkflowStatusBadge status={workflow.status} />
+                            {(workflow.status === "RUNNING" || workflow.status === "PENDING") && (
+                                <Button size="sm" variant="danger" onClick={async () => {
+                                    await apiRequest(`/workflows/${workflowId}/cancel`, "POST");
+                                    fetchWorkflow();
+                                }}>
+                                    Cancel
+                                </Button>
+                            )}
+                        </span>
+                    }
                 />
                 <CardBody>
                     <WorkflowPipeline