    return {"gpus": get_gpu_metrics()}

# ---------------- Scripts listing ----------------
@app.get("/api/scripts/list")
async def scripts_list() -> Dict[str, Any]:
    root = WORKSPACE_DIR / "scripts"
    items: List[Dict[str, Any]] = []
    if root.exists():
        for p in sorted(root.rglob("*.py")):
            if p.is_file() and not p.name.startswith('.'):
                try:
                    items.append({
                        "path": str(p.relative_to(WORKSPACE_DIR)),
                        "size": p.stat().st_size
                    })
                except Exception:
                    continue
    return {"scripts": items}

@app.get("/api/data/tree")
async def get_data_tree():
    """Get data directory structure"""
    data_dir = WORKSPACE_DIR / "data"

    def build_tree(path: Path) -> Union[Dict, str]:
        if path.is_file():
            return str(path.relative_to(WORKSPACE_DIR))

        result = {}
        for item in path.iterdir():
            if item.name.startswith('.'):
                continue
            result[item.name] = build_tree(item)
        return result

    return build_tree(data_dir)

@app.get("/api/data/preview")
async def preview_file(path: str):
    """Preview file contents"""
    full_path = WORKSPACE_DIR / path

    # Basic security check
    if not str(full_path).startswith(str(WORKSPACE_DIR)):
        raise HTTPException(status_code=403, detail="Access denied")

    try:
        with open(full_path, 'r') as f:
            # Read first 100KB for preview
            content = f.read(102400)
            return {"content": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# ---------------- Sandbox ----------------
@app.get("/api/sandbox/world")


@app.get("/api/data/tree")
async def get_data_tree():
    """Get data directory structure"""
    data_dir = WORKSPACE_DIR / "data"

    def build_tree(path: Path) -> Union[Dict, str]:
        if path.is_file():
            return str(path.relative_to(WORKSPACE_DIR))

        result = {}
        for item in path.iterdir():
            if item.name.startswith('.'):
                continue
            result[item.name] = build_tree(item)
        return result

    return build_tree(data_dir)

@app.get("/api/data/preview")
async def preview_file(path: str):
    """Preview file contents"""
    full_path = WORKSPACE_DIR / path

    # Basic security check
    if not str(full_path).startswith(str(WORKSPACE_DIR)):
        raise HTTPException(status_code=403, detail="Access denied")

    try:
        with open(full_path, 'r') as f:
            # Read first 100KB for preview
            content = f.read(102400)
            return {"content": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# ---------------- Sandbox ----------------
@app.get("/api/sandbox/world")
async def sandbox_world() -> Dict[str, Any]:
    p = WORKSPACE_DIR / "data" / "sandbox" / "tmp" / "world.json"
    if not p.exists():
        raise HTTPException(status_code=404, detail="world.json not found")
    try:
        return json.loads(p.read_text(encoding="utf-8"))
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/sandbox/snapshot")
async def sandbox_snapshot(background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    # trigger tools.world-snapshot via make
    req = TaskRequest(task="tools.world-snapshot")
    task_id = (await start_task(req, background_tasks, database)).get("task_id")
    return {"task_id": task_id}

# ============ Benchmarks: runs discovery & detail ============


@app.get("/api/data/preview")
async def preview_file(path: str):
    """Preview file contents"""
    full_path = WORKSPACE_DIR / path

    # Basic security check
    if not str(full_path).startswith(str(WORKSPACE_DIR)):
        raise HTTPException(status_code=403, detail="Access denied")

    try:
        with open(full_path, 'r') as f:
            # Read first 100KB for preview
            content = f.read(102400)
            return {"content": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# ---------------- Sandbox ----------------
@app.get("/api/sandbox/world")
async def sandbox_world() -> Dict[str, Any]:
    p = WORKSPACE_DIR / "data" / "sandbox" / "tmp" / "world.json"
    if not p.exists():
        raise HTTPException(status_code=404, detail="world.json not found")
    try:
        return json.loads(p.read_text(encoding="utf-8"))
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/sandbox/snapshot")
async def sandbox_snapshot(background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    # trigger tools.world-snapshot via make
    req = TaskRequest(task="tools.world-snapshot")
    task_id = (await start_task(req, background_tasks, database)).get("task_id")
    return {"task_id": task_id}

# ============ Benchmarks: runs discovery & detail ============

def _runs_dir_for_suite(suite: str) -> Path:
    return WORKSPACE_DIR / "data" / "bench" / "runs" / suite

def _read_json_file(p: Path) -> Any:
    try:
        with p.open("r", encoding="utf-8") as f:
            return json.load(f)
    except Exception as e:
        return {"error": str(e)}


    try:
        with open(full_path, 'r') as f:
            # Read first 100KB for preview
            content = f.read(102400)
            return {"content": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# ---------------- Sandbox ----------------
@app.get("/api/sandbox/world")
async def sandbox_world() -> Dict[str, Any]:
    p = WORKSPACE_DIR / "data" / "sandbox" / "tmp" / "world.json"
    if not p.exists():
        raise HTTPException(status_code=404, detail="world.json not found")
    try:
        return json.loads(p.read_text(encoding="utf-8"))
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/sandbox/snapshot")
async def sandbox_snapshot(background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    # trigger tools.world-snapshot via make
    req = TaskRequest(task="tools.world-snapshot")
    task_id = (await start_task(req, background_tasks, database)).get("task_id")
    return {"task_id": task_id}

# ============ Benchmarks: runs discovery & detail ============

def _runs_dir_for_suite(suite: str) -> Path:
    return WORKSPACE_DIR / "data" / "bench" / "runs" / suite

def _read_json_file(p: Path) -> Any:
    try:
        with p.open("r", encoding="utf-8") as f:
            return json.load(f)
    except Exception as e:
        return {"error": str(e)}

@app.get("/api/bench/suites")
async def bench_suites() -> Dict[str, Any]:
    # derive from registry.json and dirs under runs/
    suites: List[str] = []
    try:
        reg = json.loads((WORKSPACE_DIR / "bench" / "registry.json").read_text(encoding="utf-8"))
        suites = list((reg.get("suites", {}) or {}).keys())


# ---------------- Sandbox ----------------
@app.get("/api/sandbox/world")
async def sandbox_world() -> Dict[str, Any]:
    p = WORKSPACE_DIR / "data" / "sandbox" / "tmp" / "world.json"
    if not p.exists():
        raise HTTPException(status_code=404, detail="world.json not found")
    try:
        return json.loads(p.read_text(encoding="utf-8"))
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/sandbox/snapshot")
async def sandbox_snapshot(background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    # trigger tools.world-snapshot via make
    req = TaskRequest(task="tools.world-snapshot")
    task_id = (await start_task(req, background_tasks, database)).get("task_id")
    return {"task_id": task_id}

# ============ Benchmarks: runs discovery & detail ============

def _runs_dir_for_suite(suite: str) -> Path:
    return WORKSPACE_DIR / "data" / "bench" / "runs" / suite

def _read_json_file(p: Path) -> Any:
    try:
        with p.open("r", encoding="utf-8") as f:
            return json.load(f)
    except Exception as e:
        return {"error": str(e)}

@app.get("/api/bench/suites")
async def bench_suites() -> Dict[str, Any]:
    # derive from registry.json and dirs under runs/
    suites: List[str] = []
    try:
        reg = json.loads((WORKSPACE_DIR / "bench" / "registry.json").read_text(encoding="utf-8"))
        suites = list((reg.get("suites", {}) or {}).keys())
    except Exception:
        suites = []
    # also include any present under data/bench/runs
    base = WORKSPACE_DIR / "data" / "bench" / "runs"
    if base.exists():


    p = WORKSPACE_DIR / "data" / "sandbox" / "tmp" / "world.json"
    if not p.exists():
        raise HTTPException(status_code=404, detail="world.json not found")
    try:
        return json.loads(p.read_text(encoding="utf-8"))
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/sandbox/snapshot")
async def sandbox_snapshot(background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    # trigger tools.world-snapshot via make
    req = TaskRequest(task="tools.world-snapshot")
    task_id = (await start_task(req, background_tasks, database)).get("task_id")
    return {"task_id": task_id}

# ============ Benchmarks: runs discovery & detail ============

def _runs_dir_for_suite(suite: str) -> Path:
    return WORKSPACE_DIR / "data" / "bench" / "runs" / suite

def _read_json_file(p: Path) -> Any:
    try:
        with p.open("r", encoding="utf-8") as f:
            return json.load(f)
    except Exception as e:
        return {"error": str(e)}

@app.get("/api/bench/suites")
async def bench_suites() -> Dict[str, Any]:
    # derive from registry.json and dirs under runs/
    suites: List[str] = []
    try:
        reg = json.loads((WORKSPACE_DIR / "bench" / "registry.json").read_text(encoding="utf-8"))
        suites = list((reg.get("suites", {}) or {}).keys())
    except Exception:
        suites = []
    # also include any present under data/bench/runs
    base = WORKSPACE_DIR / "data" / "bench" / "runs"
    if base.exists():
        for p in base.iterdir():
            if p.is_dir():
                name = p.name
                if name not in suites:
                    suites.append(name)
    return {"suites": sorted(suites)}


@app.post("/api/sandbox/snapshot")
async def sandbox_snapshot(background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    # trigger tools.world-snapshot via make
    req = TaskRequest(task="tools.world-snapshot")
    task_id = (await start_task(req, background_tasks, database)).get("task_id")
    return {"task_id": task_id}

# ============ Benchmarks: runs discovery & detail ============

def _runs_dir_for_suite(suite: str) -> Path:
    return WORKSPACE_DIR / "data" / "bench" / "runs" / suite

def _read_json_file(p: Path) -> Any:
    try:
        with p.open("r", encoding="utf-8") as f:
            return json.load(f)
    except Exception as e:
        return {"error": str(e)}

@app.get("/api/bench/suites")
async def bench_suites() -> Dict[str, Any]:
    # derive from registry.json and dirs under runs/
    suites: List[str] = []
    try:
        reg = json.loads((WORKSPACE_DIR / "bench" / "registry.json").read_text(encoding="utf-8"))
        suites = list((reg.get("suites", {}) or {}).keys())
    except Exception:
        suites = []
    # also include any present under data/bench/runs
    base = WORKSPACE_DIR / "data" / "bench" / "runs"
    if base.exists():
        for p in base.iterdir():
            if p.is_dir():
                name = p.name
                if name not in suites:
                    suites.append(name)
    return {"suites": sorted(suites)}

@app.post("/api/bench/run")
async def bench_run(payload: Dict[str, Any], background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    suite = str(payload.get("suite", "")).strip()
    args = payload.get("args", {})
    stop_existing = bool(payload.get("stop_existing", False))
    if not suite:
        raise HTTPException(status_code=400, detail="suite required")
    # Optionally stop existing bench tasks for this suite
    if stop_existing:
        try:
            for tid, t in list(active_tasks.items()):
                name = str(t.get("name", ""))
                if name.startswith("bench.") and (suite == "all" or suite in name) and t.get("status") in ("running", "queued"):
                    try:
                        await stop_task(tid)  # type: ignore[arg-type]
                    except Exception:
                        pass
        except Exception:
            pass


@app.get("/api/bench/suites")
async def bench_suites() -> Dict[str, Any]:
    # derive from registry.json and dirs under runs/
    suites: List[str] = []
    try:
        reg = json.loads((WORKSPACE_DIR / "bench" / "registry.json").read_text(encoding="utf-8"))
        suites = list((reg.get("suites", {}) or {}).keys())
    except Exception:
        suites = []
    # also include any present under data/bench/runs
    base = WORKSPACE_DIR / "data" / "bench" / "runs"
    if base.exists():
        for p in base.iterdir():
            if p.is_dir():
                name = p.name
                if name not in suites:
                    suites.append(name)
    return {"suites": sorted(suites)}

@app.post("/api/bench/run")
async def bench_run(payload: Dict[str, Any], background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    suite = str(payload.get("suite", "")).strip()
    args = payload.get("args", {})
    stop_existing = bool(payload.get("stop_existing", False))
    if not suite:
        raise HTTPException(status_code=400, detail="suite required")
    # Optionally stop existing bench tasks for this suite
    if stop_existing:
        try:
            for tid, t in list(active_tasks.items()):
                name = str(t.get("name", ""))
                if name.startswith("bench.") and (suite == "all" or suite in name) and t.get("status") in ("running", "queued"):
                    try:
                        await stop_task(tid)  # type: ignore[arg-type]
                    except Exception:
                        pass
        except Exception:
            pass
    # Map to make targets
    target = "bench.run.all" if suite == "all" else f"bench.run.{suite}"
    req = TaskRequest(task=target, args=args or None)
    return await start_task(req, background_tasks, database)

@app.get("/api/bench/runs")
async def list_bench_runs(suite: str, limit: int = 200) -> List[Dict[str, Any]]:
    """List available benchmark runs for a suite, newest first."""
    runs_dir = _runs_dir_for_suite(suite)
    if not runs_dir.exists():
        return []
    items: List[Dict[str, Any]] = []
    # Accept both timestamp dirs and *.postrepair dirs
    for sub in sorted(runs_dir.iterdir() if runs_dir.exists() else [], key=lambda p: p.name, reverse=True):
        if not sub.is_dir():
            continue
        run_json = sub / "run.json"
        if run_json.exists():
            obj = _read_json_file(run_json)
            # minimal projection
            items.append({
                "suite": suite,
                "dir": sub.name,
                "started_ts": obj.get("started_ts"),


    if not suite:
        raise HTTPException(status_code=400, detail="suite required")
    # Optionally stop existing bench tasks for this suite
    if stop_existing:
        try:
            for tid, t in list(active_tasks.items()):
                name = str(t.get("name", ""))
                if name.startswith("bench.") and (suite == "all" or suite in name) and t.get("status") in ("running", "queued"):
                    try:
                        await stop_task(tid)  # type: ignore[arg-type]
                    except Exception:
                        pass
        except Exception:
            pass
    # Map to make targets
    target = "bench.run.all" if suite == "all" else f"bench.run.{suite}"
    req = TaskRequest(task=target, args=args or None)
    return await start_task(req, background_tasks, database)

@app.get("/api/bench/runs")
async def list_bench_runs(suite: str, limit: int = 200) -> List[Dict[str, Any]]:
    """List available benchmark runs for a suite, newest first."""
    runs_dir = _runs_dir_for_suite(suite)
    if not runs_dir.exists():
        return []
    items: List[Dict[str, Any]] = []
    # Accept both timestamp dirs and *.postrepair dirs
    for sub in sorted(runs_dir.iterdir() if runs_dir.exists() else [], key=lambda p: p.name, reverse=True):
        if not sub.is_dir():
            continue
        run_json = sub / "run.json"
        if run_json.exists():
            obj = _read_json_file(run_json)
            # minimal projection
            items.append({
                "suite": suite,
                "dir": sub.name,
                "started_ts": obj.get("started_ts"),
                "ended_ts": obj.get("ended_ts"),
                "model": obj.get("model"),
                "metrics": obj.get("metrics", {}),
            })
    return items[:max(0, int(limit))]

@app.get("/api/bench/run_detail")
async def bench_run_detail(suite: str, run_dir: str) -> Dict[str, Any]:
    """Return the full run.json payload for a given run directory name."""
    rdir = _runs_dir_for_suite(suite) / run_dir
    run_json = rdir / "run.json"
    if not run_json.exists():
        raise HTTPException(status_code=404, detail="run.json not found")
    return _read_json_file(run_json)

@app.get("/api/bench/run_tasks")
async def bench_run_tasks(suite: str, run_dir: str) -> Dict[str, Any]:
    """Return per-task pass flags for a given run by reading the 'out' file referenced in run.json."""
    rdir = _runs_dir_for_suite(suite) / run_dir
    run_json = rdir / "run.json"
    if not run_json.exists():
        raise HTTPException(status_code=404, detail="run.json not found")
    meta = _read_json_file(run_json)
    paths = meta.get("paths", {}) if isinstance(meta, dict) else {}
    out_path = paths.get("out")


    items: List[Dict[str, Any]] = []
    # Accept both timestamp dirs and *.postrepair dirs
    for sub in sorted(runs_dir.iterdir() if runs_dir.exists() else [], key=lambda p: p.name, reverse=True):
        if not sub.is_dir():
            continue
        run_json = sub / "run.json"
        if run_json.exists():
            obj = _read_json_file(run_json)
            # minimal projection
            items.append({
                "suite": suite,
                "dir": sub.name,
                "started_ts": obj.get("started_ts"),
                "ended_ts": obj.get("ended_ts"),
                "model": obj.get("model"),
                "metrics": obj.get("metrics", {}),
            })
    return items[:max(0, int(limit))]

@app.get("/api/bench/run_detail")
async def bench_run_detail(suite: str, run_dir: str) -> Dict[str, Any]:
    """Return the full run.json payload for a given run directory name."""
    rdir = _runs_dir_for_suite(suite) / run_dir
    run_json = rdir / "run.json"
    if not run_json.exists():
        raise HTTPException(status_code=404, detail="run.json not found")
    return _read_json_file(run_json)

@app.get("/api/bench/run_tasks")
async def bench_run_tasks(suite: str, run_dir: str) -> Dict[str, Any]:
    """Return per-task pass flags for a given run by reading the 'out' file referenced in run.json."""
    rdir = _runs_dir_for_suite(suite) / run_dir
    run_json = rdir / "run.json"
    if not run_json.exists():
        raise HTTPException(status_code=404, detail="run.json not found")
    meta = _read_json_file(run_json)
    paths = meta.get("paths", {}) if isinstance(meta, dict) else {}
    out_path = paths.get("out")
    results_path = paths.get("results")
    tasks: List[Dict[str, Any]] = []
    # Prefer the 'out' JSONL which lists per-task pass1
    if isinstance(out_path, str) and out_path:
        p = WORKSPACE_DIR / out_path
        if p.exists():
            try:
                with p.open("r", encoding="utf-8") as f:
                    for line in f:


            items.append({
                "suite": suite,
                "dir": sub.name,
                "started_ts": obj.get("started_ts"),
                "ended_ts": obj.get("ended_ts"),
                "model": obj.get("model"),
                "metrics": obj.get("metrics", {}),
            })
    return items[:max(0, int(limit))]

@app.get("/api/bench/run_detail")
async def bench_run_detail(suite: str, run_dir: str) -> Dict[str, Any]:
    """Return the full run.json payload for a given run directory name."""
    rdir = _runs_dir_for_suite(suite) / run_dir
    run_json = rdir / "run.json"
    if not run_json.exists():
        raise HTTPException(status_code=404, detail="run.json not found")
    return _read_json_file(run_json)

@app.get("/api/bench/run_tasks")
async def bench_run_tasks(suite: str, run_dir: str) -> Dict[str, Any]:
    """Return per-task pass flags for a given run by reading the 'out' file referenced in run.json."""
    rdir = _runs_dir_for_suite(suite) / run_dir
    run_json = rdir / "run.json"
    if not run_json.exists():
        raise HTTPException(status_code=404, detail="run.json not found")
    meta = _read_json_file(run_json)
    paths = meta.get("paths", {}) if isinstance(meta, dict) else {}
    out_path = paths.get("out")
    results_path = paths.get("results")
    tasks: List[Dict[str, Any]] = []
    # Prefer the 'out' JSONL which lists per-task pass1
    if isinstance(out_path, str) and out_path:
        p = WORKSPACE_DIR / out_path
        if p.exists():
            try:
                with p.open("r", encoding="utf-8") as f:
                    for line in f:
                        try:
                            obj = json.loads(line.strip())
                            if obj.get("suite") == suite:
                                tasks.append({
                                    "task_id": obj.get("task_id"),
                                    "pass1": bool(obj.get("pass1", False)),
                                })
                        except Exception:
                            continue
            except Exception:
                pass
    # If not available, try results JSONL (may require mapping)
    if not tasks and isinstance(results_path, str) and results_path:
        p = WORKSPACE_DIR / results_path
        if p.exists():
            try:
                with p.open("r", encoding="utf-8") as f:
                    for line in f:
                        try:
                            obj = json.loads(line.strip())
                            tasks.append({
                                "task_id": obj.get("task_id"),
                                "pass1": bool(obj.get("passed", False)),
                            })
                        except Exception:
                            continue
            except Exception:
                pass
    return {"suite": suite, "run_dir": run_dir, "tasks": tasks}

# ============ Benchmarks: control (execute/list/stop) ============

@app.post("/api/bench/exec")
async def bench_exec(payload: Dict[str, Any], background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    """Unified benchmark control: run/gate/autotrain/selfheal/selfheal_gate for a suite.

    Payload: { action: str, suite: str, args?: dict, stop_existing?: bool }
    Actions map to Makefile targets: bench.run.% | bench.gate.% | bench.autotrain.% | bench.selfheal.% | bench.selfheal.gate.%
    """
    action = str(payload.get("action", "")).strip()
    suite = str(payload.get("suite", "")).strip()
    args = payload.get("args", {})
    stop_existing = bool(payload.get("stop_existing", False))
    if action not in {"run", "gate", "autotrain", "selfheal", "selfheal_gate"}:
        raise HTTPException(status_code=400, detail="invalid action")
    if not suite:
        raise HTTPException(status_code=400, detail="suite required")
    # Optionally stop existing tasks for this suite
    if stop_existing:


        p = WORKSPACE_DIR / results_path
        if p.exists():
            try:
                with p.open("r", encoding="utf-8") as f:
                    for line in f:
                        try:
                            obj = json.loads(line.strip())
                            tasks.append({
                                "task_id": obj.get("task_id"),
                                "pass1": bool(obj.get("passed", False)),
                            })
                        except Exception:
                            continue
            except Exception:
                pass
    return {"suite": suite, "run_dir": run_dir, "tasks": tasks}

# ============ Benchmarks: control (execute/list/stop) ============

@app.post("/api/bench/exec")
async def bench_exec(payload: Dict[str, Any], background_tasks: BackgroundTasks, database: Session = Depends(db.get_db)) -> Dict[str, Any]:
    """Unified benchmark control: run/gate/autotrain/selfheal/selfheal_gate for a suite.

    Payload: { action: str, suite: str, args?: dict, stop_existing?: bool }
    Actions map to Makefile targets: bench.run.% | bench.gate.% | bench.autotrain.% | bench.selfheal.% | bench.selfheal.gate.%
    """
    action = str(payload.get("action", "")).strip()
    suite = str(payload.get("suite", "")).strip()
    args = payload.get("args", {})
    stop_existing = bool(payload.get("stop_existing", False))
    if action not in {"run", "gate", "autotrain", "selfheal", "selfheal_gate"}:
        raise HTTPException(status_code=400, detail="invalid action")
    if not suite:
        raise HTTPException(status_code=400, detail="suite required")
    # Optionally stop existing tasks for this suite
    if stop_existing:
        try:
            for tid, t in list(active_tasks.items()):
                name = str(t.get("name", ""))
                if name.startswith("bench.") and suite in name and t.get("status") in ("running", "queued"):
                    try:
                        await stop_task(tid)  # type: ignore[arg-type]
                    except Exception:
                        pass
        except Exception:
            pass
    # Map to make target
    if action == "run" and suite == "all":
        target = "bench.run.all"
    else:
        suffix = suite
        prefix = {
            "run": "bench.run",
            "gate": "bench.gate",
            "autotrain": "bench.autotrain",
            "selfheal": "bench.selfheal",
            "selfheal_gate": "bench.selfheal.gate",
        }[action]
        target = f"{prefix}.{suffix}"
    req = TaskRequest(task=target, args=args or None)
    return await start_task(req, background_tasks, database)

@app.get("/api/bench/active")
async def bench_active() -> List[Dict[str, Any]]:
    """List active benchmark-related tasks."""
    items: List[Dict[str, Any]] = []
    try:
        for tid, t in active_tasks.items():
            name = str(t.get("name", ""))
            if name.startswith("bench."):
                items.append({"task_id": tid, **t})
    except Exception:
        pass
    return items

@app.post("/api/bench/stop")
async def bench_stop(payload: Dict[str, Any]) -> Dict[str, Any]:
    """Stop running benchmark tasks. Payload may include { suite?: str } to filter."""
    suite = str(payload.get("suite", "")).strip()
    stopped: List[str] = []
    try:


                        pass
        except Exception:
            pass
    # Map to make target
    if action == "run" and suite == "all":
        target = "bench.run.all"
    else:
        suffix = suite
        prefix = {
            "run": "bench.run",
            "gate": "bench.gate",
            "autotrain": "bench.autotrain",
            "selfheal": "bench.selfheal",
            "selfheal_gate": "bench.selfheal.gate",
        }[action]
        target = f"{prefix}.{suffix}"
    req = TaskRequest(task=target, args=args or None)
    return await start_task(req, background_tasks, database)

@app.get("/api/bench/active")
async def bench_active() -> List[Dict[str, Any]]:
    """List active benchmark-related tasks."""
    items: List[Dict[str, Any]] = []
    try:
        for tid, t in active_tasks.items():
            name = str(t.get("name", ""))
            if name.startswith("bench."):
                items.append({"task_id": tid, **t})
    except Exception:
        pass
    return items

@app.post("/api/bench/stop")
async def bench_stop(payload: Dict[str, Any]) -> Dict[str, Any]:
    """Stop running benchmark tasks. Payload may include { suite?: str } to filter."""
    suite = str(payload.get("suite", "")).strip()
    stopped: List[str] = []
    try:
        for tid, t in list(active_tasks.items()):
            name = str(t.get("name", ""))
            if not name.startswith("bench."):
                continue
            if suite and suite not in name:
                continue
            if t.get("status") not in ("running", "queued"):
                continue
            try:
                await stop_task(tid)  # type: ignore[arg-type]
                stopped.append(tid)
            except Exception:
                continue


            "selfheal_gate": "bench.selfheal.gate",
        }[action]
        target = f"{prefix}.{suffix}"
    req = TaskRequest(task=target, args=args or None)
    return await start_task(req, background_tasks, database)

@app.get("/api/bench/active")
async def bench_active() -> List[Dict[str, Any]]:
    """List active benchmark-related tasks."""
    items: List[Dict[str, Any]] = []
    try:
        for tid, t in active_tasks.items():
            name = str(t.get("name", ""))
            if name.startswith("bench."):
                items.append({"task_id": tid, **t})
    except Exception:
        pass
    return items

@app.post("/api/bench/stop")
async def bench_stop(payload: Dict[str, Any]) -> Dict[str, Any]:
    """Stop running benchmark tasks. Payload may include { suite?: str } to filter."""
    suite = str(payload.get("suite", "")).strip()
    stopped: List[str] = []
    try:
        for tid, t in list(active_tasks.items()):
            name = str(t.get("name", ""))
            if not name.startswith("bench."):
                continue
            if suite and suite not in name:
                continue
            if t.get("status") not in ("running", "queued"):
                continue
            try:
                await stop_task(tid)  # type: ignore[arg-type]
                stopped.append(tid)
            except Exception:
                continue
    except Exception:
        pass
    return {"stopped": stopped}

@app.get("/api/bench/metrics_series")
async def bench_metrics_series(suite: str) -> Dict[str, Any]:
    """Return timeseries for pass@k and latency metrics from run artifacts."""
    runs = await list_bench_runs(suite)
    xs: List[str] = []
    metrics: Dict[str, List[Optional[float]]] = {
        "pass1_rate": [],
        "pass@k": [],  # From pass_at_k if available
        "p50_latency": [],
        "p90_latency": [],
    }

    for r in sorted(runs, key=lambda x: (x.get("ended_ts") or 0)):
        xs.append(r.get("dir"))
        m = r.get("metrics", {}) or {}

        # Extract all available metrics
        try:
            metrics["pass1_rate"].append(float(m.get("pass1_rate")) if m.get("pass1_rate") is not None else None)


    stopped: List[str] = []
    try:
        for tid, t in list(active_tasks.items()):
            name = str(t.get("name", ""))
            if not name.startswith("bench."):
                continue
            if suite and suite not in name:
                continue
            if t.get("status") not in ("running", "queued"):
                continue
            try:
                await stop_task(tid)  # type: ignore[arg-type]
                stopped.append(tid)
            except Exception:
                continue
    except Exception:
        pass
    return {"stopped": stopped}

@app.get("/api/bench/metrics_series")
async def bench_metrics_series(suite: str) -> Dict[str, Any]:
    """Return timeseries for pass@k and latency metrics from run artifacts."""
    runs = await list_bench_runs(suite)
    xs: List[str] = []
    metrics: Dict[str, List[Optional[float]]] = {
        "pass1_rate": [],
        "pass@k": [],  # From pass_at_k if available
        "p50_latency": [],
        "p90_latency": [],
    }

    for r in sorted(runs, key=lambda x: (x.get("ended_ts") or 0)):
        xs.append(r.get("dir"))
        m = r.get("metrics", {}) or {}

        # Extract all available metrics
        try:
            metrics["pass1_rate"].append(float(m.get("pass1_rate")) if m.get("pass1_rate") is not None else None)
        except Exception:
            metrics["pass1_rate"].append(None)

        try:
            # Get highest pass@k if available
            pak = m.get("pass_at_k", {})
            if isinstance(pak, dict):
                max_k = max((float(k) for k in pak.keys() if str(k).isdigit()), default=0)
                metrics["pass@k"].append(float(pak.get(str(max_k), 0)))
            else:
                metrics["pass@k"].append(None)
        except Exception:
            metrics["pass@k"].append(None)

        try:
            metrics["p50_latency"].append(float(m.get("p50_latency", m.get("latency_p50", None))))
        except Exception:
            metrics["p50_latency"].append(None)

        try:
            metrics["p90_latency"].append(float(m.get("p90_latency", m.get("latency_p90", None))))
        except Exception:
            metrics["p90_latency"].append(None)

    return {"x": xs, **metrics}

@app.get("/api/system/status")
async def get_system_status():
    """Get current system status"""
    # Get resource usage
    resource_usage = resource_manager.get_usage()

    return {
        "cpu_percent": psutil.cpu_percent(),
        "memory_percent": psutil.virtual_memory().percent,
        "active_jobs": len([t for t in active_tasks.values() if t["status"] == "running"]),
        "queued_jobs": len([t for t in active_tasks.values() if t["status"] == "queued"]),
        "completed_jobs": len([t for t in active_tasks.values() if t["status"] in ["completed", "failed", "stopped"]]),
        "resources": {
            "cpu": {
                "total": resource_usage["cpu"]["total"],
                "used": resource_usage["cpu"]["used"],
                "available": resource_usage["cpu"]["total"] - resource_usage["cpu"]["used"]
            },
            "memory": {


                max_k = max((float(k) for k in pak.keys() if str(k).isdigit()), default=0)
                metrics["pass@k"].append(float(pak.get(str(max_k), 0)))
            else:
                metrics["pass@k"].append(None)
        except Exception:
            metrics["pass@k"].append(None)

        try:
            metrics["p50_latency"].append(float(m.get("p50_latency", m.get("latency_p50", None))))
        except Exception:
            metrics["p50_latency"].append(None)

        try:
            metrics["p90_latency"].append(float(m.get("p90_latency", m.get("latency_p90", None))))
        except Exception:
            metrics["p90_latency"].append(None)

    return {"x": xs, **metrics}

@app.get("/api/system/status")
async def get_system_status():
    """Get current system status"""
    # Get resource usage
    resource_usage = resource_manager.get_usage()

    return {
        "cpu_percent": psutil.cpu_percent(),
        "memory_percent": psutil.virtual_memory().percent,
        "active_jobs": len([t for t in active_tasks.values() if t["status"] == "running"]),
        "queued_jobs": len([t for t in active_tasks.values() if t["status"] == "queued"]),
        "completed_jobs": len([t for t in active_tasks.values() if t["status"] in ["completed", "failed", "stopped"]]),
        "resources": {
            "cpu": {
                "total": resource_usage["cpu"]["total"],
                "used": resource_usage["cpu"]["used"],
                "available": resource_usage["cpu"]["total"] - resource_usage["cpu"]["used"]
            },
            "memory": {
                "total": resource_usage["memory"]["total"],
                "used": resource_usage["memory"]["used"],
                "available": resource_usage["memory"]["total"] - resource_usage["memory"]["used"]
            },
            "gpu": {
                "total": resource_usage["gpu"]["total"],
                "used_gpus": resource_usage["gpu"]["used_gpus"],
                "available_gpus": [i for i in range(resource_usage["gpu"]["total"])
                                 if i not in resource_usage["gpu"]["used_gpus"]],
                "memory": {
                    "total": resource_usage["gpu"]["memory"]["total"],
                    "used": sum(resource_usage["gpu"]["memory"]["allocations"].values())
                }
            }
        }
    }

@app.post("/api/system/cleanup")
async def cleanup_system():
    """Cleanup old tasks and cached models"""
    # Remove completed/failed/stopped tasks older than 24h
    now = datetime.now()
    for task_id in list(active_tasks.keys()):
        task = active_tasks[task_id]
        if task["status"] in ["completed", "failed", "stopped"]:
            completed_at = datetime.fromisoformat(task["completed_at"])
            if (now - completed_at) > timedelta(days=1):
                del active_tasks[task_id]

    # Stop tasks beyond TTL (if set)
    now_iso = datetime.now().isoformat()
    for task_id, t in list(active_tasks.items()):
        try:
            ttl = int(t.get("ttl_seconds") or 0)
            created = t.get("created_at") or t.get("created_at_iso")
            if ttl and created:


            },
            "memory": {
                "total": resource_usage["memory"]["total"],
                "used": resource_usage["memory"]["used"],
                "available": resource_usage["memory"]["total"] - resource_usage["memory"]["used"]
            },
            "gpu": {
                "total": resource_usage["gpu"]["total"],
                "used_gpus": resource_usage["gpu"]["used_gpus"],
                "available_gpus": [i for i in range(resource_usage["gpu"]["total"])
                                 if i not in resource_usage["gpu"]["used_gpus"]],
                "memory": {
                    "total": resource_usage["gpu"]["memory"]["total"],
                    "used": sum(resource_usage["gpu"]["memory"]["allocations"].values())
                }
            }
        }
    }

@app.post("/api/system/cleanup")
async def cleanup_system():
    """Cleanup old tasks and cached models"""
    # Remove completed/failed/stopped tasks older than 24h
    now = datetime.now()
    for task_id in list(active_tasks.keys()):
        task = active_tasks[task_id]
        if task["status"] in ["completed", "failed", "stopped"]:
            completed_at = datetime.fromisoformat(task["completed_at"])
            if (now - completed_at) > timedelta(days=1):
                del active_tasks[task_id]

    # Stop tasks beyond TTL (if set)
    now_iso = datetime.now().isoformat()
    for task_id, t in list(active_tasks.items()):
        try:
            ttl = int(t.get("ttl_seconds") or 0)
            created = t.get("created_at") or t.get("created_at_iso")
            if ttl and created:
                started = datetime.fromisoformat(str(created)) if isinstance(created, str) else created
                if (datetime.now() - started) > timedelta(seconds=ttl) and t.get("status") in ("running", "queued"):
                    try:
                        await stop_task(task_id)
                    except Exception:
                        pass
        except Exception:
            pass
    # Clear model cache
    model_cache.clear()

    return {"status": "cleanup completed"}

# Secure file download (allowlist: under data/ and api_server.log)
@app.get("/api/file/download")
async def file_download(path: str) -> FileResponse:
    full = (WORKSPACE_DIR / path).resolve()
    # allow only data/* or api log
    if not (str(full).startswith(str((WORKSPACE_DIR / "data").resolve())) or full.name == "api_server.log"):
        raise HTTPException(status_code=403, detail="Download not allowed")
    if not full.exists() or not full.is_file():
        raise HTTPException(status_code=404, detail="File not found")
    return FileResponse(str(full), filename=full.name)

# Benchmark comparison endpoint
@app.get("/api/bench/solution_diff")
async def get_solution_diff(suite: str, run_a: str, run_b: str, task_id: str) -> Dict[str, Any]:
    """Get solution diff for a specific task between two runs"""
    # Load run metas
    def _load_solutions(rd: str) -> Dict[str, str]:
        solutions = {}
        rdir = _runs_dir_for_suite(suite) / rd


    for task_id, t in list(active_tasks.items()):
        try:
            ttl = int(t.get("ttl_seconds") or 0)
            created = t.get("created_at") or t.get("created_at_iso")
            if ttl and created:
                started = datetime.fromisoformat(str(created)) if isinstance(created, str) else created
                if (datetime.now() - started) > timedelta(seconds=ttl) and t.get("status") in ("running", "queued"):
                    try:
                        await stop_task(task_id)
                    except Exception:
                        pass
        except Exception:
            pass
    # Clear model cache
    model_cache.clear()

    return {"status": "cleanup completed"}

# Secure file download (allowlist: under data/ and api_server.log)
@app.get("/api/file/download")
async def file_download(path: str) -> FileResponse:
    full = (WORKSPACE_DIR / path).resolve()
    # allow only data/* or api log
    if not (str(full).startswith(str((WORKSPACE_DIR / "data").resolve())) or full.name == "api_server.log"):
        raise HTTPException(status_code=403, detail="Download not allowed")
    if not full.exists() or not full.is_file():
        raise HTTPException(status_code=404, detail="File not found")
    return FileResponse(str(full), filename=full.name)

# Benchmark comparison endpoint
@app.get("/api/bench/solution_diff")
async def get_solution_diff(suite: str, run_a: str, run_b: str, task_id: str) -> Dict[str, Any]:
    """Get solution diff for a specific task between two runs"""
    # Load run metas
    def _load_solutions(rd: str) -> Dict[str, str]:
        solutions = {}
        rdir = _runs_dir_for_suite(suite) / rd
        rj = rdir / "run.json"
        if rj.exists():
            meta = _read_json_file(rj)
            paths = meta.get("paths", {}) if isinstance(meta, dict) else {}
            results_path = paths.get("results")
            if isinstance(results_path, str) and results_path:
                p = WORKSPACE_DIR / results_path
                if p.exists():
                    try:
                        with p.open("r") as f:
                            for line in f:


        except Exception:
            pass
    # Clear model cache
    model_cache.clear()

    return {"status": "cleanup completed"}

# Secure file download (allowlist: under data/ and api_server.log)
@app.get("/api/file/download")
async def file_download(path: str) -> FileResponse:
    full = (WORKSPACE_DIR / path).resolve()
    # allow only data/* or api log
    if not (str(full).startswith(str((WORKSPACE_DIR / "data").resolve())) or full.name == "api_server.log"):
        raise HTTPException(status_code=403, detail="Download not allowed")
    if not full.exists() or not full.is_file():
        raise HTTPException(status_code=404, detail="File not found")
    return FileResponse(str(full), filename=full.name)

# Benchmark comparison endpoint
@app.get("/api/bench/solution_diff")
async def get_solution_diff(suite: str, run_a: str, run_b: str, task_id: str) -> Dict[str, Any]:
    """Get solution diff for a specific task between two runs"""
    # Load run metas
    def _load_solutions(rd: str) -> Dict[str, str]:
        solutions = {}
        rdir = _runs_dir_for_suite(suite) / rd
        rj = rdir / "run.json"
        if rj.exists():
            meta = _read_json_file(rj)
            paths = meta.get("paths", {}) if isinstance(meta, dict) else {}
            results_path = paths.get("results")
            if isinstance(results_path, str) and results_path:
                p = WORKSPACE_DIR / results_path
                if p.exists():
                    try:
                        with p.open("r") as f:
                            for line in f:
                                try:
                                    obj = json.loads(line.strip())
                                    if obj.get("task_id") == task_id:
                                        solutions[task_id] = obj.get("solution", "")
                                except Exception:
                                    continue
                    except Exception:
                        pass
        return solutions

    solutions_a = _load_solutions(run_a)
    solutions_b = _load_solutions(run_b)

    return {
        "task_id": task_id,
        "solution_a": solutions_a.get(task_id),
        "solution_b": solutions_b.get(task_id)
    }

@app.get("/api/bench/compare")
async def bench_compare(suite: str, run_a: str, run_b: str) -> Dict[str, Any]:
    # Load run metas
    def _load_meta(rd: str) -> Dict[str, Any]:
        p = _runs_dir_for_suite(suite) / rd / "run.json"
        return _read_json_file(p) if p.exists() else {}
    a_meta = _load_meta(run_a)
    b_meta = _load_meta(run_b)
    # Load per-task flags using same logic as bench_run_tasks
    async def _load_tasks(rd: str) -> Dict[str, bool]:
        rdir = _runs_dir_for_suite(suite) / rd
        rj = rdir / "run.json"
        passed: Dict[str, bool] = {}
        if rj.exists():
            meta = _read_json_file(rj)
            paths = meta.get("paths", {}) if isinstance(meta, dict) else {}
            out_path = paths.get("out")
            results_path = paths.get("results")
            if isinstance(out_path, str) and out_path:


                                try:
                                    obj = json.loads(line.strip())
                                    if obj.get("task_id") == task_id:
                                        solutions[task_id] = obj.get("solution", "")
                                except Exception:
                                    continue
                    except Exception:
                        pass
        return solutions

    solutions_a = _load_solutions(run_a)
    solutions_b = _load_solutions(run_b)

    return {
        "task_id": task_id,
        "solution_a": solutions_a.get(task_id),
        "solution_b": solutions_b.get(task_id)
    }

@app.get("/api/bench/compare")
async def bench_compare(suite: str, run_a: str, run_b: str) -> Dict[str, Any]:
    # Load run metas
    def _load_meta(rd: str) -> Dict[str, Any]:
        p = _runs_dir_for_suite(suite) / rd / "run.json"
        return _read_json_file(p) if p.exists() else {}
    a_meta = _load_meta(run_a)
    b_meta = _load_meta(run_b)
    # Load per-task flags using same logic as bench_run_tasks
    async def _load_tasks(rd: str) -> Dict[str, bool]:
        rdir = _runs_dir_for_suite(suite) / rd
        rj = rdir / "run.json"
        passed: Dict[str, bool] = {}
        if rj.exists():
            meta = _read_json_file(rj)
            paths = meta.get("paths", {}) if isinstance(meta, dict) else {}
            out_path = paths.get("out")
            results_path = paths.get("results")
            if isinstance(out_path, str) and out_path:
                p = WORKSPACE_DIR / out_path
                if p.exists():
                    try:
                        with p.open("r", encoding="utf-8") as f:
                            for line in f:
                                try:
                                    obj = json.loads(line.strip())
                                    if obj.get("suite") == suite:
                                        passed[str(obj.get("task_id"))] = bool(obj.get("pass1", False))
                                except Exception:
                                    continue
                    except Exception:
                        pass
            if not passed and isinstance(results_path, str) and results_path:
                p = WORKSPACE_DIR / results_path
                if p.exists():
                    try:
                        with p.open("r", encoding="utf-8") as f:
                            for line in f:
                                try:
                                    obj = json.loads(line.strip())
                                    passed[str(obj.get("task_id"))] = bool(obj.get("passed", False))
                                except Exception:
                                    continue
                    except Exception:
                        pass
        return passed
    a_tasks = await _load_tasks(run_a)
    b_tasks = await _load_tasks(run_b)
    all_tids = sorted(set(a_tasks.keys()) | set(b_tasks.keys()))
    diffs = []
    changed = 0
    for tid in all_tids:
        a = bool(a_tasks.get(tid, False))
        b = bool(b_tasks.get(tid, False))
        if a != b:
            changed += 1
        diffs.append({"task_id": tid, "run_a": a, "run_b": b, "delta": (1 if b and not a else (-1 if a and not b else 0))})
    # Aggregate deltas
    def _get_pass1(meta: Dict[str, Any]) -> Optional[float]:
        m = meta.get("metrics", {}) if isinstance(meta, dict) else {}
        v = m.get("pass1_rate") if isinstance(m, dict) else None
        try:
            return float(v) if v is not None else None
        except Exception:
            return None
    pass1_a = _get_pass1(a_meta)
    pass1_b = _get_pass1(b_meta)
    delta_pass1 = None
    if pass1_a is not None and pass1_b is not None:
        delta_pass1 = pass1_b - pass1_a
    return {
        "suite": suite,
        "run_a": {"dir": run_a, "pass1_rate": pass1_a},
        "run_b": {"dir": run_b, "pass1_rate": pass1_b},
        "delta_pass1": delta_pass1,
        "changed": changed,
        "diffs": diffs,
    }

if __name__ == "__main__":
    import uvicorn
    uvicorn.run("main:app", host="127.0.0.1", port=8000, reload=True)

# ================= Model Registry APIs =================

@app.get("/api/models")
async def list_models() -> Dict[str, Any]:
    """List loaded and available models"""
    # List models from model_cache
    loaded = [{"id": k, "params": v.get("params", {})} for k, v in model_cache.items()]

    # List available models from registry paths
    available = []
    model_paths = [
        WORKSPACE_DIR / "models" / "wm_mlp",
        WORKSPACE_DIR / "models" / "verifier_calib",
        WORKSPACE_DIR / "models" / "coder_rl",
        WORKSPACE_DIR / "models" / "coder_nearmiss",


    pass1_b = _get_pass1(b_meta)
    delta_pass1 = None
    if pass1_a is not None and pass1_b is not None:
        delta_pass1 = pass1_b - pass1_a
    return {
        "suite": suite,
        "run_a": {"dir": run_a, "pass1_rate": pass1_a},
        "run_b": {"dir": run_b, "pass1_rate": pass1_b},
        "delta_pass1": delta_pass1,
        "changed": changed,
        "diffs": diffs,
    }

if __name__ == "__main__":
    import uvicorn
    uvicorn.run("main:app", host="127.0.0.1", port=8000, reload=True)

# ================= Model Registry APIs =================

@app.get("/api/models")
async def list_models() -> Dict[str, Any]:
    """List loaded and available models"""
    # List models from model_cache
    loaded = [{"id": k, "params": v.get("params", {})} for k, v in model_cache.items()]

    # List available models from registry paths
    available = []
    model_paths = [
        WORKSPACE_DIR / "models" / "wm_mlp",
        WORKSPACE_DIR / "models" / "verifier_calib",
        WORKSPACE_DIR / "models" / "coder_rl",
        WORKSPACE_DIR / "models" / "coder_nearmiss",
    ]
    for p in model_paths:
        if p.exists() and (p / "model.json").exists():
            try:
                meta = json.loads((p / "model.json").read_text())
                available.append({
                    "id": p.name,
                    "path": str(p.relative_to(WORKSPACE_DIR)),
                    "meta": meta
                })
            except Exception:
                pass
    # Inject lightweight built-in debug models
    try:
        available.extend([
            {"id": "debug/echo", "path": "(built-in)", "meta": {"title": "Echo", "desc": "Returns the prompt or last chat turn. No deps."}},
            {"id": "debug/upper", "path": "(built-in)", "meta": {"title": "Upper", "desc": "Uppercases the input. No deps."}},
            {"id": "debug/repeat", "path": "(built-in)", "meta": {"title": "Repeat", "desc": "Repeats input a few times. No deps."}},
        ])
    except Exception:
        pass

    return {
        "loaded": loaded,
        "available": available
    }

@app.post("/api/models/load")
async def load_model(request: ModelRequest) -> Dict[str, str]:
    """Load a model via core HF client and cache the client instance."""
    if request.model_id in model_cache:
        raise HTTPException(status_code=400, detail="Model already loaded")

    try:
        # Instantiate client (respects env overrides for device/dtype)
        if request.model_id.startswith("debug/"):
            client = DebugClient(request.model_id)
        else:
            client = HFClient.get_cached(request.model_id)
        # Optionally attach adapter if model dir exists with adapter
        try:
            adapter_dir = str((WORKSPACE_DIR / "models" / request.model_id / "adapter").resolve())
            if os.path.isdir(adapter_dir):
                client.attach_adapter(adapter_dir)
        except Exception:
            pass


                    "meta": meta
                })
            except Exception:
                pass
    # Inject lightweight built-in debug models
    try:
        available.extend([
            {"id": "debug/echo", "path": "(built-in)", "meta": {"title": "Echo", "desc": "Returns the prompt or last chat turn. No deps."}},
            {"id": "debug/upper", "path": "(built-in)", "meta": {"title": "Upper", "desc": "Uppercases the input. No deps."}},
            {"id": "debug/repeat", "path": "(built-in)", "meta": {"title": "Repeat", "desc": "Repeats input a few times. No deps."}},
        ])
    except Exception:
        pass

    return {
        "loaded": loaded,
        "available": available
    }

@app.post("/api/models/load")
async def load_model(request: ModelRequest) -> Dict[str, str]:
    """Load a model via core HF client and cache the client instance."""
    if request.model_id in model_cache:
        raise HTTPException(status_code=400, detail="Model already loaded")

    try:
        # Instantiate client (respects env overrides for device/dtype)
        if request.model_id.startswith("debug/"):
            client = DebugClient(request.model_id)
        else:
            client = HFClient.get_cached(request.model_id)
        # Optionally attach adapter if model dir exists with adapter
        try:
            adapter_dir = str((WORKSPACE_DIR / "models" / request.model_id / "adapter").resolve())
            if os.path.isdir(adapter_dir):
                client.attach_adapter(adapter_dir)
        except Exception:
            pass
        model_cache[request.model_id] = {
            "client": client,
            "params": request.params.dict() if getattr(request, "params", None) else {},
            "loaded_at": datetime.now().isoformat(),
        }
        return {"status": "loaded"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"load_failed: {e}")

@app.post("/api/models/unload")
async def unload_model(request: ModelRequest) -> Dict[str, str]:
    """Unload a model from cache"""
    if request.model_id not in model_cache:
        raise HTTPException(status_code=404, detail="Model not loaded")

    try:
        # Remove from cache (in practice, you'd free the model here)
        del model_cache[request.model_id]
        return {"status": "unloaded"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/models/infer")
async def model_infer(request: InferRequest) -> Dict[str, Any]:
    """Run inference using the first loaded model client."""
    if not model_cache:
        # Attempt to load default model from env for chat convenience
        try:


            client = DebugClient(request.model_id)
        else:
            client = HFClient.get_cached(request.model_id)
        # Optionally attach adapter if model dir exists with adapter
        try:
            adapter_dir = str((WORKSPACE_DIR / "models" / request.model_id / "adapter").resolve())
            if os.path.isdir(adapter_dir):
                client.attach_adapter(adapter_dir)
        except Exception:
            pass
        model_cache[request.model_id] = {
            "client": client,
            "params": request.params.dict() if getattr(request, "params", None) else {},
            "loaded_at": datetime.now().isoformat(),
        }
        return {"status": "loaded"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"load_failed: {e}")

@app.post("/api/models/unload")
async def unload_model(request: ModelRequest) -> Dict[str, str]:
    """Unload a model from cache"""
    if request.model_id not in model_cache:
        raise HTTPException(status_code=404, detail="Model not loaded")

    try:
        # Remove from cache (in practice, you'd free the model here)
        del model_cache[request.model_id]
        return {"status": "unloaded"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/models/infer")
async def model_infer(request: InferRequest) -> Dict[str, Any]:
    """Run inference using the first loaded model client."""
    if not model_cache:
        # Attempt to load default model from env for chat convenience
        try:
            default_model = os.environ.get("AGI_DEFAULT_MODEL") or os.environ.get("AGI_CHAT_MODEL")
            if default_model:
                _ = await load_model(ModelRequest(model_id=default_model))  # type: ignore[name-defined]
        except Exception:
            pass
        if not model_cache:
            raise HTTPException(status_code=400, detail="No model loaded. Use /api/models/load first or set AGI_DEFAULT_MODEL.")
    # Choose the most recently loaded model
    try:
        # Stable selection: the last inserted key
        model_id = list(model_cache.keys())[-1]
        entry = model_cache[model_id]
        client = entry.get("client")


            "loaded_at": datetime.now().isoformat(),
        }
        return {"status": "loaded"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"load_failed: {e}")

@app.post("/api/models/unload")
async def unload_model(request: ModelRequest) -> Dict[str, str]:
    """Unload a model from cache"""
    if request.model_id not in model_cache:
        raise HTTPException(status_code=404, detail="Model not loaded")

    try:
        # Remove from cache (in practice, you'd free the model here)
        del model_cache[request.model_id]
        return {"status": "unloaded"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/models/infer")
async def model_infer(request: InferRequest) -> Dict[str, Any]:
    """Run inference using the first loaded model client."""
    if not model_cache:
        # Attempt to load default model from env for chat convenience
        try:
            default_model = os.environ.get("AGI_DEFAULT_MODEL") or os.environ.get("AGI_CHAT_MODEL")
            if default_model:
                _ = await load_model(ModelRequest(model_id=default_model))  # type: ignore[name-defined]
        except Exception:
            pass
        if not model_cache:
            raise HTTPException(status_code=400, detail="No model loaded. Use /api/models/load first or set AGI_DEFAULT_MODEL.")
    # Choose the most recently loaded model
    try:
        # Stable selection: the last inserted key
        model_id = list(model_cache.keys())[-1]
        entry = model_cache[model_id]
        client = entry.get("client")
        if client is None:
            # Backward-compat: instantiate if only metadata present
            if model_id.startswith("debug/"):
                client = DebugClient(model_id)
            else:
                client = HFClient.get_cached(model_id)
            entry["client"] = client
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"no_client: {e}")

    params = request.params or {}
    max_new_tokens = int(params.get("max_new_tokens", 128) or 128)
    temperature = float(params.get("temperature", 0.0) or 0.0)
    top_p = params.get("top_p")
    top_k = params.get("top_k")
    stop = params.get("stop")
    grammar = params.get("grammar")

    import time as _time
    t0 = _time.time()
    try:
        text = client.generate(
            prompt=request.prompt,
            max_new_tokens=max_new_tokens,
            temperature=temperature,
            top_p=float(top_p) if top_p is not None else None,
            top_k=int(top_k) if top_k is not None else None,
            stop=list(stop) if isinstance(stop, (list, tuple)) else None,
            grammar=str(grammar) if grammar is not None else None,
        )
        took_ms = int((_time.time() - t0) * 1000)
        return {"output": text, "model": model_id, "took_ms": took_ms}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"infer_failed: {e}")

# ============== Thin wrappers over core services ==============

@app.post("/api/planner/plan")
async def api_planner_plan(req: PlannerRequest) -> Dict[str, Any]:
    pl_cfg = PlannerConfig(
        model=req.model,
        backend="hf",
        timeout_sec=30,
        adapter_dir=None,
        structured_mode=req.structured,
        candidates=int(req.candidates or 1),
        use_tot=bool(req.use_tot),
    )
    vf_model = req.verifier_model or req.model
    vf_cfg = PlannerVerifierConfig(model=vf_model, backend="hf", adapter_dir=None, structured_mode=req.structured)
    wm_cfg = WMConfig(enabled=bool(req.wm_path), model_path=req.wm_path, horizon=int(req.wm_horizon or 1), plan_rank=bool(req.wm_path))
    ctx = ContextAugment(use_memory=bool(req.use_memory), index_k=int(req.index_k or 0), inject_cli_policy=True, inject_dom_policy=True, inject_caps=True)
    plan, info, obs_aug, mem_snips, mem_ms = plan_with_context(req.obs, req.domain, pl_cfg, vf_cfg, wm_cfg, ctx, critic_fallback_threshold=None, log_prompts=bool(req.log_prompts))
    return {"plan": plan, "info": info, "obs_aug": obs_aug, "mem": mem_snips, "mem_ms": mem_ms}


    import time as _time
    t0 = _time.time()
    try:
        text = client.generate(
            prompt=request.prompt,
            max_new_tokens=max_new_tokens,
            temperature=temperature,
            top_p=float(top_p) if top_p is not None else None,
            top_k=int(top_k) if top_k is not None else None,
            stop=list(stop) if isinstance(stop, (list, tuple)) else None,
            grammar=str(grammar) if grammar is not None else None,
        )
        took_ms = int((_time.time() - t0) * 1000)
        return {"output": text, "model": model_id, "took_ms": took_ms}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"infer_failed: {e}")

# ============== Thin wrappers over core services ==============

@app.post("/api/planner/plan")
async def api_planner_plan(req: PlannerRequest) -> Dict[str, Any]:
    pl_cfg = PlannerConfig(
        model=req.model,
        backend="hf",
        timeout_sec=30,
        adapter_dir=None,
        structured_mode=req.structured,
        candidates=int(req.candidates or 1),
        use_tot=bool(req.use_tot),
    )
    vf_model = req.verifier_model or req.model
    vf_cfg = PlannerVerifierConfig(model=vf_model, backend="hf", adapter_dir=None, structured_mode=req.structured)
    wm_cfg = WMConfig(enabled=bool(req.wm_path), model_path=req.wm_path, horizon=int(req.wm_horizon or 1), plan_rank=bool(req.wm_path))
    ctx = ContextAugment(use_memory=bool(req.use_memory), index_k=int(req.index_k or 0), inject_cli_policy=True, inject_dom_policy=True, inject_caps=True)
    plan, info, obs_aug, mem_snips, mem_ms = plan_with_context(req.obs, req.domain, pl_cfg, vf_cfg, wm_cfg, ctx, critic_fallback_threshold=None, log_prompts=bool(req.log_prompts))
    return {"plan": plan, "info": info, "obs_aug": obs_aug, "mem": mem_snips, "mem_ms": mem_ms}

@app.post("/api/wm/rollout")
async def api_wm_rollout(req: RolloutRequest) -> Dict[str, Any]:
    if req.wm_checkpoint:
        svc = WorldModelService.load_if_exists(req.wm_checkpoint)
    else:
        svc = WorldModelService.load_if_exists(str((WORKSPACE_DIR / "models" / "wm_mlp" / "wm_mlp.joblib").resolve()))
    if svc is None:
        raise HTTPException(status_code=404, detail="world_model_not_found")
    res = svc.rollout(req.obs, req.plan, req.actions, horizon=max(1, int(req.horizon or 1)))
    if res is None:
        raise HTTPException(status_code=500, detail="rollout_failed")
    return res

@app.post("/api/actuator/execute")
async def api_actuator_execute(req: ActuatorRequest) -> Dict[str, Any]:
    repo_dir = str(Path(req.repo).resolve())
    if not os.path.isdir(repo_dir):
        raise HTTPException(status_code=400, detail="invalid_repo")
    return execute_code_action(repo_dir, req.action)


@app.post("/api/planner/plan")
async def api_planner_plan(req: PlannerRequest) -> Dict[str, Any]:
    pl_cfg = PlannerConfig(
        model=req.model,
        backend="hf",
        timeout_sec=30,
        adapter_dir=None,
        structured_mode=req.structured,
        candidates=int(req.candidates or 1),
        use_tot=bool(req.use_tot),
    )
    vf_model = req.verifier_model or req.model
    vf_cfg = PlannerVerifierConfig(model=vf_model, backend="hf", adapter_dir=None, structured_mode=req.structured)
    wm_cfg = WMConfig(enabled=bool(req.wm_path), model_path=req.wm_path, horizon=int(req.wm_horizon or 1), plan_rank=bool(req.wm_path))
    ctx = ContextAugment(use_memory=bool(req.use_memory), index_k=int(req.index_k or 0), inject_cli_policy=True, inject_dom_policy=True, inject_caps=True)
    plan, info, obs_aug, mem_snips, mem_ms = plan_with_context(req.obs, req.domain, pl_cfg, vf_cfg, wm_cfg, ctx, critic_fallback_threshold=None, log_prompts=bool(req.log_prompts))
    return {"plan": plan, "info": info, "obs_aug": obs_aug, "mem": mem_snips, "mem_ms": mem_ms}

@app.post("/api/wm/rollout")
async def api_wm_rollout(req: RolloutRequest) -> Dict[str, Any]:
    if req.wm_checkpoint:
        svc = WorldModelService.load_if_exists(req.wm_checkpoint)
    else:
        svc = WorldModelService.load_if_exists(str((WORKSPACE_DIR / "models" / "wm_mlp" / "wm_mlp.joblib").resolve()))
    if svc is None:
        raise HTTPException(status_code=404, detail="world_model_not_found")
    res = svc.rollout(req.obs, req.plan, req.actions, horizon=max(1, int(req.horizon or 1)))
    if res is None:
        raise HTTPException(status_code=500, detail="rollout_failed")
    return res

@app.post("/api/actuator/execute")
async def api_actuator_execute(req: ActuatorRequest) -> Dict[str, Any]:
    repo_dir = str(Path(req.repo).resolve())
    if not os.path.isdir(repo_dir):
        raise HTTPException(status_code=400, detail="invalid_repo")
    return execute_code_action(repo_dir, req.action)

@app.post("/api/actuator/select")
async def api_actuator_select(req: ActuatorSelectRequest) -> Dict[str, Any]:
    cfg = ActCfg(
        mode=req.mode,
        t5_model=req.t5_model,
        il_path=req.il_path,
        learned_router=bool(req.learned_router),
        router_model_path=req.router_model_path,
        router_threshold=float(req.router_threshold),
        router_use_packed_threshold=bool(req.router_use_packed_threshold),
        router_thresholds_json=req.router_thresholds_json,
        dom_structured=bool(req.dom_structured),


    vf_cfg = PlannerVerifierConfig(model=vf_model, backend="hf", adapter_dir=None, structured_mode=req.structured)
    wm_cfg = WMConfig(enabled=bool(req.wm_path), model_path=req.wm_path, horizon=int(req.wm_horizon or 1), plan_rank=bool(req.wm_path))
    ctx = ContextAugment(use_memory=bool(req.use_memory), index_k=int(req.index_k or 0), inject_cli_policy=True, inject_dom_policy=True, inject_caps=True)
    plan, info, obs_aug, mem_snips, mem_ms = plan_with_context(req.obs, req.domain, pl_cfg, vf_cfg, wm_cfg, ctx, critic_fallback_threshold=None, log_prompts=bool(req.log_prompts))
    return {"plan": plan, "info": info, "obs_aug": obs_aug, "mem": mem_snips, "mem_ms": mem_ms}

@app.post("/api/wm/rollout")
async def api_wm_rollout(req: RolloutRequest) -> Dict[str, Any]:
    if req.wm_checkpoint:
        svc = WorldModelService.load_if_exists(req.wm_checkpoint)
    else:
        svc = WorldModelService.load_if_exists(str((WORKSPACE_DIR / "models" / "wm_mlp" / "wm_mlp.joblib").resolve()))
    if svc is None:
        raise HTTPException(status_code=404, detail="world_model_not_found")
    res = svc.rollout(req.obs, req.plan, req.actions, horizon=max(1, int(req.horizon or 1)))
    if res is None:
        raise HTTPException(status_code=500, detail="rollout_failed")
    return res

@app.post("/api/actuator/execute")
async def api_actuator_execute(req: ActuatorRequest) -> Dict[str, Any]:
    repo_dir = str(Path(req.repo).resolve())
    if not os.path.isdir(repo_dir):
        raise HTTPException(status_code=400, detail="invalid_repo")
    return execute_code_action(repo_dir, req.action)

@app.post("/api/actuator/select")
async def api_actuator_select(req: ActuatorSelectRequest) -> Dict[str, Any]:
    cfg = ActCfg(
        mode=req.mode,
        t5_model=req.t5_model,
        il_path=req.il_path,
        learned_router=bool(req.learned_router),
        router_model_path=req.router_model_path,
        router_threshold=float(req.router_threshold),
        router_use_packed_threshold=bool(req.router_use_packed_threshold),
        router_thresholds_json=req.router_thresholds_json,
        dom_structured=bool(req.dom_structured),
    )
    extras = ActRouterExtras(domain=req.domain)
    vcfg = None
    if req.verifier_model:
        vcfg = ActRouterVerifierCfg(
            model=req.verifier_model,
            backend=req.verifier_backend,


async def api_wm_rollout(req: RolloutRequest) -> Dict[str, Any]:
    if req.wm_checkpoint:
        svc = WorldModelService.load_if_exists(req.wm_checkpoint)
    else:
        svc = WorldModelService.load_if_exists(str((WORKSPACE_DIR / "models" / "wm_mlp" / "wm_mlp.joblib").resolve()))
    if svc is None:
        raise HTTPException(status_code=404, detail="world_model_not_found")
    res = svc.rollout(req.obs, req.plan, req.actions, horizon=max(1, int(req.horizon or 1)))
    if res is None:
        raise HTTPException(status_code=500, detail="rollout_failed")
    return res

@app.post("/api/actuator/execute")
async def api_actuator_execute(req: ActuatorRequest) -> Dict[str, Any]:
    repo_dir = str(Path(req.repo).resolve())
    if not os.path.isdir(repo_dir):
        raise HTTPException(status_code=400, detail="invalid_repo")
    return execute_code_action(repo_dir, req.action)

@app.post("/api/actuator/select")
async def api_actuator_select(req: ActuatorSelectRequest) -> Dict[str, Any]:
    cfg = ActCfg(
        mode=req.mode,
        t5_model=req.t5_model,
        il_path=req.il_path,
        learned_router=bool(req.learned_router),
        router_model_path=req.router_model_path,
        router_threshold=float(req.router_threshold),
        router_use_packed_threshold=bool(req.router_use_packed_threshold),
        router_thresholds_json=req.router_thresholds_json,
        dom_structured=bool(req.dom_structured),
    )
    extras = ActRouterExtras(domain=req.domain)
    vcfg = None
    if req.verifier_model:
        vcfg = ActRouterVerifierCfg(
            model=req.verifier_model,
            backend=req.verifier_backend,
            adapter_dir=req.verifier_adapter_dir,
            adapter_bank=req.verifier_adapter_bank,
            structured_mode=req.verifier_structured_mode,
            timeout_sec=int(req.verifier_timeout_sec),
        )
    wpcfg = ActWMPriorCfg(enabled=bool(req.wm_prior_enabled), model_path=req.wm_prior_model_path)
    wscfg = ActWMScreenCfg(enabled=bool(req.wm_screen_enabled), threshold=float(req.wm_screen_threshold))
    rcfg = ActRepairCfg(domain=req.domain, prefer_obs_args=bool(req.prefer_obs_args), default_url=req.default_url, default_selector=req.default_selector) if req.repair else None
    action, router = actuator_select_action(req.obs, req.plan, cfg, extras, verifier_cfg=vcfg, wm_prior_cfg=wpcfg, wm_screen_cfg=wscfg, repair_cfg=rcfg)
    return {"action": action, "router": router}

@app.post("/api/verifier/verify")
async def api_verifier_verify(req: VerifyRequest) -> Dict[str, Any]:
    cfg = VerifierServiceConfig(
        model=str(req.model),
        backend=str(req.backend),
        adapter_dir=req.adapter_dir,
        adapter_bank=req.adapter_bank,
        structured_mode=str(req.structured_mode),
        timeout_sec=int(req.timeout_sec),
        strict=bool(req.strict),
        calibrate=bool(req.calibrate),
        calib_model=req.calib_model,
        log_prompts=bool(req.log_prompts),
    )
    return verifier_run(req.trace, cfg)

@app.post("/api/updater/run")
async def api_updater_run(req: UpdaterRequest) -> Dict[str, Any]:
    try:


        dom_structured=bool(req.dom_structured),
    )
    extras = ActRouterExtras(domain=req.domain)
    vcfg = None
    if req.verifier_model:
        vcfg = ActRouterVerifierCfg(
            model=req.verifier_model,
            backend=req.verifier_backend,
            adapter_dir=req.verifier_adapter_dir,
            adapter_bank=req.verifier_adapter_bank,
            structured_mode=req.verifier_structured_mode,
            timeout_sec=int(req.verifier_timeout_sec),
        )
    wpcfg = ActWMPriorCfg(enabled=bool(req.wm_prior_enabled), model_path=req.wm_prior_model_path)
    wscfg = ActWMScreenCfg(enabled=bool(req.wm_screen_enabled), threshold=float(req.wm_screen_threshold))
    rcfg = ActRepairCfg(domain=req.domain, prefer_obs_args=bool(req.prefer_obs_args), default_url=req.default_url, default_selector=req.default_selector) if req.repair else None
    action, router = actuator_select_action(req.obs, req.plan, cfg, extras, verifier_cfg=vcfg, wm_prior_cfg=wpcfg, wm_screen_cfg=wscfg, repair_cfg=rcfg)
    return {"action": action, "router": router}

@app.post("/api/verifier/verify")
async def api_verifier_verify(req: VerifyRequest) -> Dict[str, Any]:
    cfg = VerifierServiceConfig(
        model=str(req.model),
        backend=str(req.backend),
        adapter_dir=req.adapter_dir,
        adapter_bank=req.adapter_bank,
        structured_mode=str(req.structured_mode),
        timeout_sec=int(req.timeout_sec),
        strict=bool(req.strict),
        calibrate=bool(req.calibrate),
        calib_model=req.calib_model,
        log_prompts=bool(req.log_prompts),
    )
    return verifier_run(req.trace, cfg)

@app.post("/api/updater/run")
async def api_updater_run(req: UpdaterRequest) -> Dict[str, Any]:
    try:
        upd = Updater(repo_root=WORKSPACE_DIR, fast=bool(req.fast))
        # Run in background thread to avoid blocking
        def _run() -> None:
            try:
                upd.run()
            except Exception:
                pass
        threading.Thread(target=_run, daemon=True).start()
        return {"ok": True, "started": True}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# ================= Additional central management APIs =================

# File system tree (repo/data)
def _safe_root(base: str) -> Path:


    action, router = actuator_select_action(req.obs, req.plan, cfg, extras, verifier_cfg=vcfg, wm_prior_cfg=wpcfg, wm_screen_cfg=wscfg, repair_cfg=rcfg)
    return {"action": action, "router": router}

@app.post("/api/verifier/verify")
async def api_verifier_verify(req: VerifyRequest) -> Dict[str, Any]:
    cfg = VerifierServiceConfig(
        model=str(req.model),
        backend=str(req.backend),
        adapter_dir=req.adapter_dir,
        adapter_bank=req.adapter_bank,
        structured_mode=str(req.structured_mode),
        timeout_sec=int(req.timeout_sec),
        strict=bool(req.strict),
        calibrate=bool(req.calibrate),
        calib_model=req.calib_model,
        log_prompts=bool(req.log_prompts),
    )
    return verifier_run(req.trace, cfg)

@app.post("/api/updater/run")
async def api_updater_run(req: UpdaterRequest) -> Dict[str, Any]:
    try:
        upd = Updater(repo_root=WORKSPACE_DIR, fast=bool(req.fast))
        # Run in background thread to avoid blocking
        def _run() -> None:
            try:
                upd.run()
            except Exception:
                pass
        threading.Thread(target=_run, daemon=True).start()
        return {"ok": True, "started": True}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# ================= Additional central management APIs =================

# File system tree (repo/data)
def _safe_root(base: str) -> Path:
    if base == "data":
        return WORKSPACE_DIR / "data"
    return WORKSPACE_DIR

def _build_tree(path: Path, rel_to: Path, depth: int = 0, max_depth: int = 4, max_entries: int = 200) -> Union[Dict[str, Any], str]:
    if depth > max_depth:
        return "…"
    if path.is_file():
        return str(path.relative_to(rel_to))
    entries: Dict[str, Any] = {}
    try:
        children = sorted([p for p in path.iterdir() if not p.name.startswith('.')])
    except Exception:
        return entries
    count = 0


    return verifier_run(req.trace, cfg)

@app.post("/api/updater/run")
async def api_updater_run(req: UpdaterRequest) -> Dict[str, Any]:
    try:
        upd = Updater(repo_root=WORKSPACE_DIR, fast=bool(req.fast))
        # Run in background thread to avoid blocking
        def _run() -> None:
            try:
                upd.run()
            except Exception:
                pass
        threading.Thread(target=_run, daemon=True).start()
        return {"ok": True, "started": True}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# ================= Additional central management APIs =================

# File system tree (repo/data)
def _safe_root(base: str) -> Path:
    if base == "data":
        return WORKSPACE_DIR / "data"
    return WORKSPACE_DIR

def _build_tree(path: Path, rel_to: Path, depth: int = 0, max_depth: int = 4, max_entries: int = 200) -> Union[Dict[str, Any], str]:
    if depth > max_depth:
        return "…"
    if path.is_file():
        return str(path.relative_to(rel_to))
    entries: Dict[str, Any] = {}
    try:
        children = sorted([p for p in path.iterdir() if not p.name.startswith('.')])
    except Exception:
        return entries
    count = 0
    for child in children:
        # Skip bulky dirs
        if child.name in {"node_modules", "venv", "__pycache__", "data", ".git", "data/conda_envs"}:
            continue
        entries[child.name] = _build_tree(child, rel_to, depth + 1, max_depth, max_entries)
        count += 1
        if count >= max_entries:
            entries["…more…"] = "truncated"


        upd = Updater(repo_root=WORKSPACE_DIR, fast=bool(req.fast))
        # Run in background thread to avoid blocking
        def _run() -> None:
            try:
                upd.run()
            except Exception:
                pass
        threading.Thread(target=_run, daemon=True).start()
        return {"ok": True, "started": True}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# ================= Additional central management APIs =================

# File system tree (repo/data)
def _safe_root(base: str) -> Path:
    if base == "data":
        return WORKSPACE_DIR / "data"
    return WORKSPACE_DIR

def _build_tree(path: Path, rel_to: Path, depth: int = 0, max_depth: int = 4, max_entries: int = 200) -> Union[Dict[str, Any], str]:
    if depth > max_depth:
        return "…"
    if path.is_file():
        return str(path.relative_to(rel_to))
    entries: Dict[str, Any] = {}
    try:
        children = sorted([p for p in path.iterdir() if not p.name.startswith('.')])
    except Exception:
        return entries
    count = 0
    for child in children:
        # Skip bulky dirs
        if child.name in {"node_modules", "venv", "__pycache__", "data", ".git", "data/conda_envs"}:
            continue
        entries[child.name] = _build_tree(child, rel_to, depth + 1, max_depth, max_entries)
        count += 1
        if count >= max_entries:
            entries["…more…"] = "truncated"
            break
    return entries

@app.get("/api/repo/tree")
async def repo_tree(base: str = "repo", path: str = "") -> Dict[str, Any]:
    root = _safe_root(base)
    target = (root / path).resolve()
    if not str(target).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    return {"base": base, "path": str(target.relative_to(root)), "tree": _build_tree(target, root)}

@app.get("/api/fs/preview")
async def fs_preview(path: str, base: str = "repo", limit: int = 102400) -> Dict[str, Any]:
    root = _safe_root(base)
    full_path = (root / path).resolve()
    if not str(full_path).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    try:
        if full_path.stat().st_size > 5 * 1024 * 1024:
            return {"path": str(full_path), "error": "file too large"}
        with open(full_path, "r", encoding="utf-8", errors="replace") as f:
            content = f.read(limit)


    if path.is_file():
        return str(path.relative_to(rel_to))
    entries: Dict[str, Any] = {}
    try:
        children = sorted([p for p in path.iterdir() if not p.name.startswith('.')])
    except Exception:
        return entries
    count = 0
    for child in children:
        # Skip bulky dirs
        if child.name in {"node_modules", "venv", "__pycache__", "data", ".git", "data/conda_envs"}:
            continue
        entries[child.name] = _build_tree(child, rel_to, depth + 1, max_depth, max_entries)
        count += 1
        if count >= max_entries:
            entries["…more…"] = "truncated"
            break
    return entries

@app.get("/api/repo/tree")
async def repo_tree(base: str = "repo", path: str = "") -> Dict[str, Any]:
    root = _safe_root(base)
    target = (root / path).resolve()
    if not str(target).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    return {"base": base, "path": str(target.relative_to(root)), "tree": _build_tree(target, root)}

@app.get("/api/fs/preview")
async def fs_preview(path: str, base: str = "repo", limit: int = 102400) -> Dict[str, Any]:
    root = _safe_root(base)
    full_path = (root / path).resolve()
    if not str(full_path).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    try:
        if full_path.stat().st_size > 5 * 1024 * 1024:
            return {"path": str(full_path), "error": "file too large"}
        with open(full_path, "r", encoding="utf-8", errors="replace") as f:
            content = f.read(limit)
        return {"path": str(full_path.relative_to(root)), "content": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# Simple code search (regex)
@app.post("/api/code/search")
async def code_search(payload: Dict[str, Any]) -> Dict[str, Any]:
    pattern = str(payload.get("pattern", "")).strip()


    for child in children:
        # Skip bulky dirs
        if child.name in {"node_modules", "venv", "__pycache__", "data", ".git", "data/conda_envs"}:
            continue
        entries[child.name] = _build_tree(child, rel_to, depth + 1, max_depth, max_entries)
        count += 1
        if count >= max_entries:
            entries["…more…"] = "truncated"
            break
    return entries

@app.get("/api/repo/tree")
async def repo_tree(base: str = "repo", path: str = "") -> Dict[str, Any]:
    root = _safe_root(base)
    target = (root / path).resolve()
    if not str(target).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    return {"base": base, "path": str(target.relative_to(root)), "tree": _build_tree(target, root)}

@app.get("/api/fs/preview")
async def fs_preview(path: str, base: str = "repo", limit: int = 102400) -> Dict[str, Any]:
    root = _safe_root(base)
    full_path = (root / path).resolve()
    if not str(full_path).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    try:
        if full_path.stat().st_size > 5 * 1024 * 1024:
            return {"path": str(full_path), "error": "file too large"}
        with open(full_path, "r", encoding="utf-8", errors="replace") as f:
            content = f.read(limit)
        return {"path": str(full_path.relative_to(root)), "content": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# Simple code search (regex)
@app.post("/api/code/search")
async def code_search(payload: Dict[str, Any]) -> Dict[str, Any]:
    pattern = str(payload.get("pattern", "")).strip()
    base = str(payload.get("base", "repo"))
    subpath = str(payload.get("path", ""))
    max_results = int(payload.get("max_results", 200))
    if not pattern:
        raise HTTPException(status_code=400, detail="pattern required")
    try:
        regex = re.compile(pattern)
    except re.error as e:
        raise HTTPException(status_code=400, detail=f"invalid regex: {e}")
    root = _safe_root(base)
    start = (root / subpath).resolve()
    if not str(start).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    results: List[Dict[str, Any]] = []
    skip_dirs = {".git", "node_modules", "__pycache__", "data/conda_envs"}


        raise HTTPException(status_code=403, detail="Access denied")
    return {"base": base, "path": str(target.relative_to(root)), "tree": _build_tree(target, root)}

@app.get("/api/fs/preview")
async def fs_preview(path: str, base: str = "repo", limit: int = 102400) -> Dict[str, Any]:
    root = _safe_root(base)
    full_path = (root / path).resolve()
    if not str(full_path).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    try:
        if full_path.stat().st_size > 5 * 1024 * 1024:
            return {"path": str(full_path), "error": "file too large"}
        with open(full_path, "r", encoding="utf-8", errors="replace") as f:
            content = f.read(limit)
        return {"path": str(full_path.relative_to(root)), "content": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# Simple code search (regex)
@app.post("/api/code/search")
async def code_search(payload: Dict[str, Any]) -> Dict[str, Any]:
    pattern = str(payload.get("pattern", "")).strip()
    base = str(payload.get("base", "repo"))
    subpath = str(payload.get("path", ""))
    max_results = int(payload.get("max_results", 200))
    if not pattern:
        raise HTTPException(status_code=400, detail="pattern required")
    try:
        regex = re.compile(pattern)
    except re.error as e:
        raise HTTPException(status_code=400, detail=f"invalid regex: {e}")
    root = _safe_root(base)
    start = (root / subpath).resolve()
    if not str(start).startswith(str(root.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    results: List[Dict[str, Any]] = []
    skip_dirs = {".git", "node_modules", "__pycache__", "data/conda_envs"}
    for dirpath, dirnames, filenames in os.walk(start):
        # prune dirs
        dirnames[:] = [d for d in dirnames if d not in skip_dirs and not d.startswith('.')]
        for fname in filenames:
            if fname.startswith('.'):
                continue
            p = Path(dirpath) / fname
            # Skip large files
            try:
                if p.stat().st_size > 2 * 1024 * 1024:
                    continue
            except Exception:
                continue
            try:
                with p.open("r", encoding="utf-8", errors="ignore") as f:
                    for i, line in enumerate(f, start=1):
                        if regex.search(line):
                            results.append({
                                "file": str(p.relative_to(root)),
                                "line": i,
                                "text": line.rstrip("\n")
                            })
                            if len(results) >= max_results:
                                return {"results": results, "truncated": True}
            except Exception:
                continue
    return {"results": results, "truncated": False}

# ---------------- QA over codebase (stub) ----------------
@app.post("/api/qa/query")
async def qa_query(payload: Dict[str, Any]) -> Dict[str, Any]:
    question = str(payload.get("question", "")).strip()
    base = str(payload.get("base", "repo"))
    path = str(payload.get("path", ""))
    pattern = str(payload.get("pattern", question or "."))
    max_results = int(payload.get("max_results", 50))
    # Reuse code_search to retrieve snippets, then summarize via scripts/qa/answer_summarize.py
    hits = await code_search({"pattern": pattern, "base": base, "path": path, "max_results": max_results})
    snippets = []
    for h in hits.get("results", [])[:max_results]:
        try:
            snippets.append({"path": h.get("file"), "start": h.get("line", 1), "end": h.get("line", 1), "content": h.get("text", "")})
        except Exception:
            continue
    # Write snippets to temp file and invoke summarizer
    try:
        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json", encoding="utf-8") as tf:


                    continue
            except Exception:
                continue
            try:
                with p.open("r", encoding="utf-8", errors="ignore") as f:
                    for i, line in enumerate(f, start=1):
                        if regex.search(line):
                            results.append({
                                "file": str(p.relative_to(root)),
                                "line": i,
                                "text": line.rstrip("\n")
                            })
                            if len(results) >= max_results:
                                return {"results": results, "truncated": True}
            except Exception:
                continue
    return {"results": results, "truncated": False}

# ---------------- QA over codebase (stub) ----------------
@app.post("/api/qa/query")
async def qa_query(payload: Dict[str, Any]) -> Dict[str, Any]:
    question = str(payload.get("question", "")).strip()
    base = str(payload.get("base", "repo"))
    path = str(payload.get("path", ""))
    pattern = str(payload.get("pattern", question or "."))
    max_results = int(payload.get("max_results", 50))
    # Reuse code_search to retrieve snippets, then summarize via scripts/qa/answer_summarize.py
    hits = await code_search({"pattern": pattern, "base": base, "path": path, "max_results": max_results})
    snippets = []
    for h in hits.get("results", [])[:max_results]:
        try:
            snippets.append({"path": h.get("file"), "start": h.get("line", 1), "end": h.get("line", 1), "content": h.get("text", "")})
        except Exception:
            continue
    # Write snippets to temp file and invoke summarizer
    try:
        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json", encoding="utf-8") as tf:
            tf.write(json.dumps(snippets, ensure_ascii=False))
            tf_path = tf.name
        script = WORKSPACE_DIR / "scripts" / "qa" / "answer_summarize.py"
        if script.exists():
            proc = await asyncio.create_subprocess_exec(
                "python3", str(script), "--question", question, "--snippets", tf_path,
                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
            )
            out, err = await proc.communicate()
            if proc.returncode == 0:
                try:
                    return json.loads(out.decode("utf-8", errors="replace"))
                except Exception:
                    pass
        # Fallback: simple concatenation
        answer = "\n\n".join([s.get("content", "") for s in snippets])[:2000]
        return {"answer": answer, "citations": [{"path": s.get("path"), "start": s.get("start"), "end": s.get("end")} for s in snippets]}
    finally:
        try:
            os.unlink(tf_path)  # type: ignore[name-defined]
        except Exception:
            pass

# ---------------- Docs browsing ----------------
@app.get("/api/docs/list")
async def docs_list() -> Dict[str, Any]:
    root = WORKSPACE_DIR / "docs"
    items: List[Dict[str, Any]] = []
    if root.exists():
        for p in sorted(root.rglob("*")):
            if p.is_file() and not p.name.startswith('.'):
                try:
                    items.append({
                        "path": str(p.relative_to(WORKSPACE_DIR)),
                        "size": p.stat().st_size
                    })
                except Exception:
                    continue
    # include top-level README.md if present
    readme = WORKSPACE_DIR / "README.md"
    if readme.exists():
        items.insert(0, {"path": str(readme.relative_to(WORKSPACE_DIR)), "size": readme.stat().st_size})


                "python3", str(script), "--question", question, "--snippets", tf_path,
                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
            )
            out, err = await proc.communicate()
            if proc.returncode == 0:
                try:
                    return json.loads(out.decode("utf-8", errors="replace"))
                except Exception:
                    pass
        # Fallback: simple concatenation
        answer = "\n\n".join([s.get("content", "") for s in snippets])[:2000]
        return {"answer": answer, "citations": [{"path": s.get("path"), "start": s.get("start"), "end": s.get("end")} for s in snippets]}
    finally:
        try:
            os.unlink(tf_path)  # type: ignore[name-defined]
        except Exception:
            pass

# ---------------- Docs browsing ----------------
@app.get("/api/docs/list")
async def docs_list() -> Dict[str, Any]:
    root = WORKSPACE_DIR / "docs"
    items: List[Dict[str, Any]] = []
    if root.exists():
        for p in sorted(root.rglob("*")):
            if p.is_file() and not p.name.startswith('.'):
                try:
                    items.append({
                        "path": str(p.relative_to(WORKSPACE_DIR)),
                        "size": p.stat().st_size
                    })
                except Exception:
                    continue
    # include top-level README.md if present
    readme = WORKSPACE_DIR / "README.md"
    if readme.exists():
        items.insert(0, {"path": str(readme.relative_to(WORKSPACE_DIR)), "size": readme.stat().st_size})
    return {"docs": items}

@app.get("/api/docs/page")
async def docs_page(path: str) -> Dict[str, Any]:
    full = (WORKSPACE_DIR / path).resolve()
    if not str(full).startswith(str(WORKSPACE_DIR.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    if not full.exists() or not full.is_file():
        raise HTTPException(status_code=404, detail="File not found")
    if full.suffix.lower() not in {".md", ".markdown"}:
        raise HTTPException(status_code=400, detail="Not a Markdown file")
    try:
        content = full.read_text(encoding="utf-8", errors="replace")
        html = markdown.markdown(content, extensions=['tables', 'fenced_code'])
        return {"path": str(full.relative_to(WORKSPACE_DIR)), "html": html, "raw": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# Traces and datasets
@app.get("/api/traces/list")
async def traces_list(offset: int = 0, limit: int = 100) -> Dict[str, Any]:


async def docs_list() -> Dict[str, Any]:
    root = WORKSPACE_DIR / "docs"
    items: List[Dict[str, Any]] = []
    if root.exists():
        for p in sorted(root.rglob("*")):
            if p.is_file() and not p.name.startswith('.'):
                try:
                    items.append({
                        "path": str(p.relative_to(WORKSPACE_DIR)),
                        "size": p.stat().st_size
                    })
                except Exception:
                    continue
    # include top-level README.md if present
    readme = WORKSPACE_DIR / "README.md"
    if readme.exists():
        items.insert(0, {"path": str(readme.relative_to(WORKSPACE_DIR)), "size": readme.stat().st_size})
    return {"docs": items}

@app.get("/api/docs/page")
async def docs_page(path: str) -> Dict[str, Any]:
    full = (WORKSPACE_DIR / path).resolve()
    if not str(full).startswith(str(WORKSPACE_DIR.resolve())):
        raise HTTPException(status_code=403, detail="Access denied")
    if not full.exists() or not full.is_file():
        raise HTTPException(status_code=404, detail="File not found")
    if full.suffix.lower() not in {".md", ".markdown"}:
        raise HTTPException(status_code=400, detail="Not a Markdown file")
    try:
        content = full.read_text(encoding="utf-8", errors="replace")
        html = markdown.markdown(content, extensions=['tables', 'fenced_code'])
        return {"path": str(full.relative_to(WORKSPACE_DIR)), "html": html, "raw": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# Traces and datasets
@app.get("/api/traces/list")
async def traces_list(offset: int = 0, limit: int = 100) -> Dict[str, Any]:
    base = WORKSPACE_DIR / "data" / "traces"
    items: List[Dict[str, Any]] = []
    if base.exists():
        for p in sorted(base.rglob("*.jsonl")):
            try:
                sz = p.stat().st_size
                # Quick peek at first line for metadata
                meta = {}
                try:
                    with p.open("r") as f:
                        first = f.readline().strip()
                        if first:
                            meta = json.loads(first)
                except Exception:
                    pass
                items.append({