eval/basic_chat_preflight/run_preflight.py

#!/usr/bin/env python3
"""Basic chat/API preflight. This is not a tool-use or agent evaluation."""
from __future__ import annotations

import json
import os
import urllib.request
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
CASES = Path(__file__).with_name("cases.json")
OUT = ROOT / "results" / "basic_chat_preflight.json"

def chat(base: str, key: str | None, content: str, model: str) -> str:
    headers = {"Content-Type": "application/json"}
    if key:
        headers["Authorization"] = f"Bearer {key}"
    req = urllib.request.Request(
        base.rstrip("/") + "/chat/completions",
        data=json.dumps({
            "model": model,
            "messages": [{"role": "user","content": content}],
            "temperature": 0,
            "max_tokens": 256,
        }).encode(),
        headers=headers,
        method="POST",
    )
    with urllib.request.urlopen(req, timeout=180) as r:
        body = json.loads(r.read().decode())
    return body["choices"][0]["message"]["content"]

def main() -> int:
    base = os.environ.get("OPENAI_BASE_URL", "http://127.0.0.1:8001/v1")
    key = os.environ.get("OPENAI_API_KEY")
    model = os.environ.get("SMOKE_MODEL", "local-qwen3-coder-next")
    suite = json.loads(CASES.read_text())
    results = []
    for c in suite["cases"]:
        try:
            text = chat(base, key, c["prompt"], model)
            if "expect_exact" in c:
                ok = text.strip() == c["expect_exact"]
            else:
                ok = all(s.lower() in text.lower() for s in c.get("expect_substr", []))
            results.append({"id": c["id"], "ok": ok, "text": text[:800]})
        except Exception as e:
            results.append({"id": c["id"], "ok": False, "error": str(e)})
    payload = {
        "schema_version": 1,
        "suite": suite["suite"],
        "pack": suite["pack"],
        "status": "measured_preflight",
        "measured_at_utc": datetime.now(timezone.utc).isoformat(),
        "endpoint": base,
        "model": model,
        "passed": sum(1 for r in results if r.get("ok")),
        "total": len(results),
        "results": results,
        "note": "Basic chat/API preflight only; no tools are supplied or executed and this is not agent evidence.",
    }
    OUT.parent.mkdir(parents=True, exist_ok=True)
    OUT.write_text(json.dumps(payload, indent=2) + "\n")
    print(json.dumps(payload, indent=2))
    return 0 if payload["passed"] == payload["total"] else 1

if __name__ == "__main__":
    raise SystemExit(main())