#!/usr/bin/env python3 """Basic chat/API preflight. This is not a tool-use or agent evaluation.""" from __future__ import annotations import json import os import urllib.request from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[2] CASES = Path(__file__).with_name("cases.json") OUT = ROOT / "results" / "basic_chat_preflight.json" def chat(base: str, key: str | None, content: str, model: str) -> str: headers = {"Content-Type": "application/json"} if key: headers["Authorization"] = f"Bearer {key}" req = urllib.request.Request( base.rstrip("/") + "/chat/completions", data=json.dumps({ "model": model, "messages": [{"role": "user","content": content}], "temperature": 0, "max_tokens": 256, }).encode(), headers=headers, method="POST", ) with urllib.request.urlopen(req, timeout=180) as r: body = json.loads(r.read().decode()) return body["choices"][0]["message"]["content"] def main() -> int: base = os.environ.get("OPENAI_BASE_URL", "http://127.0.0.1:8001/v1") key = os.environ.get("OPENAI_API_KEY") model = os.environ.get("SMOKE_MODEL", "local-qwen3-coder-next") suite = json.loads(CASES.read_text()) results = [] for c in suite["cases"]: try: text = chat(base, key, c["prompt"], model) if "expect_exact" in c: ok = text.strip() == c["expect_exact"] else: ok = all(s.lower() in text.lower() for s in c.get("expect_substr", [])) results.append({"id": c["id"], "ok": ok, "text": text[:800]}) except Exception as e: results.append({"id": c["id"], "ok": False, "error": str(e)}) payload = { "schema_version": 1, "suite": suite["suite"], "pack": suite["pack"], "status": "measured_preflight", "measured_at_utc": datetime.now(timezone.utc).isoformat(), "endpoint": base, "model": model, "passed": sum(1 for r in results if r.get("ok")), "total": len(results), "results": results, "note": "Basic chat/API preflight only; no tools are supplied or executed and this is not agent evidence.", } OUT.parent.mkdir(parents=True, exist_ok=True) OUT.write_text(json.dumps(payload, indent=2) + "\n") print(json.dumps(payload, indent=2)) return 0 if payload["passed"] == payload["total"] else 1 if __name__ == "__main__": raise SystemExit(main())