File size: 2,569 Bytes
0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad 9a5e3ef 0927dad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | #!/usr/bin/env python3
"""Basic chat/API preflight. This is not a tool-use or agent evaluation."""
from __future__ import annotations
import json
import os
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
CASES = Path(__file__).with_name("cases.json")
OUT = ROOT / "results" / "basic_chat_preflight.json"
def chat(base: str, key: str | None, content: str, model: str) -> str:
headers = {"Content-Type": "application/json"}
if key:
headers["Authorization"] = f"Bearer {key}"
req = urllib.request.Request(
base.rstrip("/") + "/chat/completions",
data=json.dumps({
"model": model,
"messages": [{"role": "user","content": content}],
"temperature": 0,
"max_tokens": 256,
}).encode(),
headers=headers,
method="POST",
)
with urllib.request.urlopen(req, timeout=180) as r:
body = json.loads(r.read().decode())
return body["choices"][0]["message"]["content"]
def main() -> int:
base = os.environ.get("OPENAI_BASE_URL", "http://127.0.0.1:8001/v1")
key = os.environ.get("OPENAI_API_KEY")
model = os.environ.get("SMOKE_MODEL", "local-qwen3-coder-next")
suite = json.loads(CASES.read_text())
results = []
for c in suite["cases"]:
try:
text = chat(base, key, c["prompt"], model)
if "expect_exact" in c:
ok = text.strip() == c["expect_exact"]
else:
ok = all(s.lower() in text.lower() for s in c.get("expect_substr", []))
results.append({"id": c["id"], "ok": ok, "text": text[:800]})
except Exception as e:
results.append({"id": c["id"], "ok": False, "error": str(e)})
payload = {
"schema_version": 1,
"suite": suite["suite"],
"pack": suite["pack"],
"status": "measured_preflight",
"measured_at_utc": datetime.now(timezone.utc).isoformat(),
"endpoint": base,
"model": model,
"passed": sum(1 for r in results if r.get("ok")),
"total": len(results),
"results": results,
"note": "Basic chat/API preflight only; no tools are supplied or executed and this is not agent evidence.",
}
OUT.parent.mkdir(parents=True, exist_ok=True)
OUT.write_text(json.dumps(payload, indent=2) + "\n")
print(json.dumps(payload, indent=2))
return 0 if payload["passed"] == payload["total"] else 1
if __name__ == "__main__":
raise SystemExit(main())
|