Spaces:
Sleeping
Sleeping
Download tests/test_api.py from Quantum927/tinymetatron-slm: direct link, hf CLI and curl.
- Browser
- Download file 11.1 kB
-
https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tests/test_api.py
- Command line
-
hf download hf://spaces/Quantum927/tinymetatron-slm/tests/test_api.py
-
curl -L -o test_api.py https://huggingface.co/spaces/Quantum927/tinymetatron-slm/resolve/main/tests/test_api.py
11.1 kB
| """ | |
| tests/test_api.py | |
| ================ | |
| ECOSYSTEM pytest suite for the FastAPI surface (contract section 2). | |
| Covers (contract section 5 / test descriptions): | |
| * GET /data/stats -> 200 | |
| * GET /model/info -> returns config | |
| * POST /data/add -> 200 with added count (private-training mode + key) | |
| * POST /generate -> 200 (untrained model is fine) returns | |
| text + tokens_generated | |
| * GET /train/status -> 200 with status fields | |
| * Deploy-mode gating: /data/add and /train/start return 403 in demo mode | |
| and when the X-API-Key is missing in private mode. | |
| The API DB path is steered to a temp file via the TMT_DB_PATH env var so the | |
| real metatron.db is never touched (contract rule 0.6). Deploy mode + API key | |
| are read at request time by api.py, so the fixtures set them via monkeypatch. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import sys | |
| _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| if _REPO_ROOT not in sys.path: | |
| sys.path.insert(0, _REPO_ROOT) | |
| import pytest | |
| # Import the app after the env var is set in the fixture below. | |
| from fastapi.testclient import TestClient | |
| # Texts long enough to clear the quality threshold (>=0.4). | |
| _ADD_TEXTS = [ | |
| "The firewall enforces a zero-trust policy with mfa and tls encryption.", | |
| "Sparse attention masks reduce the quadratic cost of self-attention.", | |
| "Mixture of experts routes tokens to specialized feed forward networks.", | |
| "The transformer model uses attention, embeddings and a softmax over logits.", | |
| ] | |
| _ADMIN_HEADERS = {"X-API-Key": "testkey"} | |
| def client(tmp_path, monkeypatch): | |
| """Private-training mode client: admin endpoints enabled with a test key.""" | |
| db_path = str(tmp_path / "api_test.db") | |
| monkeypatch.setenv("TMT_DB_PATH", db_path) | |
| monkeypatch.setenv("TMT_DEPLOY_MODE", "private-training") | |
| monkeypatch.setenv("TMT_API_KEY", "testkey") | |
| import api as api_mod | |
| with TestClient(api_mod.app) as c: | |
| yield c | |
| def demo_client(tmp_path, monkeypatch): | |
| """Demo mode client: admin endpoints disabled regardless of key.""" | |
| db_path = str(tmp_path / "api_demo.db") | |
| monkeypatch.setenv("TMT_DB_PATH", db_path) | |
| monkeypatch.setenv("TMT_DEPLOY_MODE", "demo") | |
| monkeypatch.setenv("TMT_API_KEY", "testkey") | |
| import api as api_mod | |
| with TestClient(api_mod.app) as c: | |
| yield c | |
| # ββ GET /health -> 200 ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def test_health_ok(client: TestClient) -> None: | |
| r = client.get("/health") | |
| assert r.status_code == 200, f"status {r.status_code}: {r.text}" | |
| assert r.json() == {"status": "ok"} | |
| # ββ GET /data/stats -> 200, contract keys present βββββββββββββββββββββββββββββ | |
| def test_data_stats_ok(client: TestClient) -> None: | |
| r = client.get("/data/stats") | |
| assert r.status_code == 200, f"status {r.status_code}: {r.text}" | |
| body = r.json() | |
| for k in ("total", "by_domain", "avg_quality", "used_in_training"): | |
| assert k in body, f"/data/stats missing key {k}" | |
| # ββ GET /model/info -> returns config βββββββββββββββββββββββββββββββββββββββββ | |
| def test_model_info_returns_config(client: TestClient) -> None: | |
| r = client.get("/model/info") | |
| assert r.status_code == 200, f"status {r.status_code}: {r.text}" | |
| info = r.json() | |
| assert "config" in info and isinstance(info["config"], dict) | |
| from config import CONFIG | |
| assert info["config"].get("vocab_size") == CONFIG["vocab_size"] | |
| assert "active_checkpoint" in info | |
| # ββ POST /data/add -> 200 with added count (private mode + key) ββββββββββββββββ | |
| def test_data_add_returns_added_count(client: TestClient) -> None: | |
| r = client.post("/data/add", json={ | |
| "texts": _ADD_TEXTS, "domain": "cybersecurity", | |
| "quality_threshold": 0.4, | |
| }, headers=_ADMIN_HEADERS) | |
| assert r.status_code == 200, f"status {r.status_code}: {r.text}" | |
| body = r.json() | |
| assert body["added"] >= 1, f"added>=1, got {body['added']}" | |
| assert body["added"] + body["rejected"] == len(_ADD_TEXTS) | |
| assert body["domain"] == "cybersecurity" | |
| # ββ POST /generate -> 200, returns text + tokens_generated (untrained ok) βββββ | |
| def test_generate_returns_text_and_tokens(client: TestClient) -> None: | |
| r = client.post("/generate", json={ | |
| "prompt": "firewall ", "max_length": 8, "temperature": 0.7, | |
| }) | |
| assert r.status_code == 200, f"status {r.status_code}: {r.text}" | |
| gen = r.json() | |
| assert "text" in gen and isinstance(gen["text"], str) | |
| assert "tokens_generated" in gen | |
| assert isinstance(gen["tokens_generated"], int) | |
| assert gen["tokens_generated"] >= 0 | |
| # step==0 for an untrained model (no checkpoint in a fresh temp DB) | |
| assert gen.get("step") == 0 | |
| # ββ /generate input bounds: prompt over 2000 chars -> 422 βββββββββββββββββββββ | |
| def test_generate_rejects_oversized_prompt(client: TestClient) -> None: | |
| r = client.post("/generate", json={ | |
| "prompt": "x" * 2001, "max_length": 4, "temperature": 0.7, | |
| }) | |
| assert r.status_code == 422, f"expected 422, got {r.status_code}: {r.text}" | |
| # ββ /generate output bound: max_length over 256 -> 422 ββββββββββββββββββββββββ | |
| def test_generate_rejects_oversized_max_length(client: TestClient) -> None: | |
| r = client.post("/generate", json={ | |
| "prompt": "ok", "max_length": 257, "temperature": 0.7, | |
| }) | |
| assert r.status_code == 422, f"expected 422, got {r.status_code}: {r.text}" | |
| # ββ GET /train/status -> 200 with status fields βββββββββββββββββββββββββββββββ | |
| def test_train_status_ok(client: TestClient) -> None: | |
| r = client.get("/train/status") | |
| assert r.status_code == 200, f"status {r.status_code}: {r.text}" | |
| st = r.json() | |
| for k in ("is_training", "current_step", "total_steps", "current_loss"): | |
| assert k in st, f"/train/status missing {k}" | |
| assert st["is_training"] is False | |
| # ββ Admin gate: private mode but no key -> 403 ββββββββββββββββββββββββββββββββ | |
| def test_data_add_requires_key(client: TestClient) -> None: | |
| r = client.post("/data/add", json={ | |
| "texts": _ADD_TEXTS, "domain": "cybersecurity", | |
| "quality_threshold": 0.4, | |
| }) # no X-API-Key header | |
| assert r.status_code == 403, f"expected 403, got {r.status_code}: {r.text}" | |
| def test_data_add_wrong_key(client: TestClient) -> None: | |
| r = client.post("/data/add", json={ | |
| "texts": _ADD_TEXTS, "domain": "cybersecurity", | |
| "quality_threshold": 0.4, | |
| }, headers={"X-API-Key": "wrong"}) | |
| assert r.status_code == 403, f"expected 403, got {r.status_code}: {r.text}" | |
| # ββ Demo mode: admin endpoints disabled even with a valid key -> 403 ββββββββββ | |
| def test_data_add_forbidden_in_demo(demo_client: TestClient) -> None: | |
| r = demo_client.post("/data/add", json={ | |
| "texts": _ADD_TEXTS, "domain": "cybersecurity", | |
| "quality_threshold": 0.4, | |
| }, headers=_ADMIN_HEADERS) | |
| assert r.status_code == 403, f"expected 403, got {r.status_code}: {r.text}" | |
| def test_train_start_forbidden_in_demo(demo_client: TestClient) -> None: | |
| r = demo_client.post("/train/start", json={ | |
| "steps": 1, "domain": "general", "min_quality": 0.4, | |
| }, headers=_ADMIN_HEADERS) | |
| assert r.status_code == 403, f"expected 403, got {r.status_code}: {r.text}" | |
| # ββ Demo mode: read-only endpoints still public ββββββββββββββββββββββββββββββ | |
| def test_readonly_endpoints_ok_in_demo(demo_client: TestClient) -> None: | |
| assert demo_client.get("/health").status_code == 200 | |
| assert demo_client.get("/data/stats").status_code == 200 | |
| assert demo_client.get("/model/info").status_code == 200 | |
| assert demo_client.get("/train/status").status_code == 200 | |
| assert demo_client.post("/generate", json={ | |
| "prompt": "hi", "max_length": 4, "temperature": 0.7, | |
| }).status_code == 200 | |
| # ββ POST /ask -> contract shape {decision, route, evidence, answer} βββββββββββ | |
| def ask_client(tmp_path, monkeypatch): | |
| """Private-training client with a minimal quantum corpus DB for /ask.""" | |
| db_path = str(tmp_path / "api_test.db") | |
| qdb_path = str(tmp_path / "quantum_corpus.db") | |
| monkeypatch.setenv("TMT_DB_PATH", db_path) | |
| monkeypatch.setenv("TMT_DEPLOY_MODE", "private-training") | |
| monkeypatch.setenv("TMT_API_KEY", "testkey") | |
| monkeypatch.setenv("TMT_QUANTUM_CORPUS_DB", qdb_path) | |
| # Build a tiny corpus with train+val records so _get_ask_retriever succeeds. | |
| from quantum_corpus import schema as _qschema | |
| _qschema.init_db(qdb_path) | |
| _qschema.write_records(qdb_path, [ | |
| _qschema.Record( | |
| source_type="manifest", project="wormhole", doc_id="w:1", | |
| text="Circuit 1: OTOC Lyapunov Exponent Measurement. " | |
| "Measures the out-of-time-ordered correlator on ibm_kingston.", | |
| split="train", sensitivity="public", | |
| ), | |
| _qschema.Record( | |
| source_type="ibm_job", project="ibm-quantum", doc_id="ibm:d5a6", | |
| text="IBM Quantum job d5a6 on backend ibm_fez, status Completed, " | |
| "program sampler, 8192 samples.", | |
| split="train", sensitivity="internal", | |
| ), | |
| _qschema.Record( | |
| source_type="repo", project="GRE", doc_id="GRE:readme", | |
| text="Sierpinski triangle quantum walk with golden silver bronze coin angles.", | |
| split="val", sensitivity="public", | |
| ), | |
| ]) | |
| import api as api_mod | |
| # Clear any cached retriever from a previous import in this process. | |
| api_mod._ask_retriever_cache.update(retriever=None, db_path=None, mtime=None) | |
| with TestClient(api_mod.app) as c: | |
| yield c | |
| def test_ask_contract_shape(ask_client: TestClient) -> None: | |
| r = ask_client.post("/ask", json={ | |
| "question": "what does the OTOC circuit measure?", | |
| "top_k": 3, | |
| "mode": "quantum-private", | |
| "sensitivity": "internal", | |
| }, headers=_ADMIN_HEADERS) | |
| assert r.status_code == 200, f"status {r.status_code}: {r.text}" | |
| body = r.json() | |
| for k in ("decision", "route", "evidence", "answer"): | |
| assert k in body, f"/ask response missing contract key {k}" | |
| assert body["route"] in ("retrieval", "structured", "declined") | |
| assert isinstance(body["evidence"], list) | |
| assert isinstance(body["answer"], str) | |
| # The user-facing answer must NOT come from the 32-token sampler; the | |
| # contract response does not include a "generated" field produced by it. | |
| assert "generated" not in body | |