devmandan's picture
Claude Opus 5.5
Facts from each code at its own size; the design test's server, not yet on the page
5b3cd1e verified
Raw History Blame Contribute Delete
14.2 kB
"""The design test: a picture marked as the original graphic is sent again, rendered beside a plain code with the same
content, version, error correction and mask in 20 simulated everyday scenes, and read there by zxing-cpp and WeChat's
scanner: one test at a time across all visitors, in a low-priority worker of its own (design_worker.py) that never
takes a reading slot. When the visitor's browser has a scanner the test can use, its page also fetches each scene's
pictures and reads them itself; what it reads never comes back. Only answers leave the server, never reads or counts.
POST /api/design/slot[?ticket=] the worker for this page, before any picture is sent: {slot, hold_s}; busy
(503, Retry-After) with a ticket that keeps the page's place; or off (503)
POST /api/design?slot=[&frames=jpeg|png] the picture, under /api/measure's rules: 202 {test, n, scenes, frames,
version, ec_level, mask, truths}, or 200 {verdict: "cant", why}
GET /api/design/<test>[?frames=0] {state: "running", done}, {state: "done", scanners, ms} or {state: "late"};
from frames=0 on, no pictures are kept for the page
GET /api/design/<test>/scene/<s> a scene's box and its two frames' SHA-256, once it has rendered
GET /api/design/<test>/scene/<s>/jpeg the design's capture JPEG; .../patch, the plain code's patch as PNG
GET /api/design/<test>/frame/<k> a whole frame as PNG: k = 2s the plain code's, 2s + 1 the design's
DELETE /api/design/<id> the page leaves a test, a slot (and any test started under it) or a ticket
The state lives in memory only (design_queue.py), under random ids; nothing about a test is logged or stored. For
acceptance runs only, QSD_DESIGN_FAIL_SELFTEST=1 turns the test off as a failed self-test does.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import re
import tempfile
import threading
import time
from pathlib import Path
from anyio import CapacityLimiter, to_thread
from starlette.requests import ClientDisconnect, Request
from starlette.responses import JSONResponse, Response
from starlette.routing import Route
from .design_queue import Queue
from .design_rules import PINS, answer, judge, picture_size, settings
from .design_worker import SCENE_S, SELFTEST_S, SETUP_S, DesignWorker
__all__ = ["answer", "design_health", "design_routes", "start_design", "stop_design"]
ID = re.compile(r"[0-9a-f]{32}")
INDEX = re.compile(r"0|[1-9][0-9]?")
UNREADABLE = "This picture could not be opened." # the reading's own words for such a picture
BUILDS = CapacityLimiter(2) # fallback frames built at once; the rest wait, never in the threads readings share
_S: dict = {"queue": None, "worker": None, "cfg": None, "helpers": {}, "same_frames": None, "closing": False}
def start_design(data_dir: Path, make_worker=DesignWorker, now=time.monotonic) -> None:
"""Start the test, after the reading's own self-test: its settings, its worker and its start-up self-test. Never
raises: on any failure the test is off, /api/health says so, and the page starts no test."""
stop_design()
_S["closing"], _S["same_frames"], worker = False, None, None
try:
cfg = settings(Path(data_dir) / "test.json", Path(data_dir) / "sizes.json")
pins = json.loads(PINS.read_text(encoding="utf-8"))
worker = make_worker(cfg["nice"])
got = worker.ask(("selftest", cfg["check"]["png"], cfg["check"]["jpeg"], pins["seeds"], cfg["worker"]),
SELFTEST_S)
ok, _S["same_frames"] = judge(got[1] if got and got[0] == "selftest" else None, pins,
cfg["check"]["jpeg_grey_sha256"])
if not ok or os.environ.get("QSD_DESIGN_FAIL_SELFTEST") == "1":
raise RuntimeError("its start-up self-test failed")
except Exception as e: # noqa: BLE001 - the test is off; the reading runs as before
if worker is not None:
worker.close()
logging.getLogger("uvicorn.error").warning("The design test is off: %s", e)
return
_S.update(cfg=cfg, worker=worker, queue=Queue(cfg["queue"], now))
def stop_design() -> None:
q, w = _S["queue"], _S["worker"]
_S.update(closing=True, queue=None, worker=None)
if q is not None:
q.on = False
if w is not None:
w.close()
def design_health() -> dict:
q = _S["queue"]
return {"on": bool(q is not None and q.on), "busy": bool(q is not None and q.busy()),
"same_frames": _S["same_frames"]}
# ---- a test, in threads of its own -------------------------------------------------------------------------------
def _replace(q: Queue, w: DesignWorker) -> None:
"""A worker that went silent or died is replaced; when none can start, the test is off."""
try:
if not _S["closing"]:
w.replace()
except Exception: # noqa: BLE001 - no worker: the test is off
q.on = False
q.release()
def _setup(q: Queue, w: DesignWorker, test, data: bytes, suffix: str) -> tuple:
"""Setup, and the state it leads to, made here, whether or not the request still waits for it: the worker's
reply, ("failed",) when it went silent or died, or ("gone",) when the page left during setup."""
try:
with tempfile.NamedTemporaryFile(prefix="qsd-", suffix=suffix) as f: # deleted once setup has read it
f.write(data)
f.flush()
got = w.ask(("setup", f.name, test.mode, _S["cfg"]["worker"]), min(SETUP_S, test.deadline - q.now()))
except Exception: # noqa: BLE001 - a worker that couldn't start again: as one that died
got = None
if got is None or got[0] not in ("set", "cant", "unreadable"):
q.lose(test)
threading.Thread(target=_replace, args=(q, w), daemon=True).start()
return ("failed",)
if got[0] != "set":
q.no_test(test)
elif not q.started(test, got[1]): # cleared before it is freed, so that no next setup reaches it first
w.tell(("clear",))
q.release()
return ("gone",)
else:
threading.Thread(target=_run, args=(q, w, _S["cfg"]["rule"], test), daemon=True).start()
return got
def _run(q: Queue, w: DesignWorker, rule: dict, test) -> None:
"""The test's scenes, one at a time, then the rule; late past the deadline or when the worker fails. Once its page
has left it, the scene in hand is the last, and nothing is kept."""
try:
while True:
step = q.next_scene(test)
if step is None:
w.tell(("clear",))
q.release()
return
got = w.ask(("scene", *step), min(SCENE_S, test.deadline - q.now()))
if got is None or got[0] != "scene" or got[1] != step[0] or not isinstance(got[2], dict):
break
if q.scene_done(test, step[0], got[2]):
w.tell(("clear",)) # before finish() frees the worker, as above
q.finish(test, {name: answer(*counts, **rule) for name, counts in test.counts.items()})
return
except Exception: # noqa: BLE001 - a fault ends the test as late, never the server
pass
q.lose(test)
_replace(q, w)
# ---- the routes ---------------------------------------------------------------------------------------------------
def _json(data, status: int = 200, headers: dict | None = None) -> JSONResponse:
return JSONResponse(data, status_code=status, headers={"Cache-Control": "no-store", **(headers or {})})
def _err(status: int, code: str, headers: dict | None = None) -> JSONResponse:
return _json({"error": code}, status, headers)
def _picture(data: bytes, kind: str) -> Response:
return Response(data, media_type=f"image/{kind}", headers={"Cache-Control": "no-store"})
async def slot(request: Request) -> JSONResponse:
q, ticket = _S["queue"], request.query_params.get("ticket")
if q is None:
return _err(503, "off")
waiting = bool(_S["cfg"]["readings_switch"] and _S["helpers"]["readings_waiting"]())
got = q.slot(ticket if ticket and ID.fullmatch(ticket) else None, waiting)
if got[0] == "go":
return _json({"slot": got[1], "hold_s": got[2]})
if got[0] == "off":
return _err(503, "off")
return _json({"error": "busy", "ticket": got[1], "retry_s": got[2]}, 503, {"Retry-After": str(got[2])})
async def upload(request: Request) -> Response:
q, h = _S["queue"], _S["helpers"]
mode, slot_id = request.query_params.get("frames"), request.query_params.get("slot", "")
if mode is not None and mode not in ("jpeg", "png"):
return _err(400, "bad request")
if q is None or not ID.fullmatch(slot_id) or not q.upload(slot_id):
return _err(409, "slot")
test, retry = None, False
try:
try:
async with asyncio.timeout(h["body_s"]):
data = await h["body"](request)
except TimeoutError:
return h["problem"](408, h["messages"][408])
except ClientDisconnect:
return h["problem"](400, h["messages"][400])
if data is None:
return h["problem"](413, h["messages"][413])
suffix = h["sniff"](data)
if suffix is None:
retry = True
return h["problem"](415, h["messages"][415])
size = picture_size(data)
if size is None or size[0] * size[1] > _S["cfg"]["worker"]["max_pixels"]: # never opened as float
return _json({"verdict": "cant", "why": "big"}) if size else h["problem"](422, UNREADABLE)
test = q.arrive(slot_id, mode)
return _err(409, "slot") if test is None else await _start(q, test, data, suffix)
finally:
if test is None:
q.refuse(slot_id, retry)
async def _start(q: Queue, test, data: bytes, suffix: str) -> Response:
got = await to_thread.run_sync(_setup, q, _S["worker"], test, data, suffix)
if got[0] == "failed":
return _err(503, "failed")
if got[0] == "gone": # its page left during setup
return _err(409, "slot")
if got[0] == "unreadable":
return _S["helpers"]["problem"](422, UNREADABLE)
if got[0] == "cant":
return _json({"verdict": "cant", "why": got[1]})
f, n = got[1], _S["cfg"]["n"]
return _json({"test": test.id, "n": n, "scenes": n if test.mode else 0, "frames": test.mode,
"version": f["version"], "ec_level": f["ec_level"], "mask": f["mask"],
"truths": [f["raw_sha256"], f["text_sha256"]]}, 202)
async def poll(request: Request) -> JSONResponse:
q, tid = _S["queue"], request.path_params["test"]
got = q.poll(tid, request.query_params.get("frames") == "0") if q is not None and ID.fullmatch(tid) else None
return _err(404, "gone") if got is None else _json(got)
async def delete(request: Request) -> Response:
q, any_id = _S["queue"], request.path_params["test"]
if q is not None and ID.fullmatch(any_id):
q.delete(any_id)
return Response(status_code=204, headers={"Cache-Control": "no-store"})
async def _held(request: Request, key: str = "s", per: int = 1):
"""(what the scene holds, the test's mode, the index) once it has rendered, after waiting up to scene_wait_s for
it; else the 404 or 503 to answer."""
q, tid, k = _S["queue"], request.path_params["test"], request.path_params[key]
if q is None or not ID.fullmatch(tid) or not INDEX.fullmatch(k) or int(k) >= per * _S["cfg"]["n"]:
return _err(404, "gone")
end = time.monotonic() + _S["cfg"]["scene_wait_s"]
while (got := q.scene(tid, int(k) // per)) is not None and got[0] == "wait":
if time.monotonic() >= end:
return _err(503, "rendering", {"Retry-After": "1"})
await asyncio.sleep(0.05)
return _err(404, "gone") if got is None else (got[1], got[2], int(k))
async def scene(request: Request) -> Response:
got = await _held(request)
return got if isinstance(got, Response) else _json({"box": got[0]["box"], "sha256": got[0]["sha256"]})
async def scene_jpeg(request: Request) -> Response:
got = await _held(request)
if isinstance(got, Response):
return got
return _picture(got[0]["jpeg"], "jpeg") if got[1] == "jpeg" and got[0]["jpeg"] else _err(404, "gone")
async def scene_patch(request: Request) -> Response:
got = await _held(request)
if isinstance(got, Response):
return got
return _picture(got[0]["patch"], "png") if got[0]["patch"] else _err(404, "gone")
def _fallback(held: dict, plain: bool) -> bytes | None:
from . import design_frames # OpenCV and numpy reach the server only with its first fallback frame
return design_frames.fallback(held, plain)
async def frame(request: Request) -> Response:
got = await _held(request, "k", 2)
if isinstance(got, Response):
return got
held, mode, k = got
if mode == "png" and k % 2: # in a test with PNG frames, the design's comes from the worker
return _picture(held["png"], "png")
png = await to_thread.run_sync(_fallback, held, k % 2 == 0, limiter=BUILDS)
return _picture(png, "png") if png else _err(500, "frame")
def design_routes(helpers: dict) -> list[Route]:
"""The routes, for main.py to put before its static files. helpers: /api/measure's own body(), sniff() and
problem(), its body_s and the messages it answers with, and readings_waiting(), whether pictures wait to be read."""
_S["helpers"] = helpers
return [
Route("/api/design/slot", slot, methods=["POST"]),
Route("/api/design", upload, methods=["POST"]),
Route("/api/design/{test}", poll, methods=["GET"]),
Route("/api/design/{test}", delete, methods=["DELETE"]),
Route("/api/design/{test}/scene/{s}", scene, methods=["GET"]),
Route("/api/design/{test}/scene/{s}/jpeg", scene_jpeg, methods=["GET"]),
Route("/api/design/{test}/scene/{s}/patch", scene_patch, methods=["GET"]),
Route("/api/design/{test}/frame/{k}", frame, methods=["GET"]),
]