"""The design test: a picture marked as the original graphic is sent again, rendered beside a plain code with the same content, version, error correction and mask in 20 simulated everyday scenes, and read there by zxing-cpp and WeChat's scanner: one test at a time across all visitors, in a low-priority worker of its own (design_worker.py) that never takes a reading slot. When the visitor's browser has a scanner the test can use, its page also fetches each scene's pictures and reads them itself; what it reads never comes back. Only answers leave the server, never reads or counts. POST /api/design/slot[?ticket=] the worker for this page, before any picture is sent: {slot, hold_s}; busy (503, Retry-After) with a ticket that keeps the page's place; or off (503) POST /api/design?slot=[&frames=jpeg|png] the picture, under /api/measure's rules: 202 {test, n, scenes, frames, version, ec_level, mask, truths}, or 200 {verdict: "cant", why} GET /api/design/[?frames=0] {state: "running", done}, {state: "done", scanners, ms} or {state: "late"}; from frames=0 on, no pictures are kept for the page GET /api/design//scene/ a scene's box and its two frames' SHA-256, once it has rendered GET /api/design//scene//jpeg the design's capture JPEG; .../patch, the plain code's patch as PNG GET /api/design//frame/ a whole frame as PNG: k = 2s the plain code's, 2s + 1 the design's DELETE /api/design/ the page leaves a test, a slot (and any test started under it) or a ticket The state lives in memory only (design_queue.py), under random ids; nothing about a test is logged or stored. For acceptance runs only, QSD_DESIGN_FAIL_SELFTEST=1 turns the test off as a failed self-test does. """ from __future__ import annotations import asyncio import json import logging import os import re import tempfile import threading import time from pathlib import Path from anyio import CapacityLimiter, to_thread from starlette.requests import ClientDisconnect, Request from starlette.responses import JSONResponse, Response from starlette.routing import Route from .design_queue import Queue from .design_rules import PINS, answer, judge, picture_size, settings from .design_worker import SCENE_S, SELFTEST_S, SETUP_S, DesignWorker __all__ = ["answer", "design_health", "design_routes", "start_design", "stop_design"] ID = re.compile(r"[0-9a-f]{32}") INDEX = re.compile(r"0|[1-9][0-9]?") UNREADABLE = "This picture could not be opened." # the reading's own words for such a picture BUILDS = CapacityLimiter(2) # fallback frames built at once; the rest wait, never in the threads readings share _S: dict = {"queue": None, "worker": None, "cfg": None, "helpers": {}, "same_frames": None, "closing": False} def start_design(data_dir: Path, make_worker=DesignWorker, now=time.monotonic) -> None: """Start the test, after the reading's own self-test: its settings, its worker and its start-up self-test. Never raises: on any failure the test is off, /api/health says so, and the page starts no test.""" stop_design() _S["closing"], _S["same_frames"], worker = False, None, None try: cfg = settings(Path(data_dir) / "test.json", Path(data_dir) / "sizes.json") pins = json.loads(PINS.read_text(encoding="utf-8")) worker = make_worker(cfg["nice"]) got = worker.ask(("selftest", cfg["check"]["png"], cfg["check"]["jpeg"], pins["seeds"], cfg["worker"]), SELFTEST_S) ok, _S["same_frames"] = judge(got[1] if got and got[0] == "selftest" else None, pins, cfg["check"]["jpeg_grey_sha256"]) if not ok or os.environ.get("QSD_DESIGN_FAIL_SELFTEST") == "1": raise RuntimeError("its start-up self-test failed") except Exception as e: # noqa: BLE001 - the test is off; the reading runs as before if worker is not None: worker.close() logging.getLogger("uvicorn.error").warning("The design test is off: %s", e) return _S.update(cfg=cfg, worker=worker, queue=Queue(cfg["queue"], now)) def stop_design() -> None: q, w = _S["queue"], _S["worker"] _S.update(closing=True, queue=None, worker=None) if q is not None: q.on = False if w is not None: w.close() def design_health() -> dict: q = _S["queue"] return {"on": bool(q is not None and q.on), "busy": bool(q is not None and q.busy()), "same_frames": _S["same_frames"]} # ---- a test, in threads of its own ------------------------------------------------------------------------------- def _replace(q: Queue, w: DesignWorker) -> None: """A worker that went silent or died is replaced; when none can start, the test is off.""" try: if not _S["closing"]: w.replace() except Exception: # noqa: BLE001 - no worker: the test is off q.on = False q.release() def _setup(q: Queue, w: DesignWorker, test, data: bytes, suffix: str) -> tuple: """Setup, and the state it leads to, made here, whether or not the request still waits for it: the worker's reply, ("failed",) when it went silent or died, or ("gone",) when the page left during setup.""" try: with tempfile.NamedTemporaryFile(prefix="qsd-", suffix=suffix) as f: # deleted once setup has read it f.write(data) f.flush() got = w.ask(("setup", f.name, test.mode, _S["cfg"]["worker"]), min(SETUP_S, test.deadline - q.now())) except Exception: # noqa: BLE001 - a worker that couldn't start again: as one that died got = None if got is None or got[0] not in ("set", "cant", "unreadable"): q.lose(test) threading.Thread(target=_replace, args=(q, w), daemon=True).start() return ("failed",) if got[0] != "set": q.no_test(test) elif not q.started(test, got[1]): # cleared before it is freed, so that no next setup reaches it first w.tell(("clear",)) q.release() return ("gone",) else: threading.Thread(target=_run, args=(q, w, _S["cfg"]["rule"], test), daemon=True).start() return got def _run(q: Queue, w: DesignWorker, rule: dict, test) -> None: """The test's scenes, one at a time, then the rule; late past the deadline or when the worker fails. Once its page has left it, the scene in hand is the last, and nothing is kept.""" try: while True: step = q.next_scene(test) if step is None: w.tell(("clear",)) q.release() return got = w.ask(("scene", *step), min(SCENE_S, test.deadline - q.now())) if got is None or got[0] != "scene" or got[1] != step[0] or not isinstance(got[2], dict): break if q.scene_done(test, step[0], got[2]): w.tell(("clear",)) # before finish() frees the worker, as above q.finish(test, {name: answer(*counts, **rule) for name, counts in test.counts.items()}) return except Exception: # noqa: BLE001 - a fault ends the test as late, never the server pass q.lose(test) _replace(q, w) # ---- the routes --------------------------------------------------------------------------------------------------- def _json(data, status: int = 200, headers: dict | None = None) -> JSONResponse: return JSONResponse(data, status_code=status, headers={"Cache-Control": "no-store", **(headers or {})}) def _err(status: int, code: str, headers: dict | None = None) -> JSONResponse: return _json({"error": code}, status, headers) def _picture(data: bytes, kind: str) -> Response: return Response(data, media_type=f"image/{kind}", headers={"Cache-Control": "no-store"}) async def slot(request: Request) -> JSONResponse: q, ticket = _S["queue"], request.query_params.get("ticket") if q is None: return _err(503, "off") waiting = bool(_S["cfg"]["readings_switch"] and _S["helpers"]["readings_waiting"]()) got = q.slot(ticket if ticket and ID.fullmatch(ticket) else None, waiting) if got[0] == "go": return _json({"slot": got[1], "hold_s": got[2]}) if got[0] == "off": return _err(503, "off") return _json({"error": "busy", "ticket": got[1], "retry_s": got[2]}, 503, {"Retry-After": str(got[2])}) async def upload(request: Request) -> Response: q, h = _S["queue"], _S["helpers"] mode, slot_id = request.query_params.get("frames"), request.query_params.get("slot", "") if mode is not None and mode not in ("jpeg", "png"): return _err(400, "bad request") if q is None or not ID.fullmatch(slot_id) or not q.upload(slot_id): return _err(409, "slot") test, retry = None, False try: try: async with asyncio.timeout(h["body_s"]): data = await h["body"](request) except TimeoutError: return h["problem"](408, h["messages"][408]) except ClientDisconnect: return h["problem"](400, h["messages"][400]) if data is None: return h["problem"](413, h["messages"][413]) suffix = h["sniff"](data) if suffix is None: retry = True return h["problem"](415, h["messages"][415]) size = picture_size(data) if size is None or size[0] * size[1] > _S["cfg"]["worker"]["max_pixels"]: # never opened as float return _json({"verdict": "cant", "why": "big"}) if size else h["problem"](422, UNREADABLE) test = q.arrive(slot_id, mode) return _err(409, "slot") if test is None else await _start(q, test, data, suffix) finally: if test is None: q.refuse(slot_id, retry) async def _start(q: Queue, test, data: bytes, suffix: str) -> Response: got = await to_thread.run_sync(_setup, q, _S["worker"], test, data, suffix) if got[0] == "failed": return _err(503, "failed") if got[0] == "gone": # its page left during setup return _err(409, "slot") if got[0] == "unreadable": return _S["helpers"]["problem"](422, UNREADABLE) if got[0] == "cant": return _json({"verdict": "cant", "why": got[1]}) f, n = got[1], _S["cfg"]["n"] return _json({"test": test.id, "n": n, "scenes": n if test.mode else 0, "frames": test.mode, "version": f["version"], "ec_level": f["ec_level"], "mask": f["mask"], "truths": [f["raw_sha256"], f["text_sha256"]]}, 202) async def poll(request: Request) -> JSONResponse: q, tid = _S["queue"], request.path_params["test"] got = q.poll(tid, request.query_params.get("frames") == "0") if q is not None and ID.fullmatch(tid) else None return _err(404, "gone") if got is None else _json(got) async def delete(request: Request) -> Response: q, any_id = _S["queue"], request.path_params["test"] if q is not None and ID.fullmatch(any_id): q.delete(any_id) return Response(status_code=204, headers={"Cache-Control": "no-store"}) async def _held(request: Request, key: str = "s", per: int = 1): """(what the scene holds, the test's mode, the index) once it has rendered, after waiting up to scene_wait_s for it; else the 404 or 503 to answer.""" q, tid, k = _S["queue"], request.path_params["test"], request.path_params[key] if q is None or not ID.fullmatch(tid) or not INDEX.fullmatch(k) or int(k) >= per * _S["cfg"]["n"]: return _err(404, "gone") end = time.monotonic() + _S["cfg"]["scene_wait_s"] while (got := q.scene(tid, int(k) // per)) is not None and got[0] == "wait": if time.monotonic() >= end: return _err(503, "rendering", {"Retry-After": "1"}) await asyncio.sleep(0.05) return _err(404, "gone") if got is None else (got[1], got[2], int(k)) async def scene(request: Request) -> Response: got = await _held(request) return got if isinstance(got, Response) else _json({"box": got[0]["box"], "sha256": got[0]["sha256"]}) async def scene_jpeg(request: Request) -> Response: got = await _held(request) if isinstance(got, Response): return got return _picture(got[0]["jpeg"], "jpeg") if got[1] == "jpeg" and got[0]["jpeg"] else _err(404, "gone") async def scene_patch(request: Request) -> Response: got = await _held(request) if isinstance(got, Response): return got return _picture(got[0]["patch"], "png") if got[0]["patch"] else _err(404, "gone") def _fallback(held: dict, plain: bool) -> bytes | None: from . import design_frames # OpenCV and numpy reach the server only with its first fallback frame return design_frames.fallback(held, plain) async def frame(request: Request) -> Response: got = await _held(request, "k", 2) if isinstance(got, Response): return got held, mode, k = got if mode == "png" and k % 2: # in a test with PNG frames, the design's comes from the worker return _picture(held["png"], "png") png = await to_thread.run_sync(_fallback, held, k % 2 == 0, limiter=BUILDS) return _picture(png, "png") if png else _err(500, "frame") def design_routes(helpers: dict) -> list[Route]: """The routes, for main.py to put before its static files. helpers: /api/measure's own body(), sniff() and problem(), its body_s and the messages it answers with, and readings_waiting(), whether pictures wait to be read.""" _S["helpers"] = helpers return [ Route("/api/design/slot", slot, methods=["POST"]), Route("/api/design", upload, methods=["POST"]), Route("/api/design/{test}", poll, methods=["GET"]), Route("/api/design/{test}", delete, methods=["DELETE"]), Route("/api/design/{test}/scene/{s}", scene, methods=["GET"]), Route("/api/design/{test}/scene/{s}/jpeg", scene_jpeg, methods=["GET"]), Route("/api/design/{test}/scene/{s}/patch", scene_patch, methods=["GET"]), Route("/api/design/{test}/frame/{k}", frame, methods=["GET"]), ]