File size: 8,913 Bytes
b43c3a8
 
 
 
a59c3c4
 
 
 
 
 
 
 
5b3cd1e
 
 
 
 
b43c3a8
 
 
 
 
 
 
 
 
 
 
 
 
6c26597
b43c3a8
 
 
 
5b3cd1e
b43c3a8
6f28f9e
b43c3a8
 
 
 
 
 
 
 
 
5b3cd1e
 
 
 
 
 
b43c3a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6c26597
b43c3a8
6c26597
 
b43c3a8
6c26597
b43c3a8
 
 
 
 
 
 
 
 
 
a59c3c4
 
6f28f9e
b43c3a8
 
 
 
 
 
 
 
 
 
 
 
5b3cd1e
6c26597
5b3cd1e
b43c3a8
5b3cd1e
b43c3a8
 
5b3cd1e
b43c3a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5b3cd1e
b43c3a8
 
 
 
 
 
 
 
5b3cd1e
b43c3a8
5b3cd1e
b43c3a8
 
 
 
6f28f9e
b43c3a8
 
 
 
 
 
 
 
 
 
 
 
 
 
6f28f9e
 
 
 
 
b43c3a8
 
 
 
 
 
 
 
5b3cd1e
 
b43c3a8
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
"""QR Scan Difficulty Check: the measured bell, and a place on it for your own pictures.

GET  /                 the page (web/)
POST /api/measure      one picture (JPEG, PNG or WebP, at most 25 MB) as the request body; returns its reads by
                       the five scanners at the seven frame sizes, the decodes the many-codes rule left out
                       ("not_run"), its place on the scale, its level and how sure that level is (rating.py), the
                       picture-quality checks (quality.py; null when nothing could be measured), which are
                       diagnostics only, zxing-cpp's facts about the code (its unused error correction "uec", the
                       lowest over its reads of the content in the frame "uec_frame", and from one of those reads
                       "ec_level", "mask", the "squares" it sampled, "raw_sha256", "raw_bytes" and the "copies" of
                       its code there; see reading_facts.chosen) and the content's digests (content.sha256 and
                       content.sha256s, for pairing pictures in the page)
GET  /api/health       what is running, and whether the design test is on ("design": on, busy, same_frames)
POST /api/design/slot  the design test (design.py), one at a time across all visitors, in a worker of its own: a slot
                       for this page, then POST /api/design?slot= with the picture marked as the original graphic,
                       GET /api/design/<test> for its answer (and its scenes and frames when the page reads them
                       too), and DELETE /api/design/<id> when the page leaves it
Pictures are read from a temporary file that is deleted when the reading ends; nothing is stored, and no access log is
kept.
"""

import asyncio
import os
import tempfile
from contextlib import asynccontextmanager
from importlib.metadata import version
from pathlib import Path

from starlette.applications import Starlette
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect, Request
from starlette.responses import JSONResponse
from starlette.routing import Mount, Route
from starlette.staticfiles import StaticFiles

from .design import design_health, design_routes, start_design, stop_design
from .place import place
from .rating import rating
from .reading import FRAMES, SCANNERS, PictureError, Reader

WEB = Path(__file__).resolve().parent.parent / "web"
MAX_BYTES = 25 * 1024 * 1024
SLOTS = int(os.environ.get("QSD_SLOTS", "2"))   # pictures read at once: one decoder worker each
MAX_WAITING = 16      # pictures waiting beyond those (arriving, or waiting for a slot); more are turned away
SLOT_WAIT_S = 10      # the longest a picture waits for a slot, so that a request ends within a minute
BODY_S = 60           # the longest a picture may take to arrive; a stalled upload doesn't keep its place
BUSY = "Many pictures are being read right now. Try again in a minute."
UPLOAD = {   # the answers to a picture that didn't arrive whole, for /api/measure and the design test alike
    400: "The picture did not arrive.",
    408: "This picture took too long to arrive. Try again, or send a smaller copy.",
    413: "This picture is over 25 MB.",
    415: "Send a JPEG, PNG or WebP picture.",
}
CSP = "; ".join([
    "default-src 'self'", "img-src 'self' blob: data:", "style-src 'self' 'unsafe-inline'", "script-src 'self'",
    "connect-src 'self'", "object-src 'none'", "base-uri 'none'", "form-action 'none'",
    "frame-ancestors 'self' https://huggingface.co",
])
STATE: dict = {}


def sniff(data: bytes) -> str | None:
    """The picture's format from its first bytes: only JPEG, PNG and WebP are opened here (the page converts others)."""
    if data[:3] == b"\xff\xd8\xff":
        return ".jpg"
    if data[:8] == b"\x89PNG\r\n\x1a\n":
        return ".png"
    if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
        return ".webp"
    return None


def problem(status: int, message: str) -> JSONResponse:
    return JSONResponse({"error": message}, status_code=status, headers={"Cache-Control": "no-store"})


async def body(request: Request) -> bytes | None:
    """The request body, or None past MAX_BYTES (read no further)."""
    n = request.headers.get("content-length", "")
    if n.isdigit() and int(n) > MAX_BYTES:
        return None
    buf = bytearray()   # one buffer, so a picture costs its own size in memory however small its chunks
    async for chunk in request.stream():
        buf += chunk
        if len(buf) > MAX_BYTES:
            return None
    return bytes(buf)


def read_picture(data: bytes, suffix: str) -> dict:
    with tempfile.NamedTemporaryFile(prefix="qsd-", suffix=suffix) as f:
        f.write(data)
        f.flush()
        result = STATE["reader"].read(f.name)
    result["order"] = FRAMES   # the frame size behind each position of "reads" and "frames"
    result["place"] = place(result["reads"], FRAMES)
    result["count1280"] = sum(result["reads"][k][FRAMES.index(1280)] for k in SCANNERS)
    result["rating"] = rating(result["place"], result["count1280"], result["partial"], result["size"],
                              bool(result.get("not_run")))
    result.setdefault("checks", None)   # passed through as the reader gives them; null when nothing was measured
    return result


async def measure(request: Request) -> JSONResponse:
    if STATE["busy"] >= SLOTS + MAX_WAITING:
        return problem(503, BUSY)
    STATE["busy"] += 1   # counted from the first byte, so at most SLOTS + MAX_WAITING pictures are held in memory
    try:
        try:
            async with asyncio.timeout(BODY_S):
                data = await body(request)
        except TimeoutError:
            return problem(408, UPLOAD[408])
        except ClientDisconnect:   # the visitor left mid-upload: nobody will read this answer
            return problem(400, UPLOAD[400])
        if data is None:
            return problem(413, UPLOAD[413])
        suffix = sniff(data)
        if suffix is None:
            return problem(415, UPLOAD[415])
        try:
            await asyncio.wait_for(STATE["gate"].acquire(), SLOT_WAIT_S)
        except TimeoutError:
            return problem(503, BUSY)
        try:
            result = await run_in_threadpool(read_picture, data, suffix)
        finally:
            STATE["gate"].release()
    except PictureError as e:
        return problem(422, str(e))
    finally:
        STATE["busy"] -= 1
    return JSONResponse(result, headers={"Cache-Control": "no-store"})


async def health(request: Request) -> JSONResponse:
    return JSONResponse({
        "ok": True, "scanners": SCANNERS, "frames": FRAMES, "slots": SLOTS,
        "versions": {"opencv-contrib": version("opencv-contrib-python-headless"), "zxing-cpp": version("zxing-cpp"),
                     "zxing": "3.5.2"},
        "design": design_health(),
    }, headers={"Cache-Control": "no-store"})


@asynccontextmanager
async def lifespan(app):
    STATE["reader"] = await run_in_threadpool(Reader, SLOTS)
    STATE["gate"] = asyncio.Semaphore(SLOTS)
    STATE["busy"] = 0
    await run_in_threadpool(start_design, WEB / "data")   # never raises: on a failure the design test is off
    yield
    stop_design()
    STATE["reader"].close()


class SecurityHeaders:
    """Plain ASGI middleware: the same headers on every response, and no-cache on the page's own files."""

    def __init__(self, app):
        self.app = app

    async def __call__(self, scope, receive, send):
        if scope["type"] != "http":
            return await self.app(scope, receive, send)

        async def send_with_headers(message):
            if message["type"] == "http.response.start":
                message.setdefault("headers", [])
                message["headers"] += [(b"content-security-policy", CSP.encode()),
                                       (b"x-content-type-options", b"nosniff"),
                                       (b"referrer-policy", b"strict-origin-when-cross-origin")]
                # the page's own files are checked on every visit (a cheap 304), so an update never pairs an old
                # cached module with a new one
                path = scope.get("path", "")
                if path == "/" or path.endswith((".html", ".js", ".css", ".json")):
                    message["headers"].append((b"cache-control", b"no-cache"))
            await send(message)

        return await self.app(scope, receive, send_with_headers)


app = SecurityHeaders(Starlette(lifespan=lifespan, routes=[
    Route("/api/measure", measure, methods=["POST"]),
    Route("/api/health", health, methods=["GET"]),
    *design_routes({"body": body, "sniff": sniff, "problem": problem, "body_s": BODY_S, "messages": UPLOAD,
                    "readings_waiting": lambda: STATE["busy"] > SLOTS}),
    Mount("/", StaticFiles(directory=WEB, html=True), name="web"),
]))