File size: 18,133 Bytes
9df1f37
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
"""Web server for the Image Preprocessing Lab.

All image logic lives in image_lab.ipynb. This file runs that notebook (skipping cells tagged "demo") and
serves its functions to the browser. It only adds web plumbing: a gallery, a cache, and image encoding.

Run:  uvicorn server:app --port 8000     (from the backend folder)

Environment:
  ALLOWED_ORIGINS  comma-separated origins allowed to call the API (default "*"), e.g. https://your-app.vercel.app
"""

import ast
import hashlib
import json
import os
import re
import threading
import time
from collections import OrderedDict
from contextlib import asynccontextmanager
from pathlib import Path

import cv2
import nbformat
import numpy as np
from fastapi import Body, FastAPI, File, HTTPException, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, Response

HERE = Path(__file__).resolve().parent
DATA = HERE / "data"
UPLOADS = HERE / "uploads"
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
UPLOAD_MAX_SIDE = 1024


# ---------------------------------------------------------------- 1. load the notebook

def load_notebook(path):
    """Run every non-demo code cell into one namespace; also keep each function's source text."""
    nb = nbformat.read(path, as_version=4)
    lab = {"__file__": str(path), "__name__": "image_lab"}
    sources = {}
    for i, cell in enumerate(nb.cells):
        if cell.cell_type != "code" or "demo" in cell.metadata.get("tags", []):
            continue
        exec(compile(cell.source, f"{path.name} (cell {i})", "exec"), lab)
        for node in ast.parse(cell.source).body:
            if isinstance(node, ast.FunctionDef):
                sources[node.name] = ast.get_source_segment(cell.source, node)
    return lab, sources


LAB, SOURCES = load_notebook(HERE / "image_lab.ipynb")
STEPS = LAB["STEPS"]


def step_source(step_id):
    """The step function's source, preceded by any private helper it calls (e.g. _gray)."""
    main = SOURCES[step_id]
    helpers = [SOURCES[n] for n in dict.fromkeys(re.findall(r"\b(_[a-z]\w*)\(", main)) if n in SOURCES]
    return "\n\n\n".join([*helpers, main])


def call_line(step_id, params):
    """The exact call the pipeline makes, e.g. img = rotate(img, angle=45)."""
    args = ["img", *(f"{k}={v!r}" for k, v in params.items())]
    return f"img = {step_id}({', '.join(args)})"


# ---------------------------------------------------------------- 2. images

def gallery():
    path = DATA / "manifest.json"
    return json.loads(path.read_text())["images"] if path.exists() else []


def image_meta(image_id):
    if image_id.startswith("upload-") and (UPLOADS / f"{image_id[7:]}.png").exists():
        # Uploads are never listed in the gallery: only the person who uploaded one knows its id.
        return {"id": image_id, "use_case": "upload", "file": str(UPLOADS / f"{image_id[7:]}.png"),
                "title": "Your image", "license": "Your image", "source": "Uploaded"}
    for item in gallery():
        if item["id"] == image_id:
            return item
    raise HTTPException(404, f"No image called {image_id!r}")


def image_path(image_id):
    file = Path(image_meta(image_id)["file"])
    return file if file.is_absolute() else DATA / file


RAW = OrderedDict()  # image id -> the BGR array from load_image


def load(image_id):
    if image_id not in RAW:
        if not re.fullmatch(r"[\w-]+", image_id):
            raise HTTPException(404, "Bad image id")
        img = LAB["load_image"](image_path(image_id))
        img.setflags(write=False)  # shared by every run: steps must return new arrays
        RAW[image_id] = img
        while len(RAW) > 64:
            RAW.popitem(last=False)
    return RAW[image_id]


# ---------------------------------------------------------------- 3. turning arrays into pictures and numbers

def is_chw(a):
    return a.ndim == 3 and a.shape[0] in (1, 3) and a.shape[2] not in (1, 3, 4)


def to_display(a):
    """RGB or gray uint8 for the browser, plus a note when values had to be rescaled to be visible."""
    if is_chw(a):
        a = a.transpose(1, 2, 0)
    if a.dtype == np.uint8:
        return a, None
    lo, hi = float(a.min()), float(a.max())
    scaled = cv2.normalize(a.astype(np.float32), None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8)
    return scaled, f"{a.dtype} values {lo:.2f} to {hi:.2f}, stretched to 0..255 so you can see them"


def stats(a):
    return {"shape": list(a.shape), "dtype": str(a.dtype), "min": round(float(a.min()), 3),
            "max": round(float(a.max()), 3),
            "layout": "CHW" if is_chw(a) else ("HW" if a.ndim == 2 else "HWC")}


def encode(img, fmt="png", width=None):
    if width and img.shape[1] > width:
        img = cv2.resize(img, (width, round(img.shape[0] * width / img.shape[1])), interpolation=cv2.INTER_AREA)
    bgr = img[:, :, ::-1] if img.ndim == 3 else img
    ok, buf = cv2.imencode("." + fmt, np.ascontiguousarray(bgr), [cv2.IMWRITE_JPEG_QUALITY, 80] if fmt == "jpg" else [])
    return buf.tobytes()


# ---------------------------------------------------------------- 4. running a line, with a cache

# Each step result is cached under a key built from the image and every step up to it, so moving one
# slider only recomputes that step and the ones after it.
CACHE = OrderedDict()  # key -> dict(array, error, ms)
CACHE_LOCK = threading.Lock()
MODELS = {}
MODEL_LOCKS = {"resnet50": threading.Lock(), "yolo": threading.Lock()}


def remember(key, entry):
    with CACHE_LOCK:
        CACHE[key] = entry
        while len(CACHE) > 600:
            CACHE.popitem(last=False)  # drop the oldest
    return entry


def recall(key):
    with CACHE_LOCK:
        if key in CACHE:
            CACHE.move_to_end(key)
            return CACHE[key]
    return None


KEY_LOCKS = {}  # cache key -> lock, while that result is being computed


def once(key, compute):
    """Recall key, or compute it exactly once even when several requests ask at the same time.

    A slider drag sends many requests and an aborted fetch does not stop the server's work, so without this the
    same slow step (denoise takes about a second) could run several times in parallel."""
    entry = recall(key)
    if entry is not None:
        return entry
    with CACHE_LOCK:
        lock = KEY_LOCKS.setdefault(key, threading.Lock())
    with lock:
        try:
            entry = recall(key)
            return entry if entry is not None else remember(key, compute())
        finally:
            with CACHE_LOCK:
                KEY_LOCKS.pop(key, None)


def with_model(name, fn):
    """Run fn(model) while holding that model's lock: one inference at a time per model."""
    with MODEL_LOCKS[name]:
        if name not in MODELS:
            MODELS[name] = LAB["load_resnet50"]() if name == "resnet50" else LAB["load_yolo"]()
        return fn(MODELS[name])


def make_key(*parts):
    return hashlib.sha1("|".join(parts).encode()).hexdigest()[:16]


def histogram(display, bins=64):
    """Brightness histogram of the displayed picture: `bins` counts over 0..255."""
    gray = cv2.cvtColor(display, cv2.COLOR_RGB2GRAY) if display.ndim == 3 and display.shape[2] == 3 else display
    if gray.ndim == 3:
        gray = gray[:, :, 0]
    counts = cv2.calcHist([np.ascontiguousarray(gray)], [0], None, [bins], [0, 256])
    return [int(c) for c in counts.ravel()]


def describe(key, entry, **extra):
    if "hist" not in entry:  # computed once per cached result
        display, note = to_display(entry["array"])
        entry["hist"], entry["note"] = histogram(display), note
    return {**extra, "key": key, "error": entry.get("error"), "ms": entry.get("ms", 0),
            "stats": stats(entry["array"]), "display_note": entry["note"], "hist": entry["hist"]}


def run_steps(image_id, steps):
    """Run a line's steps through the cache. Returns (final key, final entry, one description per step)."""
    key = make_key("load", image_id)
    current = recall(key) or remember(key, {"array": load(image_id)})
    out = [describe(key, current, uid="load", op="load", name="Load image", kind="load", enabled=True, params={},
                    call="img = load_image(path)")]
    for step in steps:
        step_id = step.get("op")
        if step_id not in STEPS:
            continue
        enabled = bool(step.get("enabled", True))
        params = LAB["clean_params"](step_id, step.get("params"))
        key = make_key(key, json.dumps([step_id, params, enabled], sort_keys=True))
        entry = once(key, lambda: compute_step(current, step_id, params, enabled))
        # uid and rev are echoed untouched so the app can tell which step, and which edit, a result belongs to.
        out.append(describe(key, entry, uid=step.get("uid"), rev=step.get("rev"), op=step_id,
                            name=STEPS[step_id]["name"], kind=STEPS[step_id]["kind"], enabled=enabled, params=params,
                            call=call_line(step_id, params)))
        current = entry
    return key, current, out


def compute_step(current, step_id, params, enabled):
    if not enabled:
        return {"array": current["array"]}
    t0 = time.perf_counter()
    try:
        entry = {"array": np.asarray(LAB["run_step"](current["array"], step_id, params))}
    except Exception as exc:  # show the error on the step; the image passes through unchanged
        msg = str(exc).strip().splitlines()[-1] if str(exc).strip() else ""
        entry = {"array": current["array"], "error": f"{type(exc).__name__}: {msg}"}
    entry["ms"] = round((time.perf_counter() - t0) * 1000, 1)
    return entry


def clean_version(steps):
    """The same line with every preparation step on at its defaults and every break-it and fix-it step off."""
    return [{"op": s.get("op"), "enabled": STEPS[s["op"]]["kind"] == "prep"} for s in steps if s.get("op") in STEPS]


def run_line(image_id, steps, task):
    key, final, out = run_steps(image_id, steps)
    result = run_task(task, key, final)
    if result is not None:
        bkey, bfinal, _ = run_steps(image_id, clean_version(steps))
        result["baseline"] = run_task(task, bkey, bfinal)
        if result["type"] == "classify":
            # How sure the model is NOW of what it answered on the clean image: this is what drops.
            base, pk = result["baseline"], result["probs_key"]
            result["baseline_label_now"] = None
            if base["top5"] and pk in CACHE_PROBS:
                label = base["top5"][0]["label"]
                result["baseline_label_now"] = round(float(CACHE_PROBS[pk][LAB["LABELS"].index(label)]), 4)
    return {"image": image_meta(image_id), "steps": out, "result": result}


CACHE_PROBS = {}  # probs key -> the 1000 class probabilities (small, kept apart from the image cache)


def run_task(task, key, final):
    kind = (task or {}).get("type", "none")
    if kind == "classify":
        rkey = make_key(key, "classify")
        cached = recall(rkey)
        if cached is None:
            t0 = time.perf_counter()
            a = final["array"]
            result = {"type": "classify", "model": "ResNet50", "input_shape": [1, *a.shape], "input_dtype": str(a.dtype),
                      "input_range": [round(float(a.min()), 2), round(float(a.max()), 2)], "probs_key": rkey,
                      "error": None, "top5": []}
            try:
                probs = with_model("resnet50", lambda m: LAB["classify_resnet50"](a, m))
                CACHE_PROBS[rkey] = probs
                while len(CACHE_PROBS) > 2000:
                    CACHE_PROBS.pop(next(iter(CACHE_PROBS)))
                result["top5"] = [{"label": label, "prob": p} for label, p in LAB["top_k"](probs, 5)]
            except Exception as exc:  # e.g. channels last: PyTorch refuses, and that is the lesson
                msg = str(exc).strip().splitlines()[0] if str(exc).strip() else type(exc).__name__
                result["error"] = msg
                result["probs_key"] = None
            result["ms"] = round((time.perf_counter() - t0) * 1000, 1)
            cached = remember(rkey, {"array": a, "result": result})
        pk = cached["result"]["probs_key"]
        if pk is not None and pk not in CACHE_PROBS:  # evicted: recompute on the next run
            with CACHE_LOCK:
                CACHE.pop(rkey, None)
            return run_task(task, key, final)
        return dict(cached["result"])
    if kind == "segment":
        conf = min(0.95, max(0.05, float(task.get("conf", 0.25))))
        rkey = make_key(key, "segment", f"{conf:.3f}")
        cached = recall(rkey)
        if cached is None:
            t0 = time.perf_counter()
            img = LAB["as_uint8_rgb"](final["array"])
            overlay, instances = with_model("yolo", lambda m: LAB["segment_yolo"](img, m, conf))
            counts = {}
            for name, _ in instances:
                counts[name] = counts.get(name, 0) + 1
            # class name -> the mask color segment_yolo drew, so the app's legend matches the picture
            colors = {n: "#%02x%02x%02x" % LAB["class_color"](i) for i, n in MODELS["yolo"].names.items()}
            cached = remember(rkey, {"array": overlay, "result": {
                "type": "segment", "model": "YOLO11s-seg", "conf": conf, "total": len(instances),
                "counts": dict(sorted(counts.items(), key=lambda kv: -kv[1])), "overlay_key": rkey,
                "colors": colors,
                "instances": [{"label": n, "conf": c} for n, c in instances], "input_shape": list(img.shape),
                "notes": [] if final["array"].dtype == np.uint8 else ["The pipeline output was converted to uint8 RGB first."],
                "ms": round((time.perf_counter() - t0) * 1000, 1)}})
        return dict(cached["result"])
    return None


# ---------------------------------------------------------------- 5. HTTP routes

def warm_up():
    """Load both models, then run every gallery image through its clean line, so first clicks are instant."""
    with_model("resnet50", lambda m: LAB["classify_resnet50"](np.zeros((3, 224, 224), np.float32), m))
    with_model("yolo", lambda m: LAB["segment_yolo"](np.zeros((640, 640, 3), np.uint8), m))
    lines = {line["id"]: line for line in LAB["LINES"]}
    for item in gallery():
        line = lines[item["use_case"]]
        try:
            run_line(item["id"], line["steps"], line["task"])
        except Exception as exc:  # a missing file should not stop the server
            print("warm-up skipped", item["id"], exc)
    # The default photo is what the presenter breaks and fixes first: run every break, alone and with each fix
    # meant for it (denoise takes about a second), so those first clicks are instant too.
    for line in LAB["LINES"]:
        for s in line["steps"]:
            if STEPS.get(s["op"], {}).get("kind") != "break":
                continue
            fixes = [f for f, d in STEPS.items() if d["kind"] == "fix" and s["op"] in d["helps"]]
            for on in [{s["op"]}, *({s["op"], f} for f in fixes)]:
                steps = [{**x, "enabled": True} if x["op"] in on else x for x in line["steps"]]
                try:
                    run_line(line["default_image"], steps, line["task"])
                except Exception as exc:
                    print("warm-up skipped", line["id"], sorted(on), exc)


@asynccontextmanager
async def lifespan(_):
    threading.Thread(target=warm_up, daemon=True).start()
    yield


app = FastAPI(title="Image Preprocessing Lab", lifespan=lifespan)
app.add_middleware(CORSMiddleware, allow_origins=[o.strip() for o in os.environ.get("ALLOWED_ORIGINS", "*").split(",")],
                   allow_methods=["GET", "POST"], allow_headers=["*"])
FOREVER = {"Cache-Control": "public, max-age=31536000, immutable"}


@app.get("/api/health")
def health():
    return {"ok": True, "models_ready": len(MODELS) == 2}


@app.get("/api/steps")
def steps():
    return {
        "kinds": LAB["KINDS"],
        "steps": [{k: v for k, v in s.items() if k != "fn"} | {"source": step_source(s["id"])} for s in STEPS.values()],
        "load_source": SOURCES["load_image"],
        "model_sources": {"classify": "\n\n\n".join(SOURCES[n] for n in ("classify_resnet50", "top_k")),
                          "segment": "\n\n\n".join(SOURCES[n] for n in ("draw_label", "segment_yolo"))},
    }


@app.get("/api/lines")
def lines():
    return {"lines": LAB["LINES"]}


@app.get("/api/images")
def images():
    return {"images": gallery()}


@app.get("/api/images/{image_id}/thumb")
def image_thumb(image_id: str):
    return Response(encode(load(image_id)[:, :, ::-1], "jpg", width=200), media_type="image/jpeg", headers=FOREVER)


@app.post("/api/upload")
async def upload(file: UploadFile = File(...)):
    data = await file.read(MAX_UPLOAD_BYTES + 1)
    if len(data) > MAX_UPLOAD_BYTES:
        raise HTTPException(413, "Images up to 10 MB, please.")
    arr = cv2.imdecode(np.frombuffer(data, np.uint8), cv2.IMREAD_COLOR)
    if arr is None:
        raise HTTPException(400, "That file is not an image OpenCV can read.")
    h, w = arr.shape[:2]
    s = UPLOAD_MAX_SIDE / max(h, w)
    if s < 1:
        arr = cv2.resize(arr, (round(w * s), round(h * s)), interpolation=cv2.INTER_AREA)
    UPLOADS.mkdir(exist_ok=True)
    digest = hashlib.sha1(data).hexdigest()[:12]
    cv2.imwrite(str(UPLOADS / f"{digest}.png"), arr)
    return {"id": f"upload-{digest}"}


@app.post("/api/run")
def run(body: dict = Body(...)):
    return run_line(str(body["image"]), body.get("steps", []), body.get("task", {}))


@app.get("/api/img/{key}")
def step_image(key: str, w: int | None = None):
    entry = recall(key)
    if entry is None:
        raise HTTPException(404, "Image expired from the cache; run the line again")
    display, _ = to_display(entry["array"])
    if w:
        return Response(encode(display, "jpg", width=min(w, 2048)), media_type="image/jpeg", headers=FOREVER)
    return Response(encode(display, "png"), media_type="image/png", headers=FOREVER)


@app.exception_handler(Exception)
def crash(_, exc):
    return JSONResponse({"detail": f"{type(exc).__name__}: {exc}"}, status_code=500)