"""Web server for the Image Preprocessing Lab. All image logic lives in image_lab.ipynb. This file runs that notebook (skipping cells tagged "demo") and serves its functions to the browser. It only adds web plumbing: a gallery, a cache, and image encoding. Run: uvicorn server:app --port 8000 (from the backend folder) Environment: ALLOWED_ORIGINS comma-separated origins allowed to call the API (default "*"), e.g. https://your-app.vercel.app """ import ast import hashlib import json import os import re import threading import time from collections import OrderedDict from contextlib import asynccontextmanager from pathlib import Path import cv2 import nbformat import numpy as np from fastapi import Body, FastAPI, File, HTTPException, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, Response HERE = Path(__file__).resolve().parent DATA = HERE / "data" UPLOADS = HERE / "uploads" MAX_UPLOAD_BYTES = 10 * 1024 * 1024 UPLOAD_MAX_SIDE = 1024 # ---------------------------------------------------------------- 1. load the notebook def load_notebook(path): """Run every non-demo code cell into one namespace; also keep each function's source text.""" nb = nbformat.read(path, as_version=4) lab = {"__file__": str(path), "__name__": "image_lab"} sources = {} for i, cell in enumerate(nb.cells): if cell.cell_type != "code" or "demo" in cell.metadata.get("tags", []): continue exec(compile(cell.source, f"{path.name} (cell {i})", "exec"), lab) for node in ast.parse(cell.source).body: if isinstance(node, ast.FunctionDef): sources[node.name] = ast.get_source_segment(cell.source, node) return lab, sources LAB, SOURCES = load_notebook(HERE / "image_lab.ipynb") STEPS = LAB["STEPS"] def step_source(step_id): """The step function's source, preceded by any private helper it calls (e.g. _gray).""" main = SOURCES[step_id] helpers = [SOURCES[n] for n in dict.fromkeys(re.findall(r"\b(_[a-z]\w*)\(", main)) if n in SOURCES] return "\n\n\n".join([*helpers, main]) def call_line(step_id, params): """The exact call the pipeline makes, e.g. img = rotate(img, angle=45).""" args = ["img", *(f"{k}={v!r}" for k, v in params.items())] return f"img = {step_id}({', '.join(args)})" # ---------------------------------------------------------------- 2. images def gallery(): path = DATA / "manifest.json" return json.loads(path.read_text())["images"] if path.exists() else [] def image_meta(image_id): if image_id.startswith("upload-") and (UPLOADS / f"{image_id[7:]}.png").exists(): # Uploads are never listed in the gallery: only the person who uploaded one knows its id. return {"id": image_id, "use_case": "upload", "file": str(UPLOADS / f"{image_id[7:]}.png"), "title": "Your image", "license": "Your image", "source": "Uploaded"} for item in gallery(): if item["id"] == image_id: return item raise HTTPException(404, f"No image called {image_id!r}") def image_path(image_id): file = Path(image_meta(image_id)["file"]) return file if file.is_absolute() else DATA / file RAW = OrderedDict() # image id -> the BGR array from load_image def load(image_id): if image_id not in RAW: if not re.fullmatch(r"[\w-]+", image_id): raise HTTPException(404, "Bad image id") img = LAB["load_image"](image_path(image_id)) img.setflags(write=False) # shared by every run: steps must return new arrays RAW[image_id] = img while len(RAW) > 64: RAW.popitem(last=False) return RAW[image_id] # ---------------------------------------------------------------- 3. turning arrays into pictures and numbers def is_chw(a): return a.ndim == 3 and a.shape[0] in (1, 3) and a.shape[2] not in (1, 3, 4) def to_display(a): """RGB or gray uint8 for the browser, plus a note when values had to be rescaled to be visible.""" if is_chw(a): a = a.transpose(1, 2, 0) if a.dtype == np.uint8: return a, None lo, hi = float(a.min()), float(a.max()) scaled = cv2.normalize(a.astype(np.float32), None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8) return scaled, f"{a.dtype} values {lo:.2f} to {hi:.2f}, stretched to 0..255 so you can see them" def stats(a): return {"shape": list(a.shape), "dtype": str(a.dtype), "min": round(float(a.min()), 3), "max": round(float(a.max()), 3), "layout": "CHW" if is_chw(a) else ("HW" if a.ndim == 2 else "HWC")} def encode(img, fmt="png", width=None): if width and img.shape[1] > width: img = cv2.resize(img, (width, round(img.shape[0] * width / img.shape[1])), interpolation=cv2.INTER_AREA) bgr = img[:, :, ::-1] if img.ndim == 3 else img ok, buf = cv2.imencode("." + fmt, np.ascontiguousarray(bgr), [cv2.IMWRITE_JPEG_QUALITY, 80] if fmt == "jpg" else []) return buf.tobytes() # ---------------------------------------------------------------- 4. running a line, with a cache # Each step result is cached under a key built from the image and every step up to it, so moving one # slider only recomputes that step and the ones after it. CACHE = OrderedDict() # key -> dict(array, error, ms) CACHE_LOCK = threading.Lock() MODELS = {} MODEL_LOCKS = {"resnet50": threading.Lock(), "yolo": threading.Lock()} def remember(key, entry): with CACHE_LOCK: CACHE[key] = entry while len(CACHE) > 600: CACHE.popitem(last=False) # drop the oldest return entry def recall(key): with CACHE_LOCK: if key in CACHE: CACHE.move_to_end(key) return CACHE[key] return None KEY_LOCKS = {} # cache key -> lock, while that result is being computed def once(key, compute): """Recall key, or compute it exactly once even when several requests ask at the same time. A slider drag sends many requests and an aborted fetch does not stop the server's work, so without this the same slow step (denoise takes about a second) could run several times in parallel.""" entry = recall(key) if entry is not None: return entry with CACHE_LOCK: lock = KEY_LOCKS.setdefault(key, threading.Lock()) with lock: try: entry = recall(key) return entry if entry is not None else remember(key, compute()) finally: with CACHE_LOCK: KEY_LOCKS.pop(key, None) def with_model(name, fn): """Run fn(model) while holding that model's lock: one inference at a time per model.""" with MODEL_LOCKS[name]: if name not in MODELS: MODELS[name] = LAB["load_resnet50"]() if name == "resnet50" else LAB["load_yolo"]() return fn(MODELS[name]) def make_key(*parts): return hashlib.sha1("|".join(parts).encode()).hexdigest()[:16] def histogram(display, bins=64): """Brightness histogram of the displayed picture: `bins` counts over 0..255.""" gray = cv2.cvtColor(display, cv2.COLOR_RGB2GRAY) if display.ndim == 3 and display.shape[2] == 3 else display if gray.ndim == 3: gray = gray[:, :, 0] counts = cv2.calcHist([np.ascontiguousarray(gray)], [0], None, [bins], [0, 256]) return [int(c) for c in counts.ravel()] def describe(key, entry, **extra): if "hist" not in entry: # computed once per cached result display, note = to_display(entry["array"]) entry["hist"], entry["note"] = histogram(display), note return {**extra, "key": key, "error": entry.get("error"), "ms": entry.get("ms", 0), "stats": stats(entry["array"]), "display_note": entry["note"], "hist": entry["hist"]} def run_steps(image_id, steps): """Run a line's steps through the cache. Returns (final key, final entry, one description per step).""" key = make_key("load", image_id) current = recall(key) or remember(key, {"array": load(image_id)}) out = [describe(key, current, uid="load", op="load", name="Load image", kind="load", enabled=True, params={}, call="img = load_image(path)")] for step in steps: step_id = step.get("op") if step_id not in STEPS: continue enabled = bool(step.get("enabled", True)) params = LAB["clean_params"](step_id, step.get("params")) key = make_key(key, json.dumps([step_id, params, enabled], sort_keys=True)) entry = once(key, lambda: compute_step(current, step_id, params, enabled)) # uid and rev are echoed untouched so the app can tell which step, and which edit, a result belongs to. out.append(describe(key, entry, uid=step.get("uid"), rev=step.get("rev"), op=step_id, name=STEPS[step_id]["name"], kind=STEPS[step_id]["kind"], enabled=enabled, params=params, call=call_line(step_id, params))) current = entry return key, current, out def compute_step(current, step_id, params, enabled): if not enabled: return {"array": current["array"]} t0 = time.perf_counter() try: entry = {"array": np.asarray(LAB["run_step"](current["array"], step_id, params))} except Exception as exc: # show the error on the step; the image passes through unchanged msg = str(exc).strip().splitlines()[-1] if str(exc).strip() else "" entry = {"array": current["array"], "error": f"{type(exc).__name__}: {msg}"} entry["ms"] = round((time.perf_counter() - t0) * 1000, 1) return entry def clean_version(steps): """The same line with every preparation step on at its defaults and every break-it and fix-it step off.""" return [{"op": s.get("op"), "enabled": STEPS[s["op"]]["kind"] == "prep"} for s in steps if s.get("op") in STEPS] def run_line(image_id, steps, task): key, final, out = run_steps(image_id, steps) result = run_task(task, key, final) if result is not None: bkey, bfinal, _ = run_steps(image_id, clean_version(steps)) result["baseline"] = run_task(task, bkey, bfinal) if result["type"] == "classify": # How sure the model is NOW of what it answered on the clean image: this is what drops. base, pk = result["baseline"], result["probs_key"] result["baseline_label_now"] = None if base["top5"] and pk in CACHE_PROBS: label = base["top5"][0]["label"] result["baseline_label_now"] = round(float(CACHE_PROBS[pk][LAB["LABELS"].index(label)]), 4) return {"image": image_meta(image_id), "steps": out, "result": result} CACHE_PROBS = {} # probs key -> the 1000 class probabilities (small, kept apart from the image cache) def run_task(task, key, final): kind = (task or {}).get("type", "none") if kind == "classify": rkey = make_key(key, "classify") cached = recall(rkey) if cached is None: t0 = time.perf_counter() a = final["array"] result = {"type": "classify", "model": "ResNet50", "input_shape": [1, *a.shape], "input_dtype": str(a.dtype), "input_range": [round(float(a.min()), 2), round(float(a.max()), 2)], "probs_key": rkey, "error": None, "top5": []} try: probs = with_model("resnet50", lambda m: LAB["classify_resnet50"](a, m)) CACHE_PROBS[rkey] = probs while len(CACHE_PROBS) > 2000: CACHE_PROBS.pop(next(iter(CACHE_PROBS))) result["top5"] = [{"label": label, "prob": p} for label, p in LAB["top_k"](probs, 5)] except Exception as exc: # e.g. channels last: PyTorch refuses, and that is the lesson msg = str(exc).strip().splitlines()[0] if str(exc).strip() else type(exc).__name__ result["error"] = msg result["probs_key"] = None result["ms"] = round((time.perf_counter() - t0) * 1000, 1) cached = remember(rkey, {"array": a, "result": result}) pk = cached["result"]["probs_key"] if pk is not None and pk not in CACHE_PROBS: # evicted: recompute on the next run with CACHE_LOCK: CACHE.pop(rkey, None) return run_task(task, key, final) return dict(cached["result"]) if kind == "segment": conf = min(0.95, max(0.05, float(task.get("conf", 0.25)))) rkey = make_key(key, "segment", f"{conf:.3f}") cached = recall(rkey) if cached is None: t0 = time.perf_counter() img = LAB["as_uint8_rgb"](final["array"]) overlay, instances = with_model("yolo", lambda m: LAB["segment_yolo"](img, m, conf)) counts = {} for name, _ in instances: counts[name] = counts.get(name, 0) + 1 # class name -> the mask color segment_yolo drew, so the app's legend matches the picture colors = {n: "#%02x%02x%02x" % LAB["class_color"](i) for i, n in MODELS["yolo"].names.items()} cached = remember(rkey, {"array": overlay, "result": { "type": "segment", "model": "YOLO11s-seg", "conf": conf, "total": len(instances), "counts": dict(sorted(counts.items(), key=lambda kv: -kv[1])), "overlay_key": rkey, "colors": colors, "instances": [{"label": n, "conf": c} for n, c in instances], "input_shape": list(img.shape), "notes": [] if final["array"].dtype == np.uint8 else ["The pipeline output was converted to uint8 RGB first."], "ms": round((time.perf_counter() - t0) * 1000, 1)}}) return dict(cached["result"]) return None # ---------------------------------------------------------------- 5. HTTP routes def warm_up(): """Load both models, then run every gallery image through its clean line, so first clicks are instant.""" with_model("resnet50", lambda m: LAB["classify_resnet50"](np.zeros((3, 224, 224), np.float32), m)) with_model("yolo", lambda m: LAB["segment_yolo"](np.zeros((640, 640, 3), np.uint8), m)) lines = {line["id"]: line for line in LAB["LINES"]} for item in gallery(): line = lines[item["use_case"]] try: run_line(item["id"], line["steps"], line["task"]) except Exception as exc: # a missing file should not stop the server print("warm-up skipped", item["id"], exc) # The default photo is what the presenter breaks and fixes first: run every break, alone and with each fix # meant for it (denoise takes about a second), so those first clicks are instant too. for line in LAB["LINES"]: for s in line["steps"]: if STEPS.get(s["op"], {}).get("kind") != "break": continue fixes = [f for f, d in STEPS.items() if d["kind"] == "fix" and s["op"] in d["helps"]] for on in [{s["op"]}, *({s["op"], f} for f in fixes)]: steps = [{**x, "enabled": True} if x["op"] in on else x for x in line["steps"]] try: run_line(line["default_image"], steps, line["task"]) except Exception as exc: print("warm-up skipped", line["id"], sorted(on), exc) @asynccontextmanager async def lifespan(_): threading.Thread(target=warm_up, daemon=True).start() yield app = FastAPI(title="Image Preprocessing Lab", lifespan=lifespan) app.add_middleware(CORSMiddleware, allow_origins=[o.strip() for o in os.environ.get("ALLOWED_ORIGINS", "*").split(",")], allow_methods=["GET", "POST"], allow_headers=["*"]) FOREVER = {"Cache-Control": "public, max-age=31536000, immutable"} @app.get("/api/health") def health(): return {"ok": True, "models_ready": len(MODELS) == 2} @app.get("/api/steps") def steps(): return { "kinds": LAB["KINDS"], "steps": [{k: v for k, v in s.items() if k != "fn"} | {"source": step_source(s["id"])} for s in STEPS.values()], "load_source": SOURCES["load_image"], "model_sources": {"classify": "\n\n\n".join(SOURCES[n] for n in ("classify_resnet50", "top_k")), "segment": "\n\n\n".join(SOURCES[n] for n in ("draw_label", "segment_yolo"))}, } @app.get("/api/lines") def lines(): return {"lines": LAB["LINES"]} @app.get("/api/images") def images(): return {"images": gallery()} @app.get("/api/images/{image_id}/thumb") def image_thumb(image_id: str): return Response(encode(load(image_id)[:, :, ::-1], "jpg", width=200), media_type="image/jpeg", headers=FOREVER) @app.post("/api/upload") async def upload(file: UploadFile = File(...)): data = await file.read(MAX_UPLOAD_BYTES + 1) if len(data) > MAX_UPLOAD_BYTES: raise HTTPException(413, "Images up to 10 MB, please.") arr = cv2.imdecode(np.frombuffer(data, np.uint8), cv2.IMREAD_COLOR) if arr is None: raise HTTPException(400, "That file is not an image OpenCV can read.") h, w = arr.shape[:2] s = UPLOAD_MAX_SIDE / max(h, w) if s < 1: arr = cv2.resize(arr, (round(w * s), round(h * s)), interpolation=cv2.INTER_AREA) UPLOADS.mkdir(exist_ok=True) digest = hashlib.sha1(data).hexdigest()[:12] cv2.imwrite(str(UPLOADS / f"{digest}.png"), arr) return {"id": f"upload-{digest}"} @app.post("/api/run") def run(body: dict = Body(...)): return run_line(str(body["image"]), body.get("steps", []), body.get("task", {})) @app.get("/api/img/{key}") def step_image(key: str, w: int | None = None): entry = recall(key) if entry is None: raise HTTPException(404, "Image expired from the cache; run the line again") display, _ = to_display(entry["array"]) if w: return Response(encode(display, "jpg", width=min(w, 2048)), media_type="image/jpeg", headers=FOREVER) return Response(encode(display, "png"), media_type="image/png", headers=FOREVER) @app.exception_handler(Exception) def crash(_, exc): return JSONResponse({"detail": f"{type(exc).__name__}: {exc}"}, status_code=500)