painting-vision-robotics-kit / console_server.py
constructelligence's picture
Update code: real-machine G-code adapter, commissioning gate, machine_job runner, pump watchdog firmware, twin completion gate, realism and masking modules (card and weights unchanged)
4eecd03 verified
Raw History Blame Contribute Delete
118 kB
#!/usr/bin/env python3 """Operator console: photo in, reviewed and approved paint work order out. A local web app for the person standing next to the robot. Upload (or take) a photo of the wall; the kit shows where it will paint, what it keeps out of, which damage needs prep first, and why the safety gate would stop. The operator can: * toggle layers (paint area, keep-out, uncertain wall, touch-up, damage, tool paths); * correct the plan with a brush: **paint here** / **do not paint**; * sign the confirmations the gate allows a person to make (not wallpaper, depth waived, uncertain areas reviewed), each recorded with their name; * re-plan, then **approve**. An approved order is frozen: the job keeps the photo, masks, edits and work order with their sha256, and an entry is appended to the hash-linked ``registry.jsonl`` (``run_registry.py verify`` checks it). Production notes: binds to 127.0.0.1 unless ``--host`` says otherwise; one model loaded once, inference serialised behind a lock; uploads capped (``MAX_UPLOAD``) and decoded before anything is written; job ids are random and file serving is an allowlist, so no path from a request reaches the filesystem. stdlib + the kit's own dependencies only: nothing extra to install on the robot PC. Usage:: python3 console_server.py --checkpoint artifacts/best.pt --jobs-dir jobs # open http://127.0.0.1:8765 """ from __future__ import annotations import argparse import base64 import hashlib import io import json import re import secrets import threading import time from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib.parse import parse_qs, urlparse import numpy as np from PIL import Image from work_order import CONFIRMATIONS, EDIT_ADD, EDIT_EXCLUDE, apply_edits, build_work_order, scale_from_prediction VERSION = "1.0" STATIC = Path(__file__).with_name("console") MAX_UPLOAD = 30 * 1024 * 1024 MAX_SIDE = 2048 # frames are normalised to this long side before analysis JOB_ID = re.compile(r"^[0-9a-f]{12}$") LAYER_COLOURS = {"paint": (46, 204, 113), "keepout": (231, 76, 60), "uncertain": (241, 196, 15), "touchup": (0, 210, 255)} JOB_FILES = {"photo.jpg": "image/jpeg", "demo_paint.png": "image/png", "demo_regions.png": "image/png", "work_order.json": "application/json", "result.json": "application/json", "approval.json": "application/json", "layer_edits.png": "image/png", **{f"layer_{k}.png": "image/png" for k in LAYER_COLOURS}, **{f"layer_{k}_edge.png": "image/png" for k in LAYER_COLOURS}, "layer_floor.png": "image/png", "layer_floor_edge.png": "image/png", "floor.json": "application/json", "dronetwin_heatmap.png": "image/png", "dronetwin.json": "application/json", "dronetwin_geojson.json": "application/geo+json"} FLOOR_COLOUR = (176, 125, 79) VIDEO_ID = re.compile(r"^v[0-9a-f]{12}$") VIDEO_EXT = {"mp4": "video/mp4", "mov": "video/quicktime", "webm": "video/webm", "m4v": "video/mp4"} VIDEO_FILE = re.compile(r"^(manifest\.json|track\.json|plan_\d+\.json|frame_\d+\.png|paint_\d+\.png|photo_\d+\.jpg|video\.(mp4|mov|webm|m4v))$") MAX_VIDEO = 200 * 1024 * 1024 MAX_DRAWING = 40 * 1024 * 1024 MAX_DEPTH = 60 * 1024 * 1024 MAX_SCAN = 150 * 1024 * 1024 MAX_VIEWS = 14 ZONE_KINDS = ("keepout", "paint", "note", "window", "door", "fixture", "wall") CORRECT_CLASS = {"window": 8, "door": 7, "fixture": 9} # operator corrections: "this is a ..." (fixture = wall obstacle) CORRECTION_KINDS = ("window", "door", "fixture", "wall") ZONE_SHAPES = ("rect", "poly", "pin") MAX_ZONES = 200 INFO_FIELDS = ("client", "site", "room", "spec", "notes") DRAWING_ID = re.compile(r"^d[0-9a-f]{8}$") DRAWING_SIDE = 2400 # a drawing is kept at this long side: enough to click a dimension on STATIC_FILES = {"index.html": "text/html; charset=utf-8", "app.js": "text/javascript; charset=utf-8", "style.css": "text/css; charset=utf-8", "demo.html": "text/html; charset=utf-8", "demo.js": "text/javascript; charset=utf-8", "demo.css": "text/css; charset=utf-8", "demo_video.js": "text/javascript; charset=utf-8", "zones.js": "text/javascript; charset=utf-8", "zones.css": "text/css; charset=utf-8", "floor.js": "text/javascript; charset=utf-8", "dronetwin.js": "text/javascript; charset=utf-8", "robot.js": "text/javascript; charset=utf-8", "collapse.js": "text/javascript; charset=utf-8", "collapse.css": "text/css; charset=utf-8", "live.js": "text/javascript; charset=utf-8", "twin.js": "text/javascript; charset=utf-8", "prompt.js": "text/javascript; charset=utf-8"} def _points(pts): """Two clicked [x, y] points, or None: stored so the drawn lines come back when a drawing is reopened.""" try: (a, b), (c, d) = pts return [[float(a), float(b)], [float(c), float(d)]] except (TypeError, ValueError): return None def sha256_bytes(data): return hashlib.sha256(data).hexdigest() def now(): return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) def save_mask(path, mask): Image.fromarray(np.asarray(mask, bool).astype(np.uint8) * 255).save(path) def load_mask(path): return np.asarray(Image.open(path).convert("L")) > 127 def layer_png(mask, colour, alpha=150): mask = np.asarray(mask, bool) rgba = np.zeros(mask.shape + (4,), np.uint8) rgba[mask] = (*colour, alpha) return Image.fromarray(rgba) def edge_png(mask, colour, width=3): """The outline of a layer, legible on ANY wall colour: a ``width`` px ring in the layer's colour, a 2 px white halo on both sides of it, and a thin dark edge outside the halo (the way a map outlines a road). A green fill vanishes on a green wall; this ring does not. Tiny specks are dropped.""" from scipy import ndimage mask = np.asarray(mask, bool) if mask.any(): mask = ndimage.binary_opening(mask, iterations=1) inner = ndimage.binary_erosion(mask, iterations=width) if mask.any() else mask ring = mask & ~inner halo = ndimage.binary_dilation(ring, iterations=2) & ~ring dark = ndimage.binary_dilation(ring, iterations=3) & ~ndimage.binary_dilation(ring, iterations=2) rgba = np.zeros(mask.shape + (4,), np.uint8) rgba[dark] = (24, 26, 32, 175) rgba[halo] = (255, 255, 255, 235) rgba[ring] = (*colour, 255) return Image.fromarray(rgba) def decode_edits(data_url, shape): """Edit canvas (PNG data URL) -> 0/1/2 layer: green = paint here, red = do not paint.""" if not data_url: return None match = re.match(r"^data:image/png;base64,([A-Za-z0-9+/=]+)$", data_url) if not match: raise ValueError("edits must be a PNG data URL") image = Image.open(io.BytesIO(base64.b64decode(match.group(1)))).convert("RGBA") if image.size != (shape[1], shape[0]): image = image.resize((shape[1], shape[0]), Image.Resampling.NEAREST) rgba = np.asarray(image) painted = rgba[..., 3] > 0 edits = np.zeros(shape, np.uint8) edits[painted & (rgba[..., 1] > rgba[..., 0])] = EDIT_ADD edits[painted & (rgba[..., 0] >= rgba[..., 1])] = EDIT_EXCLUDE return edits def window_gate(rgb, classes): """Turn window blobs that are pictures, posters, mirrors, screens or boards (or have no frame) into uncertain wall: window_negatives.py. Fails open to the ungated map, with the reason, if the general model cannot load.""" try: import window_negatives as wn gated, reasons = wn.filter_windows(np.asarray(classes), rgb, wn.ade_maps(rgb)) return gated, {"applied": True, "rejected": reasons} except Exception as exc: # no transformers / no network for the first download return classes, {"applied": False, "reason": str(exc)[:200]} class Console: """Jobs on disk + the analyser. ``analyze_fn(rgb) -> (summary, masks)`` is injected (tests fake it).""" def __init__(self, jobs_dir, analyze_fn, model_info, defaults=None): self.jobs = Path(jobs_dir) self.jobs.mkdir(parents=True, exist_ok=True) self.analyze_fn = analyze_fn self.model_info = model_info self.defaults = defaults or {} self.lock = threading.Lock() # one inference at a time; also serialises job writes # ---------------------------------------------------------------- jobs def job_dir(self, job_id): if not JOB_ID.match(job_id or ""): raise KeyError(job_id) path = self.jobs / job_id if not path.is_dir(): raise KeyError(job_id) return path def read_job(self, job_id): path = self.job_dir(job_id) job = json.loads((path / "job.json").read_text()) order_path = path / "work_order.json" job["work_order"] = json.loads(order_path.read_text()) if order_path.is_file() else None result = json.loads((path / "result.json").read_text()) job["regions"] = (result.get("surface_condition") or {}).get("regions", []) job["measurements"] = result.get("measurements") job["completion"] = result.get("completion") job["paint_color"] = result.get("paint_color") job["site_objects"] = result.get("site_objects") job["has_edits"] = (path / "layer_edits.png").is_file() return job # ---------------------------------------------------------------- operator zones, notes and job information def _clean_zones(self, path, request): """Validate what the console sent: zones are polygons / rectangles / pins in photo pixels.""" job = json.loads((path / "job.json").read_text()) w, h = float(job["width"]), float(job["height"]) zones = [] for z in (request.get("zones") or [])[:MAX_ZONES]: kind, shape = z.get("kind"), z.get("shape") if kind not in ZONE_KINDS or shape not in ZONE_SHAPES: raise ValueError(f"unknown zone kind/shape: {kind!r}/{shape!r}") pts = [[min(max(float(x), 0.0), w), min(max(float(y), 0.0), h)] for x, y in z.get("pts") or []] need = {"pin": 1, "rect": 2, "poly": 3}[shape] if len(pts) < need or len(pts) > 400: raise ValueError(f"a {shape} zone needs {need}{'+' if shape == 'poly' else ''} points") if shape == "pin" and kind != "note": raise ValueError("a pin can only be a note") zones.append({"id": str(z.get("id") or secrets.token_hex(3))[:12], "kind": kind, "shape": shape, "pts": pts[:2] if shape == "rect" else pts, "label": str(z.get("label") or "")[:60], "text": str(z.get("text") or "")[:600], "visible": bool(z.get("visible", True))}) info = {k: str((request.get("info") or {}).get(k) or "")[:600] for k in INFO_FIELDS} return zones, info def get_zones(self, job_id): path = self.job_dir(job_id) f = path / "zones.json" return json.loads(f.read_text()) if f.is_file() else {"zones": [], "info": {k: "" for k in INFO_FIELDS}} def set_zones(self, job_id, request): path = self.job_dir(job_id) if json.loads((path / "job.json").read_text()).get("approved"): raise PermissionError("job is approved and frozen; start a new job to change zones") zones, info = self._clean_zones(path, request) data = {"zones": zones, "info": info, "updated": now()} (path / "zones.json").write_text(json.dumps(data, indent=1)) return data @staticmethod def zone_edits(zones, shape): """Rasterise keep-out / paint-only zones into an edits map (the same codes as the brush); notes do nothing.""" import cv2 out = np.zeros(shape, np.uint8) # "plain wall" corrections free the area to paint; windows / doors / fixtures keep paint out; keep-out wins last for kinds, code in ((("paint", "wall"), EDIT_ADD), (("keepout", "window", "door", "fixture"), EDIT_EXCLUDE)): layer = np.zeros(shape, np.uint8) for z in zones: if z["kind"] not in kinds or z["shape"] == "pin": continue pts = np.array(z["pts"], np.float32) if z["shape"] == "rect": (x0, y0), (x1, y1) = pts pts = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], np.float32) cv2.fillPoly(layer, [np.round(pts).astype(np.int32)], 1) out[layer > 0] = code return out @staticmethod def corrected_classes(classes, zones): """The model's class map with the operator's corrections painted in: "plain wall" zones erase detections, window / door / fixture zones add them. Same ids as the segmenter (demo_plan.CLASS_IDS).""" import cv2 out = np.asarray(classes).copy() for kinds in (("wall",), tuple(CORRECT_CLASS)): for z in zones: if z["kind"] not in kinds or z["shape"] == "pin" or z.get("visible", True) is False: continue pts = np.array(z["pts"], np.float32) if z["shape"] == "rect": (x0, y0), (x1, y1) = pts pts = np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]], np.float32) m = np.zeros(out.shape, np.uint8) cv2.fillPoly(m, [np.round(pts).astype(np.int32)], 1) m = m.astype(bool) if z["kind"] == "wall": out[m & (out >= 4) & (out <= 9)] = 2 # a detection removed: it is ordinary painted wall else: out[m] = CORRECT_CLASS[z["kind"]] return out def objects(self, job_id): """Detected windows, doors, sockets and other protected objects as polygons (photo pixels), after the operator's corrections, so the console can show them and let the operator say a detection is wrong.""" import cv2 from demo_plan import CLASS_IDS, REGION_STYLE, refine_classes path = self.job_dir(job_id) classes = np.asarray(Image.open(path / "classes.png")) classes = refine_classes(self.corrected_classes(classes, self.get_zones(job_id)["zones"])) names = {v: k for k, v in CLASS_IDS.items()} out = [] for cid, name in sorted(names.items()): mask = (classes == cid).astype(np.uint8) contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) for c in sorted(contours, key=cv2.contourArea, reverse=True)[:60]: area = float(cv2.contourArea(c)) if area < 30: continue poly = cv2.approxPolyDP(c, max(1.5, 0.008 * cv2.arcLength(c, True)), True).reshape(-1, 2) if len(poly) < 3: continue out.append({"id": f"{name}-{len(out)}", "class": name, "label": REGION_STYLE[name][0], "area_px": area, "polygon": [[int(x), int(y)] for x, y in poly]}) return {"objects": out, "width": int(classes.shape[1]), "height": int(classes.shape[0])} def floor(self, job_id): """Floor mask + wall/floor line for a job, computed on first request (off by default in the console) and cached until the operator's zones change. Uses the wall model's class map (with corrections) to keep the floor below the wall and off windows / doors.""" path = self.job_dir(job_id) zf = path / "zones.json" stamp = zf.stat().st_mtime if zf.is_file() else 0 cache = path / "floor.json" if cache.is_file() and (path / "layer_floor.png").is_file(): data = json.loads(cache.read_text()) if data.get("zones_stamp") == stamp: return data import floor_detect rgb = np.asarray(Image.open(path / "photo.jpg").convert("RGB")) classes = None if (path / "classes.png").is_file(): classes = self.corrected_classes(np.asarray(Image.open(path / "classes.png")), self.get_zones(job_id)["zones"]) with self.lock: started = time.time() mask, info = floor_detect.detect_floor(rgb, classes) info["seconds"] = round(time.time() - started, 1) layer_png(mask, FLOOR_COLOUR, 110).save(path / "layer_floor.png") edge_png(mask, FLOOR_COLOUR).save(path / "layer_floor_edge.png") info["zones_stamp"] = stamp info["width"], info["height"] = int(mask.shape[1]), int(mask.shape[0]) cache.write_text(json.dumps(info)) return info # ---------------------------------------------------------------- drone digital twin (drone_twin.py) TWIN_DEFAULTS = {"tip": "517", "pressure_psi": 2000, "standoff_mm": 300, "boom_m": 0.5, "masked_openings": True, "home": None, "a_en": None, "b_en": None, "ground_alt_m": 0.0} def _twin_setup(self, path, request=None): f = path / "dronetwin_setup.json" setup = {**self.TWIN_DEFAULTS, **(json.loads(f.read_text()) if f.is_file() else {})} if request: for k in self.TWIN_DEFAULTS: if k in request: setup[k] = request[k] for k in ("home", "a_en", "b_en"): v = setup[k] if v is not None: setup[k] = [float(x) for x in v][: 3 if k == "home" else 2] if len(setup[k]) < (2 if k != "home" else 2): raise ValueError(f"{k} needs numbers") setup["masked_openings"] = bool(setup["masked_openings"]) f.write_text(json.dumps(setup, indent=1)) return setup def _twin_spray(self, setup): import application tip = application.SprayTip.parse(str(setup["tip"])) sp = application.spray_settings(tip.flow_lps(float(setup["pressure_psi"])), tip.fan_angle_deg, float(setup["standoff_mm"]), 38, 0.38) return {**sp, "tip": str(setup["tip"]), "pressure_psi": float(setup["pressure_psi"])} # the audit needs both for the spray force def _scale_mm_per_px(self, path): saved = json.loads((path / "scale.json").read_text()) if (path / "scale.json").is_file() else {} if saved.get("mm_per_px"): return float(saved["mm_per_px"]), saved.get("source") prediction = json.loads((path / "result.json").read_text()) mm, source = scale_from_prediction(prediction) if not mm: raise ValueError("this job has no scale yet: set one from a drawing, or plan it first") return float(mm), source def dronetwin(self, job_id, request=None, create=True): """The job's digital twin: built from the photo's class map (operator corrections applied) on first use, saved in the job folder, and kept in step with later zone corrections. Returns (twin, setup, spray).""" import drone_twin path = self.job_dir(job_id) setup = self._twin_setup(path, request) spray = self._twin_spray(setup) tf = path / "dronetwin.json" classes = self.corrected_classes(np.asarray(Image.open(path / "classes.png")), self.get_zones(job_id)["zones"]) mm, source = self._scale_mm_per_px(path) h, w = classes.shape a_en = tuple(setup["a_en"]) if setup["a_en"] else (0.0, 0.0) b_en = tuple(setup["b_en"]) if setup["b_en"] else (w * mm / 1000.0, 0.0) if tf.is_file() and not (request and any(k in request for k in ("a_en", "b_en", "masked_openings"))): twin = drone_twin.Twin.from_json(tf.read_text()) fac = twin.facades["wall"] if fac.classes.shape == classes.shape and not np.array_equal(fac.classes, classes): changed = fac.classes != classes fac.classes = classes.copy() twin._log("classes_updated", "wall", {"pixels_changed": int(changed.sum())}) fac.mm_per_px = mm elif create: twin = drone_twin.Twin(job_id) twin.add_facade("wall", classes, mm, a_en, b_en, bool(setup["masked_openings"])) else: raise KeyError(job_id) twin.scale_source = source return twin, setup, spray def save_dronetwin(self, job_id, twin): path = self.job_dir(job_id) (path / "dronetwin.json").write_text(twin.to_json()) from PIL import Image as _I _I.fromarray(twin.heatmap("wall")).save(path / "dronetwin_heatmap.png") (path / "dronetwin_geojson.json").write_text(json.dumps(twin.geojson())) twin._loaded_revision = twin.revision # the files now match this revision def dronetwin_status(self, job_id, request=None): twin, setup, spray = self.dronetwin(job_id, request) path = self.job_dir(job_id) if getattr(twin, "_loaded_revision", None) != twin.revision or not (path / "dronetwin_heatmap.png").is_file(): self.save_dronetwin(job_id, twin) # write only when the twin changed (or has no files yet) st = twin.status("wall", spray["fan_width_mm"]) fac = twin.facades["wall"] return {"status": st, "setup": setup, "spray": {k: spray[k] for k in ("fan_width_mm", "pass_spacing_mm", "traverse_speed_mm_s")}, "facade": {"width_m": round(fac.classes.shape[1] * fac.mm_per_px / 1000.0, 2), "height_m": round(fac.classes.shape[0] * fac.mm_per_px / 1000.0, 2), "mm_per_px": round(fac.mm_per_px, 3), "scale_source": getattr(twin, "scale_source", None)}, "calibration": twin.calibration, "events": twin.events[-12:], "sorties": sum(1 for e in twin.events if e["kind"] == "sortie"), "history": twin.history("wall"), "observations": sum(1 for e in twin.events if e["kind"] == "observe"), "spray_audit": twin.last_audit("wall"), "revision": twin.revision, "width": int(fac.classes.shape[1]), "height": int(fac.classes.shape[0])} def dronetwin_observe(self, job_id, data): """A re-scan photo of the wall: segment it, line it up with the job photo, fuse its paint read into the belief. Pixels the new photo does not show, or does not call wall, are left unobserved (never assumed bare).""" import drone_align from PIL import ImageOps twin, setup, spray = self.dronetwin(job_id) try: img = ImageOps.exif_transpose(Image.open(io.BytesIO(data))).convert("RGB") except Exception as exc: raise ValueError(f"not a readable photo ({exc})") from exc if max(img.size) > MAX_SIDE: img.thumbnail((MAX_SIDE, MAX_SIDE), Image.Resampling.LANCZOS) new_rgb = np.asarray(img) job_rgb = np.asarray(Image.open(self.job_dir(job_id) / "photo.jpg").convert("RGB")) H, quality = drone_align.homography(job_rgb, new_rgb) with self.lock: _, masks = self.analyze_fn(new_rgb) fac = twin.facades["wall"] warped, covered = drone_align.warp_classes(np.asarray(masks["mask"], np.uint8), H, fac.classes.shape) valid = covered & np.isin(warped, (1, 2, 3)) & fac.target if valid.mean() < 0.02: raise ValueError("the re-scan shows almost none of the wall to be painted: frame the same wall as the job photo") twin.observe("wall", warped == 2, valid=valid) self.save_dronetwin(job_id, twin) return {"alignment": quality, "observed_share_of_target": round(float(valid.sum() / max(fac.target.sum(), 1)), 3), **self.dronetwin_status(job_id)} def dronetwin_next(self, job_id): """The next sortie: passes over what the belief says is still bare, as photo-pixel strokes (for the overlay) and as a QGroundControl mission (needs the launch point). None when nothing worth a flight is left. Fail-closed: when the plan's gate refuses the flight (wind, positioning, standoff, ceiling), the gate reasons are returned and **no mission is emitted**, so the console has nothing to download. The passes are still returned so the operator can see what the sortie would have done. """ import drone_plan twin, setup, spray = self.dronetwin(job_id) fac = twin.facades["wall"] nxt = twin.next_sortie("wall", spray["fan_width_mm"]) if nxt is None: return {"strokes": [], "mission": None, "note": "nothing left that a sortie would cover"} strokes = nxt["plan"]["strokes"] h = fac.classes.shape[0] out = {"strokes": strokes, "passes": len(strokes), "mission": None} if setup["home"]: mm = fac.mm_per_px strokes_mm = [[(x * mm, (h - y) * mm) for x, y in s] for s in strokes] width_mm, height_mm = fac.classes.shape[1] * mm, h * mm home = tuple(setup["home"]) + ((0.0,) if len(setup["home"]) == 2 else ()) plan = drone_plan.build(strokes_mm, width_mm, height_mm, fac.a_en, fac.b_en, home, spray, standoff_m=float(setup["standoff_mm"]) / 1000.0, boom_m=float(setup["boom_m"])) out["gate"] = plan["gate"] out["sorties"] = plan["sorties"] out["reaction"] = plan.get("reaction") if plan["gate"]["go"]: along, outward = plan["_frame"] out["mission"] = drone_plan.qgc_mission(plan["_sorties"], home, fac.a_en, along, outward, plan["standoff_setpoint_mm"] / 1000.0 + float(setup["boom_m"]), plan["speed_ms"]) else: out["refused"] = True out["note"] = "flight plan withheld: " + "; ".join(plan["gate"]["reasons"]) return out def dronetwin_whatif(self, job_id, positioning_m=0.02, wind_ms=2.0, trials=3): """What this facade would be predicted to get under a different position error and wind. The twin's own answer from its plan and deposit model - a prediction, not a measurement - so the operator can see what a windier day or a sloppier hold costs before flying it. """ twin, setup, spray = self.dronetwin(job_id) positioning_m = min(max(float(positioning_m), 0.002), 0.30) wind_ms = min(max(float(wind_ms), 0.0), 12.0) trials = max(1, min(int(trials), 6)) out = twin.what_if("wall", spray, positioning_m, wind_ms, trials=trials) out.update({"positioning_m": positioning_m, "wind_ms": wind_ms, "trials": trials, "masked_openings": bool(twin.facades["wall"].masked_openings)}) return out def dronetwin_log(self, job_id, text, sortie=None): """Ingest a flight log (CSV) into the job's twin; returns what the flight did and where the facade stands.""" twin, setup, spray = self.dronetwin(job_id) if not setup["home"]: raise ValueError("set the home position first (lat, lon): the facade's east/north coordinates are measured from it") result = twin.ingest_log("wall", text, spray["fan_width_mm"], spray["pass_spacing_mm"], target_standoff_m=float(setup["standoff_mm"]) / 1000.0, boom_m=float(setup["boom_m"]), home=tuple(setup["home"]), ground_alt_m=float(setup["ground_alt_m"]), sortie=sortie, spray=spray) self.save_dronetwin(job_id, twin) return {"ingest": result, **self.dronetwin_status(job_id)} # ---------------------------------------------------------------- job drawings def _drawings(self, job_id): path = self.job_dir(job_id) / "drawings" index = path / "index.json" return path, (json.loads(index.read_text()) if index.is_file() else []) def add_drawing(self, job_id, data, filename=""): """Attach a plan / elevation to a job: an image, or the first page of a PDF, kept as a PNG to click on.""" if len(data) > MAX_DRAWING: raise ValueError(f"drawing larger than {MAX_DRAWING // (1024 * 1024)} MB") path, items = self._drawings(job_id) ext = Path(filename).suffix.lower() pages = 1 if data[:5] == b"%PDF-" or ext == ".pdf": try: import pypdfium2 as pdfium pdf = pdfium.PdfDocument(data) pages = len(pdf) page = pdf[0] w, h = page.get_size() image = page.render(scale=min(4.0, DRAWING_SIDE / max(w, h))).to_pil().convert("RGB") except ImportError as exc: raise ValueError("PDF drawings need pypdfium2 installed on the server; upload a PNG or JPG instead") from exc except Exception as exc: raise ValueError(f"could not read that PDF: {exc}") from exc else: try: image = Image.open(io.BytesIO(data)) image.load() except Exception as exc: raise ValueError(f"not a readable drawing (use PNG, JPG or PDF): {exc}") from exc from PIL import ImageOps image = ImageOps.exif_transpose(image).convert("RGB") if max(image.size) > DRAWING_SIDE: image.thumbnail((DRAWING_SIDE, DRAWING_SIDE), Image.Resampling.LANCZOS) path.mkdir(exist_ok=True) did = "d" + secrets.token_hex(4) image.save(path / f"{did}.png") item = {"id": did, "filename": (filename or "drawing")[:120], "width": image.width, "height": image.height, "pages": pages, "created": now(), "calibration": None, "measurements": []} items.append(item) (path / "index.json").write_text(json.dumps(items, indent=1)) return item def list_drawings(self, job_id): return self._drawings(job_id)[1] def update_drawing(self, job_id, drawing_id, request): """Store the operator's calibration (two clicked points + the real length) and measured lines.""" path, items = self._drawings(job_id) item = next((d for d in items if d["id"] == drawing_id), None) if item is None: raise KeyError(drawing_id) if "calibration" in request: c = request["calibration"] if c is not None: mm = float(c["length_mm"]) px = float(c["length_px"]) if not (mm > 0 and px > 0): raise ValueError("a calibration needs a positive length and two different points") c = {"length_mm": mm, "length_px": px, "mm_per_px": mm / px, "label": str(c.get("label", ""))[:40], "pts": _points(c.get("pts"))} item["calibration"] = c if "measurements" in request: item["measurements"] = [{"id": str(m.get("id", ""))[:12], "length_px": float(m["length_px"]), "label": str(m.get("label", ""))[:60], "pts": _points(m.get("pts"))} for m in request["measurements"]][:60] (path / "index.json").write_text(json.dumps(items, indent=1)) return item def drawing_file(self, job_id, drawing_id): if not DRAWING_ID.match(drawing_id): raise KeyError(drawing_id) f = self.job_dir(job_id) / "drawings" / f"{drawing_id}.png" if not f.is_file(): raise KeyError(drawing_id) return f def set_scale_from_drawing(self, job_id, request): """A length measured on a drawing (the wall's width or height) fixes the photo's scale: the wall's extent in the photo's pixels is known from its mask, so mm per pixel follows. Saved for every later plan of this job.""" path = self.job_dir(job_id) if json.loads((path / "job.json").read_text()).get("approved"): raise PermissionError("job is approved and frozen; start a new job to change the scale") length_mm = float(request["length_mm"]) axis = request.get("axis") if axis not in ("width", "height") or not 100 <= length_mm <= 60000: raise ValueError("give the wall's width or height, between 0.1 m and 60 m") wall = load_mask(path / "wall.png") ys, xs = np.nonzero(wall) if not len(xs): raise ValueError("no wall was found in the photo, so there is nothing to scale") extent = (xs.max() - xs.min() + 1) if axis == "width" else (ys.max() - ys.min() + 1) operator = request.get("source") == "operator" # a length the operator measured on the real wall, not read off a drawing saved = {"mm_per_px": length_mm / float(extent), "source": (f"operator measured: wall {axis} = {length_mm / 1000:.2f} m" if operator else f"drawing: {str(request.get('label') or axis)[:60]} = {length_mm / 1000:.2f} m ({axis})"), "axis": axis, "length_mm": length_mm, "at": now()} (path / "scale.json").write_text(json.dumps(saved, indent=1)) self._record_anchor(path, ("measured_" + axis) if operator else "drawing", saved["mm_per_px"], saved["source"]) saved["evidence"] = self.evidence(job_id) return saved def generate_drawing(self, job_id, request): """Draw this job's wall to scale from its photo (drawing_make.py) and file it with the job's drawings, ready to calibrate-check, match, measure and download as SVG, DXF and JSON. The scale comes from, in order: a wall height or width the operator gives, the job's saved scale, or an assumed 8 ft ceiling, and the sheet says which in its title block. It is a DRAFT.""" import cv2 import drawing_make as dmk from wall_measure import measure_wall path, items = self._drawings(job_id) job = self.job_dir(job_id) classes = job / "classes.png" if not classes.is_file(): raise ValueError("this job has no saved class mask; re-run the photo through the current model") labels = np.asarray(Image.open(classes)) kw = {} height_ft, width_ft = request.get("wall_height_ft"), request.get("wall_width_ft") if height_ft not in (None, ""): height_ft = float(height_ft) if not 4 <= height_ft <= 40: raise ValueError("wall height must be between 4 and 40 ft") kw["wall_height_ft"] = height_ft if width_ft not in (None, ""): width_ft = float(width_ft) if not 1 <= width_ft <= 200: raise ValueError("wall width must be between 1 and 200 ft") kw["wall_width_ft"] = width_ft explicit = bool(kw) # an height or width typed for THIS drawing means "draw from the photo with that size" scale = job / "scale.json" if not kw and scale.is_file(): kw["mm_per_px"] = float(json.loads(scale.read_text())["mm_per_px"]) info = json.loads((job / "job.json").read_text()) spec = None if request.get("from_twin", True) and not explicit and (job / "twin.npz").is_file(): # a twin's whole-wall size and fused openings beat one photo _, twin = self._twin_load(job_id) spec = dmk.make_elevation_from_twin(twin, info.get("wall_id") or info.get("filename") or "Wall") if spec is None: measured = measure_wall(labels, find_corners=False, **kw) if not measured["faces"]: raise ValueError("no measurable wall in this photo, so there is nothing to draw") face = max(measured["faces"], key=lambda f: f["width_ft"] * f["height_ft"]) spec = dmk.make_elevation(labels, face, info.get("wall_id") or info.get("filename") or "Wall", source_note="drawn from one photo" + (f" ({len(measured['faces'])} wall faces seen; the largest is drawn)" if len(measured["faces"]) > 1 else "")) img, mm_per_px = dmk.to_png(spec) did = "d" + secrets.token_hex(4) path.mkdir(exist_ok=True) cv2.imwrite(str(path / f"{did}.png"), img) (path / f"{did}.json").write_text(dmk.dumps(spec)) (path / f"{did}.svg").write_text(dmk.to_svg(spec)) (path / f"{did}.dxf").write_text(dmk.to_dxf(spec)) item = {"id": did, "filename": f"{spec['name']} - generated elevation (draft).png", "width": int(img.shape[1]), "height": int(img.shape[0]), "pages": 1, "created": now(), "generated": True, "calibration": {"length_mm": 100 * mm_per_px, "length_px": 100.0, "mm_per_px": mm_per_px, "label": "generated sheet", "pts": None}, # no clicked points: nothing to draw over the sheet "measurements": [], "provenance": spec["provenance"], "openings": len(spec["openings"]), "schedule": dmk.schedule(spec), "wall_mm": [spec["wall"]["width_mm"], spec["wall"]["height_mm"]]} items.append(item) (path / "index.json").write_text(json.dumps(items, indent=1)) return item def drawing_export(self, job_id, drawing_id, ext): if not DRAWING_ID.match(drawing_id) or ext not in ("svg", "dxf", "json"): raise KeyError(drawing_id) f = self.job_dir(job_id) / "drawings" / f"{drawing_id}.{ext}" if not f.is_file(): raise KeyError(drawing_id) return f # ------------------------------------------------------------------ evidence: what supports this job's measurement (evidence.py) def _record_anchor(self, path, kind, mm_per_px, label): """Remember every independent way the scale was fixed (so agreement or conflict between them can be shown), latest per kind.""" f = path / "anchors.json" anchors = json.loads(f.read_text()) if f.is_file() else {} anchors[kind] = {"mm_per_px": float(mm_per_px), "label": str(label)[:120], "at": now()} f.write_text(json.dumps(anchors, indent=1)) def _views(self, path): f = path / "views.json" return json.loads(f.read_text()) if f.is_file() else [] def _main_arrays(self, path): import cv2 gray = cv2.imread(str(path / "photo.jpg"), cv2.IMREAD_GRAYSCALE) return gray, np.asarray(Image.open(path / "classes.png")) def _refresh_coverage(self, path): import evidence_inputs as ei gray, labels = self._main_arrays(path) regs = [] for v in self._views(path): if v.get("registered") and (path / "views" / f"{v['id']}.jpg").is_file(): import cv2 regs.append((cv2.imread(str(path / "views" / f"{v['id']}.jpg"), cv2.IMREAD_GRAYSCALE), np.asarray(Image.open(path / "views" / f"{v['id']}_classes.png")))) cov = ei.coverage(gray, labels, regs) (path / "coverage.json").write_text(json.dumps(cov, indent=1)) return cov def _add_one_view(self, path, rgb, kind, name): """Segment one extra image, try to register it to the job's own photo, record the result. Never raises for a view that fails to register.""" import cv2 import evidence_inputs as ei views = self._views(path) if len(views) >= MAX_VIEWS: raise ValueError(f"at most {MAX_VIEWS} extra views per wall") gray_main, labels_main = self._main_arrays(path) h, w = gray_main.shape rgb = cv2.resize(np.asarray(rgb), (w, h), interpolation=cv2.INTER_AREA) with self.lock: summary, masks = self.analyze_fn(rgb) labels = np.asarray(masks["mask"], np.uint8) if masks.get("mask") is not None else None if labels is None: raise ValueError("the model returned no class mask for this view") gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY) counted = [cv2.imread(str(path / "views" / f"{v['id']}.jpg"), cv2.IMREAD_GRAYSCALE) for v in views if v.get("registered") and (path / "views" / f"{v['id']}.jpg").is_file()] ok, inliers, why = ei.register_view(gray_main, labels_main, gray, labels, existing=counted) vid = "v" + secrets.token_hex(4) rec = {"id": vid, "kind": kind, "name": str(name)[:100], "registered": bool(ok), "inliers": int(inliers), "reason": why, "wall_pixels": int(summary.get("wall_pixels", 0)), "at": now()} if ok: (path / "views").mkdir(exist_ok=True) cv2.imwrite(str(path / "views" / f"{vid}.jpg"), cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 90]) Image.fromarray(labels).save(path / "views" / f"{vid}_classes.png") views.append(rec) (path / "views.json").write_text(json.dumps(views, indent=1)) return rec def add_view(self, job_id, data, filename=""): """Another photo or a video of the same wall. Each image must register to the wall's own photo to count; a video contributes up to six sharp, distinct keyframes (each counts half, they are correlated). The result lists what registered and what did not, and why.""" import cv2 from multiview import video_keyframes path = self.job_dir(job_id) if json.loads((path / "job.json").read_text()).get("approved"): raise PermissionError("job is approved and frozen; start a new job to add evidence") ext = Path(filename).suffix.lower().lstrip(".") added = [] if ext in VIDEO_EXT: if len(data) > MAX_VIDEO: raise ValueError(f"video larger than {MAX_VIDEO // (1024 * 1024)} MB") tmp = path / f"_upload.{ext}" tmp.write_bytes(data) try: frames = video_keyframes(tmp, every_sec=1.0, max_frames=6) finally: tmp.unlink(missing_ok=True) if len(frames) < 1: raise ValueError("no sharp frames could be read from that video") for i, bgr in enumerate(frames): added.append(self._add_one_view(path, cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB), "keyframe", f"{filename or 'video'}#{i}")) else: if len(data) > MAX_UPLOAD: raise ValueError(f"image larger than {MAX_UPLOAD // (1024 * 1024)} MB") try: image = Image.open(io.BytesIO(data)) image.load() except Exception as exc: raise ValueError(f"not a readable photo or video: {exc}") from exc from PIL import ImageOps image = ImageOps.exif_transpose(image).convert("RGB") if max(image.size) > MAX_SIDE: image.thumbnail((MAX_SIDE, MAX_SIDE), Image.Resampling.LANCZOS) added.append(self._add_one_view(path, np.asarray(image), "photo", filename or "photo")) cov = self._refresh_coverage(path) return {"added": added, "registered": sum(1 for a in added if a["registered"]), "tried": len(added), "coverage": cov, "evidence": self.evidence(job_id)} def add_depth(self, job_id, data, filename="", fx=None, hfov_deg=None): """A depth / LiDAR map aligned with the job's photo: checked against the wall mask, and its metric scale is recorded as a scale anchor. The focal length (fx in depth-image pixels, or the horizontal field of view) decides whether that scale is trusted.""" import evidence_inputs as ei path = self.job_dir(job_id) if json.loads((path / "job.json").read_text()).get("approved"): raise PermissionError("job is approved and frozen; start a new job to add evidence") if len(data) > MAX_DEPTH: raise ValueError(f"depth map larger than {MAX_DEPTH // (1024 * 1024)} MB") depth = ei.load_depth_bytes(data, filename) wall, keep = load_mask(path / "wall.png"), load_mask(path / "keepout.png") job = json.loads((path / "job.json").read_text()) rep = ei.depth_check(depth, wall, keep, (job["width"], job["height"]), fx=fx, hfov_deg=hfov_deg) rep["filename"], rep["at"] = Path(filename).name[:120], now() (path / "depth_report.json").write_text(json.dumps(rep, indent=1)) if rep["plane_ok"]: # the relief (mm from the wall plane) per photo pixel feeds the twin's relief layer import twin_sources as ts resid, valid, rinfo = ts.relief_from_depth(depth, (job["width"], job["height"]), fx_px=fx, hfov_deg=hfov_deg) np.savez_compressed(path / "depth_relief.npz", resid=resid.astype(np.float16), valid=valid) if rep.get("mm_per_px") and rep["plane_ok"]: kind = "depth" if rep["intrinsics_source"] == "known" else "assumed_focal" self._record_anchor(path, kind, rep["mm_per_px"], f"depth scan ({rep['intrinsics_source']} focal length)") (path / "scale.json").write_text(json.dumps({"mm_per_px": rep["mm_per_px"], "source": f"depth {rep['intrinsics_source']} focal", "axis": "depth", "length_mm": None, "at": now()}, indent=1)) return {"report": {k: rep.get(k) for k in ("plane_ok", "wall_fraction_on_plane", "mm_per_px", "intrinsics_source", "fx_px", "warnings", "points")}, "evidence": self.evidence(job_id)} def evidence(self, job_id): """The evidence score for this job and what would raise it, from what is recorded on disk (evidence.py explains the rules).""" import evidence path = self.job_dir(job_id) views = self._views(path) photos = [v for v in views if v["kind"] == "photo" and v["registered"]] keyframes = [v for v in views if v["kind"] == "keyframe" and v["registered"]] cov = json.loads((path / "coverage.json").read_text()) if (path / "coverage.json").is_file() else self._refresh_coverage(path) anchors = json.loads((path / "anchors.json").read_text()) if (path / "anchors.json").is_file() else {} facts = {"views": 1 + len(photos), "video_keyframes": len(keyframes), "wall_complete": bool(cov.get("complete")), "scale": "assumed"} if anchors: facts["anchors"] = [{"kind": k, "mm_per_px": a["mm_per_px"]} for k, a in anchors.items()] drawings = [d for d in self._drawings(job_id)[1] if not d.get("generated")] # a drawing made FROM the photo is not independent evidence if any((d.get("auto_match") or {}).get("usable_for_scale") for d in drawings): facts["drawing"] = "matched" elif any(d.get("calibration") for d in drawings): facts["drawing"] = "calibrated" dr = json.loads((path / "depth_report.json").read_text()) if (path / "depth_report.json").is_file() else None facts["depth_plane_ok"] = bool(dr and dr.get("plane_ok")) out = evidence.assess(facts) out["facts"] = facts out["views"] = [{k: v.get(k) for k in ("id", "kind", "name", "registered", "inliers", "reason")} for v in views] out["depth"] = None if not dr else {k: dr.get(k) for k in ("plane_ok", "wall_fraction_on_plane", "mm_per_px", "intrinsics_source", "warnings", "filename")} out["coverage"] = cov return out # ------------------------------------------------------------------ live drone feed (live_feed.py): advisory only, one session at a time def list_videos(self): """Uploaded videos (newest first): what the live panel can replay as a feed.""" out = [] for d in sorted(self.jobs.iterdir(), key=lambda p: p.stat().st_mtime, reverse=True): if VIDEO_ID.match(d.name) and any(d.glob("video.*")): out.append({"id": d.name, "modified": time.strftime("%Y-%m-%d %H:%M", time.localtime(d.stat().st_mtime))}) return out[:30] def _person_detector(self): """The person detector this machine can run, built once and reused. None means not configured - never 'clear'.""" if not hasattr(self, "_people_det"): import people try: self._people_det = people.make_detector() except Exception: self._people_det = None return self._people_det def live_start(self, request): """Open a camera URL (rtsp/http), or play one of this console's own uploaded videos as if it were live. Replaces any running session.""" import live_feed as lf self.live_stop() if request.get("video_id"): vid = str(request["video_id"]) if not VIDEO_ID.match(vid): raise ValueError("not a video id") clip = next((p for p in sorted((self.jobs / vid).glob("video.*")) if p.suffix.lstrip(".") in VIDEO_EXT), None) if (self.jobs / vid).is_dir() else None if clip is None: raise KeyError(vid) grabber, label = lf.CaptureGrabber(str(clip), realtime=True, loop=True), f"recorded video {vid} (played as live)" else: url = lf.parse_source(request.get("source")) grabber, label = lf.CaptureGrabber(url), "camera stream" def analyze(rgb): with self.lock: return self.analyze_fn(rgb) twin_job = request.get("twin_job") sink = None if twin_job: if not JOB_ID.match(str(twin_job)): raise ValueError("not a job id") self._twin_load(twin_job) # raises KeyError (404) when that job has no twin yet sink = lambda rgb, masks, summary: self.twin_observe_live(twin_job, rgb, masks, summary) # noqa: E731 session = lf.LiveSession(analyze, grabber, interval_s=1.0, on_frame=sink, people_fn=self._person_detector() if request.get("people", True) else None) session.start() self.live = {"session": session, "label": label, "started": now(), "twin_job": twin_job} return self.live_state() def live_state(self): if not getattr(self, "live", None): return {"running": False, "disclaimer": None, "people_detector": getattr(self._person_detector(), "name", "none")} out = self.live["session"].snapshot() out["source"], out["started"] = self.live["label"], self.live["started"] out["people_detector"] = getattr(getattr(self, "live", {}).get("session").people_fn, "name", "none") or "none" return out def live_frame(self): live = getattr(self, "live", None) jpg = live["session"].annotated if live else None if not jpg: raise KeyError("no live frame yet") return jpg def live_arm(self, request): live = getattr(self, "live", None) if not live: raise ValueError("no live session is running") if request.get("armed"): live["session"].arm(request.get("operator", "")) else: live["session"].disarm() return self.live_state() def live_stop(self): live = getattr(self, "live", None) if live: live["session"].stop() self.live = None return {"running": False} # ------------------------------------------------------------------ digital twin (twin.py): a per-job wall model fed by every observation TWIN_LAYERS = ("painted", "confidence", "plan", "surface", "relief") def _twin_clock(self, path): """Observation time in seconds since this twin was built: a monotonic clock that survives restarts (stored as the build time).""" m = json.loads((path / "twin_meta.json").read_text()) return max(0.0, time.time() - float(m["epoch"])) def _anchor_from_dims(self, path, kind, dims_mm, label): """Record a laser-derived size as a scale anchor, in the SAME definition as the console's other anchors (mm per pixel = a real length over the wall mask's extent in the photo), and make it the job's scale. Returns mm per pixel.""" from wall_measure import measure_wall wall = load_mask(path / "wall.png") ys, xs = np.nonzero(wall) if not len(xs): raise ValueError("no wall was found in the photo, so a size cannot be tied to it") faces = measure_wall(np.asarray(Image.open(path / "classes.png")), find_corners=False)["faces"] if (path / "classes.png").is_file() else [] if faces and {"top", "bottom"} & set(max(faces, key=lambda f: f["width_ft"] * f["height_ft"])["clipped_sides"]): return None # the photo cuts the wall off vertically: its pixel height is not the wall's height, so it cannot anchor a scale extent_h = float(ys.max() - ys.min() + 1) mm_per_px = float(dims_mm[1]) / extent_h self._record_anchor(path, kind, mm_per_px, label) (path / "scale.json").write_text(json.dumps({"mm_per_px": mm_per_px, "source": label, "axis": "height", "length_mm": float(dims_mm[1]), "at": now()}, indent=1)) return mm_per_px def add_laser(self, job_id, request): """Laser distance meter readings of the wall (width, height, optionally the diagonal): validated, stored, and used as the twin's size on the next build.""" import twin_sources as ts path = self.job_dir(job_id) if json.loads((path / "job.json").read_text()).get("approved"): raise PermissionError("job is approved and frozen; start a new job to add evidence") got = ts.laser_dimensions(request.get("width_mm"), request.get("height_mm"), request.get("diagonal_mm"), request.get("accuracy_mm")) rec = {"width_mm": got["dims_mm"][0], "height_mm": got["dims_mm"][1], "accuracy_mm": got["accuracy_mm"], "diagonal_check": got["diagonal_check"], "at": now()} (path / "laser.json").write_text(json.dumps(rec, indent=1)) anchored = self._anchor_from_dims(path, "laser", got["dims_mm"], f"laser distance meter: {got['dims_mm'][0] / 1000:.3f} x {got['dims_mm'][1] / 1000:.3f} m") return {"laser": rec, "scale_anchored": anchored is not None, "evidence": self.evidence(job_id), "note": "used as the twin's size on the axes the photos show end to end, the next time it is built" + ("" if anchored else "; the photo cuts the wall off vertically, so it cannot set the photo's scale yet")} def add_scan(self, job_id, data, filename="", up_axis="z", flip_x=False): """A laser-scan point cloud (PLY / XYZ / CSV, metres) of the wall: plane-fitted for the wall's size and its relief. The scan's vertical axis and left-right order are the operator's statement (the cloud's frame is arbitrary) and the result repeats that assumption.""" import twin_sources as ts path = self.job_dir(job_id) if json.loads((path / "job.json").read_text()).get("approved"): raise PermissionError("job is approved and frozen; start a new job to add evidence") if not data or len(data) > MAX_SCAN: raise ValueError(f"a scan must be between 1 byte and {MAX_SCAN // (1024 * 1024)} MB") pts = ts.parse_point_cloud(data, filename) fit = ts.fit_wall_from_cloud(pts, up_axis=up_axis, flip_x=flip_x) meta = {k: fit[k] for k in ("dims_mm", "plane_rms_mm", "inlier_fraction", "points", "uncertainty_mm", "warnings", "alignment", "normal")} meta.update(filename=Path(filename).name[:120], at=now()) np.savez_compressed(path / "scan.npz", relief=np.nan_to_num(fit["relief_mm"]).astype(np.float16), weight=fit["relief_w"].astype(np.float16), meta=np.frombuffer(json.dumps(meta).encode(), np.uint8)) anchored = self._anchor_from_dims(path, "laser_scan", fit["dims_mm"], f"laser scan {Path(filename).name[:40]}: {fit['dims_mm'][0] / 1000:.3f} x {fit['dims_mm'][1] / 1000:.3f} m") return {"scan": meta, "relief_cells": int((fit["relief_w"] > 0).sum()), "scale_anchored": anchored is not None, "evidence": self.evidence(job_id), "note": "used as the twin's size (on axes the photos show end to end) and relief the next time it is built"} def _scan_load(self, path): f = path / "scan.npz" if not f.is_file(): return None z = np.load(f, allow_pickle=False) return {"relief": z["relief"].astype(np.float32), "weight": z["weight"].astype(np.float32), "meta": json.loads(bytes(z["meta"]).decode())} def twin_build(self, job_id, request): """Build (or rebuild) this job's twin from ALL the evidence on file, deterministically: the main photo, every registered extra view and video keyframe (fused, so the wall is measured whole), laser distance readings, a laser scan, and a depth map's relief. The wall's SIZE comes from the most accurate source present (scan, then laser meter, then photo geometry) and the others are reported against it, never averaged into it.""" import cv2 import evidence_inputs as ei import twin as tn import twin_sources as ts from wall_measure import measure_wall path = self.job_dir(job_id) if not (path / "classes.png").is_file(): raise ValueError("this job has no saved class mask; re-run the photo through the current model") gray_main, labels = self._main_arrays(path) kw = {} for key in ("wall_height_ft", "wall_width_ft"): v = request.get(key) if v not in (None, ""): kw[key] = float(v) laser = json.loads((path / "laser.json").read_text()) if (path / "laser.json").is_file() else None scan = self._scan_load(path) if not kw and (path / "scale.json").is_file() and not (laser or scan): kw["mm_per_px"] = float(json.loads((path / "scale.json").read_text())["mm_per_px"]) # extra views that register to the main photo, with their homographies regs = [] if request.get("fuse_views", True): for v in self._views(path): f = path / "views" / f"{v['id']}.jpg" if v.get("registered") and f.is_file(): g = cv2.imread(str(f), cv2.IMREAD_GRAYSCALE) lab = np.asarray(Image.open(path / "views" / f"{v['id']}_classes.png")) H, inl, why = ei.register_homography(gray_main, labels, g, lab) if H is not None: regs.append((v, g, lab, H)) if regs: face, finfo = ts.fuse_views_face(gray_main, labels, [(g, lab) for _, g, lab, _ in regs], kw) else: faces = measure_wall(labels, find_corners=False, **kw)["faces"] if not faces: raise ValueError("no measurable wall in this photo, so there is nothing to build a twin of") face = max(faces, key=lambda f: f["width_ft"] * f["height_ft"]) finfo = {"views_used": 1, "complete": not face["clipped_sides"], "clipped_sides": list(face["clipped_sides"])} prov = {"job": job_id, "built": now(), "views_fused": finfo["views_used"], "wall_complete": finfo["complete"], "clipped_sides": finfo["clipped_sides"]} dims = None if laser: prov["laser"] = {"accuracy_mm": laser["accuracy_mm"], "at": laser["at"]} if laser.get("diagonal_check"): prov["diagonal_check"] = laser["diagonal_check"] dims = (laser["width_mm"], laser["height_mm"]) if scan: sm = scan["meta"] prov["laser_scan"] = {"uncertainty_mm": sm["uncertainty_mm"], "plane_rms_mm": sm["plane_rms_mm"], "inlier_fraction": sm["inlier_fraction"], "file": sm.get("filename"), "alignment": sm["alignment"], "warnings": sm["warnings"], "note": sm["alignment"]["assumption"]} if laser: prov["scan_vs_laser_pct"] = {"width": round(100 * (sm["dims_mm"][0] - laser["width_mm"]) / laser["width_mm"], 2), "height": round(100 * (sm["dims_mm"][1] - laser["height_mm"]) / laser["height_mm"], 2)} dims = tuple(sm["dims_mm"]) # the scan is the most accurate source present info = json.loads((path / "job.json").read_text()) twin = tn.WallTwin.from_face(labels, face, info.get("wall_id") or info.get("filename") or "Wall", cell_mm=float(request.get("cell_mm") or 50.0), t=0.0, provenance=prov, dims_mm=dims) if (path / "touchup.png").is_file(): # the photo's damage / touch-up regions, placed on the wall tm = np.asarray(Image.open(path / "touchup.png").convert("L")) > 0 if tm.shape == labels.shape: twin.damage = np.where(twin._to_grid(tm.astype(np.uint8), np.eye(3)) == 1, 1, twin.damage).astype(np.uint8) observed = [] for k, (v, g, lab, H) in enumerate(regs, start=1): twin.observe_view(lab, H, float(k), v.get("name") or v["id"]) observed.append(v["id"]) if scan: twin.observe_relief_grid(scan["relief"], scan["weight"], 0.0, "laser scan", weight=3.0) if (path / "depth_relief.npz").is_file(): z = np.load(path / "depth_relief.npz", allow_pickle=False) twin.observe_relief_photo(z["resid"].astype(np.float32), z["valid"], np.eye(3), 0.0, "depth map", weight=1.0) self._twin_set_plan(path, twin, labels.shape) (path / "twin.npz").write_bytes(twin.to_bytes()) (path / "twin_meta.json").write_text(json.dumps({"epoch": time.time() - float(len(regs) + 1), "observed_views": observed, "built": now(), "sources": {"views": len(regs), "laser": bool(laser), "scan": bool(scan), "depth_relief": (path / "depth_relief.npz").is_file()}}, indent=1)) return self.twin_state(job_id) def twin_build_v1(self, job_id, request): """Build (or rebuild) this job's twin from its photo: the wall's size comes from the same measurement a generated drawing uses, so a measured height gives true millimetres and an assumed one is flagged on the twin. The photo's planned strokes become the twin's plan.""" import cv2 import twin as tn from wall_measure import measure_wall path = self.job_dir(job_id) if not (path / "classes.png").is_file(): raise ValueError("this job has no saved class mask; re-run the photo through the current model") labels = np.asarray(Image.open(path / "classes.png")) kw = {} for key in ("wall_height_ft", "wall_width_ft"): v = request.get(key) if v not in (None, ""): kw[key] = float(v) if not kw and (path / "scale.json").is_file(): kw["mm_per_px"] = float(json.loads((path / "scale.json").read_text())["mm_per_px"]) faces = measure_wall(labels, find_corners=False, **kw)["faces"] if not faces: raise ValueError("no measurable wall in this photo, so there is nothing to build a twin of") face = max(faces, key=lambda f: f["width_ft"] * f["height_ft"]) info = json.loads((path / "job.json").read_text()) twin = tn.WallTwin.from_face(labels, face, info.get("wall_id") or info.get("filename") or "Wall", cell_mm=float(request.get("cell_mm") or 50.0), t=0.0, provenance={"job": job_id, "built": now()}) self._twin_set_plan(path, twin, labels.shape) (path / "twin.npz").write_bytes(twin.to_bytes()) (path / "twin_meta.json").write_text(json.dumps({"epoch": time.time(), "observed_views": [], "built": now()}, indent=1)) return self.twin_state(job_id) def _twin_set_plan(self, path, twin, shape): import twin as tn f = path / "work_order.json" if not f.is_file(): return steps = (json.loads(f.read_text()).get("steps") or []) for st in steps: if st.get("step") == "paint" and st.get("plan") and st["plan"].get("plan"): plan = st["plan"]["plan"] twin.set_plan(tn.rasterise_plan(plan.get("strokes"), float(plan.get("stroke_width_units") or 8), shape)) return def _twin_load(self, job_id): import twin as tn path = self.job_dir(job_id) if not (path / "twin.npz").is_file(): raise KeyError("no twin built for this job yet") return path, tn.WallTwin.from_bytes((path / "twin.npz").read_bytes()) def twin_update(self, job_id): """Fold every registered extra view of this job into the twin (each once), and refresh the plan from the current work order.""" import cv2 import evidence_inputs as ei path, twin = self._twin_load(job_id) meta = json.loads((path / "twin_meta.json").read_text()) done = set(meta.get("observed_views", [])) gray_main, labels_main = self._main_arrays(path) added = [] for v in self._views(path): if not v.get("registered") or v["id"] in done: continue f = path / "views" / f"{v['id']}.jpg" if not f.is_file(): continue g = cv2.imread(str(f), cv2.IMREAD_GRAYSCALE) lab = np.asarray(Image.open(path / "views" / f"{v['id']}_classes.png")) H, inl, why = ei.register_homography(gray_main, labels_main, g, lab) if H is None: continue twin.observe_view(lab, H, self._twin_clock(path), v.get("name") or v["id"]) done.add(v["id"]); added.append(v["id"]) self._twin_set_plan(path, twin, labels_main.shape) (path / "twin.npz").write_bytes(twin.to_bytes()) meta["observed_views"] = sorted(done) (path / "twin_meta.json").write_text(json.dumps(meta, indent=1)) out = self.twin_state(job_id) out["added_views"] = added return out # ------------------------------------------------------------------ typed instructions (prompt_cmd.py): proposals first, applied on request def prompt_context(self, job_id): path = self.job_dir(job_id) job = json.loads((path / "job.json").read_text()) wall = load_mask(path / "wall.png") ys, xs = np.nonzero(wall) box = [int(xs.min()), int(ys.min()), int(xs.max()), int(ys.max())] if len(xs) else None return {"width": job["width"], "height": job["height"], "wall_box": box, "objects": self.objects(job_id)["objects"], "has_twin": (path / "twin.npz").is_file()} def prompt(self, job_id, request, env=None): """Parse a typed instruction. Returns the proposed actions and the answers to any questions; applies NOTHING.""" import prompt_cmd as pc ctx = self.prompt_context(job_id) text = str(request.get("text") or "") plan = pc.parse(text, ctx) if not (plan["actions"] or plan["answers"] or plan["questions"] or plan["refused"]) and text.strip(): ai = pc.ai_parse(text, ctx, env) if ai is not None and (ai["actions"] or ai["questions"]): plan = ai plan["actions"], dropped = pc.validate(plan["actions"], ctx) plan["answered"] = [{"topic": a["topic"], "text": self.prompt_answer(job_id, a["topic"], ctx)} for a in plan.pop("answers", [])] plan["needs_confirm"] = [a["type"] for a in plan["actions"] if a["type"] in pc.CONFIRM] if dropped: plan["dropped"] = dropped if plan["answered"] and not plan["actions"]: plan["reply"] = plan["answered"][0]["text"] return plan def prompt_answer(self, job_id, topic, ctx): """Read-only answers, from the job's own data (never a model's opinion).""" path = self.job_dir(job_id) wo = json.loads((path / "work_order.json").read_text()) if (path / "work_order.json").is_file() else None objs = ctx["objects"] if topic == "objects": counts = {} for o in objs: counts[o["label"]] = counts.get(o["label"], 0) + 1 return ("Detected on this wall: " + ", ".join(f"{n} x {k.lower()}" for k, n in sorted(counts.items())) + ".") if counts else "No windows, doors or fixtures were detected." if topic in ("paint", "area"): src = ((wo or {}).get("scale") or {}).get("source", "unknown") assumed = "assumed" in str(src) if (path / "twin.npz").is_file(): import twin_model _, twin = self._twin_load(job_id) ev = self.evidence(job_id) unc = twin_model.size_uncertainty(twin, ev.get("typical_error_pct"), ev.get("high_error_pct")) q = twin_model.quantities(twin, uncertainty=unc) if topic == "area": return (f"The twin's wall is {twin.width_mm / 1000:.2f} x {twin.height_mm / 1000:.2f} m ({twin.width_mm * twin.height_mm / 1e6:.1f} m2 gross, " f"{q['paintable_wall_m2']:.1f} m2 paintable wall), size from {twin.accuracy()['dimension_source']}" + (f", about +/-{unc['area_pm_pct']:.0f}% in area" if unc.get("known") else "") + ".") return (f"About {q['litres']:g} L for {q['coats']} coats of the {q['to_paint_m2']:.1f} m2 that is unpainted and seen (coverage {q['coverage_m2_per_l']:g} m2/L, " f"{int(q['waste'] * 100)}% waste); {q['unobserved_m2']:.1f} m2 has not been seen. " + (q.get("warning", "") or "Use the tin's own coverage figure.")) if wo and wo.get("summary", {}).get("paint_area_ft2") is not None: a = wo["summary"]["paint_area_ft2"] return (f"The plan covers about {a:.1f} sq ft ({a * 0.0929:.1f} m2): roughly {a * 0.0929 * 0.1 * 2:.1f} L for two coats at 10 m2/L. " + ("The scale is an ASSUMED 8 ft ceiling, so this is not trustworthy: give the real wall height. " if assumed else "") + "Build the twin for a better estimate.") return "There is no plan yet for this job, so there is no area to report. Press Re-plan." if topic == "accuracy": ev = self.evidence(job_id) return f"Evidence score {ev['score']}% ({ev['label']}): {ev['expected_error']}. That is how much evidence there is, not a probability of being right." if topic == "drone": try: st = self.dronetwin_status(job_id) s = st["status"] return f"Drone twin: {st['sorties']} sortie(s) logged, verdict {s.get('verdict')}, about {round(100 * s.get('believed_painted', 0))}% believed painted. (Set up a facade first if this is zero.)" except Exception: return "No drone twin is set up for this job yet (it needs a home position and the facade's end points)." return "I can't answer that yet." def prompt_apply(self, job_id, request): """Run the proposed actions. Re-validates everything: only whitelisted verbs with sane values run; each result says what happened.""" import prompt_cmd as pc path = self.job_dir(job_id) ctx = self.prompt_context(job_id) actions, dropped = pc.validate(request.get("actions"), ctx) results, client = [], [] for a in actions: t = a["type"] try: if t == "zone": cur = self.get_zones(job_id) zones = cur["zones"] + [{"kind": a["kind"], "shape": a["shape"], "pts": a["pts"], "label": a["label"] or "from instruction", "text": "added by a typed instruction", "visible": True}] self.set_zones(job_id, {"zones": zones, "info": cur.get("info")}) results.append({"type": t, "ok": True, "detail": f"added zone '{a['label']}'; Re-plan to use it"}) elif t == "clear_zones": cur = self.get_zones(job_id) self.set_zones(job_id, {"zones": [], "info": cur.get("info")}) results.append({"type": t, "ok": True, "detail": f"removed {len(cur['zones'])} zone(s); Re-plan to use that"}) elif t == "set_height": out = self.set_scale_from_drawing(job_id, {"length_mm": a["feet"] * 304.8, "axis": "height", "source": "operator", "label": "typed instruction"}) results.append({"type": t, "ok": True, "detail": f"wall height {a['feet']:g} ft recorded as your measurement; evidence now {out['evidence']['score']}%"}) elif t == "twin": import twin_pipeline rep = twin_pipeline.run(self, job_id, {}) results.append({"type": t, "ok": rep["ok"], "detail": next((s["detail"] for s in rep["stages"] if s["name"] == "twin"), "")}) elif t == "drawing": d = self.generate_drawing(job_id, {}) results.append({"type": t, "ok": True, "detail": f"drew {d['openings']} opening(s) at {d['wall_mm'][0] / 1000:.2f} x {d['wall_mm'][1] / 1000:.2f} m (draft); see Drawings", "drawing": d["id"]}) elif t in ("layer", "replan"): client.append(a) # the browser does these with its own controls results.append({"type": t, "ok": True, "detail": a.get("why") or t}) except PermissionError as exc: results.append({"type": t, "ok": False, "detail": str(exc)}) except (ValueError, KeyError) as exc: results.append({"type": t, "ok": False, "detail": str(exc)}) return {"results": results, "client_actions": client, "dropped": dropped} def twin_cell(self, job_id, x_mm, y_mm): """What the twin holds at one point of the wall (millimetres from its top-left): the surface, paint, how well and how recently seen, relief, and any opening there.""" import twin as tn import twin_model path, twin = self._twin_load(job_id) x, y = float(x_mm), float(y_mm) if not (0 <= x < twin.width_mm and 0 <= y < twin.height_mm): raise ValueError("that point is outside the wall") c, r = int(x // twin.cell_mm), int(y // twin.cell_mm) names = {tn.UNKNOWN: "not observed", **{v: k for k, v in {"unpainted wall": tn.UNPAINTED, "painted wall": tn.PAINTED, "uncertain wall": tn.UNCERTAIN, "skirting": tn.SKIRTING, "switch / outlet": tn.SWITCH, "air conditioner": tn.AC, "door": tn.DOOR, "window": tn.WINDOW, "obstacle": tn.OBSTACLE}.items()}} now_s = self._twin_clock(path) seen = float(twin.weight[r, c]) > 0.05 or int(twin.surface[r, c]) != tn.UNKNOWN inside = [o["id"] for o in twin_model.openings(twin) if o["x_mm"] <= x < o["x_mm"] + o["w_mm"] and o["y_mm"] <= y < o["y_mm"] + o["h_mm"]] out = {"x_mm": round(x), "y_mm": round(y), "surface": names.get(int(twin.surface[r, c]), "other"), "observed": bool(seen), "painted_fraction": round(float(twin.painted[r, c]), 2) if twin.weight[r, c] > 0.05 else None, "evidence_weight": round(float(twin.weight[r, c]), 2), "age_s": round(now_s - float(twin.last_seen[r, c]), 1) if twin.last_seen[r, c] >= 0 else None, "planned": bool(twin.effective_plan()[r, c]), "damage": bool(twin.damage[r, c]), "relief_mm": round(float(twin.relief[r, c]), 1) if twin.relief_w[r, c] > 0 else None, "opening": inside[0] if inside else None} return out def twin_history(self, job_id): """The snapshots a pipeline run saved: how much paint was seen on the wall at each, and what changed between consecutive ones.""" import twin_model from twin import WALL_CLASSES path = self.job_dir(job_id) snaps = sorted((path / "twin_snapshots").glob("*.npz")) if (path / "twin_snapshots").is_dir() else [] out, prev = [], None for f in snaps: z = np.load(f, allow_pickle=False) cur = {"surface": z["surface"], "painted": z["painted"], "damage": z["damage"], "dims": tuple(z["dims"])} ft2 = (cur["dims"][2] / 304.8) ** 2 wall = np.isin(cur["surface"], WALL_CLASSES) row = {"snapshot": f.stem, "painted_ft2": round(float((cur["painted"] & wall).sum()) * ft2, 1), "paintable_ft2": round(float(wall.sum()) * ft2, 1), "observed_share": round(float((cur["surface"] != 255).mean()), 3)} if prev is not None: d = twin_model.diff(prev, cur) row["change"] = d if d.get("comparable") else {"comparable": False, "reason": d.get("reason")} out.append(row) prev = cur return {"snapshots": out} def room_twin(self, request): """Combine the twins of several jobs (walls in order around a room) into one room summary; stateless, and nothing is stored.""" import room_twin ids = request.get("jobs") or [] if not isinstance(ids, list) or not 1 <= len(ids) <= 12: raise ValueError("give between 1 and 12 job ids, in order around the room") walls = [] for jid in ids: path, twin = self._twin_load(str(jid)) info = json.loads((path / "job.json").read_text()) walls.append((info.get("wall_id") or info.get("filename") or str(jid), twin)) room = room_twin.build(walls, coats=int(request.get("coats") or 2)) if request.get("obj"): room["obj"] = room_twin.to_obj(room, str(request.get("name") or "room")) return room def twin_export(self, job_id, name): """The twin as data for other tools: its openings in wall millimetres (JSON / CSV) or a heightfield mesh (OBJ, metres).""" import twin_model path, twin = self._twin_load(job_id) if name == "model.obj": return "text/plain; charset=utf-8", twin_model.export_obj(twin) if name == "defects.json": return "application/json", {"origin": "top-left of the wall, millimetres", "defects": twin_model.defects(twin)} if name == "quantities.json": return "application/json", twin_model.quantities(twin) rows = twin_model.openings(twin) if name == "openings.csv": return "text/csv; charset=utf-8", twin_model.openings_csv(rows) return "application/json", {"wall_mm": [round(twin.width_mm), round(twin.height_mm)], "origin": "top-left of the wall, x right, y down, millimetres", "dimension_source": twin.accuracy()["dimension_source"], "openings": rows} def twin_state(self, job_id): path, twin = self._twin_load(job_id) s = twin.summary(self._twin_clock(path)) s["history"] = twin.history[-30:] s["layers"] = list(self.TWIN_LAYERS) s["live_linked"] = bool(getattr(self, "live", None) and self.live.get("twin_job") == job_id) s["sources"] = json.loads((path / "twin_meta.json").read_text()).get("sources", {}) try: ev = self.evidence(job_id) s["model_accuracy"] = {k: ev[k] for k in ("score", "label", "expected_error", "typical_error_pct", "high_error_pct", "next", "cap", "conflict", "disclaimer")} except Exception: # the accuracy block is advisory: a failure there must not hide the twin s["model_accuracy"] = None return s def twin_layer(self, job_id, layer): if layer not in self.TWIN_LAYERS: raise ValueError(f"unknown layer {layer!r}: {', '.join(self.TWIN_LAYERS)}") path, twin = self._twin_load(job_id) return twin.to_png(layer, self._twin_clock(path)) def twin_observe_live(self, job_id, rgb, masks, summary): """Called by a live session for each locked frame: register the frame to the job's photo and fold it into the twin.""" import cv2 import evidence_inputs as ei path, twin = self._twin_load(job_id) gray_main, labels_main = self._main_arrays(path) labels = masks.get("mask") if labels is None: return gray = cv2.cvtColor(np.asarray(rgb), cv2.COLOR_RGB2GRAY) H, inl, why = ei.register_homography(gray_main, labels_main, gray, np.asarray(labels, np.uint8)) if H is None: return dmg = masks.get("hazards") twin.observe_view(np.asarray(labels, np.uint8), H, self._twin_clock(path), "live frame", weight=0.8) (path / "twin.npz").write_bytes(twin.to_bytes()) def auto_match_drawing(self, job_id, drawing_id, request): """Place the job's photo on an uploaded drawing from the layout of its windows and doors (drawing_match.py), instead of by hand. Needs the drawing calibrated first (its two clicked points + real length give mm per pixel, so its openings read in mm) and the job's saved class mask. The result is stored on the drawing (``auto_match``) so the operator can review it. The scale is only written to the job when the request says ``apply`` AND the match is usable (two or more openings agree and no other placement fits as well); an ambiguous or one-opening match is reported and never applied.""" import cv2 import drawing_match as dm path, items = self._drawings(job_id) item = next((d for d in items if d["id"] == drawing_id), None) if item is None: raise KeyError(drawing_id) cal = item.get("calibration") if not cal: raise ValueError("calibrate the drawing first (two clicked points and a real length): its scale is needed to read its openings") job = self.job_dir(job_id) classes = job / "classes.png" if not classes.is_file(): raise ValueError("this job has no saved class mask; re-run the photo through the current model") labels = np.asarray(Image.open(classes)) gray = cv2.imread(str(path / f"{drawing_id}.png"), cv2.IMREAD_GRAYSCALE) try: drawing = dm.drawing_from_image(gray, float(cal["mm_per_px"])) except ValueError as exc: raise ValueError(f"could not read openings from this drawing: {exc}") from exc result = dm.match(dm.openings_from_mask(labels), drawing, labels.shape) applied = False if request.get("apply"): if json.loads((job / "job.json").read_text()).get("approved"): raise PermissionError("job is approved and frozen; start a new job to change the scale") if not (result.get("matched") and result.get("usable_for_scale")): raise ValueError("this match is not usable for scale (" + str(result.get("note") or result.get("reason") or "low confidence") + ")") saved = {"mm_per_px": result["mm_per_px_at_centre"], "source": f"drawing auto-match ({result['confidence']} confidence, " f"{len(result['matched_openings'])} openings)", "axis": "match", "length_mm": None, "at": now()} (job / "scale.json").write_text(json.dumps(saved, indent=1)) self._record_anchor(job, "drawing", result["mm_per_px_at_centre"], f"drawing auto-match ({result['confidence']})") applied = True item["auto_match"] = {k: result.get(k) for k in ("matched", "confidence", "ambiguous", "usable_for_scale", "mm_per_px_at_centre", "matched_openings", "in_photo_not_in_drawing", "in_drawing_not_in_photo", "candidates", "note", "reason", "photo_footprint_mm", "wall_mm")} | { "applied": applied, "at": now()} (path / "index.json").write_text(json.dumps(items, indent=1)) return {"match": item["auto_match"], "applied": applied} def list_jobs(self): jobs = [] for path in sorted(self.jobs.iterdir(), key=lambda p: p.stat().st_mtime, reverse=True): if JOB_ID.match(path.name) and (path / "job.json").is_file(): jobs.append(json.loads((path / "job.json").read_text())) return jobs[:200] def create_job(self, data, filename="", wall_id=""): if len(data) > MAX_UPLOAD: raise ValueError(f"image larger than {MAX_UPLOAD // (1024 * 1024)} MB") try: image = Image.open(io.BytesIO(data)) image.load() except Exception as exc: raise ValueError(f"not a readable image: {exc}") from exc from PIL import ImageOps image = ImageOps.exif_transpose(image).convert("RGB") if max(image.size) > MAX_SIDE: image.thumbnail((MAX_SIDE, MAX_SIDE), Image.Resampling.LANCZOS) rgb = np.asarray(image) job_id = secrets.token_hex(6) path = self.jobs / job_id path.mkdir() image.save(path / "photo.jpg", quality=92) with self.lock: started = time.time() summary, masks = self.analyze_fn(rgb) seconds = round(time.time() - started, 2) if masks.get("mask") is not None and self.defaults.get("window_gate", False): masks["mask"], summary["window_lookalike_gate"] = window_gate(rgb, masks["mask"]) (path / "result.json").write_text(json.dumps(summary, indent=2) + "\n") save_mask(path / "paintable.png", masks["safe_paintable"]) save_mask(path / "keepout.png", masks["hazards"]) save_mask(path / "touchup.png", masks["touch_up"]) save_mask(path / "wall.png", masks["wall"]) if masks.get("mask") is not None: Image.fromarray(np.asarray(masks["mask"], np.uint8)).save(path / "classes.png") layer_png(masks["uncertain"], LAYER_COLOURS["uncertain"]).save(path / "layer_uncertain.png") edge_png(masks["uncertain"], LAYER_COLOURS["uncertain"]).save(path / "layer_uncertain_edge.png") job = {"id": job_id, "created": now(), "filename": Path(filename).name[:120], "wall_id": wall_id[:80], "width": image.size[0], "height": image.size[1], "photo_sha256": sha256_bytes( (path / "photo.jpg").read_bytes()), "analysis_seconds": seconds, "model": self.model_info, "status": None, "approved": None} (path / "job.json").write_text(json.dumps(job, indent=2) + "\n") self.plan(job_id, {}) return self.read_job(job_id) def plan(self, job_id, request): path = self.job_dir(job_id) job = json.loads((path / "job.json").read_text()) if job.get("approved"): raise PermissionError("job is approved and frozen; start a new job to change the plan") prediction = json.loads((path / "result.json").read_text()) paintable, keepout = load_mask(path / "paintable.png"), load_mask(path / "keepout.png") touchup, wall = load_mask(path / "touchup.png"), load_mask(path / "wall.png") # edits_png absent: keep the saved edits; "" clears them; a data URL replaces them. if "edits_png" in request: edits = decode_edits(request["edits_png"], paintable.shape) if edits is not None and edits.any(): Image.fromarray(edits).save(path / "edits.png") elif (path / "edits.png").is_file(): (path / "edits.png").unlink() edits = np.asarray(Image.open(path / "edits.png")) if (path / "edits.png").is_file() else None edit_stats = None operator_zones = [z for z in self.get_zones(job_id)["zones"] if z.get("visible", True)] effective = edits if any(z["kind"] != "note" for z in operator_zones): zmap = self.zone_edits(operator_zones, paintable.shape) effective = zmap if edits is None else np.where(zmap > 0, zmap, edits).astype(edits.dtype) if effective is not None: paintable, keepout, edit_stats = apply_edits(paintable, keepout, effective, wall=wall) if edits is not None: # the brush layer only: zones are drawn from zones.json, not baked into it rgba = np.zeros(edits.shape + (4,), np.uint8) rgba[edits == EDIT_ADD] = (0, 255, 0, 255) rgba[edits == EDIT_EXCLUDE] = (255, 0, 0, 255) Image.fromarray(rgba).save(path / "layer_edits.png") if edits is None and (path / "layer_edits.png").is_file(): (path / "layer_edits.png").unlink() operator = (request.get("operator") or "").strip()[:80] confirmations = {} for name in request.get("confirmations") or []: if name not in CONFIRMATIONS: raise ValueError(f"unknown confirmation {name!r}") if not operator: raise ValueError("enter the operator's name to sign a confirmation") confirmations[name] = {"by": operator, "at": now()} depth = request.get("depth_report") if depth is None and (path / "depth_report.json").is_file(): depth = json.loads((path / "depth_report.json").read_text()) # a depth scan uploaded for this job (evidence.py) saved = json.loads((path / "scale.json").read_text()) if (path / "scale.json").is_file() else {} mm_per_px = request.get("mm_per_px") or saved.get("mm_per_px") or self.defaults.get("mm_per_px") source = "operator" if request.get("mm_per_px") else (saved.get("source") if saved.get("mm_per_px") else ("default" if mm_per_px else None)) if not mm_per_px: mm_per_px, source = scale_from_prediction(prediction) tools_mm = {**{"roller": 230.0, "trim": 60.0, "prep": 125.0, "margin": 40.0}, **(request.get("tools_mm") or {})} order, _ = build_work_order(prediction, paintable, keepout, touchup, depth=depth, require_depth=self.defaults.get("require_depth", True), mm_per_px=float(mm_per_px) if mm_per_px else None, scale_source=source, tools_mm=tools_mm, confirmations=confirmations, edit_stats=edit_stats) order["job_id"], order["planned_at"], order["model"] = job_id, now(), self.model_info zdata = self.get_zones(job_id) order["operator_zones"] = [{"kind": z["kind"], "shape": z["shape"], "label": z["label"], "text": z["text"], "points_px": z["pts"], "applied": z["kind"] != "note" and z.get("visible", True), "correction": z["kind"] in CORRECTION_KINDS} for z in zdata["zones"]] order["job_info"] = zdata["info"] (path / "work_order.json").write_text(json.dumps(order, indent=2) + "\n") layer_png(paintable & ~keepout, LAYER_COLOURS["paint"], 120).save(path / "layer_paint.png") layer_png(keepout & wall, LAYER_COLOURS["keepout"], 120).save(path / "layer_keepout.png") edge_png(paintable & ~keepout, LAYER_COLOURS["paint"]).save(path / "layer_paint_edge.png") edge_png(keepout & wall, LAYER_COLOURS["keepout"]).save(path / "layer_keepout_edge.png") # Touch-up patches only matter on a signed-off wall; otherwise the coat covers them. signed_off = (prediction.get("completion") or {}).get("status") == "complete" layer_png(touchup & ~keepout & signed_off, LAYER_COLOURS["touchup"], 170).save(path / "layer_touchup.png") edge_png(touchup & ~keepout & signed_off, LAYER_COLOURS["touchup"]).save(path / "layer_touchup_edge.png") job.update(status=order["status"], summary=order.get("summary"), updated=now()) (path / "job.json").write_text(json.dumps(job, indent=2) + "\n") return self.read_job(job_id) def demo(self, job_id): """Tape + paint plan as drawable geometry (see demo_plan.py) for the phone demo page.""" from demo_plan import build_demo_plan path = self.job_dir(job_id) if not (path / "classes.png").is_file(): raise ValueError("this job predates the demo plan: upload the photo again") prediction = json.loads((path / "result.json").read_text()) classes = self.corrected_classes(np.asarray(Image.open(path / "classes.png")), self.get_zones(job_id)["zones"]) from scipy import ndimage from demo_plan import CLASS_IDS, WALL_IDS protect = np.isin(classes, list(CLASS_IDS.values())) paintable = np.isin(classes, WALL_IDS) & ~ndimage.binary_dilation(protect, iterations=3) layer_png(paintable, (255, 255, 255), 255).save(path / "demo_paint.png") from demo_plan import regions_rgba Image.fromarray(regions_rgba(classes)[0]).save(path / "demo_regions.png") gray = np.asarray(Image.open(path / "photo.jpg").convert("L")) mm_per_px, source = scale_from_prediction(prediction) plan = build_demo_plan(classes, paintable, mm_per_px, source, gray) plan["job_id"], plan["status"] = job_id, json.loads((path / "job.json").read_text()).get("status") return plan def create_video(self, data, filename=""): """Save an upload and analyse keyframes (video_demo.py). Synchronous: the model is the slow part.""" from video_demo import process_video ext = Path(filename).suffix.lstrip(".").lower() if ext not in VIDEO_EXT: raise ValueError(f"video must be one of: {', '.join(sorted(VIDEO_EXT))}") if not data or len(data) > MAX_VIDEO: raise ValueError(f"video must be between 1 byte and {MAX_VIDEO // (1024 * 1024)} MB") vid = "v" + secrets.token_hex(6) path = self.jobs / vid path.mkdir() (path / f"video.{ext}").write_bytes(data) try: with self.lock: manifest = process_video(path / f"video.{ext}", self.analyze_fn, path) except ValueError: import shutil shutil.rmtree(path, ignore_errors=True) raise manifest.update(id=vid, video=f"video.{ext}") (path / "manifest.json").write_text(json.dumps(manifest, indent=1)) return manifest def approve(self, job_id, request): from run_registry import append operator = (request.get("operator") or "").strip()[:80] if not operator: raise ValueError("enter the operator's name to approve") with self.lock: path = self.job_dir(job_id) job = json.loads((path / "job.json").read_text()) if job.get("approved"): raise PermissionError("already approved") order_bytes = (path / "work_order.json").read_bytes() order = json.loads(order_bytes) if order["status"] == "blocked": raise PermissionError("the safety gate blocks this order: " + "; ".join(order["gate"]["reasons"])) hashes = {name: sha256_bytes((path / name).read_bytes()) for name in ("photo.jpg", "result.json", "paintable.png", "keepout.png", "edits.png") if (path / name).is_file()} approval = {"job_id": job_id, "operator": operator, "note": (request.get("note") or "")[:500], "approved_at": now(), "work_order_sha256": sha256_bytes(order_bytes), "inputs_sha256": hashes, "status": order["status"], "confirmations": order.get("confirmations"), "operator_edits": order.get("operator_edits"), "model": self.model_info} record = append(self.jobs / "registry.jsonl", "work_order_approval", checkpoint_sha256=(self.model_info or {}).get("sha256"), run_id=job_id, config={"work_order_sha256": approval["work_order_sha256"]}, extra={"operator": operator, "work_order_sha256": approval["work_order_sha256"]}) approval["registry_hash"] = record["hash"] (path / "approval.json").write_text(json.dumps(approval, indent=2) + "\n") job.update(approved={"by": operator, "at": approval["approved_at"], "registry_hash": record["hash"]}) (path / "job.json").write_text(json.dumps(job, indent=2) + "\n") return self.read_job(job_id) def make_handler(console): class Handler(BaseHTTPRequestHandler): server_version = f"PaintingVisionConsole/{VERSION}" server_console = console def log_message(self, fmt, *args): # quieter, one line per request print(f"{self.address_string()} {fmt % args}", flush=True) def send(self, status, body, content_type="application/json", extra=None): if isinstance(body, (dict, list)): body = json.dumps(body).encode() elif isinstance(body, str): body = body.encode() self.send_response(status) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(body))) self.send_header("Cache-Control", "no-store") self.send_header("X-Content-Type-Options", "nosniff") self.send_header("Content-Security-Policy", "default-src 'self'; img-src 'self' data: blob:; " "style-src 'self'; script-src 'self'; media-src 'self'") for key, value in (extra or {}).items(): self.send_header(key, value) self.end_headers() self.wfile.write(body) def send_file(self, path, name): """Serve a job file with Range support (Safari will not play a video without it).""" ext = name.rsplit(".", 1)[-1] ctype = VIDEO_EXT.get(ext) or {"json": "application/json", "png": "image/png", "jpg": "image/jpeg"}[ext] size = path.stat().st_size rng = re.match(r"^bytes=(\d*)-(\d*)$", self.headers.get("Range") or "") start, end, status = 0, size - 1, 200 if rng and (rng.group(1) or rng.group(2)): if rng.group(1): start = int(rng.group(1)) end = int(rng.group(2)) if rng.group(2) else size - 1 else: start = max(0, size - int(rng.group(2))) end = min(end, size - 1) if start > end: self.send_response(416) self.send_header("Content-Range", f"bytes */{size}") self.end_headers() return status = 206 with open(path, "rb") as f: f.seek(start) body = f.read(end - start + 1) self.send_response(status) self.send_header("Content-Type", ctype) self.send_header("Content-Length", str(len(body))) self.send_header("Accept-Ranges", "bytes") self.send_header("Cache-Control", "no-store") self.send_header("X-Content-Type-Options", "nosniff") if status == 206: self.send_header("Content-Range", f"bytes {start}-{end}/{size}") self.end_headers() self.wfile.write(body) def error(self, status, message): self.send(status, {"error": message}) def body(self, limit=MAX_UPLOAD): length = int(self.headers.get("Content-Length") or 0) if length > limit: raise ValueError("request too large") return self.rfile.read(length) if length else b"" def do_GET(self): url = urlparse(self.path) parts = [p for p in url.path.split("/") if p] try: if not parts: return self.send(200, (STATIC / "index.html").read_bytes(), STATIC_FILES["index.html"]) if parts == ["api", "assist", "status"]: import ai_assist return self.send(200, ai_assist.status()) if parts == ["api", "health"]: return self.send(200, {"ok": True, "version": VERSION, "model": console.model_info, "confirmations": {k: v[1] for k, v in CONFIRMATIONS.items()}}) if parts == ["api", "jobs"]: return self.send(200, console.list_jobs()) if len(parts) == 3 and parts[:2] == ["api", "jobs"]: return self.send(200, console.read_job(parts[2])) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "zones": return self.send(200, console.get_zones(parts[2])) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "dronetwin": return self.send(200, console.dronetwin_status(parts[2])) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "dronetwinnext": return self.send(200, console.dronetwin_next(parts[2])) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "dronetwinwhatif": q = parse_qs(url.query) return self.send(200, console.dronetwin_whatif(parts[2], float(q.get("positioning", [0.02])[0]), float(q.get("wind", [2.0])[0]), int(q.get("trials", [3])[0]))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "floor": return self.send(200, console.floor(parts[2])) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "objects": return self.send(200, console.objects(parts[2])) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "drawings": return self.send(200, console.list_drawings(parts[2])) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "drawings" and parts[4].rsplit(".", 1)[-1] in ("svg", "dxf", "json"): did, ext = parts[4].rsplit(".", 1) mime = {"svg": "image/svg+xml", "dxf": "application/dxf", "json": "application/json"}[ext] f = console.drawing_export(parts[2], did, ext) return self.send(200, f.read_bytes(), mime, {"Content-Disposition": f'attachment; filename="{parts[2]}_{did}.{ext}"'} if "download" in parse_qs(url.query) else None) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "drawings": return self.send(200, console.drawing_file(parts[2], parts[4].removesuffix(".png")).read_bytes(), "image/png") if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "twin": try: return self.send(200, console.twin_state(parts[2]) | {"built": True}) except KeyError as exc: if "no twin" not in str(exc): raise return self.send(200, {"built": False}) # not an error: most jobs simply have no twin yet if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "twin" and parts[4] == "pipeline": import twin_pipeline return self.send(200, twin_pipeline.last(console, parts[2])) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "twin" and parts[4] == "history": return self.send(200, console.twin_history(parts[2])) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "twin" and parts[4] == "cell": q = parse_qs(url.query) return self.send(200, console.twin_cell(parts[2], q.get("x", ["nan"])[0], q.get("y", ["nan"])[0])) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "twin" and parts[4] in ("openings.json", "openings.csv", "model.obj", "defects.json", "quantities.json"): kind, body = console.twin_export(parts[2], parts[4]) return self.send(200, body, kind, {"Content-Disposition": f'attachment; filename="{parts[2]}_twin_{parts[4]}"'} if "download" in parse_qs(url.query) else None) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "twin" and parts[4].endswith(".png"): return self.send(200, console.twin_layer(parts[2], parts[4][:-4]), "image/png") if parts == ["api", "videos"]: return self.send(200, console.list_videos()) if parts == ["api", "live"]: return self.send(200, console.live_state()) if parts == ["api", "live", "frame.jpg"]: return self.send(200, console.live_frame(), "image/jpeg") if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "evidence": return self.send(200, console.evidence(parts[2])) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "demo": return self.send(200, console.demo(parts[2])) if len(parts) == 4 and parts[:2] == ["api", "videos"] and VIDEO_ID.match(parts[2]) \ and VIDEO_FILE.match(parts[3]): path = self.server_console.jobs / parts[2] / parts[3] if not path.is_file(): return self.error(404, "not found") return self.send_file(path, parts[3]) if parts == ["demo"]: return self.send(200, (STATIC / "demo.html").read_bytes(), STATIC_FILES["demo.html"]) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] in JOB_FILES: path = console.job_dir(parts[2]) / parts[3] if not path.is_file() and parts[3].endswith("_edge.png"): # a job made before outlines existed: derive the ring from its fill layer's own pixels name = parts[3][len("layer_"):-len("_edge.png")] fill = path.parent / f"layer_{name}.png" if name in LAYER_COLOURS and fill.is_file(): edge_png(np.asarray(Image.open(fill).convert("RGBA"))[..., 3] > 0, LAYER_COLOURS[name]).save(path) if not path.is_file(): return self.error(404, "not found") extra = ({"Content-Disposition": f'attachment; filename="{parts[2]}_{parts[3]}"'} if "download" in parse_qs(url.query) else None) return self.send(200, path.read_bytes(), JOB_FILES[parts[3]], extra) if len(parts) == 1 and parts[0] in STATIC_FILES: return self.send(200, (STATIC / parts[0]).read_bytes(), STATIC_FILES[parts[0]]) return self.error(404, "not found") except KeyError: return self.error(404, "no such job") except (ValueError, json.JSONDecodeError) as exc: # a bad parameter in a GET is the caller's error, not a dropped connection return self.error(400, str(exc)) except Exception as exc: import traceback traceback.print_exc() return self.error(500, f"{type(exc).__name__}: {exc}") def do_POST(self): url = urlparse(self.path) parts = [p for p in url.path.split("/") if p] try: if parts == ["api", "videos"]: query = parse_qs(url.query) return self.send(201, console.create_video(self.body(MAX_VIDEO), query.get("filename", [""])[0])) if parts == ["api", "jobs"]: query = parse_qs(url.query) job = console.create_job(self.body(), query.get("filename", [""])[0], query.get("wall_id", [""])[0]) return self.send(201, job) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "laser": return self.send(201, console.add_laser(parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "scan": q = parse_qs(url.query) return self.send(201, console.add_scan(parts[2], self.body(MAX_SCAN), q.get("filename", [""])[0], q.get("up", ["z"])[0], q.get("flip", ["0"])[0] in ("1", "true"))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "twin": return self.send(201, console.twin_build(parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "twin" and parts[4] == "pipeline": import twin_pipeline return self.send(200, twin_pipeline.run(console, parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "twin" and parts[4] == "update": return self.send(200, console.twin_update(parts[2])) if parts == ["api", "live"]: return self.send(201, console.live_start(json.loads(self.body() or b"{}"))) if parts == ["api", "live", "arm"]: return self.send(200, console.live_arm(json.loads(self.body() or b"{}"))) if parts == ["api", "live", "stop"]: return self.send(200, console.live_stop()) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "views": return self.send(201, console.add_view(parts[2], self.body(MAX_VIDEO), parse_qs(url.query).get("filename", [""])[0])) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "depth": q = parse_qs(url.query) num = lambda k: (float(q[k][0]) if q.get(k) and q[k][0] != "" else None) # noqa: E731 return self.send(201, console.add_depth(parts[2], self.body(MAX_DEPTH), q.get("filename", [""])[0], fx=num("fx"), hfov_deg=num("hfov_deg"))) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "drawings" and parts[4] == "generate": return self.send(201, console.generate_drawing(parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "drawings": query = parse_qs(url.query) return self.send(201, console.add_drawing(parts[2], self.body(MAX_DRAWING), query.get("filename", [""])[0])) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "drawings" and DRAWING_ID.match(parts[4]): return self.send(200, console.update_drawing(parts[2], parts[4], json.loads(self.body() or b"{}"))) if len(parts) == 6 and parts[:2] == ["api", "jobs"] and parts[3] == "drawings" and DRAWING_ID.match(parts[4]) \ and parts[5] == "match": return self.send(200, console.auto_match_drawing(parts[2], parts[4], json.loads(self.body() or b"{}"))) if parts == ["api", "assist"]: # Optional AI helper: advisory text only, and only with an API key in the server's environment (ai_assist.py). import ai_assist request = json.loads(self.body() or b"{}") if not isinstance(request, dict): raise ValueError("expected a JSON object") return self.send(200, ai_assist.ask(request.get("context"), request.get("question", ""))) if parts == ["api", "room"]: return self.send(200, console.room_twin(json.loads(self.body() or b"{}"))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "prompt": return self.send(200, console.prompt(parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 5 and parts[:2] == ["api", "jobs"] and parts[3] == "prompt" and parts[4] == "apply": return self.send(200, console.prompt_apply(parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "zones": return self.send(200, console.set_zones(parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "dronetwin": return self.send(200, console.dronetwin_status(parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "dronetwinobserve": return self.send(200, console.dronetwin_observe(parts[2], self.body())) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "dronetwinlog": sortie = parse_qs(url.query).get("sortie", [None])[0] return self.send(200, console.dronetwin_log(parts[2], self.body(MAX_DRAWING).decode("utf-8", "replace"), sortie)) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] == "scale": return self.send(200, console.set_scale_from_drawing(parts[2], json.loads(self.body() or b"{}"))) if len(parts) == 4 and parts[:2] == ["api", "jobs"] and parts[3] in ("plan", "approve"): request = json.loads(self.body() or b"{}") if not isinstance(request, dict): raise ValueError("expected a JSON object") action = console.plan if parts[3] == "plan" else console.approve return self.send(200, action(parts[2], request)) return self.error(404, "not found") except KeyError: return self.error(404, "no such job") except PermissionError as exc: return self.error(409, str(exc)) except (ValueError, json.JSONDecodeError) as exc: return self.error(400, str(exc)) except Exception as exc: # keep the console up; the operator sees the error import traceback traceback.print_exc() return self.error(500, f"{type(exc).__name__}: {exc}") return Handler def model_analyzer(checkpoint_path, device_name=None): """Load the checkpoint once; return ``(analyze_fn, model_info)``.""" import cv2 import torch from hub import resolve_checkpoint from predict import analyze, build_parser from train import load_checkpoint_model path = Path(resolve_checkpoint(str(checkpoint_path))) device = torch.device(device_name or ("cuda" if torch.cuda.is_available() else "cpu")) model, checkpoint = load_checkpoint_model(path, device) args = build_parser().parse_args(["unused.jpg"]) args.depth_map, args.focal_px = None, None warnings = [] if checkpoint.get("completion_threshold") is None: warnings.append("sign-off is disabled: this checkpoint has no calibrated completion threshold") if not checkpoint.get("condition_operating_point"): warnings.append("damage flags use uncalibrated defaults (run calibrate_condition.py --write)") if "temperature" not in checkpoint: warnings.append("confidences are not temperature-calibrated (run train.py --calibrate-checkpoint)") info = {"checkpoint": path.name, "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), "arch": checkpoint.get("arch"), "device": str(device), "completion_threshold": checkpoint.get("completion_threshold"), "condition_operating_point": checkpoint.get("condition_operating_point"), "warnings": warnings} def analyze_fn(rgb): return analyze(cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR), model, checkpoint, device, args) return analyze_fn, info def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--checkpoint", default="artifacts/best.pt", help="local .pt or hf://owner/repo") ap.add_argument("--jobs-dir", type=Path, default=Path("jobs")) ap.add_argument("--host", default="127.0.0.1", help="0.0.0.0 exposes the console to the network: only on a " "trusted site network") ap.add_argument("--port", type=int, default=8765) ap.add_argument("--device") ap.add_argument("--mm-per-px", type=float, help="default scale when a job gives none") ap.add_argument("--no-window-gate", action="store_true", help="keep every window the wall model finds (skip the picture/mirror/screen look-alike check)") ap.add_argument("--no-require-depth", action="store_true", help="do not require depth confirmation (operators can still waive it per job)") args = ap.parse_args() analyze_fn, info = model_analyzer(args.checkpoint, args.device) console = Console(args.jobs_dir, analyze_fn, info, {"mm_per_px": args.mm_per_px, "require_depth": not args.no_require_depth, "window_gate": not args.no_window_gate}) server = ThreadingHTTPServer((args.host, args.port), make_handler(console)) for warning in info["warnings"]: print("warning:", warning) print(f"console on http://{args.host}:{args.port} (model {info['checkpoint']}, jobs in {args.jobs_dir})", flush=True) try: server.serve_forever() except KeyboardInterrupt: pass if __name__ == "__main__": main()
#!/usr/bin/env python3
"""Operator console: photo in, reviewed and approved paint work order out.
A local web app for the person standing next to the robot. Upload (or take) a photo
of the wall; the kit shows where it will paint, what it keeps out of, which damage
needs prep first, and why the safety gate would stop. The operator can:
* toggle layers (paint area, keep-out, uncertain wall, touch-up, damage, tool paths);
* correct the plan with a brush: **paint here** / **do not paint**;
* sign the confirmations the gate allows a person to make (not wallpaper, depth
waived, uncertain areas reviewed), each recorded with their name;
* re-plan, then **approve**. An approved order is frozen: the job keeps the photo,
masks, edits and work order with their sha256, and an entry is appended to the
hash-linked ``registry.jsonl`` (``run_registry.py verify`` checks it).
Production notes: binds to 127.0.0.1 unless ``--host`` says otherwise; one model
loaded once, inference serialised behind a lock; uploads capped (``MAX_UPLOAD``)
and decoded before anything is written; job ids are random and file serving is an
allowlist, so no path from a request reaches the filesystem. stdlib + the kit's own
dependencies only: nothing extra to install on the robot PC.
Usage::
python3 console_server.py --checkpoint artifacts/best.pt --jobs-dir jobs
# open http://127.0.0.1:8765
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import io
import json
import re
import secrets
import threading
import time
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, urlparse
import numpy as np
from PIL import Image
from work_order import CONFIRMATIONS, EDIT_ADD, EDIT_EXCLUDE, apply_edits, build_work_order, scale_from_prediction
VERSION = "1.0"
STATIC = Path(__file__).with_name("console")
MAX_UPLOAD = 30 * 1024 * 1024
MAX_SIDE = 2048 # frames are normalised to this long side before analysis
JOB_ID = re.compile(r"^[0-9a-f]{12}$")
LAYER_COLOURS = {"paint": (46, 204, 113), "keepout": (231, 76, 60), "uncertain": (241, 196, 15),
"touchup": (0, 210, 255)}
JOB_FILES = {"photo.jpg": "image/jpeg", "demo_paint.png": "image/png", "demo_regions.png": "image/png", "work_order.json": "application/json", "result.json": "application/json",
"approval.json": "application/json", "layer_edits.png": "image/png",
**{f"layer_{k}.png": "image/png" for k in LAYER_COLOURS},
**{f"layer_{k}_edge.png": "image/png" for k in LAYER_COLOURS},
"layer_floor.png": "image/png", "layer_floor_edge.png": "image/png", "floor.json": "application/json",
"dronetwin_heatmap.png": "image/png", "dronetwin.json": "application/json", "dronetwin_geojson.json": "application/geo+json"}
FLOOR_COLOUR = (176, 125, 79)
VIDEO_ID = re.compile(r"^v[0-9a-f]{12}$")
VIDEO_EXT = {"mp4": "video/mp4", "mov": "video/quicktime", "webm": "video/webm", "m4v": "video/mp4"}
VIDEO_FILE = re.compile(r"^(manifest\.json|track\.json|plan_\d+\.json|frame_\d+\.png|paint_\d+\.png|photo_\d+\.jpg|video\.(mp4|mov|webm|m4v))$")
MAX_VIDEO = 200 * 1024 * 1024
MAX_DRAWING = 40 * 1024 * 1024
MAX_DEPTH = 60 * 1024 * 1024
MAX_SCAN = 150 * 1024 * 1024
MAX_VIEWS = 14
ZONE_KINDS = ("keepout", "paint", "note", "window", "door", "fixture", "wall")
CORRECT_CLASS = {"window": 8, "door": 7, "fixture": 9} # operator corrections: "this is a ..." (fixture = wall obstacle)
CORRECTION_KINDS = ("window", "door", "fixture", "wall")
ZONE_SHAPES = ("rect", "poly", "pin")
MAX_ZONES = 200
INFO_FIELDS = ("client", "site", "room", "spec", "notes")
DRAWING_ID = re.compile(r"^d[0-9a-f]{8}$")
DRAWING_SIDE = 2400 # a drawing is kept at this long side: enough to click a dimension on
STATIC_FILES = {"index.html": "text/html; charset=utf-8", "app.js": "text/javascript; charset=utf-8",
"style.css": "text/css; charset=utf-8", "demo.html": "text/html; charset=utf-8",
"demo.js": "text/javascript; charset=utf-8", "demo.css": "text/css; charset=utf-8",
"demo_video.js": "text/javascript; charset=utf-8", "zones.js": "text/javascript; charset=utf-8",
"zones.css": "text/css; charset=utf-8", "floor.js": "text/javascript; charset=utf-8", "dronetwin.js": "text/javascript; charset=utf-8", "robot.js": "text/javascript; charset=utf-8", "collapse.js": "text/javascript; charset=utf-8", "collapse.css": "text/css; charset=utf-8", "live.js": "text/javascript; charset=utf-8", "twin.js": "text/javascript; charset=utf-8", "prompt.js": "text/javascript; charset=utf-8"}
def _points(pts):
"""Two clicked [x, y] points, or None: stored so the drawn lines come back when a drawing is reopened."""
try:
(a, b), (c, d) = pts
return [[float(a), float(b)], [float(c), float(d)]]
except (TypeError, ValueError):
return None
def sha256_bytes(data):
return hashlib.sha256(data).hexdigest()
def now():
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())