Spaces:
Running
Running
| """Checks that a generated view still matches the geometry of its conditioning render. | |
| Instruction-edit models sometimes re-frame the picture: they zoom out of a close-up, re-center the subject or | |
| redraw the pose. Projecting such an image paints the wrong colours onto the mesh, so every generated view is | |
| scored and rejected (then retried with another seed) when it does not line up. | |
| * ``silhouette_iou``: objects/characters on a plain background. The foreground of the generated image is | |
| segmented against the background colour and compared with the rendered mask. | |
| * ``edge_agreement``: any view. This is the share of geometric edges of the render (silhouettes, occlusion | |
| boundaries, creases from the normal map) that have an image edge within a few pixels, minus the same share | |
| for randomly shifted edges (chance level). | |
| * ``edges_moved``: steep gap views. Share of the render's edges the output drew elsewhere (optical flow, beyond a | |
| global zoom/shift), for a subject redrawn from another angle with nearly the same outline. | |
| """ | |
| from __future__ import annotations | |
| import cv2 | |
| import numpy as np | |
| def _fill_holes(mask: np.ndarray) -> np.ndarray: | |
| m = mask.astype(np.uint8) | |
| h, w = m.shape | |
| flood = m.copy() | |
| ff = np.zeros((h + 2, w + 2), np.uint8) | |
| for seed in ((0, 0), (w - 1, 0), (0, h - 1), (w - 1, h - 1)): | |
| if flood[seed[1], seed[0]] == 0: | |
| cv2.floodFill(flood, ff, seed, 1) | |
| return (m | (1 - flood)).astype(bool) | |
| def foreground_mask(img: np.ndarray, depth: np.ndarray, thresh: float = 0.07) -> np.ndarray | None: | |
| """Pixels of a generated object view that are *not* background (colour far from the border background). | |
| None when the render has no visible background (environments, extreme close-ups).""" | |
| fg_r = depth > 0 | |
| border = np.zeros_like(fg_r) | |
| border[:4], border[-4:], border[:, :4], border[:, -4:] = True, True, True, True | |
| bg_px = border & ~fg_r | |
| if bg_px.sum() < 50 or fg_r.mean() < 0.005: | |
| return None | |
| bgc = np.median(img[bg_px], axis=0) | |
| diff = np.abs(cv2.GaussianBlur(img, (0, 0), 1.0) - bgc[None, None]).max(-1) | |
| fg_g = diff > thresh | |
| fg_g = cv2.morphologyEx(fg_g.astype(np.uint8), cv2.MORPH_OPEN, np.ones((3, 3), np.uint8)) > 0 | |
| # a re-framed output leaves background near the silhouette's edge; deep inside it, background-like colours are | |
| # paint (misty blue-grey mountains on a mug were cut out, and another view's mountains showed through) | |
| depth_in = cv2.distanceTransform(fg_r.astype(np.uint8), cv2.DIST_L2, 5) | |
| return _fill_holes(fg_g) | (depth_in > 0.025 * max(img.shape[:2])) | |
| def silhouette_iou(img: np.ndarray, depth: np.ndarray) -> float | None: | |
| fg_r = depth > 0 | |
| border = np.zeros_like(fg_r) | |
| border[:4], border[-4:], border[:, :4], border[:, -4:] = True, True, True, True | |
| bg_px = border & ~fg_r | |
| if bg_px.sum() < 50 or fg_r.mean() < 0.005: | |
| return None # no visible background (environments, extreme close-ups) | |
| bgc = np.median(img[bg_px], axis=0) | |
| diff = np.abs(cv2.GaussianBlur(img, (0, 0), 1.0) - bgc[None, None]).max(-1) | |
| fg_g = diff > 0.07 | |
| fg_g = cv2.morphologyEx(fg_g.astype(np.uint8), cv2.MORPH_OPEN, np.ones((3, 3), np.uint8)) > 0 | |
| fg_g = _fill_holes(fg_g) | |
| inter = (fg_g & fg_r).sum() | |
| union = (fg_g | fg_r).sum() | |
| return float(inter / max(union, 1)) | |
| def geometry_edges(depth: np.ndarray, normal: np.ndarray, crease_deg: float = 35.0) -> np.ndarray: | |
| fg = depth > 0 | |
| inv = np.where(fg, 1.0 / np.maximum(depth, 1e-12), 0.0) | |
| ip = np.pad(inv, 1, mode="edge") | |
| d2 = np.maximum(np.abs(ip[1:-1, 2:] + ip[1:-1, :-2] - 2 * inv), np.abs(ip[2:, 1:-1] + ip[:-2, 1:-1] - 2 * inv)) | |
| occl = (d2 / np.maximum(inv, 1e-12)) > 0.05 | |
| n = normal / np.maximum(np.linalg.norm(normal, axis=-1, keepdims=True), 1e-9) | |
| c = np.cos(np.radians(crease_deg)) | |
| crease = np.zeros_like(fg) | |
| crease[:, :-1] |= (n[:, :-1] * n[:, 1:]).sum(-1) < c | |
| crease[:-1] |= (n[:-1] * n[1:]).sum(-1) < c | |
| sil = cv2.morphologyEx(fg.astype(np.uint8), cv2.MORPH_GRADIENT, np.ones((3, 3), np.uint8)) > 0 | |
| e = (occl & fg) | (crease & fg) | sil | |
| # only fairly long edges are informative | |
| n_lab, lab, stats, _ = cv2.connectedComponentsWithStats(e.astype(np.uint8), connectivity=8) | |
| keep = np.zeros(n_lab, bool) | |
| keep[1:] = stats[1:, cv2.CC_STAT_AREA] >= max(8, depth.shape[0] // 64) | |
| return keep[lab] | |
| def edge_agreement(img: np.ndarray, depth: np.ndarray, normal: np.ndarray, tol_px: float | None = None) -> float | None: | |
| h, w = depth.shape | |
| s = 512 / max(h, w) | |
| if s < 1: | |
| img = cv2.resize(img, (int(w * s), int(h * s)), interpolation=cv2.INTER_AREA) | |
| depth = cv2.resize(depth, (int(w * s), int(h * s)), interpolation=cv2.INTER_NEAREST) | |
| normal = cv2.resize(normal, (int(w * s), int(h * s)), interpolation=cv2.INTER_NEAREST) | |
| ge = geometry_edges(depth, normal) | |
| if ge.sum() < max(100, 0.004 * ge.size): | |
| return None # too little geometric structure to judge alignment | |
| gray = (np.clip(cv2.GaussianBlur(img.mean(-1), (0, 0), 1.0), 0, 1) * 255).astype(np.uint8) | |
| ie = cv2.Canny(gray, 20, 60) > 0 | |
| tol = tol_px or max(3.0, 0.008 * max(depth.shape)) | |
| dist = cv2.distanceTransform((~ie).astype(np.uint8), cv2.DIST_L2, 5) | |
| hit = float((dist[ge] <= tol).mean()) | |
| # chance level: the same edge pixels shifted by a large offset | |
| chance = [] | |
| for dx, dy in ((37, 0), (0, 41), (-29, 23)): | |
| sh = np.roll(np.roll(ge, dy, 0), dx, 1) | |
| chance.append(float((dist[sh] <= tol).mean())) | |
| return round(hit - float(np.mean(chance)), 4) | |
| def edges_moved(img: np.ndarray, ref: np.ndarray, depth: np.ndarray, normal: np.ndarray, size: int = 512, | |
| moved: float = 0.03) -> float | None: | |
| """Share of the model's edges that the output drew elsewhere: optical flow from the conditioning render to the | |
| output, at the geometry's edges, minus the best global zoom/shift (a slightly re-framed but faithful view is | |
| fine), counted where it is still more than ``moved`` of the image size. | |
| From high above, a model can redraw a boxy subject from its usual angle (a bus seen from 65 degrees came back | |
| as a raised side view, its upper windows where the render shows the roof). The outline barely changes and | |
| chance edges keep edge_agreement up, but the edge between roof and side moves with respect to the outline. | |
| Gap views above 50 degrees: 30-76 % for such views (two buses, a van), at most 11 % for views that followed the | |
| geometry. Both images are contrast-normalised, so a new pattern does not count. Unreliable on renders with | |
| few textured edges (the first views of a smooth clay pot), hence only used on gap views. | |
| """ | |
| h, w = depth.shape | |
| s = size / max(h, w) | |
| sz = (max(1, int(round(w * s))), max(1, int(round(h * s)))) | |
| img, ref = cv2.resize(img, sz, interpolation=cv2.INTER_AREA), cv2.resize(ref, sz, interpolation=cv2.INTER_AREA) | |
| depth = cv2.resize(depth, sz, interpolation=cv2.INTER_NEAREST) | |
| normal = cv2.resize(normal, sz, interpolation=cv2.INTER_NEAREST) | |
| ge = cv2.dilate(geometry_edges(depth, normal).astype(np.uint8), np.ones((3, 3), np.uint8)) > 0 | |
| if ge.sum() < 200: | |
| return None | |
| def lcn(x: np.ndarray) -> np.ndarray: | |
| g = (x[..., :3] @ np.array([0.299, 0.587, 0.114], np.float32)).astype(np.float32) * 255 | |
| m = cv2.GaussianBlur(g, (0, 0), 8) | |
| sd = np.sqrt(cv2.GaussianBlur((g - m) ** 2, (0, 0), 8)) + 4 | |
| return np.clip(128 + 40 * (g - m) / sd, 0, 255).astype(np.uint8) | |
| flow = cv2.DISOpticalFlow_create(cv2.DISOPTICAL_FLOW_PRESET_MEDIUM).calc(lcn(ref), lcn(img), None) | |
| ys, xs = np.nonzero(ge) | |
| p = np.stack([xs, ys], 1).astype(np.float32) | |
| q = p + flow[ys, xs] | |
| cv2.setRNGSeed(0) | |
| M, _ = cv2.estimateAffinePartial2D(p, q, method=cv2.RANSAC, ransacReprojThreshold=4.0, maxIters=2000) | |
| if M is None: | |
| M = np.eye(2, 3, dtype=np.float32) | |
| res = np.linalg.norm(q - (p @ M[:, :2].T + M[:, 2]), axis=1) / max(sz) | |
| return float((res > moved).mean()) | |
| def score_view(img: np.ndarray, depth: np.ndarray, normal: np.ndarray, environment: bool) -> dict: | |
| out = {"edge_agreement": edge_agreement(img, depth, normal)} | |
| if not environment: | |
| iou = silhouette_iou(img, depth) | |
| out["silhouette_iou"] = None if iou is None else round(iou, 4) | |
| return out | |
| def painted_agreement(img: np.ndarray, cond: dict, spare_grazed: bool = False) -> dict: | |
| """Does the new view agree with the texture already painted by earlier views where both overlap? | |
| Silhouette and edge checks cannot tell a roof plan from a facade on a boxy model: a top view re-imagined as a | |
| front elevation keeps a plausible outline and plenty of edges, and projecting it paints windows on the roof. | |
| Where the conditioning render already shows projected texture, the output must match it at low frequency | |
| (blurred, per-channel gain matched). Returns painted_frac (share of the object already painted), and when | |
| there is enough of it painted_corr (luminance correlation) and painted_diff (mean colour difference). | |
| ``spare_grazed``: leave out what earlier views only grazed (best cosine < 0.4) and this view sees far more | |
| head-on: from the side, the underside of a skateboard deck gets smears of the rails' colours, which a view from | |
| below that paints its graphic would otherwise be judged against. | |
| """ | |
| ref, cover, depth = cond["image"], cond.get("cover_px"), cond["depth"] | |
| if cover is None: | |
| return {} | |
| h, w = img.shape[:2] | |
| if ref.shape[:2] != (h, w): | |
| ref = cv2.resize(ref, (w, h), interpolation=cv2.INTER_AREA) | |
| if cover.shape != (h, w): | |
| cover = cv2.resize(cover, (w, h), interpolation=cv2.INTER_NEAREST) | |
| depth = cv2.resize(depth, (w, h), interpolation=cv2.INTER_NEAREST) | |
| fg = depth > 0 | |
| painted = (cover > 0.9) & fg | |
| if spare_grazed and cond.get("cos_px") is not None and cond.get("bestcos_px") is not None: | |
| cs, bc = cond["cos_px"], cond["bestcos_px"] | |
| if cs.shape != (h, w): | |
| cs = cv2.resize(cs, (w, h), interpolation=cv2.INTER_NEAREST) | |
| bc = cv2.resize(bc, (w, h), interpolation=cv2.INTER_NEAREST) | |
| painted &= ~((bc < 0.4) & (cs > bc + 0.3)) | |
| m = cv2.erode(painted.astype(np.uint8), np.ones((9, 9), np.uint8)) > 0 | |
| frac = float(m.sum() / max(int(fg.sum()), 1)) | |
| out = {"painted_frac": round(frac, 3)} | |
| if m.sum() < 2000 or frac < 0.05: | |
| return out | |
| sig = 0.012 * max(h, w) | |
| a = cv2.GaussianBlur(img.astype(np.float32), (0, 0), sig)[m].astype(np.float64) | |
| b = cv2.GaussianBlur(ref.astype(np.float32), (0, 0), sig)[m].astype(np.float64) | |
| g = np.clip((a * b).sum(0) / np.maximum((a * a).sum(0), 1e-9), 0.5, 2.0) | |
| out["painted_diff"] = round(float(np.abs(a * g - b).mean()), 4) | |
| la, lb = a @ np.array([0.299, 0.587, 0.114]), b @ np.array([0.299, 0.587, 0.114]) | |
| if la.std() > 1e-6 and lb.std() > 1e-6: | |
| out["painted_corr"] = round(float(np.corrcoef(la, lb)[0, 1]), 3) | |
| out["painted_chroma_overlap"] = round(_chroma_overlap(a, b), 3) | |
| return out | |
| def contradiction_mask(img: np.ndarray, cond: dict, delta_e: float = 20.0) -> tuple[np.ndarray, float] | None: | |
| """Where an accepted view repaints, in other colours, what the conditioning render showed fully painted. | |
| A view can agree with the painted texture overall and still re-imagine part of it: a roof slope seen from above | |
| drawn as a raised view of the house, its lower half a red wall with windows. Those pixels would win the blend | |
| (the view sees the slope head-on) and paint a wall on the roof. Compared at low frequency (blurred over painted | |
| pixels only, one brightness gain): connected regions more than ``delta_e`` (CIELAB) apart, closed, hole-filled | |
| and widened so the whole re-imagined patch goes. The mask spares what this view sees much more head-on than | |
| the views that painted it (low views paint a saucer's flat ring with contact shadows; the view from above | |
| corrects them). Returns (mask 0..1 to leave out, share of the painted pixels that contradict), or None when | |
| too little is painted. | |
| """ | |
| ref, cover, depth = cond["image"], cond.get("cover_px"), cond["depth"] | |
| if cover is None: | |
| return None | |
| h, w = img.shape[:2] | |
| if ref.shape[:2] != (h, w): | |
| ref = cv2.resize(ref, (w, h), interpolation=cv2.INTER_AREA) | |
| if cover.shape != (h, w): | |
| cover = cv2.resize(cover, (w, h), interpolation=cv2.INTER_NEAREST) | |
| depth = cv2.resize(depth, (w, h), interpolation=cv2.INTER_NEAREST) | |
| fg = depth > 0 | |
| # the render mixes paint with clay below full cover: compare only what it showed as painted | |
| painted = ((cover > 0.9) & fg).astype(np.float32) | |
| m = cv2.erode(painted.astype(np.uint8), np.ones((9, 9), np.uint8)) > 0 | |
| if m.sum() < 2000: | |
| return None | |
| size = max(h, w) | |
| sig = 0.012 * size | |
| wsum = np.maximum(cv2.GaussianBlur(painted, (0, 0), sig), 1e-6)[..., None] | |
| a = cv2.GaussianBlur(img.astype(np.float32) * painted[..., None], (0, 0), sig) / wsum | |
| b = cv2.GaussianBlur(ref.astype(np.float32) * painted[..., None], (0, 0), sig) / wsum | |
| am, bm = a[m].astype(np.float64), b[m].astype(np.float64) | |
| g = np.float32(np.clip((am * bm).sum() / max((am * am).sum(), 1e-9), 0.5, 2.0)) | |
| d = np.linalg.norm(cv2.cvtColor(np.clip(a * g, 0, 1), cv2.COLOR_RGB2LAB) | |
| - cv2.cvtColor(np.clip(b, 0, 1).astype(np.float32), cv2.COLOR_RGB2LAB), axis=-1) | |
| n, lbl, st, _ = cv2.connectedComponentsWithStats((m & (d > delta_e)).astype(np.uint8), connectivity=8) | |
| keep = np.zeros(n, bool) | |
| keep[1:] = st[1:, cv2.CC_STAT_AREA] >= 0.004 * h * w | |
| c = keep[lbl] | |
| share = float(c.sum() / m.sum()) | |
| if cond.get("cos_px") is not None and cond.get("bestcos_px") is not None: | |
| rs = (lambda x: x if x.shape == (h, w) else cv2.resize(x, (w, h), interpolation=cv2.INTER_NEAREST)) | |
| c &= ~(rs(cond["cos_px"]) > rs(cond["bestcos_px"]) + 0.15) | |
| if not c.any(): | |
| return np.zeros((h, w), np.float32), round(share, 3) | |
| k = int(0.025 * size) | 1 | |
| c = cv2.morphologyEx(c.astype(np.uint8), cv2.MORPH_CLOSE, np.ones((k, k), np.uint8)) > 0 | |
| c = _fill_holes(c) | |
| k = int(0.012 * size) | 1 | |
| c = cv2.dilate(c.astype(np.uint8), np.ones((k, k), np.uint8)).astype(np.float32) | |
| c = np.clip(cv2.GaussianBlur(c, (0, 0), k / 3) * 1.5, 0, 1) * fg | |
| return c.astype(np.float32), round(share, 3) | |
| def design_continuation(img: np.ndarray, cond: dict, contra: np.ndarray, new: np.ndarray, face_material: np.ndarray, | |
| part_colors: np.ndarray, delta_e: float = 25.0) -> np.ndarray: | |
| """The part of a partial view's new paint that continues a design it drew over established paint. | |
| A partial view only paints surface nobody painted yet, but its picture is one design: a carousel horse's | |
| head-on view drew a red breastplate across a chest whose left half was already painted white, and the right | |
| half, unpainted, kept the breastplate (the later side view continued it): one side decorated, the other plain. | |
| ``contra``: where the view repainted established paint in other colours (contradiction_mask); ``new``: the | |
| pixels it may paint. New pixels whose colour (blurred) is ``delta_e`` (CIELAB) from their part's colour as | |
| painted so far (``part_colors``, NaN when unknown) and connected through such pixels to ``contra`` are returned | |
| (bool): their paint is that design's continuation.""" | |
| face = cond["face"] | |
| h, w = img.shape[:2] | |
| if face.shape != (h, w): | |
| face = cv2.resize(face.astype(np.float32), (w, h), interpolation=cv2.INTER_NEAREST).astype(np.int64) | |
| lab = np.where(face >= 0, face_material[np.clip(face, 0, len(face_material) - 1)], -1) | |
| known = np.isfinite(part_colors).all(1) | |
| pc = cv2.cvtColor(np.clip(np.nan_to_num(part_colors), 0, 1).astype(np.float32).reshape(-1, 1, 3), | |
| cv2.COLOR_RGB2LAB).reshape(-1, 3) | |
| L = cv2.cvtColor(cv2.GaussianBlur(np.clip(img, 0, 1).astype(np.float32), (0, 0), 0.004 * max(h, w)), | |
| cv2.COLOR_RGB2LAB) | |
| ok = (lab >= 0) & known[np.clip(lab, 0, len(known) - 1)] | |
| differ = ok & (np.linalg.norm(L - pc[np.clip(lab, 0, len(pc) - 1)], axis=-1) > delta_e) | |
| k = new & differ | |
| n, cc = cv2.connectedComponents((contra | k).astype(np.uint8), connectivity=8) | |
| hit = np.zeros(n, bool) | |
| hit[np.unique(cc[contra])] = True | |
| hit[0] = False | |
| return hit[cc] & k | |
| def fill_unpainted_islands(img: np.ndarray, cover: np.ndarray, face: np.ndarray, face_material: np.ndarray, | |
| max_share: float = 0.03, min_ring: float = 0.6, max_spread: float = 15.0) -> np.ndarray: | |
| """A conditioning render with its small unpainted islands filled from the paint around them. | |
| Spots the earlier views missed show in the part's guide colour (or clay) amid its paint; the image model keeps | |
| them as markings: a teddy bear's back view kept pale tan patches on its brown arms, feet and tail, which | |
| every later view then preserved. Islands smaller than ``max_share`` of the object, whose surrounding ring is | |
| painted (``min_ring``), of the same part and evenly coloured (RMS CIELAB distance to its mean, lightness at half | |
| weight, within ``max_spread``: inside a landscape or florals the smear would be kept as a blur, the guide colour | |
| gets the motifs painted) are inpainted from that ring; they stay unpainted for every check.""" | |
| h, w = img.shape[:2] | |
| if cover is None or face is None or face.shape != (h, w) or cover.shape != (h, w): | |
| return img | |
| lab = np.where(face >= 0, face_material[np.clip(face, 0, len(face_material) - 1)], -1) | |
| fg = lab >= 0 | |
| un = (fg & (cover < 0.3)).astype(np.uint8) | |
| total = int(fg.sum()) | |
| if total < 500 or not un.any(): | |
| return img | |
| n, cc, st, _ = cv2.connectedComponentsWithStats(un, connectivity=8) | |
| small = np.zeros(n, bool) | |
| small[1:] = (st[1:, cv2.CC_STAT_AREA] >= 4) & (st[1:, cv2.CC_STAT_AREA] < max_share * total) | |
| if not small.any(): | |
| return img | |
| cand = small[cc] | |
| owner = cv2.dilate(np.where(cand, cc, 0).astype(np.float32), np.ones((5, 5), np.uint8)).astype(np.int64) | |
| ring = (owner > 0) & ~cand & fg # around each island (the nearest one where two are close) | |
| painted = ring & (cover >= 0.5) | |
| n_ring = np.bincount(owner[ring], minlength=n) | |
| n_painted = np.bincount(owner[painted], minlength=n).astype(np.float64) | |
| n_lab = int(lab.max()) + 1 | |
| part = np.bincount(cc[cand] * n_lab + lab[cand], minlength=n * n_lab).reshape(n, n_lab).argmax(1) | |
| o = owner[painted] | |
| same = np.bincount(o, weights=(lab[painted] == part[o]).astype(np.float64), minlength=n) | |
| L = cv2.cvtColor(np.clip(img, 0, 1).astype(np.float32), cv2.COLOR_RGB2LAB) | |
| L[..., 0] *= 0.5 | |
| x = L[painted].astype(np.float64) | |
| den = np.maximum(n_painted, 1)[:, None] | |
| mean = np.stack([np.bincount(o, weights=x[:, c], minlength=n) for c in range(3)], 1) / den | |
| rms = np.sqrt(np.bincount(o, weights=((x - mean[o]) ** 2).sum(1), minlength=n) / den[:, 0]) | |
| ok = small & (n_ring > 0) & (n_painted >= min_ring * np.maximum(n_ring, 1)) & (same >= 0.7 * den[:, 0]) \ | |
| & (rms <= max_spread) | |
| mask = ok[cc] & cand | |
| if not mask.any(): | |
| return img | |
| m3 = (cv2.dilate(mask.astype(np.uint8), np.ones((3, 3), np.uint8)) > 0) & fg | |
| # the background next to an island is unknown too, so the fill does not pull the backdrop in at the silhouette | |
| unknown = m3 | ((cv2.dilate(m3.astype(np.uint8), np.ones((9, 9), np.uint8)) > 0) & ~fg) | |
| filled = cv2.inpaint((np.clip(img, 0, 1) * 255).astype(np.uint8), unknown.astype(np.uint8), 5, cv2.INPAINT_TELEA) | |
| out = img.copy() | |
| out[m3] = filled[m3].astype(np.float32) / 255 | |
| return out | |
| def _chroma_overlap(a: np.ndarray, b: np.ndarray, step: float = 4.0, span: float = 64.0) -> float: | |
| """Overlap of the hue/saturation distributions (CIELAB a*, b*) of two sets of pixels, brightness ignored. | |
| Same colours rearranged (a vase view that invents its own peonies) overlap; other colours (slate repainted as | |
| sage-green siding) do not, whatever the gains. Each pixel is first brought to the same luminance: a*, b* shrink | |
| with lightness, so the conditioning render's shading (a brick gable facing away from the light, at L* 29 | |
| against 52 in the evenly lit output) would otherwise read as another colour.""" | |
| def hist(x): | |
| x = np.clip(np.asarray(x, np.float32).reshape(-1, 3), 0, 1) | |
| lum = x @ np.array([0.299, 0.587, 0.114], np.float32) | |
| x = np.clip(x * (0.5 / np.maximum(lum, 0.04))[:, None], 0, 1) | |
| lab = cv2.cvtColor(x.reshape(-1, 1, 3), cv2.COLOR_RGB2LAB).reshape(-1, 3) | |
| n = int(2 * span / step) | |
| i = np.clip(((lab[:, 1:] + span) / step).astype(np.int64), 0, n - 1) | |
| h = np.bincount(i[:, 0] * n + i[:, 1], minlength=n * n).astype(np.float64).reshape(n, n) | |
| h = cv2.GaussianBlur(h, (0, 0), 1.0) | |
| return h / max(h.sum(), 1e-12) | |
| return float(np.minimum(hist(a), hist(b)).sum()) | |
| def foreign_content(img: np.ndarray, face: np.ndarray, face_material: np.ndarray, part_colors: np.ndarray, | |
| mask: bool = False) -> dict: | |
| """Share of the object painted with another part's colours where that part is not modelled. | |
| Image models put an object's signature details on plain surfaces they see first (a radio's dial and grille on | |
| its plain back, a front door on a gable). ``part_colors`` are the parts' colours as already painted (NaN when | |
| unknown). Inside each visible part, low-passed pixels far from that part's colour and much closer to another | |
| part's form "foreign" blobs; returns foreign_frac (blobs of at least 0.4 % of the image, over the object), and | |
| with ``mask`` the blobs at the image's size ("foreign_mask", bool). | |
| """ | |
| known = np.isfinite(part_colors).all(1) | |
| if known.sum() < 2: | |
| return {} | |
| h, w = img.shape[:2] | |
| sc = 256 / max(h, w) | |
| small = cv2.GaussianBlur(cv2.resize(img.astype(np.float32), (int(w * sc), int(h * sc)), interpolation=cv2.INTER_AREA), | |
| (0, 0), 1.5) | |
| f = cv2.resize(face.astype(np.float32), small.shape[1::-1], interpolation=cv2.INTER_NEAREST).astype(np.int64) | |
| lab = np.where(f >= 0, face_material[np.clip(f, 0, len(face_material) - 1)], -1) | |
| fg = lab >= 0 | |
| if fg.sum() < 500: | |
| return {} | |
| foreign = np.zeros(fg.shape, bool) | |
| for p in np.unique(lab[fg]): | |
| if p >= len(known) or not known[p]: | |
| continue | |
| m = lab == p | |
| if m.sum() < 0.02 * fg.sum(): | |
| continue | |
| others = [q for q in np.where(known)[0] if q != p and np.linalg.norm(part_colors[q] - part_colors[p]) > 0.25] | |
| if not others: | |
| continue | |
| pix = small[m] | |
| d_own = np.linalg.norm(pix - part_colors[p], axis=1) | |
| d_oth = np.min(np.stack([np.linalg.norm(pix - part_colors[q], axis=1) for q in others]), 0) | |
| foreign[m] = (d_own > 0.25) & (d_oth < 0.6 * d_own) | |
| n, cc, stats, _ = cv2.connectedComponentsWithStats(foreign.astype(np.uint8), connectivity=8) | |
| keep = np.zeros(n, bool) | |
| keep[1:] = stats[1:, cv2.CC_STAT_AREA] >= 0.004 * foreign.size | |
| out = {"foreign_frac": round(float(keep[cc].sum() / fg.sum()), 4)} | |
| if mask: | |
| out["foreign_mask"] = cv2.resize(keep[cc].astype(np.uint8), (w, h), interpolation=cv2.INTER_NEAREST) > 0 | |
| return out | |
| def merged_parts(img: np.ndarray, cond: dict, face_material: np.ndarray, guide: np.ndarray, skip: set[int], | |
| min_piece: float = 0.001, min_left: float = 25.0, max_merge: float = 12.0, | |
| min_guides: float = 25.0) -> dict: | |
| """Parts the view painted in the colour of a part they touch, although their guide colours clearly differ. | |
| The image model sometimes paints two neighbouring parts as one: a guitar's dark rosewood fretboard, a strip | |
| across the neck, came back as the maple of the neck it lies on; a bus's lower windows as its red body. Among | |
| the parts this view paints for the first time (shown in guide colours), each connected piece of at least | |
| ``min_piece`` of the object on its own (three of five maki came back with their rice the red of the tuna | |
| filling, while the nigiri rice of the same material stayed white), one counts as merged when it left its | |
| own guide colour (CIELAB, ``min_left``; lifted or plain guide, or as rendered) and came back within ``max_merge`` | |
| of a touching part whose guide differs by ``min_guides``. Surfaces facing down are left out: models paint | |
| them in shade (a dapple-grey rocking horse's belly, seen from below, came back the grey of its hooves). | |
| ``guide`` is the lifted guide colour per material (sRGB); ``skip`` are materials whose painted colour says | |
| little (mirror-like metal, glass). Returns | |
| merged_frac (their share of the object in view) and merged (pairs of material indices), or {} when nothing is | |
| shown in guide colours. | |
| """ | |
| face, cover, ref = cond["face"], cond.get("cover_px"), cond["image"] | |
| if cover is None: | |
| return {} | |
| h, w = img.shape[:2] | |
| if face.shape != (h, w): | |
| face = cv2.resize(face.astype(np.float32), (w, h), interpolation=cv2.INTER_NEAREST).astype(np.int64) | |
| cover = cv2.resize(cover, (w, h), interpolation=cv2.INTER_NEAREST) | |
| if ref.shape[:2] != (h, w): | |
| ref = cv2.resize(ref, (w, h), interpolation=cv2.INTER_AREA) | |
| lab = np.where(face >= 0, face_material[np.clip(face, 0, len(face_material) - 1)], -1) | |
| fg = lab >= 0 | |
| total = int(fg.sum()) | |
| if total < 500: | |
| return {} | |
| facing = np.ones_like(fg) | |
| if cond.get("normal") is not None: # world space, +Y up | |
| nrm = cond["normal"] | |
| if nrm.shape[:2] != (h, w): | |
| nrm = cv2.resize(nrm, (w, h), interpolation=cv2.INTER_NEAREST) | |
| facing = nrm[..., 1] > -0.3 | |
| def to_lab(c): | |
| return cv2.cvtColor(np.clip(np.asarray(c, np.float32), 0, 1).reshape(-1, 1, 3), cv2.COLOR_RGB2LAB).reshape(-1, 3) | |
| g_lab = to_lab(guide) | |
| plain_lab = to_lab(np.clip((guide - 0.22) / 0.78, 0, 1)) # the base colour before lifting | |
| k3 = np.ones((3, 3), np.uint8) | |
| unpainted = fg & (cover < 0.1) & facing | |
| counts = np.bincount(lab[unpainted], minlength=len(guide)) | |
| merged, area, mean_cache = [], 0, {} | |
| def median_lab(m): | |
| if m not in mean_cache: | |
| sel = cv2.erode((lab == m).astype(np.uint8), k3) > 0 | |
| mean_cache[m] = to_lab(np.median(img[sel], 0))[0] if sel.sum() >= 50 else None | |
| return mean_cache[m] | |
| min_px = max(30, int(min_piece * total)) | |
| for p in np.nonzero(counts >= min_px)[0]: | |
| if p in skip or p >= len(guide): | |
| continue | |
| part = (lab == p).astype(np.uint8) | |
| n_cc, cc = cv2.connectedComponents(((cv2.erode(part, k3) > 0) & unpainted).astype(np.uint8), connectivity=8) | |
| for c in range(1, n_cc): # each piece of the part on its own: a material shared by many objects (the rice | |
| core = cc == c # of nigiri and of maki) is merged on some of them only | |
| if core.sum() < min_px: | |
| continue | |
| g = to_lab(np.median(img[core], 0))[0] | |
| shown = to_lab(np.median(ref[core], 0))[0] | |
| left = min(np.linalg.norm(g - shown), np.linalg.norm(g - g_lab[p]), np.linalg.norm(g - plain_lab[p])) | |
| if left < min_left: | |
| continue | |
| ring = (cv2.dilate(core.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0) & (lab != p) & fg | |
| qs, n = np.unique(lab[ring], return_counts=True) | |
| for q in qs[n >= 10]: | |
| if q in skip or q >= len(guide) or np.linalg.norm(g_lab[p] - g_lab[q]) < min_guides: | |
| continue | |
| gq = median_lab(int(q)) | |
| if gq is not None and np.linalg.norm(g - gq) <= min(max_merge, 0.5 * left): | |
| if [int(p), int(q)] not in merged: | |
| merged.append([int(p), int(q)]) | |
| area += int(core.sum()) | |
| break | |
| return {"merged_frac": round(area / total, 4), **({"merged": merged} if merged else {})} | |
| def spilled_paint(img: np.ndarray, face: np.ndarray, face_material: np.ndarray, skip: set[int], | |
| max_spread: float = 12.0, min_apart: float = 25.0, min_core: int = 150, min_thick: float = 5.0, | |
| band: int = 30, min_contact: int = 10, l_weight: float = 0.5) -> np.ndarray | None: | |
| """Pixels where the view painted a touching part's colour onto an evenly coloured part. | |
| A view drawn slightly larger or shifted puts a part's paint on its neighbour: a xylophone's hot-pink bar | |
| spilled onto the end of the maple frame, a barn's red siding onto the stone foundation. Colours are CIELAB with | |
| lightness at ``l_weight`` (shading and wood grain are not a different colour). Two touching parts qualify when | |
| both are evenly coloured in this view (80th percentile of distances to their medians within ``max_spread``), | |
| their medians are ``min_apart`` apart, and the part painted over is not a thin strip (75th percentile of the | |
| distance to its outline at least ``min_thick`` px: a trim's best view would lose its paint to worse ones). Spill | |
| = its pixels far from its own colour and close to the other part's (its median or its colour right across the | |
| boundary), within ``band`` px of that part, in blobs at | |
| least 5 px thick that touch it along ``min_contact`` px: a flower painted on a panel, away from or barely | |
| touching a part of its colour, is not spill. Returns a bool mask (image size), None when no part qualifies. | |
| ``skip``: materials whose painted colour says little (mirror-like metal, glass). | |
| """ | |
| h, w = img.shape[:2] | |
| if face.shape != (h, w): | |
| face = cv2.resize(face.astype(np.float32), (w, h), interpolation=cv2.INTER_NEAREST).astype(np.int64) | |
| lab = np.where(face >= 0, face_material[np.clip(face, 0, len(face_material) - 1)], -1) | |
| fg = lab >= 0 | |
| if fg.sum() < 500: | |
| return None | |
| L = cv2.cvtColor(cv2.GaussianBlur(np.clip(img, 0, 1).astype(np.float32), (0, 0), 1.0), cv2.COLOR_RGB2LAB) | |
| L[..., 0] *= l_weight | |
| k3, k5 = np.ones((3, 3), np.uint8), np.ones((5, 5), np.uint8) | |
| stats = {} | |
| for m in np.unique(lab[fg]): | |
| if m in skip: | |
| continue | |
| part = (lab == m).astype(np.uint8) | |
| core = cv2.erode(part, k5) > 0 | |
| if core.sum() < min_core: | |
| continue | |
| med = np.median(L[core], 0) | |
| spread = float(np.percentile(np.linalg.norm(L[core] - med, axis=1), 80)) | |
| thick = float(np.percentile(cv2.distanceTransform(part, cv2.DIST_L2, 3)[part > 0], 75)) | |
| stats[int(m)] = (med, spread, thick) | |
| out = np.zeros((h, w), bool) | |
| any_pair = False | |
| for p, (mp, sp, tp) in stats.items(): | |
| if sp > max_spread or tp < min_thick: | |
| continue | |
| part = (lab == p).astype(np.uint8) | |
| ring = (cv2.dilate(part, k3) > 0) & (lab != p) & fg | |
| qs, n = np.unique(lab[ring], return_counts=True) | |
| d_own = None | |
| for q in qs[n >= 10]: | |
| q = int(q) | |
| if q not in stats or stats[q][1] > max_spread or np.linalg.norm(mp - stats[q][0]) < min_apart: | |
| continue | |
| any_pair = True | |
| mq, sq, _ = stats[q] | |
| if d_own is None: | |
| d_own = np.linalg.norm(L - mp, axis=-1) | |
| qm = (lab == q).astype(np.uint8) | |
| d_q = np.linalg.norm(L - mq, axis=-1) | |
| near = (qm > 0) & (cv2.distanceTransform(1 - part, cv2.DIST_L2, 3) <= 6) | |
| if near.sum() >= 30: # or the colour right across the boundary: a bar's shaded side, not its top | |
| d_q = np.minimum(d_q, np.linalg.norm(L - np.median(L[near], 0), axis=-1)) | |
| cand = (part > 0) & (d_own > max(20.0, 2 * sp)) & (d_q < max(10.0, 1.2 * sq)) | |
| cand &= cv2.distanceTransform(1 - qm, cv2.DIST_L2, 3) <= band | |
| if not cand.any(): | |
| continue | |
| n_cc, cc = cv2.connectedComponents(cand.astype(np.uint8), connectivity=8) | |
| contact = np.bincount(cc[cand & (cv2.dilate(qm, k3) > 0)], minlength=n_cc) | |
| thick = np.zeros(n_cc, bool) | |
| thick[np.unique(cc[cv2.erode(cand.astype(np.uint8), k5) > 0])] = True | |
| keep = (contact >= min_contact) & thick | |
| keep[0] = False | |
| out |= keep[cc] | |
| if not any_pair: | |
| return None | |
| return (cv2.dilate(out.astype(np.uint8), k3) > 0) & fg | |
| def accept(scores: dict, environment: bool, min_iou: float = 0.85, min_edges: float = 0.12) -> bool: | |
| iou = scores.get("silhouette_iou") | |
| ea = scores.get("edge_agreement") | |
| if not environment and iou is not None: | |
| return iou >= min_iou and (ea is None or ea >= min_edges * 0.5) | |
| return ea is None or ea >= min_edges | |
| def rescue_alignment(img: np.ndarray, cond: np.ndarray, max_scale_change: float = 0.6) -> tuple[np.ndarray, dict] | None: | |
| """Undo a global re-framing (zoom / shift / small rotation) of a generated view. | |
| SIFT features of the conditioning render and of the generated image are matched (Lowe ratio test) and a | |
| similarity transform is fitted with RANSAC; the generated image is warped back onto the render's framing. | |
| Works best when the render already shows some texture (later views); returns None when unreliable. | |
| The info dict carries ``inside``: 1 where the warped pixel comes from inside the generated image. | |
| """ | |
| h, w = cond.shape[:2] | |
| s = 768 / max(h, w) | |
| g1 = (np.clip(cv2.resize(cond, (int(w * s), int(h * s))).mean(-1), 0, 1) * 255).astype(np.uint8) | |
| g2 = (np.clip(cv2.resize(img, (int(w * s), int(h * s))).mean(-1), 0, 1) * 255).astype(np.uint8) | |
| try: | |
| sift = cv2.SIFT_create(nfeatures=4000) | |
| except AttributeError: # pragma: no cover | |
| return None | |
| k1, d1 = sift.detectAndCompute(g1, None) | |
| k2, d2 = sift.detectAndCompute(g2, None) | |
| if d1 is None or d2 is None or len(k1) < 12 or len(k2) < 12: | |
| return None | |
| matches = cv2.BFMatcher().knnMatch(d2, d1, k=2) | |
| good = [m for m, n in (x for x in matches if len(x) == 2) if m.distance < 0.75 * n.distance] | |
| if len(good) < 12: | |
| return None | |
| src = np.float32([k2[m.queryIdx].pt for m in good]) | |
| dst = np.float32([k1[m.trainIdx].pt for m in good]) | |
| M, inl = cv2.estimateAffinePartial2D(src, dst, method=cv2.RANSAC, ransacReprojThreshold=4.0, maxIters=5000) | |
| if M is None or inl is None or inl.sum() < 12: | |
| return None | |
| scale = float(np.sqrt(abs(np.linalg.det(M[:, :2])))) | |
| if abs(np.log(scale)) > np.log(1 + max_scale_change): | |
| return None | |
| if abs(scale - 1) < 0.01 and np.abs(M[:, 2]).max() < 1.0: | |
| return None # already aligned | |
| M[:, 2] /= s | |
| out = cv2.warpAffine(img, M, (w, h), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE) | |
| # pixels that come from outside the generated image are filler (edge replication): never project them | |
| inside = cv2.warpAffine(np.ones(img.shape[:2], np.float32), M, (w, h), flags=cv2.INTER_NEAREST, | |
| borderMode=cv2.BORDER_CONSTANT, borderValue=0) | |
| inside = cv2.erode((inside > 0.5).astype(np.uint8), np.ones((5, 5), np.uint8)).astype(np.float32) | |
| return out, {"inside": inside, "scale": round(scale, 4), "shift_px": [round(float(x), 1) for x in M[:, 2]], | |
| "rotation_deg": round(float(np.degrees(np.arctan2(M[1, 0], M[0, 0]))), 2), | |
| "inliers": int(inl.sum()), "matches": len(good)} | |