"""The reward: how close a submitted document is to the ground truth. Two signals, because each is blind where the other sees: - **Structure** matches submitted elements to ground-truth elements one-to-one (Hungarian assignment) and scores every pair on content, geometry and, for text, typography. It is dense, it is readable off a trajectory, and it tells you *which* element is wrong. Unmatched elements on either side count against the score, so spamming elements does not buy coverage. - **Visual** compares `render(submission)` with `render(ground_truth)` through the same renderer: multi-scale SSIM plus colour-layout error, weighted toward coarse scales and normalised so that an empty canvas scores zero rather than whatever fraction of the target happens to be white. Reference mode (the model sees the target image) uses tight tolerances. Description mode (the model only reads a description) loosens geometry, since "top left, large" cannot pin a box to the pixel, and adds a coverage term: were all the assets and all the words actually used? Every constant below is a tolerance a person can reason about: `pos_sigma` is a fraction of the canvas diagonal, `size_sigma` a log-ratio. """ from __future__ import annotations import math from dataclasses import asdict, dataclass, field, replace import numpy as np from scipy.ndimage import gaussian_filter from scipy.optimize import linear_sum_assignment from .colors import delta_e from .doc import DesignDoc, Element @dataclass(frozen=True) class ScoreConfig: mode: str pos_sigma: float size_sigma: float pixel_size: int w_structure: float w_visual: float w_coverage: float w_order: float w_background: float # (pyramid level, term, weight); level 0 is the 256 px render, each level # halves it by area averaging, so level 3 is 32 px. visual_terms: tuple = () # Precision lives in the structure term. The visual term answers "does it look # like the target", so it is weighted toward coarse scales, where a shift of a # few pixels is invisible, as it is to a person looking at the design. REFERENCE = ScoreConfig("reference", 0.025, 0.12, 256, 0.60, 0.30, 0.00, 0.05, 0.05, ((1, "ssim", 0.15), (2, "ssim", 0.30), (3, "ssim", 0.25), (3, "color", 0.30))) DESCRIPTION = ScoreConfig("description", 0.12, 0.45, 256, 0.50, 0.15, 0.25, 0.05, 0.05, ((3, "ssim", 0.30), (4, "ssim", 0.30), (4, "color", 0.40))) CONFIGS = {"reference": REFERENCE, "description": DESCRIPTION} IOU_PAD = 0.004 # fraction of the canvas diagonal # Version 3 (reference mode), calibrated against a pairwise visual judge on frontier and Qwen attempts over all # three surfaces (results/reward-design). Geometry is coarse-to-fine: half the tight kernel above, half a loose # one, so a hand-placed element in the right region earns credit while exact placement is still needed for 1.0. # Pixel-level terms only count once the structure is there, which closes colour-patch and dump exploits. V3_COARSE_MIX = 0.5 V3_COARSE_POS = 0.15 # fraction of the canvas diagonal V3_COARSE_SIZE = 0.6 # log-ratio V3_TEXT_WEIGHT = 3.0 # a piece of copy counts three times an image or shape in structure V3_GATE = 0.3 # visual, order and background scale with min(1, structure / V3_GATE) @dataclass class PairScore: gt_id: str matched_id: str | None score: float content: float = 0.0 geometry: float = 0.0 style: float | None = None note: str = "" @dataclass class Grade: total: float structure: float visual: float coverage: float order: float background: float mode: str n_gt: int n_submitted: int n_matched: int elements: list[PairScore] = field(default_factory=list) unmatched: list[str] = field(default_factory=list) def as_dict(self) -> dict: d = asdict(self) for key in ("total", "structure", "visual", "coverage", "order", "background"): d[key] = round(d[key], 4) for e in d["elements"]: for k in ("score", "content", "geometry"): e[k] = round(e[k], 4) if e["style"] is not None: e["style"] = round(e["style"], 4) return d def report(self) -> str: head = ( f"reward {self.total:.3f} = structure {self.structure:.3f}, visual {self.visual:.3f}" + (f", coverage {self.coverage:.3f}" if self.mode == "description" else "") + f", order {self.order:.3f}, background {self.background:.3f}" ) lines = [head, f"matched {self.n_matched}/{self.n_gt} target elements, " f"{self.n_submitted} submitted"] worst = sorted(self.elements, key=lambda p: p.score)[:4] for p in worst: if p.score < 0.9: lines.append(f" {p.gt_id}: {p.score:.2f} {p.note}".rstrip()) if self.unmatched: lines.append(f" extra elements: {', '.join(self.unmatched)}") return "\n".join(lines) # -------------------------------------------------------------- helpers def levenshtein(a: str, b: str) -> int: if a == b: return 0 if len(a) < len(b): a, b = b, a prev = list(range(len(b) + 1)) for i, ca in enumerate(a, 1): cur = [i] for j, cb in enumerate(b, 1): cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) prev = cur return prev[-1] def text_similarity(a: str, b: str) -> float: na = " ".join(a.split()).casefold() nb = " ".join(b.split()).casefold() if not na and not nb: return 1.0 return max(0.0, 1.0 - levenshtein(na, nb) / max(len(na), len(nb))) def _box(el: Element) -> tuple[float, float, float, float]: return el.x, el.y, el.x + el.width, el.y + el.height def iou(a: Element, b: Element) -> float: ax0, ay0, ax1, ay1 = _box(a) bx0, by0, bx1, by1 = _box(b) iw = max(0.0, min(ax1, bx1) - max(ax0, bx0)) ih = max(0.0, min(ay1, by1) - max(ay0, by0)) inter = iw * ih union = a.width * a.height + b.width * b.height - inter return inter / union if union > 0 else 0.0 def _angle_gap(a: float, b: float) -> float: d = abs(a - b) % 360 return min(d, 360 - d) def _iou_box(a, b) -> float: iw = max(0.0, min(a[2], b[2]) - max(a[0], b[0])) ih = max(0.0, min(a[3], b[3]) - max(a[1], b[1])) inter = iw * ih union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter return inter / union if union > 0 else 0.0 def geometry(a: Element, g: Element, diag: float, cfg: ScoreConfig, boxes=None) -> float: """Position, size and overlap of what is drawn. `boxes` maps element ids to the box that is actually rendered; for text that is the extent of the glyphs, not the declared box, whose width is invisible for left-aligned copy and should not be scored. """ ba = (boxes or {}).get(("a", a.id)) or (a.x, a.y, a.x + a.width, a.y + a.height) bg = (boxes or {}).get(("g", g.id)) or (g.x, g.y, g.x + g.width, g.y + g.height) wa, ha = max(1e-3, ba[2] - ba[0]), max(1e-3, ba[3] - ba[1]) wg, hg = max(1e-3, bg[2] - bg[0]), max(1e-3, bg[3] - bg[1]) dist = math.dist(((ba[0] + ba[2]) / 2, (ba[1] + ba[3]) / 2), ((bg[0] + bg[2]) / 2, (bg[1] + bg[3]) / 2)) pos = math.exp(-dist / (cfg.pos_sigma * diag)) size = math.exp(-(abs(math.log(wa / wg)) + abs(math.log(ha / hg))) / (2 * cfg.size_sigma)) rot = math.exp(-_angle_gap(a.angle, g.angle) / 10.0) # IoU on boxes padded by a sliver of the canvas: plain IoU punishes a # four-pixel nudge of a four-pixel dot as hard as a wholesale miss. pad = IOU_PAD * diag pa = (ba[0] - pad, ba[1] - pad, ba[2] + pad, ba[3] + pad) pg = (bg[0] - pad, bg[1] - pad, bg[2] + pad, bg[3] + pad) base = (_iou_box(pa, pg) + pos + size) / 3 return base * (0.85 + 0.15 * rot) def geometry_v3(a: Element, g: Element, diag: float, cfg: ScoreConfig, boxes=None) -> float: """Coarse-to-fine geometry, scaled by the square root of the (coarse-to-fine) position match: an element in the wrong place is the wrong element however well it is sized.""" m = V3_COARSE_MIX coarse = replace(cfg, pos_sigma=V3_COARSE_POS, size_sigma=V3_COARSE_SIZE) base = (1 - m) * geometry(a, g, diag, cfg, boxes) + m * geometry(a, g, diag, coarse, boxes) ba = (boxes or {}).get(("a", a.id)) or (a.x, a.y, a.x + a.width, a.y + a.height) bg = (boxes or {}).get(("g", g.id)) or (g.x, g.y, g.x + g.width, g.y + g.height) dist = math.dist(((ba[0] + ba[2]) / 2, (ba[1] + ba[3]) / 2), ((bg[0] + bg[2]) / 2, (bg[1] + bg[3]) / 2)) pos = (1 - m) * math.exp(-dist / (cfg.pos_sigma * diag)) + m * math.exp(-dist / (V3_COARSE_POS * diag)) return base * math.sqrt(pos) def text_box(el: Element, fonts) -> tuple[float, float, float, float]: from .text import layout lay = layout(el, fonts) x0 = min((ln.x for ln in lay.lines if ln.width > 0), default=0.0) x1 = max((ln.x + ln.width for ln in lay.lines if ln.width > 0), default=el.width) return el.x + x0, el.y, el.x + max(x1, x0 + 1.0), el.y + lay.height def _dominant_color(el: Element) -> str: runs = el.styled_runs() weights: dict[str, int] = {} for r in runs: weights[r.color] = weights.get(r.color, 0) + len(r.text.strip()) return max(weights.items(), key=lambda kv: kv[1])[0] if weights else (el.color or "#000000") def typography(a: Element, g: Element, salient: bool = False) -> tuple[float, list[str]]: notes = [] font = 1.0 if (a.font_family or "").lower() == (g.font_family or "").lower() else 0.0 if font < 1: notes.append(f"font {a.font_family} vs {g.font_family}") size = math.exp(-abs(math.log(a.font_size / g.font_size)) / 0.1) if size < 0.7: notes.append(f"size {a.font_size:.0f} vs {g.font_size:.0f}") de = delta_e(_dominant_color(a), _dominant_color(g)) color = math.exp(-de / 25.0) if color < 0.7: notes.append(f"color {_dominant_color(a)} vs {_dominant_color(g)}") align = 1.0 if a.align == g.align else 0.0 weight = 1.0 - 0.5 * ((a.bold != g.bold) + (a.italic != g.italic)) case = 1.0 if a.uppercase == g.uppercase or a.display_text() == g.display_text() else 0.5 spacing = math.exp(-abs(a.letter_spacing - g.letter_spacing) / max(1.0, 0.05 * g.font_size)) lines = 1.0 if len(g.lines) > 1: lines = math.exp(-abs(a.line_height - g.line_height) / 0.15) rest = (0.22 * size + 0.08 * align + 0.06 * weight + 0.04 * case + 0.05 * spacing + 0.05 * lines) / 0.50 if salient: # font and colour are what a reader notices first: they scale everything else return (0.5 + 0.5 * font) * (0.4 + 0.6 * color) * (0.5 + 0.5 * rest), notes return 0.28 * font + 0.22 * color + 0.50 * rest, notes def pair_score(a: Element, g: Element, diag: float, cfg: ScoreConfig, boxes=None, asset_sim=None, style_weight: float = 0.38, version: int = 1) -> PairScore: geometry_fn = geometry_v3 if version >= 3 else geometry if a.kind == "text" and g.kind == "text": content = text_similarity(a.display_text(), g.display_text()) if content < 0.25: return PairScore(g.id, a.id, 0.0, content, 0.0, None, "text differs") geo = geometry_fn(a, g, diag, cfg, boxes) style, notes = typography(a, g, salient=version >= 3) opacity = math.exp(-abs(a.opacity - g.opacity) / 0.2) if version >= 3: # well-set copy in the wrong place is not the copy the target has score = content * geo * ((1 - style_weight) + style_weight * style) * (0.9 + 0.1 * opacity) else: score = content * ((1 - style_weight) * geo + style_weight * style) * (0.9 + 0.1 * opacity) if content < 0.95: notes.insert(0, f'text "{a.text[:20]}" vs "{g.text[:20]}"') return PairScore(g.id, a.id, score, content, geo, style, "; ".join(notes)) if a.kind == "text" or g.kind == "text": return PairScore(g.id, a.id, 0.0, 0.0, 0.0, None, "kind mismatch") if a.kind == "image" and g.kind == "image": if a.asset_id == g.asset_id: content = 1.0 else: # A look-alike asset (two near-identical clouds) is a small mistake, # an unrelated one a large one. sim = asset_sim(a.asset_id, g.asset_id) if asset_sim else 0.0 content = 0.15 + 0.6 * sim else: content = 0.2 # a drawn shape standing in for a rendered graphic geo = geometry_fn(a, g, diag, cfg, boxes) opacity = math.exp(-abs(a.opacity - g.opacity) / 0.2) note = "" if content == 1.0 else f"asset {a.asset_id or a.shape} vs {g.asset_id}" return PairScore(g.id, a.id, content * geo * (0.9 + 0.1 * opacity), content, geo, None, note) # ---------------------------------------------------------------- pixels def _gray(rgb: np.ndarray) -> np.ndarray: return rgb[..., 0] * 0.299 + rgb[..., 1] * 0.587 + rgb[..., 2] * 0.114 def ssim(a: np.ndarray, b: np.ndarray, sigma: float = 1.5) -> float: x, y = _gray(a.astype(np.float64)), _gray(b.astype(np.float64)) c1, c2 = (0.01 * 255) ** 2, (0.03 * 255) ** 2 mx, my = gaussian_filter(x, sigma), gaussian_filter(y, sigma) sxx = gaussian_filter(x * x, sigma) - mx * mx syy = gaussian_filter(y * y, sigma) - my * my sxy = gaussian_filter(x * y, sigma) - mx * my num = (2 * mx * my + c1) * (2 * sxy + c2) den = (mx * mx + my * my + c1) * (sxx + syy + c2) return float(np.clip((num / den).mean(), 0.0, 1.0)) def pyramid(img: np.ndarray, levels: int) -> list[np.ndarray]: """Area-averaged pyramid. Rendering small directly aliases (a one-pixel canvas shift can flip a 32 px thumbnail's pixels by 150 levels); halving a 256 px render does not.""" out = [img.astype(np.float64)] for _ in range(levels): x = out[-1] h, w = (x.shape[0] // 2) * 2, (x.shape[1] // 2) * 2 if h < 4 or w < 4: break x = x[:h, :w] out.append((x[0::2, 0::2] + x[1::2, 0::2] + x[0::2, 1::2] + x[1::2, 1::2]) / 4) return out def _term(kind: str, a: np.ndarray, b: np.ndarray) -> float: if kind == "ssim": return ssim(a, b) mae = float(np.abs(a - b).mean() / 255.0) return math.exp(-mae / 0.12) def visual_score(attempt: np.ndarray, target: np.ndarray, blank: np.ndarray, terms=REFERENCE.visual_terms) -> float: """Multi-scale visual similarity, rescaled so an empty canvas scores 0. Each term is normalised against the blank canvas on its own scale, then the terms are averaged with their weights. """ if attempt.shape != target.shape: raise ValueError("renders differ in size") depth = max(level for level, _, _ in terms) pa, pt, pb = pyramid(attempt, depth), pyramid(target, depth), pyramid(blank, depth) total = weight = 0.0 for level, kind, w in terms: level = min(level, len(pt) - 1) v = _term(kind, pa[level], pt[level]) v0 = _term(kind, pb[level], pt[level]) score = (1.0 if v >= 0.999 else 0.0) if v0 >= 0.999 else (v - v0) / (1.0 - v0) total += w * float(np.clip(score, 0.0, 1.0)) weight += w return total / weight if weight else 0.0 # ----------------------------------------------------------------- grade def _overlap(a: Element, b: Element) -> bool: ax0, ay0, ax1, ay1 = _box(a) bx0, by0, bx1, by1 = _box(b) return min(ax1, bx1) > max(ax0, bx0) and min(ay1, by1) > max(ay0, by0) def _split_runs(el: Element) -> list[list]: """The element's runs cut at its line breaks: one run list per line.""" per_line: list[list] = [[]] for r in el.runs or []: parts = r.text.split("\n") for k, part in enumerate(parts): if k: per_line.append([]) if part: per_line[-1].append(r.model_copy(update={"text": part})) return per_line def explode_lines(elements: list[Element], fonts) -> list[Element]: """Reward v2 scores text line by line: a three-line box and three stacked one-line boxes that render identically are the same design, and building it either way should score the same.""" from .text import layout out = [] for e in elements: if e.kind != "text" or e.angle or len(e.lines) < 2 or fonts is None: out.append(e) continue lay = layout(e, fonts) runs = _split_runs(e) if e.runs else None for k, (line, text) in enumerate(zip(lay.lines, e.lines, strict=False)): if not text.strip(): continue out.append(e.model_copy(update={ "id": f"{e.id}#l{k}", "text": text, "y": e.y + line.top, "height": lay.line_box, "runs": (runs[k] or None) if runs and k < len(runs) else None})) return out def grade( attempt: DesignDoc, target: DesignDoc, mode: str = "reference", attempt_pixels: np.ndarray | None = None, target_pixels: np.ndarray | None = None, blank_pixels: np.ndarray | None = None, fonts=None, asset_sim=None, version: int = 1, ) -> Grade: cfg = CONFIGS[mode] if version >= 3 and mode != "reference": version = 1 # v3 is calibrated for reference mode only; description mode keeps v1 sub = [e for e in attempt.elements if e.visible and e.opacity > 0.01] gts = [e for e in target.elements if e.visible and e.opacity > 0.01] if version == 2: sub, gts = explode_lines(sub, fonts), explode_lines(gts, fonts) diag = math.hypot(target.width, target.height) boxes = {} if fonts is not None: # Only unrotated text: a rotated box's width moves its pivot, so there # the declared box is what determines where the glyphs land. boxes.update({("a", e.id): text_box(e, fonts) for e in sub if e.kind == "text" and not e.angle}) boxes.update({("g", e.id): text_box(e, fonts) for e in gts if e.kind == "text" and not e.angle}) if not sub: return Grade(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, mode, len(gts), 0, 0, [PairScore(g.id, None, 0.0, note="missing") for g in gts], []) matrix = np.zeros((len(sub), len(gts))) pairs: dict[tuple[int, int], PairScore] = {} for i, a in enumerate(sub): for j, g in enumerate(gts): # v2 weighs typography as much as placement: wrong-colour or wrong-font copy is as # visible to a reader as misplaced copy. ps = pair_score(a, g, diag, cfg, boxes, asset_sim, style_weight=0.5 if version == 2 else 0.38, version=version) matrix[i, j] = ps.score pairs[(i, j)] = ps rows, cols = linear_sum_assignment(-matrix) if gts else ([], []) matched = {} for i, j in zip(rows, cols, strict=True): if matrix[i, j] > 0: matched[j] = i elements = [] for j, g in enumerate(gts): if j in matched: elements.append(pairs[(matched[j], j)]) else: elements.append(PairScore(g.id, None, 0.0, note="missing")) used = set(matched.values()) unmatched = [a.id for i, a in enumerate(sub) if i not in used] if version >= 3: # Copy weighs more; every unmatched submission counts against the score like an average element. weights = [V3_TEXT_WEIGHT if g.kind == "text" else 1.0 for g in gts] mean_w = sum(weights) / len(weights) if weights else 1.0 denom = sum(weights) + len(unmatched) * mean_w structure = sum(w * p.score for w, p in zip(weights, elements, strict=True)) / denom if denom else 1.0 else: denom = max(len(gts), len(sub)) structure = sum(p.score for p in elements) / denom if denom else 1.0 # Stacking order only matters where elements overlap. agree = total = 0 gt_index = {g.id: k for k, g in enumerate(gts)} sub_index = {a.id: k for k, a in enumerate(sub)} matched_pairs = [(gts[j], sub[i]) for j, i in matched.items()] for p in range(len(matched_pairs)): for q in range(p + 1, len(matched_pairs)): g1, a1 = matched_pairs[p] g2, a2 = matched_pairs[q] if not _overlap(g1, g2): continue total += 1 same = (gt_index[g1.id] < gt_index[g2.id]) == (sub_index[a1.id] < sub_index[a2.id]) agree += same order = (agree / total if total else 1.0) if matched else 0.0 background = math.exp(-delta_e(attempt.background, target.background) / 20.0) visual = 0.0 if attempt_pixels is not None and target_pixels is not None and blank_pixels is not None: visual = visual_score(attempt_pixels, target_pixels, blank_pixels, cfg.visual_terms) coverage = 0.0 if mode == "description": gt_assets = {g.asset_id for g in gts if g.kind == "image"} sub_assets = {a.asset_id for a in sub if a.kind == "image"} asset_cov = len(gt_assets & sub_assets) / len(gt_assets) if gt_assets else 1.0 gt_texts = [g.display_text() for g in gts if g.kind == "text"] sub_texts = [a.display_text() for a in sub if a.kind == "text"] hits = sum( 1 for t in gt_texts if any(text_similarity(t, s) >= 0.8 for s in sub_texts) ) text_cov = hits / len(gt_texts) if gt_texts else 1.0 coverage = 0.5 * asset_cov + 0.5 * text_cov gate = min(1.0, structure / V3_GATE) if version >= 3 else 1.0 total_score = ( cfg.w_structure * structure + gate * (cfg.w_visual * visual + cfg.w_order * order + cfg.w_background * background) + cfg.w_coverage * coverage ) return Grade( total=float(min(1.0, max(0.0, total_score))), structure=float(structure), visual=float(visual), coverage=float(coverage), order=float(order), background=float(background), mode=mode, n_gt=len(gts), n_submitted=len(sub), n_matched=len(matched), elements=elements, unmatched=unmatched, )