| """How far apart are the three pictures of one design? |
| |
| For a sample of tasks, render the ground truth three ways and compare them: |
| |
| skia the server renderer: what every reward is computed from |
| browser the editor's DOM (Chromium, or --browser): what a computer-use agent sees |
| crello Crello's own shipped preview: what the design actually looked like |
| |
| skia-vs-browser is the number that matters for the GUI surface (an agent must |
| not be misled by its own screen). skia-vs-crello says how faithful the |
| renderer is to the source data. Run against a live server: |
| |
| python scripts/parity.py --server http://127.0.0.1:8007 --n 40 --out parity.json |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import io |
| import json |
| import statistics |
| import sys |
| import urllib.request |
| from pathlib import Path |
|
|
| import numpy as np |
| from PIL import Image |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parents[2])) |
|
|
| from design_canvas.core.expert import html_page |
| from design_canvas.core.pack import load_pack |
| from design_canvas.core.score import ssim |
|
|
| SIZE = 512 |
|
|
|
|
| def post(url, data: bytes, ctype): |
| req = urllib.request.Request(url, data=data, headers={"content-type": ctype}, method="POST") |
| with urllib.request.urlopen(req, timeout=20) as r: |
| return json.loads(r.read()) |
|
|
|
|
| def compare(a: Image.Image, b: Image.Image) -> dict: |
| b = b.resize(a.size, Image.LANCZOS) |
| x = np.asarray(a.convert("RGB"), dtype=np.float64) |
| y = np.asarray(b.convert("RGB"), dtype=np.float64) |
| return {"mae": float(np.abs(x - y).mean()), "ssim": ssim(x, y)} |
|
|
|
|
| def main(argv=None) -> int: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--server", default="http://127.0.0.1:8007") |
| ap.add_argument("--split", default="test") |
| ap.add_argument("--n", type=int, default=40) |
| ap.add_argument("--out", default=None) |
| ap.add_argument("--browser", default="chromium", choices=["chromium", "firefox", "webkit"]) |
| ap.add_argument("--save-examples", default=None, help="directory for side-by-side PNGs") |
| args = ap.parse_args(argv) |
|
|
| from playwright.sync_api import sync_playwright |
|
|
| pack = load_pack() |
| n = min(args.n, pack.count(args.split)) |
| step = max(1, pack.count(args.split) // n) |
| rows = [] |
| with sync_playwright() as p: |
| flags = ["--font-render-hinting=none", "--disable-lcd-text"] if args.browser == "chromium" else [] |
| browser = getattr(p, args.browser).launch(args=flags) |
| page = browser.new_page(viewport={"width": 1600, "height": 1000}, device_scale_factor=1) |
| for i in range(0, step * n, step): |
| task = pack.at(args.split, i) |
| gt = pack.gt_doc(task) |
| ep = post(f"{args.server}/api/episodes", |
| json.dumps({"task_id": task["task_id"]}).encode(), "application/json") |
| eid = ep["episode_id"] |
| post(f"{args.server}/api/episodes/{eid}/html", html_page(gt).encode(), "text/plain") |
| page.goto(f"{args.server}/editor/?episode={eid}&skin=figma") |
| page.wait_for_function("window.designEditor && window.designEditor.ready") |
| scale = SIZE / max(gt.width, gt.height) |
| page.evaluate(f"""async () => {{ |
| const a = window.__app; |
| a.store.select([]); |
| a.stage.setZoom({scale}); |
| await document.fonts.ready; |
| await a.store.whenIdle(3000); |
| document.querySelector('.overlay').style.display = 'none'; |
| document.querySelector('.artboard').style.boxShadow = 'none'; |
| }}""") |
| page.wait_for_timeout(150) |
| dom = Image.open(io.BytesIO(page.locator(".artboard").screenshot())).convert("RGB") |
| skia = Image.open(io.BytesIO(pack.target_png(task, SIZE))).convert("RGB") |
| crello = Image.open(pack.preview_path(task)).convert("RGB") |
| row = {"task_id": task["task_id"], "n_text": task["stats"]["n_text"], |
| "browser_vs_skia": compare(skia, dom), "crello_vs_skia": compare(skia, crello)} |
| rows.append(row) |
| print(f"{task['task_id']} browser/skia ssim {row['browser_vs_skia']['ssim']:.3f} " |
| f"mae {row['browser_vs_skia']['mae']:.2f} crello/skia ssim " |
| f"{row['crello_vs_skia']['ssim']:.3f} mae {row['crello_vs_skia']['mae']:.2f}") |
| if args.save_examples: |
| out = Path(args.save_examples) |
| out.mkdir(parents=True, exist_ok=True) |
| w, h = skia.size |
| sheet = Image.new("RGB", (w * 3 + 16, h), "white") |
| for k, im in enumerate((crello.resize((w, h)), skia, dom.resize((w, h)))): |
| sheet.paste(im, (k * (w + 8), 0)) |
| sheet.save(out / f"{task['task_id']}.png") |
| browser.close() |
|
|
| def summ(key, metric): |
| vals = [r[key][metric] for r in rows] |
| return {"mean": statistics.mean(vals), "min": min(vals), "max": max(vals), |
| "median": statistics.median(vals)} |
|
|
| summary = { |
| "n": len(rows), "size": SIZE, |
| "browser_vs_skia": {"ssim": summ("browser_vs_skia", "ssim"), "mae": summ("browser_vs_skia", "mae")}, |
| "crello_vs_skia": {"ssim": summ("crello_vs_skia", "ssim"), "mae": summ("crello_vs_skia", "mae")}, |
| } |
| print(json.dumps(summary, indent=2)) |
| if args.out: |
| Path(args.out).write_text(json.dumps({"summary": summary, "rows": rows}, indent=2)) |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
