File size: 5,564 Bytes
df84e45 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | """How far apart are the three pictures of one design?
For a sample of tasks, render the ground truth three ways and compare them:
skia the server renderer: what every reward is computed from
browser the editor's DOM (Chromium, or --browser): what a computer-use agent sees
crello Crello's own shipped preview: what the design actually looked like
skia-vs-browser is the number that matters for the GUI surface (an agent must
not be misled by its own screen). skia-vs-crello says how faithful the
renderer is to the source data. Run against a live server:
python scripts/parity.py --server http://127.0.0.1:8007 --n 40 --out parity.json
"""
from __future__ import annotations
import argparse
import io
import json
import statistics
import sys
import urllib.request
from pathlib import Path
import numpy as np
from PIL import Image
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from design_canvas.core.expert import html_page # noqa: E402
from design_canvas.core.pack import load_pack # noqa: E402
from design_canvas.core.score import ssim # noqa: E402
SIZE = 512
def post(url, data: bytes, ctype):
req = urllib.request.Request(url, data=data, headers={"content-type": ctype}, method="POST")
with urllib.request.urlopen(req, timeout=20) as r:
return json.loads(r.read())
def compare(a: Image.Image, b: Image.Image) -> dict:
b = b.resize(a.size, Image.LANCZOS)
x = np.asarray(a.convert("RGB"), dtype=np.float64)
y = np.asarray(b.convert("RGB"), dtype=np.float64)
return {"mae": float(np.abs(x - y).mean()), "ssim": ssim(x, y)}
def main(argv=None) -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--server", default="http://127.0.0.1:8007")
ap.add_argument("--split", default="test")
ap.add_argument("--n", type=int, default=40)
ap.add_argument("--out", default=None)
ap.add_argument("--browser", default="chromium", choices=["chromium", "firefox", "webkit"])
ap.add_argument("--save-examples", default=None, help="directory for side-by-side PNGs")
args = ap.parse_args(argv)
from playwright.sync_api import sync_playwright
pack = load_pack()
n = min(args.n, pack.count(args.split))
step = max(1, pack.count(args.split) // n)
rows = []
with sync_playwright() as p:
flags = ["--font-render-hinting=none", "--disable-lcd-text"] if args.browser == "chromium" else []
browser = getattr(p, args.browser).launch(args=flags)
page = browser.new_page(viewport={"width": 1600, "height": 1000}, device_scale_factor=1)
for i in range(0, step * n, step):
task = pack.at(args.split, i)
gt = pack.gt_doc(task)
ep = post(f"{args.server}/api/episodes",
json.dumps({"task_id": task["task_id"]}).encode(), "application/json")
eid = ep["episode_id"]
post(f"{args.server}/api/episodes/{eid}/html", html_page(gt).encode(), "text/plain")
page.goto(f"{args.server}/editor/?episode={eid}&skin=figma")
page.wait_for_function("window.designEditor && window.designEditor.ready")
scale = SIZE / max(gt.width, gt.height)
page.evaluate(f"""async () => {{
const a = window.__app;
a.store.select([]);
a.stage.setZoom({scale});
await document.fonts.ready;
await a.store.whenIdle(3000);
document.querySelector('.overlay').style.display = 'none';
document.querySelector('.artboard').style.boxShadow = 'none';
}}""")
page.wait_for_timeout(150)
dom = Image.open(io.BytesIO(page.locator(".artboard").screenshot())).convert("RGB")
skia = Image.open(io.BytesIO(pack.target_png(task, SIZE))).convert("RGB")
crello = Image.open(pack.preview_path(task)).convert("RGB")
row = {"task_id": task["task_id"], "n_text": task["stats"]["n_text"],
"browser_vs_skia": compare(skia, dom), "crello_vs_skia": compare(skia, crello)}
rows.append(row)
print(f"{task['task_id']} browser/skia ssim {row['browser_vs_skia']['ssim']:.3f} "
f"mae {row['browser_vs_skia']['mae']:.2f} crello/skia ssim "
f"{row['crello_vs_skia']['ssim']:.3f} mae {row['crello_vs_skia']['mae']:.2f}")
if args.save_examples:
out = Path(args.save_examples)
out.mkdir(parents=True, exist_ok=True)
w, h = skia.size
sheet = Image.new("RGB", (w * 3 + 16, h), "white")
for k, im in enumerate((crello.resize((w, h)), skia, dom.resize((w, h)))):
sheet.paste(im, (k * (w + 8), 0))
sheet.save(out / f"{task['task_id']}.png")
browser.close()
def summ(key, metric):
vals = [r[key][metric] for r in rows]
return {"mean": statistics.mean(vals), "min": min(vals), "max": max(vals),
"median": statistics.median(vals)}
summary = {
"n": len(rows), "size": SIZE,
"browser_vs_skia": {"ssim": summ("browser_vs_skia", "ssim"), "mae": summ("browser_vs_skia", "mae")},
"crello_vs_skia": {"ssim": summ("crello_vs_skia", "ssim"), "mae": summ("crello_vs_skia", "mae")},
}
print(json.dumps(summary, indent=2))
if args.out:
Path(args.out).write_text(json.dumps({"summary": summary, "rows": rows}, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
|