Spaces:
Running
Running
| """Draw the page's charts in the Figure Kit style of the paper figures (user, 2026-10-01: "网站上的图表也换成这个风格"). | |
| - Takeaway: the requirement-family dot plot becomes the kit's grouped bars (company colour for Spatial Composition, | |
| greys for the other three families, logos, rotated names), drawn by assets/takeaway-bars.js; the mismatch chart and | |
| the judge's checks take the kit's neutrals; the evidence scene gets the kit's company-colour chip. | |
| - Leaderboard bars: the "Text-to-Scene" qualifier beside the title, and the Qwen glyph in place of its wordmark. | |
| - Pareto: the y axis is the Text-to-Scene score (it said "Simulation Quality"); effort in title case, as in the kit. | |
| Each step asserts it found what it edits, so a re-run stops instead of half-applying. | |
| python3 scripts/kit_charts.py | |
| """ | |
| from __future__ import annotations | |
| import json, re, sys | |
| from pathlib import Path | |
| from bs4 import BeautifulSoup | |
| ROOT = Path(__file__).resolve().parent.parent | |
| KIT = ROOT.parent / "figure-kit" # ~/Downloads/code4scene-video/figure-kit (the paper figures' build) | |
| V = "20261004kit" | |
| sys.path.insert(0, str(KIT)) | |
| import build as kit # noqa: E402 (logos, colours) | |
| # paper Figure 14b: Identity & Environment, Content & Quantity, Attributes & Materials, Spatial Composition | |
| FAMILY = { | |
| "Claude Fable 5.1 (max)": (.906, .738, .670, .538), | |
| "GPT-6 Astra (max)": (.875, .715, .643, .543), | |
| "Claude Opus 5 (max)": (.803, .624, .573, .479), | |
| "GPT-5.6 Sol (high)": (.750, .660, .602, .469), | |
| "Gemini 3.8 Flash (high)": (.814, .639, .572, .473), | |
| "Muse Spark 1.3 (medium)": (.730, .641, .546, .393), | |
| "Grok 4.6 (high)": (.647, .558, .457, .334), | |
| "Qwen 3.8 27B (thinking off)": (.724, .520, .428, .300), | |
| "Qwen 3.8 27B (thinking on)": (.660, .525, .470, .307), | |
| "GLM-5.3 Flash (max)": (.467, .421, .377, .299), | |
| "Gemma 4 31B (thinking on)": (.561, .407, .387, .272), | |
| "Gemma 4 31B (thinking off)": (.612, .453, .387, .252), | |
| "Inkling (high)": (.572, .394, .361, .193), | |
| "DeepSeek V4.1 Flash (high)": (.294, .215, .259, .120), | |
| } | |
| FAMILIES = ["Identity & Environment", "Content & Quantity", "Attributes & Materials", "Spatial Composition"] | |
| GREYS = ["#dedede", "#c6c6c6", "#a9a9a9"] | |
| SC_SWATCH = ["#D97757", "#10A37F", "#7b1fa2", "#42a5f5"] | |
| MISMATCH = [("Spatial relation", 32.0), ("Distribution", 19.1), ("Composition", 4.3)] # paper Figure 6a, % | |
| def one(found, what): | |
| assert found, f"not found: {what}" | |
| return found | |
| def uniq_ids(spec, suffix): | |
| """A logo's gradient ids, suffixed, so a second copy on the page cannot point at the first one's defs.""" | |
| if "inner" not in spec: | |
| return spec | |
| s = spec["inner"] | |
| for i in set(re.findall(r'id="([^"]+)"', s)): | |
| s = s.replace(f'id="{i}"', f'id="{i}{suffix}"').replace(f"url(#{i})", f"url(#{i}{suffix})") | |
| return dict(spec, inner=s) | |
| def rows(): | |
| board = json.loads(re.search(r"const ALL = (\{.*?\});\n", (ROOT / "assets/leaderboard-bars.js").read_text(), re.S).group(1))["t2s"] | |
| out = [] | |
| for b in sorted(board, key=lambda b: -b["score"]): | |
| fam = FAMILY[b["name"]] | |
| assert min(fam) == fam[3], b["name"] # the claim the chart makes, per agent | |
| out.append({"name": b["name"], "org": b["org"], "colour": b["colour"], "fam": fam}) | |
| assert len(out) == len(FAMILY) == 14 | |
| return out | |
| TAKEAWAY_JS = r"""// Code4Scene takeaway: requirement-family scores per agent (paper Figure 14b), in the paper figures' kit style -- | |
| // grouped bars in Text-to-Scene rank order, Spatial Composition in the agent's company colour, the other families grey. | |
| // Generated by scripts/kit_charts.py; the layout follows figure-kit/takeaway.py and the leaderboard bars. | |
| import {bindRecordTooltip} from "./chart-interactions.js?v=%(v)s"; | |
| const D = %(data)s; | |
| const LOGOS = %(logos)s; | |
| await Promise.all(['400 13px Inter', '700 13px Inter', '500 12.5px "IBM Plex Mono"'].map(f => document.fonts.load(f))); | |
| const svg = document.getElementById('tk-families'), W = 1200, H = 410, NS = 'http://www.w3.org/2000/svg'; | |
| const SANS = 'Inter, Arial, sans-serif', MONO = '"IBM Plex Mono", monospace'; | |
| const G = {left: 40, right: 24, top: 34, base: 262, groupFill: 0.80, gap: 2, radius: 3.5, | |
| logo: 18, logoGap: 8, nameGap: 9, nameAngle: -60, nameSize: 13, nameLead: 14.5, nameOneLine: 97, nameLine: 93, nameTol: 12}; | |
| function el(tag, attrs, parent) { const e = document.createElementNS(NS, tag); for (const k in attrs) e.setAttribute(k, attrs[k]); (parent || svg).appendChild(e); return e; } | |
| function text(x, y, s, attrs, parent) { const t = el('text', Object.assign({x, y}, attrs), parent); t.textContent = s; return t; } | |
| const ctx = document.createElement('canvas').getContext('2d'); | |
| function width(t, font) { ctx.font = font; return ctx.measureText(t).width; } | |
| function wrap(name, font) { | |
| if (width(name, font) <= G.nameOneLine) return [name]; | |
| const w = name.split(' '); | |
| const splits = k => { const out = []; | |
| if (k === 2) for (let i = 1; i < w.length; i++) out.push([w.slice(0, i), w.slice(i)]); | |
| if (k === 3) for (let i = 1; i < w.length; i++) for (let j = i + 1; j < w.length; j++) out.push([w.slice(0, i), w.slice(i, j), w.slice(j)]); | |
| return out.map(p => p.map(x => x.join(' '))); }; | |
| for (const k of [2, 3]) { | |
| const c = splits(k).map(lines => { const ws = lines.map(l => width(l, font)); return {lines, worst: Math.max(...ws), last: ws[ws.length - 1]}; }) | |
| .filter(o => o.worst <= G.nameLine); | |
| if (!c.length) continue; | |
| const floor = Math.min(...c.map(o => o.worst)); | |
| return c.filter(o => o.worst <= floor + G.nameTol).sort((a, b) => b.last - a.last)[0].lines; | |
| } | |
| return [name]; | |
| } | |
| function logo(org, cx, ly, parent) { | |
| const L = LOGOS[org], s = G.logo, lx = cx - s / 2; | |
| if (L.wordmark) { L.wordmark.forEach((w, k) => text(cx, ly + 7.5 + k * 6.2, w, {'text-anchor': 'middle', 'font-family': SANS, 'font-weight': 500, 'font-size': 5.2, 'letter-spacing': 0.35, fill: '#4a4a4a'}, parent)); return; } | |
| let pad = 0, fill = L.fill || null; | |
| if (L.tile) { if (L.shape === 'circle') el('circle', {cx, cy: ly + s / 2, r: s / 2, fill: L.tile}, parent); else el('rect', {x: lx, y: ly, width: s, height: s, rx: 3.2, fill: L.tile}, parent); pad = 3.4; fill = '#ffffff'; } | |
| const ic = el('svg', {x: lx + pad, y: ly + pad, width: s - 2 * pad, height: s - 2 * pad, viewBox: L.viewBox}, parent); | |
| if (fill) { ic.setAttribute('fill', fill); ic.setAttribute('color', fill); } | |
| if (L.rule) ic.setAttribute('fill-rule', L.rule); | |
| ic.innerHTML = L.inner; | |
| } | |
| const y = v => G.base - v / 100 * (G.base - G.top); | |
| for (const v of [25, 50, 75, 100]) | |
| el('line', {x1: G.left - 8, x2: W - G.right, y1: y(v), y2: y(v), stroke: '#cfcfcf', 'stroke-width': 1.4, 'stroke-dasharray': '0.1 4.2', 'stroke-linecap': 'round'}); | |
| el('line', {x1: G.left - 8, x2: W - G.right, y1: G.base, y2: G.base, stroke: '#e6e6e6', 'stroke-width': 1}); | |
| const n = D.rows.length, pitch = (W - G.left - G.right) / n, gw = pitch * G.groupFill, bw = (gw - 3 * G.gap) / 4; | |
| D.rows.forEach((d, i) => { | |
| const cx = G.left + (i + 0.5) * pitch, g0 = cx - gw / 2; | |
| const node = el('g', {class: 'bar-node'}); | |
| el('rect', {x: G.left + i * pitch, y: G.top - 20, width: pitch, height: H - G.top + 20, fill: 'transparent', class: 'bar-hit'}, node); | |
| d.fam.forEach((v, j) => { | |
| const x0 = g0 + j * (bw + G.gap), top = y(v * 100), r = Math.min(G.radius, bw / 2); | |
| el('path', {class: j === 3 ? 'bar' : 'bar fam', d: `M${x0},${G.base} V${top + r} Q${x0},${top} ${x0 + r},${top} H${x0 + bw - r} Q${x0 + bw},${top} ${x0 + bw},${top + r} V${G.base} Z`, | |
| fill: j === 3 ? d.colour : D.greys[j]}, node); | |
| if (j === 3) text(x0 + bw / 2, top - 6, String(Math.round(v * 100)), {'text-anchor': 'middle', 'font-family': MONO, 'font-weight': 500, 'font-size': 12.5, fill: '#262626'}, node); | |
| }); | |
| logo(d.org, cx, G.base + G.logoGap, node); | |
| const font = `400 ${G.nameSize}px ${SANS}`, lines = wrap(d.name, font); | |
| const g = el('g', {transform: `translate(${cx + 3},${G.base + G.logoGap + G.logo + G.nameGap}) rotate(${G.nameAngle})`}, node); | |
| const off = -(lines.length - 1) / 2 * G.nameLead; | |
| lines.forEach((ln, k) => text(0, off + k * G.nameLead + 4.5, ln, {'text-anchor': 'end', 'font-family': SANS, 'font-size': G.nameSize, fill: '#262626'}, g)); | |
| bindRecordTooltip(node, svg, {name: d.name, provider: d.org, | |
| metrics: D.families.map((f, j) => ({label: f, value: d.fam[j].toFixed(3)}))}, '.bar:not(.fam)'); | |
| }); | |
| svg.dataset.ready = 'true'; | |
| """ | |
| def write_takeaway_js(): | |
| rs = rows() | |
| logos = {org: uniq_ids(spec, "-tk") for org, spec in kit.logos().items() if org in {r["org"] for r in rs}} | |
| data = {"rows": rs, "families": FAMILIES, "greys": GREYS} | |
| (ROOT / "assets/takeaway-bars.js").write_text(TAKEAWAY_JS % {"v": V, "data": json.dumps(data), "logos": json.dumps(logos)}) | |
| return rs | |
| def mismatch_svg(): | |
| """Paper Figure 6a in the kit's neutrals: the spatial-relation bar dark, the others grey, values in Plex Mono.""" | |
| Wm, x0, x1, top, rh, bh = 520, 136, 440, 10, 52, 34 | |
| X = lambda v: x0 + v / 40 * (x1 - x0) | |
| H = top + rh * len(MISMATCH) + 30 | |
| o = [f'<svg class="kit-ch" viewBox="0 0 {Wm} {H}" role="img" aria-label="Mismatch rate when the required objects are present">'] | |
| for v in (10, 20, 30, 40): | |
| o.append(f'<line x1="{X(v):.1f}" x2="{X(v):.1f}" y1="{top}" y2="{top + rh * 3}" stroke="#cfcfcf" stroke-width="1.4" stroke-dasharray="0.1 4.2" stroke-linecap="round"/>') | |
| o.append(f'<line x1="{x0}" x2="{x0}" y1="{top}" y2="{top + rh * 3}" stroke="#e6e6e6" stroke-width="1"/>') | |
| for v in (0, 10, 20, 30, 40): | |
| o.append(f'<text x="{X(v):.1f}" y="{top + rh * 3 + 20}" text-anchor="middle" font-family="IBM Plex Mono, monospace" font-weight="500" font-size="12" fill="#9a9a9a">{v}%</text>') | |
| for i, (lab, v) in enumerate(MISMATCH): | |
| yc = top + (i + .5) * rh; hot = i == 0; xe = X(v); r = min(4.5, (xe - x0) / 2) | |
| o.append(f'<g data-tip="{lab}: {v:.1f}% of checks fail although the objects are present">') | |
| o.append(f'<text x="{x0 - 14}" y="{yc + 4.6:.1f}" text-anchor="end" font-family="Inter, sans-serif" font-size="13" font-weight="{700 if hot else 400}" fill="#262626">{lab}</text>') | |
| o.append(f'<path d="M{x0},{yc - bh / 2:.1f} H{xe - r:.1f} Q{xe:.1f},{yc - bh / 2:.1f} {xe:.1f},{yc - bh / 2 + r:.1f} V{yc + bh / 2 - r:.1f} ' | |
| f'Q{xe:.1f},{yc + bh / 2:.1f} {xe - r:.1f},{yc + bh / 2:.1f} H{x0} Z" fill="{"#2b3531" if hot else "#c6c6c6"}"/>') | |
| o.append(f'<text x="{xe + 9:.1f}" y="{yc + 5.4:.1f}" font-family="IBM Plex Mono, monospace" font-weight="500" font-size="15.5" fill="#262626">{v:.1f}%</text></g>') | |
| o.append("</svg>") | |
| return "".join(o) | |
| def legend_html(rs): | |
| mean = [sum(r["fam"][j] for r in rs) / len(rs) for j in range(4)] | |
| sw = "linear-gradient(90deg," + ",".join(f"{c} {k * 25}% {(k + 1) * 25}%" for k, c in enumerate(SC_SWATCH)) + ")" | |
| items = [] | |
| for j, f in enumerate(FAMILIES): | |
| bg = GREYS[j] if j < 3 else sw | |
| lab = f if j < 3 else f"<b>{f}</b>, in the agent’s colour" | |
| items.append(f'<span><i style="background:{bg}"></i><span>{lab.replace("&", "&") if j < 3 else lab.replace(" & ", " & ")}</span><em>mean {round(mean[j] * 100)}</em></span>') | |
| return '<div class="kit-legend">' + "".join(items) + "</div>" | |
| def index(rs): | |
| p = ROOT / "index.html" | |
| soup = BeautifulSoup(p.read_text(), "html.parser") | |
| frag = lambda html: BeautifulSoup(html, "html.parser") | |
| # takeaway: the family chart | |
| tk = one(soup.find(id="takeaways"), "takeaways") | |
| figs = one(tk.find(class_="figs"), "takeaway figs") | |
| fam_fig = one(figs.find("figure", recursive=False), "family figure") | |
| fam_fig.clear() | |
| fam_fig.append(frag('<div class="fh">Requirement-family score per agent</div>' + legend_html(rs) + | |
| '<div class="bar-scroll" role="region" tabindex="0" aria-label="Requirement-family scores, all 14 configurations">' | |
| '<svg id="tk-families" class="kit-bars" viewBox="0 0 1200 410" role="group" ' | |
| 'aria-label="Requirement-family scores per agent, in Text-to-Scene rank order"></svg></div>' | |
| '<figcaption>One group per agent, in Text-to-Scene rank order: Identity & Environment, Content & Quantity, ' | |
| 'Attributes & Materials, then Spatial Composition, the lowest bar in every group. Scores ×100 (paper Figure 14b).</figcaption>')) | |
| # the mismatch chart | |
| mm = None | |
| for f in tk.find_all("figure"): | |
| h = f.find(class_="fh") | |
| if h and h.get_text(strip=True).startswith("Mismatch"): | |
| mm = f | |
| one(mm, "mismatch figure") | |
| one(mm.find("svg"), "mismatch svg").replace_with(frag(mismatch_svg())) | |
| mm.find(class_="fh").string = "Mismatch when the objects are present" | |
| mm.find("figcaption").string = "Share of checks that fail although every object they name is in the scene, across the benchmark (paper Figure 6a)." | |
| # the evidence scene: the kit's company-colour chip over the panel | |
| scene = one(tk.find(class_="live-scene", attrs={"data-agent": "sol"}), "evidence scene") | |
| if not scene.find(class_="agent-chip"): | |
| ic = kit.logos()["OpenAI"] | |
| scene.insert(0, frag(f'<span class="agent-chip" style="background:{kit.COLOUR["OpenAI"]}" aria-hidden="true">' | |
| f'<svg viewBox="{ic["viewBox"]}" fill="#fff">{ic["inner"]}</svg>GPT-5.6 Sol (high)</span>')) | |
| # leaderboard bars: the qualifier beside the title | |
| h3 = one(soup.select_one(".bar-heading h3"), "bar heading") | |
| h3.clear(); h3.append("Code4Scene "); q = soup.new_tag("span", attrs={"class": "q"}); q.string = "Text-to-Scene"; h3.append(q) | |
| # scripts and cache keys | |
| if not soup.find("script", src=re.compile(r"takeaway-bars\.js")): | |
| soup.body.append(soup.new_tag("script", attrs={"type": "module", "src": f"assets/takeaway-bars.js?v={V}"})) | |
| for t in soup.find_all(["script", "link"]): | |
| a = "src" if t.name == "script" else "href" | |
| u = t.get(a) or "" | |
| if re.match(r"assets/(buildingbench\.css|leaderboard-bars\.js|pareto\.js|takeaway-bars\.js)", u): | |
| t[a] = re.sub(r"\?v=.*$", "", u) + f"?v={V}" | |
| p.write_text(str(soup)) | |
| print("index.html: takeaway, mismatch, chip, bar heading, scripts") | |
| def bars_js(): | |
| p = ROOT / "assets/leaderboard-bars.js"; s = p.read_text() | |
| m = one(re.search(r"const LOGOS = (\{.*?\});\n", s, re.S), "LOGOS") | |
| logos = json.loads(m.group(1)) | |
| logos["Alibaba"] = kit.logos()["Alibaba"] # the Qwen glyph, as in the paper figure | |
| s = s[:m.start(1)] + json.dumps(logos) + s[m.end(1):] | |
| s, n = re.subn(r"chart-interactions\.js\?v=\w+", f"chart-interactions.js?v={V}", s); assert n == 1 | |
| p.write_text(s); print("leaderboard-bars.js: Qwen glyph") | |
| def pareto_js(): | |
| p = ROOT / "assets/pareto.js"; s = p.read_text() | |
| a = "'Simulation Quality'"; assert a in s or "'Text-to-Scene Score'" in s; s = s.replace(a, "'Text-to-Scene Score'") | |
| a = ".replace(/\\((.*?)\\)/,(_,s)=>'('+s[0].toUpperCase()+s.slice(1)+')')" | |
| assert a in s or "s.replace(/\\b\\w/g" in s, "pareto name casing" | |
| s = s.replace(a, ".replace(/\\((.*?)\\)/,(_,s)=>'('+s.replace(/\\b\\w/g,c=>c.toUpperCase())+')')") | |
| s, n = re.subn(r"chart-interactions\.js\?v=\w+", f"chart-interactions.js?v={V}", s); assert n == 1 | |
| p.write_text(s); print("pareto.js: axis title, effort casing") | |
| def interactions_js(): | |
| p = ROOT / "assets/chart-interactions.js"; s = p.read_text() | |
| if "bindRecordTooltip" in s: | |
| return | |
| a = ("export async function bindChartTooltip(node,plot,key,mode,anchorSelector='.bar,.chip'){\n" | |
| " const data=await detailPromise,record=data[mode].find(d=>d.key===key||d.name===key);if(!record)return;\n") | |
| assert a in s, "bindChartTooltip head" | |
| s = s.replace(a, "export async function bindChartTooltip(node,plot,key,mode,anchorSelector='.bar,.chip'){\n" | |
| " const data=await detailPromise,record=data[mode].find(d=>d.key===key||d.name===key);if(!record)return;\n" | |
| " bindRecordTooltip(node,plot,record,anchorSelector);\n}\n" | |
| "// the same behaviour for a chart whose record is not a leaderboard row (the takeaway's family scores)\n" | |
| "export function bindRecordTooltip(node,plot,record,anchorSelector='.bar,.chip'){\n") | |
| p.write_text(s); print("chart-interactions.js: bindRecordTooltip") | |
| CSS = """ | |
| /* kit-style charts (scripts/kit_charts.py) */ | |
| .bar-heading h3 span.q{font:inherit!important;color:#8d8d8d!important;max-width:none!important;text-align:left!important} | |
| .tk figure{background:#fff;min-width:0} | |
| .tk .figs{grid-template-columns:minmax(0,1fr)}.tk .figs.two{grid-template-columns:minmax(0,1.15fr) minmax(0,1fr)} | |
| @media(max-width:900px){.tk .figs.two{grid-template-columns:minmax(0,1fr)}} | |
| .tk figure .fh{font:400 22px/1.25 'Source Serif 4',Georgia,serif;color:#8d8d8d;margin-bottom:12px;letter-spacing:-.3px} | |
| .kit-legend{display:flex;flex-wrap:wrap;gap:6px 26px;margin:0 0 6px;font:400 13px/1.5 Inter,sans-serif;color:#262626} | |
| .kit-legend>span{display:inline-flex;align-items:center;gap:7px}.kit-legend b{font-weight:700} | |
| .kit-legend i{width:12px;height:12px;border-radius:3px;display:inline-block;flex:none} | |
| .kit-legend em{font:500 12.5px 'IBM Plex Mono',monospace;color:#8d8d8d;font-style:normal;margin-left:2px} | |
| svg.kit-ch{display:block;width:100%;height:auto;overflow:visible} | |
| .tk .chip{font:500 12.5px Inter,sans-serif;padding:5px 12px;border-radius:999px} | |
| .tk .chip.ok{color:#262626;background:#f1f1ee;border-color:#e4e4e2} | |
| .tk .chip.no{color:#fff;background:#111;border-color:#111} | |
| .tk .chip small{font:500 12px 'IBM Plex Mono',monospace;color:#8d8d8d}.tk .chip.no small{color:#bdbdbd} | |
| .agent-chip{position:absolute;z-index:4;top:12px;left:12px;display:inline-flex;align-items:center;gap:6px;padding:5px 10px;border-radius:6px;color:#fff;font:700 12.5px Inter,sans-serif;pointer-events:none} | |
| .agent-chip svg{width:14px;height:14px} | |
| @media(max-width:650px){.kit-legend{font-size:11px;gap:4px 14px}.kit-legend em{font-size:10.5px}.tk figure .fh{font-size:18px}.agent-chip{font-size:11px;top:8px;left:8px}} | |
| """ | |
| def css(): | |
| p = ROOT / "assets/buildingbench.css"; s = p.read_text() | |
| if "kit-style charts (scripts/kit_charts.py)" in s: | |
| s = s[:s.index("\n/* kit-style charts (scripts/kit_charts.py) */")] | |
| p.write_text(s.rstrip("\n") + "\n" + CSS); print("buildingbench.css: kit chart rules") | |
| if __name__ == "__main__": | |
| rs = write_takeaway_js(); interactions_js(); bars_js(); pareto_js(); css(); index(rs) | |