"""NEURAL BOY (HTML-component build) — games dreamed by a diffusion world model. This is an alternate front-end to ../space that renders the liked `mockup.html` design as a single Gradio 6 custom HTML component: gr.HTML(html_template=..., js_on_load=..., server_functions=[load_game, step]) - html_template : the mockup console + cartridge library (static shell). - css (global) : the mockup styles + Press Start 2P font. - js_on_load : controls (keyboard + d-pad/A-B), drag-drop cartridges, frame loop. - server_functions: load_game() seeds a session and step() advances one dreamed frame. Gradio passes a single arg per server call, so the js_on_load loop calls `await server.step({sid, action})` and paints the returned base64 PNG onto the console screen — no gr.Timer / gr.Image / hidden buttons. Actions are passed straight in, so no held-action TTL hack is needed. Model code (`src/`) and cartridges (`assets/games/`) are reused from ../space if not present locally, so nothing large is duplicated. """ import sys from pathlib import Path HERE = Path(__file__).resolve().parent def _resolve(rel: str) -> Path: for base in (HERE, HERE.parent / "space"): if (base / rel).exists(): return base / rel return HERE / rel SRC = _resolve("src") GAMES_DIR = _resolve("assets/games") sys.path.insert(0, str(SRC)) import base64 import glob import io import json import os import gradio as gr import numpy as np import torch from hydra.utils import instantiate from omegaconf import OmegaConf from PIL import Image from data.episode import Episode from models.diffusion import Denoiser, DiffusionSampler DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") CTX = 4 DENOISING_STEPS = 10 # diffusion steps per frame (higher = sharper, slower) # default action set for cartridges without meta.json (our Game Boy games) DEFAULT_META = { "num_actions": 6, "action_names": ["noop", "left", "right", "fire", "leftfire", "rightfire"], "display_name": None, } TOKENS = ["up", "down", "left", "right", "fire"] # Per-cartridge {game action token -> UI token}. Pong names its paddle actions # "right"/"left" but they move the paddle up/down (ALE paddle quirk), so we bind # the ↑/↓ keys instead of ◀/▶. TOKEN_REMAP = {"pong": {"right": "up", "left": "down"}} HIDDEN_GAMES = {"tobu", "pacman", "kangaroo", "crazyclimber"} # cartridges to skip (case-insensitive substring) if not OmegaConf.has_resolver("eval"): OmegaConf.register_new_resolver("eval", eval) def tokens_of(name: str, remap: dict = None): toks = [t for t in TOKENS if t in name] if remap: toks = sorted((remap.get(t, t) for t in toks), key=TOKENS.index) return toks def canonical(name: str, remap: dict = None) -> str: return "" if name == "noop" else "+".join(tokens_of(name, remap)) # ---------------- cartridges ---------------- GAME_DIRS = { p.name: p for p in sorted(GAMES_DIR.iterdir()) if p.is_dir() and not any(h in p.name.lower() for h in HIDDEN_GAMES) } if GAMES_DIR.exists() else {} GAME_NAMES = list(GAME_DIRS) _cache = {} def get_game(name: str): if name not in _cache: d = GAME_DIRS[name] meta_path = d / "meta.json" meta = json.load(open(meta_path)) if meta_path.exists() else dict(DEFAULT_META) num_actions = int(meta["num_actions"]) action_names = meta["action_names"] cfg = OmegaConf.load(d / "trainer.yaml") dcfg = instantiate(cfg.agent.denoiser) dcfg.inner_model.num_actions = num_actions denoiser = Denoiser(dcfg).to(DEVICE).eval() sd = torch.load(d / "agent.pt", map_location=DEVICE) denoiser.load_state_dict({k[len("denoiser."):]: v for k, v in sd.items() if k.startswith("denoiser.")}) samp_cfg = instantiate(cfg.world_model_env.diffusion_sampler) samp_cfg.num_steps_denoising = DENOISING_STEPS sampler = DiffusionSampler(denoiser, samp_cfg) seeds = [Episode.load(p, map_location=DEVICE) for p in sorted(glob.glob(str(d / "seeds" / "*.pt")))] assert seeds, f"no seed episodes in {d/'seeds'}" noop = action_names.index("noop") if "noop" in action_names else 0 remap = TOKEN_REMAP.get(name, {}) keymap = {canonical(n, remap): i for i, n in enumerate(action_names)} keymap.setdefault("", noop) _cache[name] = { "sampler": sampler, "seeds": seeds, "noop": noop, "keymap": keymap, "display": (meta.get("display_name") or name.replace("-", " ")).upper(), } return _cache[name] def pick_seed(seeds): """Pick a high-motion window so the dream starts somewhere interesting.""" import random best, best_motion = None, -1.0 for _ in range(30): ep = random.choice(seeds) max_start = len(ep) - CTX - 2 if max_start < 1: continue start = random.randrange(max_start) clip = ep.obs[start + CTX : start + CTX + 16] motion = (clip[1:] - clip[:-1]).abs().mean().item() if len(clip) > 1 else 0.0 if motion > best_motion: best, best_motion = (ep, start), motion return best def to_dataurl(frame: torch.Tensor) -> str: """Encode a CHW [-1,1] frame as a tiny base64 PNG (CSS upscales it pixelated).""" x = ((frame.clamp(-1, 1) + 1) / 2).permute(1, 2, 0).cpu().numpy() im = Image.fromarray((x * 255).astype(np.uint8)) buf = io.BytesIO() im.save(buf, format="PNG") return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() @torch.no_grad() def model_step(sampler, obs_buf, act_buf, action_id): act_buf[:, -1] = action_id next_obs, _ = sampler.sample(obs_buf, act_buf) next_obs = next_obs.clamp(-1, 1) obs_buf = obs_buf.roll(-1, dims=1) act_buf = act_buf.roll(-1, dims=1) obs_buf[:, -1] = next_obs return obs_buf, act_buf, next_obs # ---------------- sessions + server functions ---------------- # sid (generated client-side) -> {"game", "obs", "act"}. One frame per step() call. SESSIONS = {} # NOTE: Gradio server functions receive a SINGLE argument (the backend calls # `fn(body.data)`, and the frontend forwards only the first JS argument), so each # of these takes one dict that the JS side packs — extra positional args would be # silently dropped and padded to None. def load_game(payload): """Seed a session for a cartridge; return first frame + control keymap.""" sid, name = payload["sid"], payload["name"] game = get_game(name) ep, start = pick_seed(game["seeds"]) SESSIONS[sid] = { "game": name, "obs": ep.obs[start : start + CTX].unsqueeze(0).to(DEVICE), "act": ep.act[start : start + CTX].unsqueeze(0).to(DEVICE), } return { "frame": to_dataurl(SESSIONS[sid]["obs"][0, -1]), "keymap": game["keymap"], "name": game["display"], "noop": game["noop"], } def step(payload): """Advance one dreamed frame for the session using the held action index.""" sid = payload["sid"] s = SESSIONS.get(sid) if s is None: return {} game = get_game(s["game"]) action = payload.get("action") a = int(action) if action is not None else game["noop"] s["obs"], s["act"], next_obs = model_step(game["sampler"], s["obs"], s["act"], a) return {"frame": to_dataurl(next_obs[0])} if GAME_NAMES: get_game(GAME_NAMES[0]) # warm the first cartridge GAMES_LIST = [{"id": n, "name": n.replace("-", " ").upper()} for n in GAME_NAMES] # ---------------- UI ---------------- CSS = """ @import url('https://fonts.googleapis.com/css2?family=Press+Start+2P&display=swap'); html, body { background:#14102a; } .gradio-container { max-width:100% !important; background: radial-gradient(ellipse at 20% 0%, #3b2d63 0%, #221a3d 45%, #14102a 100%) fixed !important; } .gradio-container, .gradio-container * { font-family:'Press Start 2P', monospace !important; } footer { display:none !important; } .gradio-container .block, .gradio-container .form, .html-container, .prose, .gap { background:transparent !important; border:none !important; box-shadow:none !important; padding:0 !important; } .nb-root { color:#f5edff; display:flex; flex-direction:column; align-items:center; gap:26px; padding:24px 16px 60px; } .nb-root * { box-sizing:border-box; } .nb-root header { text-align:center; } .nb-root header h1 { font-size:30px; letter-spacing:3px; margin:0 0 12px; text-shadow:0 0 18px #c685ff88, 3px 3px 0 #1a1430; } .nb-root header p { font-size:10px; color:#bda9e8; line-height:1.9; margin:0; } .nb-root .pink { color:#ff8fc6; } /* ---------- console ---------- */ .nb-root .console { width:460px; max-width:95vw; background:linear-gradient(160deg,#bdb4cf 0%,#9a8fb4 100%); border-radius:22px 22px 70px 22px; padding:26px 26px 34px; box-shadow:0 24px 60px #00000088, inset 0 2px 0 #ffffff55, inset 0 -6px 14px #00000033; } .nb-root .bezel { position:relative; background:#2e2a3e; border-radius:14px 14px 40px 14px; padding:20px 34px; box-shadow:inset 0 4px 12px #000000aa; } .nb-root .powerled { position:absolute; left:14px; top:50%; transform:translateY(-50%); width:9px; height:9px; border-radius:50%; background:#ff3b5c; box-shadow:0 0 10px #ff3b5c; z-index:2; } .nb-root .screen { position:relative; background:#0d1a0d; border-radius:6px; overflow:hidden; aspect-ratio:1/1; } .nb-root .screen-img { position:absolute; inset:0; width:100%; height:100%; image-rendering:pixelated; display:none; } .nb-root .screen .empty { position:absolute; inset:0; display:flex; align-items:center; justify-content:center; font-size:9px; color:#3f6b3f; text-align:center; line-height:2; } .nb-root .scanlines { position:absolute; inset:0; pointer-events:none; background:repeating-linear-gradient(to bottom,#0000 0 2px,#00000022 2px 3px); } .nb-root .gamename { text-align:center; font-size:8px; color:#9be29b; margin-top:8px; min-height:12px; letter-spacing:1px; } .nb-root .brand { text-align:center; font-size:9px; color:#4a4060; letter-spacing:1px; margin:12px 0 16px; } .nb-root .brand b { color:#c2447c; font-size:11px; font-weight:normal; } /* ---------- controls ---------- */ .nb-root .controls { display:flex; align-items:center; justify-content:space-between; background:#00000022; border-radius:14px; padding:14px 22px; } .nb-root .dpad { display:grid; grid-template-columns:repeat(3,32px); grid-template-rows:repeat(3,32px); } .nb-root .pad { display:flex; align-items:center; justify-content:center; color:#cdbff0; font-size:11px; background:#3a3450; box-shadow:0 4px 0 #221d33; cursor:pointer; user-select:none; -webkit-user-select:none; touch-action:none; transition:transform .04s, box-shadow .04s, background .04s; } .nb-root .pad.on, .nb-root .pad:active { transform:translateY(3px); box-shadow:0 1px 0 #221d33; background:#473d6b; color:#fff; } .nb-root .pad.up { grid-area:1/2; border-radius:8px 8px 0 0; } .nb-root .pad.left { grid-area:2/1; border-radius:8px 0 0 8px; } .nb-root .pad.mid { grid-area:2/2; background:#3a3450; box-shadow:none; cursor:default; } .nb-root .pad.mid::after { content:''; width:11px; height:11px; border-radius:50%; background:#221d33; } .nb-root .pad.right { grid-area:2/3; border-radius:0 8px 8px 0; } .nb-root .pad.down { grid-area:3/2; border-radius:0 0 8px 8px; } .nb-root .abtns { display:flex; gap:12px; transform:rotate(-20deg); } .nb-root .ab { width:42px; height:42px; border-radius:50%; display:flex; align-items:center; justify-content:center; background:#c2447c; color:#fff; font-size:11px; cursor:pointer; user-select:none; -webkit-user-select:none; touch-action:none; box-shadow:0 5px 0 #7c2450; transition:transform .04s, box-shadow .04s; } .nb-root .ab.on, .nb-root .ab:active { transform:translateY(3px); box-shadow:0 1px 0 #7c2450; } .nb-root .restart { display:block; margin:16px auto 0; font-family:inherit; font-size:9px; color:#fff; background:#c2447c; border:none; border-radius:10px; padding:10px 22px; cursor:pointer; box-shadow:0 4px 0 #7c2450; } .nb-root .restart:active { transform:translateY(3px); box-shadow:0 1px 0 #7c2450; } /* ---------- cartridge library ---------- */ .nb-root .library { width:460px; max-width:95vw; background:#ffffff0d; border:1px solid #ffffff1a; border-radius:18px; padding:18px; display:flex; flex-direction:column; gap:16px; } .nb-root .lib-title { font-size:9px; color:#bda9e8; text-align:center; line-height:1.8; } .nb-root .lib-body { display:flex; gap:18px; align-items:flex-start; } .nb-root .rack { display:flex; flex-wrap:wrap; gap:12px; flex:1; } .nb-root .cart { position:relative; width:96px; cursor:grab; user-select:none; -webkit-user-select:none; transition:transform .12s ease, filter .12s ease; } .nb-root .cart:hover { transform:translateY(-6px); } .nb-root .cart.dragging { opacity:.35; } .nb-root .cart.active { filter:drop-shadow(0 0 10px #ff8fc6cc); } .nb-root .cart.ghost { position:fixed; left:0; top:0; z-index:9999; width:96px; margin:0; pointer-events:none; opacity:.95; transition:none; transform:translate(-50%,-50%) rotate(-5deg); filter:drop-shadow(0 14px 22px #000a); } .nb-root .cart.ghost:hover { transform:translate(-50%,-50%) rotate(-5deg); } .nb-root .cart .body { background:linear-gradient(180deg,#4a4460,#332e47); border-radius:7px 7px 4px 4px; padding:7px 7px 12px; box-shadow:0 6px 14px #00000066, inset 0 1px 0 #ffffff22; } .nb-root .cart .ridges { display:flex; gap:3px; justify-content:center; margin-bottom:6px; } .nb-root .cart .ridges i { width:8px; height:5px; background:#00000044; border-radius:1px; } .nb-root .cart .label { border-radius:4px; overflow:hidden; background:#0d1a0d; aspect-ratio:1/1; border:2px solid #1a1530; } .nb-root .cart .label svg { width:100%; height:100%; display:block; image-rendering:pixelated; } .nb-root .cart .name { text-align:center; font-size:7px; color:#cdbff0; margin-top:7px; letter-spacing:1px; } .nb-root .slot { width:130px; flex:0 0 auto; } .nb-root .slot-title { font-size:7px; color:#bda9e8; text-align:center; margin-bottom:8px; letter-spacing:1px; } .nb-root .slot-well { background:#00000033; border:2px dashed #ffffff2a; border-radius:10px; min-height:150px; display:flex; align-items:center; justify-content:center; padding:12px; text-align:center; transition:border-color .12s, background .12s; } .nb-root .slot-well.empty { font-size:7px; color:#7c6fa3; line-height:1.9; } .nb-root .slot-well.over { border-color:#ff8fc6; background:#ff8fc61a; } .nb-root .slot-well .cart { width:104px; cursor:pointer; } .nb-root .eject { font-size:6px; color:#8d80b3; text-align:center; margin-top:8px; } .nb-root .note { max-width:520px; text-align:center; font-size:8px; color:#8d80b3; line-height:2; } .nb-root .note .pink { color:#ff8fc6; } """ HTML = """
"The Matrix is everywhere. It is all around us. Even now, in this very room."