File size: 5,534 Bytes
4f47eb1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
"""Trout Pixel Studio -- interactive demo for Trout.

Trout is a 47.6M-parameter absorbing-state masked-diffusion language model for
16x16 pixel art, built and trained from scratch (architecture after
Tessera-small by TheJagStudio; trained on Angais' Pixel Art Atlas 1.4M).
A sprite is 256 tokens over a learned 512-color palette; the model unmasks all
pixels in parallel over ~16 steps, conditioned on a frozen CLIP text tower.

Runs on free CPU hardware: expect ~10-30 s per generation depending on
steps/samples.
"""

import os
import sys
import time

import torch
from huggingface_hub import hf_hub_download

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from trout import (DEFAULT_TEXT_ENCODER, FrozenTextEncoder,
                   load_model_from_checkpoint, make_grid, sample,
                   tokens_to_images, upscale)

import gradio as gr

MODEL_REPO = os.environ.get("MODEL_REPO", "FrameXlabs/Trout")
DEVICE = "cpu"

print(f"loading Trout from {MODEL_REPO} ...", flush=True)
ckpt_path = hf_hub_download(MODEL_REPO, "trout-fp16.pt")
model, palette, ck = load_model_from_checkpoint(ckpt_path, device=DEVICE, use_ema=True)
enc = FrozenTextEncoder(ck.get("text_encoder", DEFAULT_TEXT_ENCODER),
                        model.cfg.text_len, DEVICE)
print(f"Trout ready: {model.num_params():,} parameters, step {ck.get('step')}", flush=True)

EXAMPLES = [
    "red health potion bottle", "golden crown with blue gems", "green slime monster",
    "iron sword with a leather hilt", "wooden treasure chest", "pink cherry blossom tree",
    "blue magic spellbook", "orange fox head", "glowing purple crystal", "cheeseburger",
    "skull with a candle", "rocket ship", "brown leather boots", "yellow rubber duck",
    "ice cream cone with strawberry scoop", "black cat", "tiny green dragon", "gold coin",
]

HEADER = """
<div style="text-align:center; margin-bottom: 8px;">
<h1 style="margin-bottom:4px;">🐟 Trout Pixel Studio</h1>
<p style="color:#666; font-size:15px;">
A <b>diffusion language model</b> for 16×16 pixel art — each sprite is a
256-token “sentence” over a learned 512-color palette, unmasked in parallel
over ~16 steps. 47.6M parameters, built &amp; trained from scratch on
<b>Pixel Art Atlas 1.4M</b> (Angais, CC0), conditioned on a frozen
CLIP text tower. Runs on CPU — please allow ~10–30 s per batch.
</p>
</div>
"""


def generate(prompt: str, num: int, steps: int, cfg: float, temperature: float,
             seed: int, polish: int):
    t0 = time.time()
    seed = int(seed) if seed is not None else 0
    num = int(num)
    g = torch.Generator().manual_seed(seed)
    with torch.no_grad():
        states, pooled = enc.encode([prompt] * num)
        toks = sample(model, states, pooled, steps=int(steps), cfg_scale=float(cfg),
                      temperature=float(temperature), polish_steps=int(polish),
                      generator=g)
    imgs = tokens_to_images(toks, palette)
    big = [upscale(im, 16, checker=True) for im in imgs]
    grid = make_grid(imgs, ncols=max(2, num), scale=10).convert("RGB")
    dt = time.time() - t0
    info = (f"{num} sprite{'s' if num > 1 else ''} · {int(steps)} steps · "
            f"cfg {float(cfg):.1f} · {dt:.1f}s on CPU")
    return big, grid, info


def build_ui():
    with gr.Blocks(title="Trout Pixel Studio", theme=gr.themes.Soft()) as demo:
        gr.HTML(HEADER)
        with gr.Row():
            with gr.Column(scale=5):
                prompt = gr.Textbox(label="Prompt", value="red health potion bottle",
                                     placeholder="describe a 16x16 game sprite...",
                                     lines=1, max_lines=2)
                with gr.Row():
                    num = gr.Slider(1, 8, value=4, step=1, label="Samples")
                    steps = gr.Slider(4, 32, value=16, step=1, label="Diffusion steps")
                with gr.Row():
                    cfg = gr.Slider(1.0, 8.0, value=3.0, step=0.1, label="CFG scale")
                    temperature = gr.Slider(0.0, 1.5, value=1.0, step=0.05,
                                            label="Temperature")
                with gr.Row():
                    seed = gr.Number(value=1234, label="Seed", precision=0)
                    polish = gr.Slider(0, 2, value=0, step=1, label="Polish steps")
                go = gr.Button("🎣 Generate pixel art", variant="primary")
                examples = gr.Dataset(components=[gr.Textbox(visible=False)],
                                      samples=[[e] for e in EXAMPLES],
                                      label="Example prompts — click one to use it")
                examples.click(lambda s: (s[0] if isinstance(s, (list, tuple)) else s),
                               inputs=[examples], outputs=[prompt])
            with gr.Column(scale=4):
                info = gr.Markdown("")
                grid = gr.Image(label="Sprites grid", type="pil", height=420)
        gallery = gr.Gallery(label="Individual sprites (upscaled ×16)",
                             columns=4, height=420, object_fit="contain")
        go.click(generate,
                 inputs=[prompt, num, steps, cfg, temperature, seed, polish],
                 outputs=[gallery, grid, info])
        prompt.submit(generate,
                      inputs=[prompt, num, steps, cfg, temperature, seed, polish],
                      outputs=[gallery, grid, info])
    return demo


demo = build_ui()
if os.environ.get("SPACE_TEST"):
    print("SPACE_TEST: UI built OK")
else:
    demo.launch()