indepth-10M / modeling_indepth.py
DexagonAI's picture
Upload folder using huggingface_hub
3711808 verified
Raw History Blame Contribute Delete
11.6 kB
"""InDepth model definition — self-contained PyTorch implementation.
A tiny rectified-flow DiT for 16x16 RGBA sprites, conditioned on captions through a
word + hashed char-trigram subword text encoder trained jointly from scratch.
Weights: model.safetensors (EMA), config.json, vocab.json.
"""
import json
import math
from pathlib import Path
import torch
from torch import nn
from torch.nn import functional as F
class IndepthConfig:
def __init__(self, image_size=16, in_channels=4, patch_size=2, dim=256, depth=8, heads=4,
cond_dim=128, vocab_size=8192, text_dim=128, text_layers=2, text_heads=4,
max_len=16, subword_buckets=4096, **kwargs):
self.image_size = image_size
self.in_channels = in_channels
self.patch_size = patch_size
self.dim = dim
self.depth = depth
self.heads = heads
self.cond_dim = cond_dim
self.vocab_size = vocab_size
self.text_dim = text_dim
self.text_layers = text_layers
self.text_heads = text_heads
self.max_len = max_len
self.subword_buckets = subword_buckets
if dim % heads or dim % 4 or text_dim % text_heads:
raise ValueError("dims must divide head counts")
def _time_embedding(values, dim):
half = dim // 2
freqs = torch.exp(-math.log(10000) * torch.arange(half, dtype=torch.float32, device=values.device) / half)
args = values[:, None].float() * freqs[None]
return torch.cat([torch.sin(args), torch.cos(args)], dim=-1)
def _grid_embedding(height, width, dim, device):
half = dim // 4
y = torch.arange(height, device=device).float().unsqueeze(1).expand(height, width).reshape(-1)
x = torch.arange(width, device=device).float().unsqueeze(0).expand(height, width).reshape(-1)
freqs = torch.exp(-math.log(10000) * torch.arange(half, device=device).float() / half)
args_x = x[:, None] * freqs[None]
args_y = y[:, None] * freqs[None]
return torch.cat([torch.sin(args_x), torch.cos(args_x), torch.sin(args_y), torch.cos(args_y)], dim=-1)
class IndepthDiTBlock(nn.Module):
"""adaLN-Zero transformer block over 2x2 patch tokens (matches the trained weights)."""
def __init__(self, cfg):
super().__init__()
self.norm1 = nn.LayerNorm(cfg.dim, elementwise_affine=False)
self.qkv = nn.Linear(cfg.dim, cfg.dim * 3)
self.proj = nn.Linear(cfg.dim, cfg.dim)
self.norm2 = nn.LayerNorm(cfg.dim, elementwise_affine=False)
self.fc1 = nn.Linear(cfg.dim, cfg.dim * 4)
self.fc2 = nn.Linear(cfg.dim * 4, cfg.dim)
self.modulation = nn.Sequential(nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.dim * 6))
nn.init.zeros_(self.modulation[-1].weight)
nn.init.zeros_(self.modulation[-1].bias)
def forward(self, tokens, condition):
cfg = self.config_ref
batch, count, dim = tokens.shape
mod = self.modulation(condition)
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1)
normed = self.norm1(tokens) * (1.0 + scale_msa[:, None, :]) + shift_msa[:, None, :]
qkv = self.qkv(normed).reshape(batch, count, 3, cfg.heads, dim // cfg.heads).permute(2, 0, 3, 1, 4)
attended = F.scaled_dot_product_attention(qkv[0], qkv[1], qkv[2])
attended = attended.transpose(1, 2).reshape(batch, count, dim)
tokens = tokens + gate_msa[:, None, :] * self.proj(attended)
normed = self.norm2(tokens) * (1.0 + scale_mlp[:, None, :]) + shift_mlp[:, None, :]
hidden = F.gelu(self.fc1(normed), approximate="tanh")
return tokens + gate_mlp[:, None, :] * self.fc2(hidden)
class IndepthTextBlock(nn.Module):
def __init__(self, width, heads):
super().__init__()
self.heads = heads
self.head_dim = width // heads
self.norm1 = nn.LayerNorm(width)
self.qkv = nn.Linear(width, width * 3)
self.proj = nn.Linear(width, width)
self.norm2 = nn.LayerNorm(width)
self.fc1 = nn.Linear(width, width * 4)
self.fc2 = nn.Linear(width * 4, width)
def forward(self, tokens, pad_mask):
batch, count, width = tokens.shape
normalized = self.norm1(tokens)
qkv = self.qkv(normalized).reshape(batch, count, 3, self.heads, self.head_dim).permute(2, 0, 3, 1, 4)
additive = torch.zeros((batch, 1, 1, count), device=tokens.device, dtype=qkv[0].dtype)
additive.masked_fill_(pad_mask[:, None, None, :], float("-inf"))
attended = F.scaled_dot_product_attention(qkv[0], qkv[1], qkv[2], attn_mask=additive)
attended = attended.transpose(1, 2).reshape(batch, count, width)
tokens = tokens + self.proj(attended)
hidden = F.gelu(self.fc1(self.norm2(tokens)), approximate="tanh")
return tokens + self.fc2(hidden)
class IndepthTextEncoder(nn.Module):
"""Word embeddings + hashed char-trigram subword features, 2-layer transformer, masked mean."""
SUBWORD_SLOTS = 16
def __init__(self, config):
super().__init__()
self.subword_buckets = int(config.subword_buckets or 0)
self.token_embedding = nn.Embedding(config.vocab_size, config.text_dim)
self.position_embedding = nn.Parameter(torch.zeros(1, config.max_len, config.text_dim))
nn.init.normal_(self.position_embedding, std=0.02)
self.subword_embedding = None
if self.subword_buckets:
self.subword_embedding = nn.Embedding(self.subword_buckets, config.text_dim, padding_idx=0)
self.blocks = nn.ModuleList([IndepthTextBlock(config.text_dim, config.text_heads) for _ in range(config.text_layers)])
self.final_norm = nn.LayerNorm(config.text_dim)
self.out = nn.Linear(config.text_dim, config.cond_dim)
def forward(self, tokens, subwords=None):
mask = tokens != 0
hidden = self.token_embedding(tokens) + self.position_embedding[:, : tokens.shape[1]]
if self.subword_embedding is not None:
if subwords is None:
subwords = torch.zeros((*tokens.shape, self.SUBWORD_SLOTS), dtype=tokens.dtype, device=tokens.device)
counts = (subwords != 0).sum(dim=-1, keepdim=True).clamp(min=1).to(hidden.dtype)
hidden = hidden + self.subword_embedding(subwords).sum(dim=2) / counts
for block in self.blocks:
hidden = block(hidden, ~mask)
weights = mask.to(hidden.dtype)
mean = (hidden * weights[:, :, None]).sum(dim=1) / weights.sum(dim=1, keepdim=True).clamp(min=1.0)
return self.out(self.final_norm(mean))
class IndepthModel(nn.Module):
"""Rectified-flow DiT over 16x16 RGBA (4 x 16x16 in [-1, 1], alpha in {-1, 1})."""
def __init__(self, config=None):
super().__init__()
self.config = config or IndepthConfig()
cfg = self.config
self.grid_size = cfg.image_size // cfg.patch_size
self.patch_embed = nn.Conv2d(cfg.in_channels, cfg.dim, kernel_size=cfg.patch_size, stride=cfg.patch_size)
self.register_buffer("position", _grid_embedding(self.grid_size, self.grid_size, cfg.dim, torch.device("cpu")).unsqueeze(0), persistent=False)
self.time_mlp = nn.Sequential(nn.Linear(cfg.cond_dim, cfg.cond_dim), nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.cond_dim))
self.step_mlp = nn.Sequential(nn.Linear(cfg.cond_dim, cfg.cond_dim), nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.cond_dim))
self.text_encoder = IndepthTextEncoder(cfg)
self.text_proj = nn.Linear(cfg.cond_dim, cfg.cond_dim)
self.blocks = nn.ModuleList()
for _ in range(cfg.depth):
block = IndepthDiTBlock(cfg)
block.config_ref = cfg
self.blocks.append(block)
self.final_modulation = nn.Sequential(nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.dim * 2))
self.final_norm = nn.LayerNorm(cfg.dim, elementwise_affine=False)
self.output = nn.Linear(cfg.dim, cfg.patch_size * cfg.patch_size * cfg.in_channels)
def forward(self, image, time, step_size, tokens, subwords=None):
cfg = self.config
condition = self.time_mlp(_time_embedding(time, cfg.cond_dim))
condition = condition + self.step_mlp(_time_embedding(step_size, cfg.cond_dim))
condition = condition + self.text_proj(self.text_encoder(tokens, subwords))
tokens_image = self.patch_embed(image).flatten(2).transpose(1, 2)
tokens_image = tokens_image + self.position.to(dtype=tokens_image.dtype)
for block in self.blocks:
tokens_image = block(tokens_image, condition)
shift, scale = self.final_modulation(condition).chunk(2, dim=-1)
tokens_image = self.final_norm(tokens_image) * (1.0 + scale[:, None, :]) + shift[:, None, :]
patches = self.output(tokens_image)
batch = image.shape[0]
patches = patches.reshape(batch, self.grid_size, self.grid_size, cfg.in_channels, cfg.patch_size, cfg.patch_size)
return patches.permute(0, 3, 1, 4, 2, 5).reshape(batch, cfg.in_channels, cfg.image_size, cfg.image_size)
def load_indepth(model_dir):
"""Load the model + tokenizer from a local directory or the Hugging Face hub."""
import numpy as np
from safetensors.torch import load_file
directory = Path(model_dir)
if not directory.exists():
from huggingface_hub import snapshot_download
directory = Path(snapshot_download("DexagonAI/indepth-10M"))
config = IndepthConfig(**json.loads((directory / "config.json").read_text()))
model = IndepthModel(config)
try:
from safetensors.torch import load_file
sd = load_file(str(directory / "model.safetensors"))
except ImportError:
from safetensors.numpy import load_file
sd = {k: torch.from_numpy(v) for k, v in load_file(str(directory / "model.safetensors")).items()}
model.load_state_dict(sd)
return model.eval(), config, directory
@torch.no_grad()
def generate(model, config, token_rows, subword_rows, seed=13372000, steps=4, guidance=1.5, device="cpu", generator=None):
"""Rectified-flow sampling with classifier-free guidance. Returns (B, 4, 16, 16) in [-1, 1]."""
token_rows = token_rows.to(device)
subword_rows = subword_rows.to(device) if subword_rows is not None else None
batch = token_rows.shape[0]
gen = generator or torch.Generator(device=device).manual_seed(seed)
image = torch.randn((batch, config.in_channels, config.image_size, config.image_size), generator=gen, device=device)
null_rows = torch.zeros_like(token_rows)
null_sub = torch.zeros_like(subword_rows) if subword_rows is not None else None
for index in range(steps):
t = torch.full((batch,), index / steps, device=device)
d = torch.full((batch,), 1.0 / steps, device=device)
if guidance != 1.0:
v_uncond = model(image, t, d, null_rows, null_sub).float()
v_cond = model(image, t, d, token_rows, subword_rows).float()
velocity = v_uncond + guidance * (v_cond - v_uncond)
else:
velocity = model(image, t, d, token_rows, subword_rows).float()
image = image + (1.0 / steps) * velocity
image = image.clamp(-1, 1)
image[:, 3] = torch.where(image[:, 3] >= 0, 1.0, -1.0)
image[:, :3] = torch.where(image[:, 3:4] >= 0, image[:, :3], 0.0)
return image
def decode_to_rgba(image):
"""(B, 4, 16, 16) in [-1, 1] -> (B, 16, 16, 4) uint8."""
arr = ((image.cpu().permute(0, 2, 3, 1).numpy() + 1.0) * 127.5).round().clip(0, 255).astype("uint8")
return arr