"""InDepth model definition — self-contained PyTorch implementation. A tiny rectified-flow DiT for 16x16 RGBA sprites, conditioned on captions through a word + hashed char-trigram subword text encoder trained jointly from scratch. Weights: model.safetensors (EMA), config.json, vocab.json. """ import json import math from pathlib import Path import torch from torch import nn from torch.nn import functional as F class IndepthConfig: def __init__(self, image_size=16, in_channels=4, patch_size=2, dim=256, depth=8, heads=4, cond_dim=128, vocab_size=8192, text_dim=128, text_layers=2, text_heads=4, max_len=16, subword_buckets=4096, **kwargs): self.image_size = image_size self.in_channels = in_channels self.patch_size = patch_size self.dim = dim self.depth = depth self.heads = heads self.cond_dim = cond_dim self.vocab_size = vocab_size self.text_dim = text_dim self.text_layers = text_layers self.text_heads = text_heads self.max_len = max_len self.subword_buckets = subword_buckets if dim % heads or dim % 4 or text_dim % text_heads: raise ValueError("dims must divide head counts") def _time_embedding(values, dim): half = dim // 2 freqs = torch.exp(-math.log(10000) * torch.arange(half, dtype=torch.float32, device=values.device) / half) args = values[:, None].float() * freqs[None] return torch.cat([torch.sin(args), torch.cos(args)], dim=-1) def _grid_embedding(height, width, dim, device): half = dim // 4 y = torch.arange(height, device=device).float().unsqueeze(1).expand(height, width).reshape(-1) x = torch.arange(width, device=device).float().unsqueeze(0).expand(height, width).reshape(-1) freqs = torch.exp(-math.log(10000) * torch.arange(half, device=device).float() / half) args_x = x[:, None] * freqs[None] args_y = y[:, None] * freqs[None] return torch.cat([torch.sin(args_x), torch.cos(args_x), torch.sin(args_y), torch.cos(args_y)], dim=-1) class IndepthDiTBlock(nn.Module): """adaLN-Zero transformer block over 2x2 patch tokens (matches the trained weights).""" def __init__(self, cfg): super().__init__() self.norm1 = nn.LayerNorm(cfg.dim, elementwise_affine=False) self.qkv = nn.Linear(cfg.dim, cfg.dim * 3) self.proj = nn.Linear(cfg.dim, cfg.dim) self.norm2 = nn.LayerNorm(cfg.dim, elementwise_affine=False) self.fc1 = nn.Linear(cfg.dim, cfg.dim * 4) self.fc2 = nn.Linear(cfg.dim * 4, cfg.dim) self.modulation = nn.Sequential(nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.dim * 6)) nn.init.zeros_(self.modulation[-1].weight) nn.init.zeros_(self.modulation[-1].bias) def forward(self, tokens, condition): cfg = self.config_ref batch, count, dim = tokens.shape mod = self.modulation(condition) shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1) normed = self.norm1(tokens) * (1.0 + scale_msa[:, None, :]) + shift_msa[:, None, :] qkv = self.qkv(normed).reshape(batch, count, 3, cfg.heads, dim // cfg.heads).permute(2, 0, 3, 1, 4) attended = F.scaled_dot_product_attention(qkv[0], qkv[1], qkv[2]) attended = attended.transpose(1, 2).reshape(batch, count, dim) tokens = tokens + gate_msa[:, None, :] * self.proj(attended) normed = self.norm2(tokens) * (1.0 + scale_mlp[:, None, :]) + shift_mlp[:, None, :] hidden = F.gelu(self.fc1(normed), approximate="tanh") return tokens + gate_mlp[:, None, :] * self.fc2(hidden) class IndepthTextBlock(nn.Module): def __init__(self, width, heads): super().__init__() self.heads = heads self.head_dim = width // heads self.norm1 = nn.LayerNorm(width) self.qkv = nn.Linear(width, width * 3) self.proj = nn.Linear(width, width) self.norm2 = nn.LayerNorm(width) self.fc1 = nn.Linear(width, width * 4) self.fc2 = nn.Linear(width * 4, width) def forward(self, tokens, pad_mask): batch, count, width = tokens.shape normalized = self.norm1(tokens) qkv = self.qkv(normalized).reshape(batch, count, 3, self.heads, self.head_dim).permute(2, 0, 3, 1, 4) additive = torch.zeros((batch, 1, 1, count), device=tokens.device, dtype=qkv[0].dtype) additive.masked_fill_(pad_mask[:, None, None, :], float("-inf")) attended = F.scaled_dot_product_attention(qkv[0], qkv[1], qkv[2], attn_mask=additive) attended = attended.transpose(1, 2).reshape(batch, count, width) tokens = tokens + self.proj(attended) hidden = F.gelu(self.fc1(self.norm2(tokens)), approximate="tanh") return tokens + self.fc2(hidden) class IndepthTextEncoder(nn.Module): """Word embeddings + hashed char-trigram subword features, 2-layer transformer, masked mean.""" SUBWORD_SLOTS = 16 def __init__(self, config): super().__init__() self.subword_buckets = int(config.subword_buckets or 0) self.token_embedding = nn.Embedding(config.vocab_size, config.text_dim) self.position_embedding = nn.Parameter(torch.zeros(1, config.max_len, config.text_dim)) nn.init.normal_(self.position_embedding, std=0.02) self.subword_embedding = None if self.subword_buckets: self.subword_embedding = nn.Embedding(self.subword_buckets, config.text_dim, padding_idx=0) self.blocks = nn.ModuleList([IndepthTextBlock(config.text_dim, config.text_heads) for _ in range(config.text_layers)]) self.final_norm = nn.LayerNorm(config.text_dim) self.out = nn.Linear(config.text_dim, config.cond_dim) def forward(self, tokens, subwords=None): mask = tokens != 0 hidden = self.token_embedding(tokens) + self.position_embedding[:, : tokens.shape[1]] if self.subword_embedding is not None: if subwords is None: subwords = torch.zeros((*tokens.shape, self.SUBWORD_SLOTS), dtype=tokens.dtype, device=tokens.device) counts = (subwords != 0).sum(dim=-1, keepdim=True).clamp(min=1).to(hidden.dtype) hidden = hidden + self.subword_embedding(subwords).sum(dim=2) / counts for block in self.blocks: hidden = block(hidden, ~mask) weights = mask.to(hidden.dtype) mean = (hidden * weights[:, :, None]).sum(dim=1) / weights.sum(dim=1, keepdim=True).clamp(min=1.0) return self.out(self.final_norm(mean)) class IndepthModel(nn.Module): """Rectified-flow DiT over 16x16 RGBA (4 x 16x16 in [-1, 1], alpha in {-1, 1}).""" def __init__(self, config=None): super().__init__() self.config = config or IndepthConfig() cfg = self.config self.grid_size = cfg.image_size // cfg.patch_size self.patch_embed = nn.Conv2d(cfg.in_channels, cfg.dim, kernel_size=cfg.patch_size, stride=cfg.patch_size) self.register_buffer("position", _grid_embedding(self.grid_size, self.grid_size, cfg.dim, torch.device("cpu")).unsqueeze(0), persistent=False) self.time_mlp = nn.Sequential(nn.Linear(cfg.cond_dim, cfg.cond_dim), nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.cond_dim)) self.step_mlp = nn.Sequential(nn.Linear(cfg.cond_dim, cfg.cond_dim), nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.cond_dim)) self.text_encoder = IndepthTextEncoder(cfg) self.text_proj = nn.Linear(cfg.cond_dim, cfg.cond_dim) self.blocks = nn.ModuleList() for _ in range(cfg.depth): block = IndepthDiTBlock(cfg) block.config_ref = cfg self.blocks.append(block) self.final_modulation = nn.Sequential(nn.SiLU(), nn.Linear(cfg.cond_dim, cfg.dim * 2)) self.final_norm = nn.LayerNorm(cfg.dim, elementwise_affine=False) self.output = nn.Linear(cfg.dim, cfg.patch_size * cfg.patch_size * cfg.in_channels) def forward(self, image, time, step_size, tokens, subwords=None): cfg = self.config condition = self.time_mlp(_time_embedding(time, cfg.cond_dim)) condition = condition + self.step_mlp(_time_embedding(step_size, cfg.cond_dim)) condition = condition + self.text_proj(self.text_encoder(tokens, subwords)) tokens_image = self.patch_embed(image).flatten(2).transpose(1, 2) tokens_image = tokens_image + self.position.to(dtype=tokens_image.dtype) for block in self.blocks: tokens_image = block(tokens_image, condition) shift, scale = self.final_modulation(condition).chunk(2, dim=-1) tokens_image = self.final_norm(tokens_image) * (1.0 + scale[:, None, :]) + shift[:, None, :] patches = self.output(tokens_image) batch = image.shape[0] patches = patches.reshape(batch, self.grid_size, self.grid_size, cfg.in_channels, cfg.patch_size, cfg.patch_size) return patches.permute(0, 3, 1, 4, 2, 5).reshape(batch, cfg.in_channels, cfg.image_size, cfg.image_size) def load_indepth(model_dir): """Load the model + tokenizer from a local directory or the Hugging Face hub.""" import numpy as np from safetensors.torch import load_file directory = Path(model_dir) if not directory.exists(): from huggingface_hub import snapshot_download directory = Path(snapshot_download("DexagonAI/indepth-10M")) config = IndepthConfig(**json.loads((directory / "config.json").read_text())) model = IndepthModel(config) try: from safetensors.torch import load_file sd = load_file(str(directory / "model.safetensors")) except ImportError: from safetensors.numpy import load_file sd = {k: torch.from_numpy(v) for k, v in load_file(str(directory / "model.safetensors")).items()} model.load_state_dict(sd) return model.eval(), config, directory @torch.no_grad() def generate(model, config, token_rows, subword_rows, seed=13372000, steps=4, guidance=1.5, device="cpu", generator=None): """Rectified-flow sampling with classifier-free guidance. Returns (B, 4, 16, 16) in [-1, 1].""" token_rows = token_rows.to(device) subword_rows = subword_rows.to(device) if subword_rows is not None else None batch = token_rows.shape[0] gen = generator or torch.Generator(device=device).manual_seed(seed) image = torch.randn((batch, config.in_channels, config.image_size, config.image_size), generator=gen, device=device) null_rows = torch.zeros_like(token_rows) null_sub = torch.zeros_like(subword_rows) if subword_rows is not None else None for index in range(steps): t = torch.full((batch,), index / steps, device=device) d = torch.full((batch,), 1.0 / steps, device=device) if guidance != 1.0: v_uncond = model(image, t, d, null_rows, null_sub).float() v_cond = model(image, t, d, token_rows, subword_rows).float() velocity = v_uncond + guidance * (v_cond - v_uncond) else: velocity = model(image, t, d, token_rows, subword_rows).float() image = image + (1.0 / steps) * velocity image = image.clamp(-1, 1) image[:, 3] = torch.where(image[:, 3] >= 0, 1.0, -1.0) image[:, :3] = torch.where(image[:, 3:4] >= 0, image[:, :3], 0.0) return image def decode_to_rgba(image): """(B, 4, 16, 16) in [-1, 1] -> (B, 16, 16, 4) uint8.""" arr = ((image.cpu().permute(0, 2, 3, 1).numpy() + 1.0) * 127.5).round().clip(0, 255).astype("uint8") return arr