Download train.py from YesWhyNot/alosha-painter: direct link, hf CLI and curl.
- Browser
- Download file 9.69 kB
-
https://huggingface.co/YesWhyNot/alosha-painter/resolve/main/train.py
- Command line
-
hf download hf://YesWhyNot/alosha-painter/train.py
-
curl -L -o train.py https://huggingface.co/YesWhyNot/alosha-painter/resolve/main/train.py
9.69 kB
| import os | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import torch.optim as optim | |
| from torchvision import datasets, transforms | |
| from torchvision.utils import save_image | |
| IMAGE_SIZE = 64 | |
| CHANNELS = 3 | |
| BATCH_SIZE = 64 | |
| EPOCHS = 1000 | |
| SAMPLE_STEPS = 20 | |
| LR = 2e-4 | |
| SAVE_EVERY = 10 | |
| DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| CHECKPOINT_DIR = "weights" | |
| OUTPUT_DIR = "outputs" | |
| LATEST_CHECKPOINT = os.path.join(CHECKPOINT_DIR, "checkpoint_latest.pth") | |
| # --- 1. EMA Helper --- | |
| class EMA: | |
| def __init__(self, model, decay=0.999): | |
| self.decay = decay | |
| self.shadow = {k: v.clone().detach() for k, v in model.state_dict().items()} | |
| def update(self, model): | |
| with torch.no_grad(): | |
| for k, v in model.state_dict().items(): | |
| self.shadow[k].data.mul_(self.decay).add_(v.data, alpha=1 - self.decay) | |
| def apply_shadow(self, model): | |
| self.backup = {k: v.clone().detach() for k, v in model.state_dict().items()} | |
| model.load_state_dict(self.shadow) | |
| def restore(self, model): | |
| model.load_state_dict(self.backup) | |
| # --- 2. Continuous Time Position Embeddings --- | |
| class SinusoidalPositionEmbeddings(nn.Module): | |
| def __init__(self, dim): | |
| super().__init__() | |
| self.dim = dim | |
| def forward(self, time): | |
| device = time.device | |
| half_dim = self.dim // 2 | |
| embeddings = math.log(10000) / (half_dim - 1) | |
| embeddings = torch.exp(torch.arange(half_dim, device=device) * -embeddings) | |
| embeddings = time[:, None] * embeddings[None, :] | |
| embeddings = torch.cat((embeddings.sin(), embeddings.cos()), dim=-1) | |
| return embeddings | |
| # --- 3. Residual Block with GroupNorm --- | |
| class ResBlock(nn.Module): | |
| def __init__(self, in_ch, out_ch, time_emb_dim): | |
| super().__init__() | |
| self.time_mlp = nn.Linear(time_emb_dim, out_ch) | |
| self.gn1 = nn.GroupNorm(8 if in_ch >= 8 else 1, in_ch) | |
| self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=1) | |
| self.gn2 = nn.GroupNorm(8 if out_ch >= 8 else 1, out_ch) | |
| self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1) | |
| if in_ch != out_ch: | |
| self.shortcut = nn.Conv2d(in_ch, out_ch, 1) | |
| else: | |
| self.shortcut = nn.Identity() | |
| def forward(self, x, t_emb): | |
| h = F.silu(self.gn1(x)) | |
| h = self.conv1(h) | |
| # Add time conditioning | |
| t_proj = F.silu(self.time_mlp(t_emb))[(..., None, None)] | |
| h = h + t_proj | |
| h = F.silu(self.gn2(h)) | |
| h = self.conv2(h) | |
| return h + self.shortcut(x) | |
| class FlowMatchingUNet(nn.Module): | |
| def __init__(self, in_channels=CHANNELS, time_emb_dim=128): | |
| super().__init__() | |
| self.time_mlp = nn.Sequential( | |
| SinusoidalPositionEmbeddings(time_emb_dim), | |
| nn.Linear(time_emb_dim, time_emb_dim), | |
| nn.SiLU(), | |
| nn.Linear(time_emb_dim, time_emb_dim), | |
| ) | |
| self.in_conv = nn.Conv2d(in_channels, 64, 3, padding=1) | |
| # Downsampling path | |
| self.down1 = ResBlock(64, 64, time_emb_dim) | |
| self.down2_conv = nn.Conv2d(64, 128, 4, stride=2, padding=1) | |
| self.down2 = ResBlock(128, 128, time_emb_dim) | |
| self.down3_conv = nn.Conv2d(128, 256, 4, stride=2, padding=1) | |
| self.down3 = ResBlock(256, 256, time_emb_dim) | |
| # Bottleneck | |
| self.bot1 = ResBlock(256, 256, time_emb_dim) | |
| self.bot2 = ResBlock(256, 256, time_emb_dim) | |
| # Upsampling path | |
| self.up1_conv = nn.ConvTranspose2d(256, 128, 4, stride=2, padding=1) | |
| self.up1 = ResBlock(256, 128, time_emb_dim) | |
| self.up2_conv = nn.ConvTranspose2d(128, 64, 4, stride=2, padding=1) | |
| self.up2 = ResBlock(128, 64, time_emb_dim) | |
| self.out_gn = nn.GroupNorm(8, 64) | |
| self.out_conv = nn.Conv2d(64, in_channels, 3, padding=1) | |
| def forward(self, x, t): | |
| t_emb = self.time_mlp(t) | |
| x_in = self.in_conv(x) | |
| # Downsample | |
| x1 = self.down1(x_in, t_emb) # [B, 64, 64, 64] | |
| x2 = self.down2(self.down2_conv(x1), t_emb) # [B, 128, 32, 32] | |
| x3 = self.down3(self.down3_conv(x2), t_emb) # [B, 256, 16, 16] | |
| # Bottleneck | |
| h = self.bot1(x3, t_emb) | |
| h = self.bot2(h, t_emb) | |
| # Upsample with Skip Connections | |
| h = self.up1_conv(h) # [B, 128, 32, 32] | |
| h = self.up1(torch.cat([h, x2], dim=1), t_emb) # [B, 256, 32, 32] -> [B, 128, 32, 32] | |
| h = self.up2_conv(h) # [B, 64, 64, 64] | |
| h = self.up2(torch.cat([h, x1], dim=1), t_emb) # [B, 128, 64, 64] -> [B, 64, 64, 64] | |
| h = F.silu(self.out_gn(h)) | |
| return self.out_conv(h) | |
| # --- 4. Euler ODE Sampler --- | |
| def sample_images(model, num_samples=16, steps=SAMPLE_STEPS): | |
| model.eval() | |
| x_t = torch.randn(num_samples, CHANNELS, IMAGE_SIZE, IMAGE_SIZE, device=DEVICE) | |
| dt = 1.0 / steps | |
| for step in range(steps): | |
| t_val = step / steps | |
| t_batch = torch.full((num_samples,), t_val, device=DEVICE, dtype=torch.float32) | |
| v_pred = model(x_t, t_batch) | |
| x_t = x_t + v_pred * dt | |
| # Step clamp to prevent trajectory drift | |
| x_t = x_t.clamp(-1.5, 1.5) | |
| return (x_t.clamp(-1.0, 1.0) + 1.0) / 2.0 | |
| if __name__ == '__main__': | |
| if DEVICE.type == "cuda": | |
| torch.backends.cudnn.benchmark = True | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| os.makedirs(CHECKPOINT_DIR, exist_ok=True) | |
| transform = transforms.Compose([ | |
| transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)), | |
| transforms.CenterCrop(IMAGE_SIZE), | |
| transforms.RandomHorizontalFlip(p=0.5), | |
| transforms.ToTensor(), | |
| transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), | |
| ]) | |
| dataset = datasets.ImageFolder(root="data", transform=transform) | |
| dataloader = torch.utils.data.DataLoader( | |
| dataset, | |
| batch_size=BATCH_SIZE, | |
| shuffle=True, | |
| drop_last=True, | |
| num_workers=4 if os.cpu_count() >= 4 else 2, | |
| pin_memory=True if DEVICE.type == "cuda" else False, | |
| persistent_workers=True, | |
| prefetch_factor=2 | |
| ) | |
| model = FlowMatchingUNet().to(DEVICE) | |
| ema = EMA(model, decay=0.999) | |
| optimizer = optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4) | |
| scaler = torch.amp.GradScaler('cuda', enabled=(DEVICE.type == "cuda")) | |
| start_epoch = 0 | |
| if os.path.exists(LATEST_CHECKPOINT): | |
| print(f"Loading checkpoint from {LATEST_CHECKPOINT}...") | |
| try: | |
| ckpt = torch.load(LATEST_CHECKPOINT, map_location=DEVICE, weights_only=True) | |
| model.load_state_dict(ckpt['model']) | |
| optimizer.load_state_dict(ckpt['optimizer']) | |
| scaler.load_state_dict(ckpt['scaler']) | |
| if 'ema' in ckpt: | |
| ema.shadow = ckpt['ema'] | |
| start_epoch = ckpt['epoch'] + 1 | |
| print(f"Resumed successfully at Epoch {start_epoch + 1}.") | |
| except (KeyError, RuntimeError) as e: | |
| print(f"Incompatible checkpoint detected ({e}). Starting fresh from Epoch 1...") | |
| start_epoch = 0 | |
| print("Starting Flow Matching Training Loop...") | |
| for epoch in range(start_epoch, EPOCHS): | |
| model.train() | |
| total_loss = 0.0 | |
| for i, (x1, _) in enumerate(dataloader): | |
| x1 = x1.to(DEVICE, non_blocking=True) | |
| b_size = x1.size(0) | |
| x0 = torch.randn_like(x1) | |
| t = torch.rand(b_size, device=DEVICE) | |
| t_expanded = t[:, None, None, None] | |
| x_t = (1.0 - t_expanded) * x0 + t_expanded * x1 | |
| v_target = x1 - x0 | |
| optimizer.zero_grad(set_to_none=True) | |
| with torch.amp.autocast('cuda', enabled=(DEVICE.type == "cuda")): | |
| v_pred = model(x_t, t) | |
| loss = F.mse_loss(v_pred, v_target) | |
| scaler.scale(loss).backward() | |
| # Gradient clipping to prevent exploding velocity vectors | |
| scaler.unscale_(optimizer) | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) | |
| scaler.step(optimizer) | |
| scaler.update() | |
| # Update EMA weights | |
| ema.update(model) | |
| total_loss += loss.item() | |
| avg_loss = total_loss / len(dataloader) | |
| if (epoch + 1) % SAVE_EVERY == 0 or (epoch + 1) == EPOCHS: | |
| print(f"Epoch [{epoch+1}/{EPOCHS}] | Velocity MSE Loss: {avg_loss:.5f}") | |
| # Sample using EMA weights | |
| ema.apply_shadow(model) | |
| samples = sample_images(model, num_samples=16, steps=SAMPLE_STEPS) | |
| ema.restore(model) | |
| save_image(samples, os.path.join(OUTPUT_DIR, f"epoch_{epoch+1:03d}.png"), nrow=4) | |
| # Save state including EMA | |
| checkpoint_data = { | |
| 'epoch': epoch, | |
| 'model': model.state_dict(), | |
| 'ema': ema.shadow, | |
| 'optimizer': optimizer.state_dict(), | |
| 'scaler': scaler.state_dict(), | |
| } | |
| torch.save(checkpoint_data, LATEST_CHECKPOINT) | |
| torch.save(model.state_dict(), os.path.join(CHECKPOINT_DIR, f"model_epoch_{epoch+1:03d}.pth")) | |
| print("Training finished.") |