import os import math import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torchvision import datasets, transforms from torchvision.utils import save_image IMAGE_SIZE = 64 CHANNELS = 3 BATCH_SIZE = 64 EPOCHS = 1000 SAMPLE_STEPS = 20 LR = 2e-4 SAVE_EVERY = 10 DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") CHECKPOINT_DIR = "weights" OUTPUT_DIR = "outputs" LATEST_CHECKPOINT = os.path.join(CHECKPOINT_DIR, "checkpoint_latest.pth") # --- 1. EMA Helper --- class EMA: def __init__(self, model, decay=0.999): self.decay = decay self.shadow = {k: v.clone().detach() for k, v in model.state_dict().items()} def update(self, model): with torch.no_grad(): for k, v in model.state_dict().items(): self.shadow[k].data.mul_(self.decay).add_(v.data, alpha=1 - self.decay) def apply_shadow(self, model): self.backup = {k: v.clone().detach() for k, v in model.state_dict().items()} model.load_state_dict(self.shadow) def restore(self, model): model.load_state_dict(self.backup) # --- 2. Continuous Time Position Embeddings --- class SinusoidalPositionEmbeddings(nn.Module): def __init__(self, dim): super().__init__() self.dim = dim def forward(self, time): device = time.device half_dim = self.dim // 2 embeddings = math.log(10000) / (half_dim - 1) embeddings = torch.exp(torch.arange(half_dim, device=device) * -embeddings) embeddings = time[:, None] * embeddings[None, :] embeddings = torch.cat((embeddings.sin(), embeddings.cos()), dim=-1) return embeddings # --- 3. Residual Block with GroupNorm --- class ResBlock(nn.Module): def __init__(self, in_ch, out_ch, time_emb_dim): super().__init__() self.time_mlp = nn.Linear(time_emb_dim, out_ch) self.gn1 = nn.GroupNorm(8 if in_ch >= 8 else 1, in_ch) self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=1) self.gn2 = nn.GroupNorm(8 if out_ch >= 8 else 1, out_ch) self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1) if in_ch != out_ch: self.shortcut = nn.Conv2d(in_ch, out_ch, 1) else: self.shortcut = nn.Identity() def forward(self, x, t_emb): h = F.silu(self.gn1(x)) h = self.conv1(h) # Add time conditioning t_proj = F.silu(self.time_mlp(t_emb))[(..., None, None)] h = h + t_proj h = F.silu(self.gn2(h)) h = self.conv2(h) return h + self.shortcut(x) class FlowMatchingUNet(nn.Module): def __init__(self, in_channels=CHANNELS, time_emb_dim=128): super().__init__() self.time_mlp = nn.Sequential( SinusoidalPositionEmbeddings(time_emb_dim), nn.Linear(time_emb_dim, time_emb_dim), nn.SiLU(), nn.Linear(time_emb_dim, time_emb_dim), ) self.in_conv = nn.Conv2d(in_channels, 64, 3, padding=1) # Downsampling path self.down1 = ResBlock(64, 64, time_emb_dim) self.down2_conv = nn.Conv2d(64, 128, 4, stride=2, padding=1) self.down2 = ResBlock(128, 128, time_emb_dim) self.down3_conv = nn.Conv2d(128, 256, 4, stride=2, padding=1) self.down3 = ResBlock(256, 256, time_emb_dim) # Bottleneck self.bot1 = ResBlock(256, 256, time_emb_dim) self.bot2 = ResBlock(256, 256, time_emb_dim) # Upsampling path self.up1_conv = nn.ConvTranspose2d(256, 128, 4, stride=2, padding=1) self.up1 = ResBlock(256, 128, time_emb_dim) self.up2_conv = nn.ConvTranspose2d(128, 64, 4, stride=2, padding=1) self.up2 = ResBlock(128, 64, time_emb_dim) self.out_gn = nn.GroupNorm(8, 64) self.out_conv = nn.Conv2d(64, in_channels, 3, padding=1) def forward(self, x, t): t_emb = self.time_mlp(t) x_in = self.in_conv(x) # Downsample x1 = self.down1(x_in, t_emb) # [B, 64, 64, 64] x2 = self.down2(self.down2_conv(x1), t_emb) # [B, 128, 32, 32] x3 = self.down3(self.down3_conv(x2), t_emb) # [B, 256, 16, 16] # Bottleneck h = self.bot1(x3, t_emb) h = self.bot2(h, t_emb) # Upsample with Skip Connections h = self.up1_conv(h) # [B, 128, 32, 32] h = self.up1(torch.cat([h, x2], dim=1), t_emb) # [B, 256, 32, 32] -> [B, 128, 32, 32] h = self.up2_conv(h) # [B, 64, 64, 64] h = self.up2(torch.cat([h, x1], dim=1), t_emb) # [B, 128, 64, 64] -> [B, 64, 64, 64] h = F.silu(self.out_gn(h)) return self.out_conv(h) # --- 4. Euler ODE Sampler --- @torch.no_grad() def sample_images(model, num_samples=16, steps=SAMPLE_STEPS): model.eval() x_t = torch.randn(num_samples, CHANNELS, IMAGE_SIZE, IMAGE_SIZE, device=DEVICE) dt = 1.0 / steps for step in range(steps): t_val = step / steps t_batch = torch.full((num_samples,), t_val, device=DEVICE, dtype=torch.float32) v_pred = model(x_t, t_batch) x_t = x_t + v_pred * dt # Step clamp to prevent trajectory drift x_t = x_t.clamp(-1.5, 1.5) return (x_t.clamp(-1.0, 1.0) + 1.0) / 2.0 if __name__ == '__main__': if DEVICE.type == "cuda": torch.backends.cudnn.benchmark = True os.makedirs(OUTPUT_DIR, exist_ok=True) os.makedirs(CHECKPOINT_DIR, exist_ok=True) transform = transforms.Compose([ transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)), transforms.CenterCrop(IMAGE_SIZE), transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ]) dataset = datasets.ImageFolder(root="data", transform=transform) dataloader = torch.utils.data.DataLoader( dataset, batch_size=BATCH_SIZE, shuffle=True, drop_last=True, num_workers=4 if os.cpu_count() >= 4 else 2, pin_memory=True if DEVICE.type == "cuda" else False, persistent_workers=True, prefetch_factor=2 ) model = FlowMatchingUNet().to(DEVICE) ema = EMA(model, decay=0.999) optimizer = optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4) scaler = torch.amp.GradScaler('cuda', enabled=(DEVICE.type == "cuda")) start_epoch = 0 if os.path.exists(LATEST_CHECKPOINT): print(f"Loading checkpoint from {LATEST_CHECKPOINT}...") try: ckpt = torch.load(LATEST_CHECKPOINT, map_location=DEVICE, weights_only=True) model.load_state_dict(ckpt['model']) optimizer.load_state_dict(ckpt['optimizer']) scaler.load_state_dict(ckpt['scaler']) if 'ema' in ckpt: ema.shadow = ckpt['ema'] start_epoch = ckpt['epoch'] + 1 print(f"Resumed successfully at Epoch {start_epoch + 1}.") except (KeyError, RuntimeError) as e: print(f"Incompatible checkpoint detected ({e}). Starting fresh from Epoch 1...") start_epoch = 0 print("Starting Flow Matching Training Loop...") for epoch in range(start_epoch, EPOCHS): model.train() total_loss = 0.0 for i, (x1, _) in enumerate(dataloader): x1 = x1.to(DEVICE, non_blocking=True) b_size = x1.size(0) x0 = torch.randn_like(x1) t = torch.rand(b_size, device=DEVICE) t_expanded = t[:, None, None, None] x_t = (1.0 - t_expanded) * x0 + t_expanded * x1 v_target = x1 - x0 optimizer.zero_grad(set_to_none=True) with torch.amp.autocast('cuda', enabled=(DEVICE.type == "cuda")): v_pred = model(x_t, t) loss = F.mse_loss(v_pred, v_target) scaler.scale(loss).backward() # Gradient clipping to prevent exploding velocity vectors scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) scaler.step(optimizer) scaler.update() # Update EMA weights ema.update(model) total_loss += loss.item() avg_loss = total_loss / len(dataloader) if (epoch + 1) % SAVE_EVERY == 0 or (epoch + 1) == EPOCHS: print(f"Epoch [{epoch+1}/{EPOCHS}] | Velocity MSE Loss: {avg_loss:.5f}") # Sample using EMA weights ema.apply_shadow(model) samples = sample_images(model, num_samples=16, steps=SAMPLE_STEPS) ema.restore(model) save_image(samples, os.path.join(OUTPUT_DIR, f"epoch_{epoch+1:03d}.png"), nrow=4) # Save state including EMA checkpoint_data = { 'epoch': epoch, 'model': model.state_dict(), 'ema': ema.shadow, 'optimizer': optimizer.state_dict(), 'scaler': scaler.state_dict(), } torch.save(checkpoint_data, LATEST_CHECKPOINT) torch.save(model.state_dict(), os.path.join(CHECKPOINT_DIR, f"model_epoch_{epoch+1:03d}.pth")) print("Training finished.")