alosha-painter / train.py
YesWhyNot's picture
Upload folder using huggingface_hub
fa69b72 verified
Raw History Blame Contribute Delete
9.69 kB
import os
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torchvision.utils import save_image
IMAGE_SIZE = 64
CHANNELS = 3
BATCH_SIZE = 64
EPOCHS = 1000
SAMPLE_STEPS = 20
LR = 2e-4
SAVE_EVERY = 10
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
CHECKPOINT_DIR = "weights"
OUTPUT_DIR = "outputs"
LATEST_CHECKPOINT = os.path.join(CHECKPOINT_DIR, "checkpoint_latest.pth")
# --- 1. EMA Helper ---
class EMA:
def __init__(self, model, decay=0.999):
self.decay = decay
self.shadow = {k: v.clone().detach() for k, v in model.state_dict().items()}
def update(self, model):
with torch.no_grad():
for k, v in model.state_dict().items():
self.shadow[k].data.mul_(self.decay).add_(v.data, alpha=1 - self.decay)
def apply_shadow(self, model):
self.backup = {k: v.clone().detach() for k, v in model.state_dict().items()}
model.load_state_dict(self.shadow)
def restore(self, model):
model.load_state_dict(self.backup)
# --- 2. Continuous Time Position Embeddings ---
class SinusoidalPositionEmbeddings(nn.Module):
def __init__(self, dim):
super().__init__()
self.dim = dim
def forward(self, time):
device = time.device
half_dim = self.dim // 2
embeddings = math.log(10000) / (half_dim - 1)
embeddings = torch.exp(torch.arange(half_dim, device=device) * -embeddings)
embeddings = time[:, None] * embeddings[None, :]
embeddings = torch.cat((embeddings.sin(), embeddings.cos()), dim=-1)
return embeddings
# --- 3. Residual Block with GroupNorm ---
class ResBlock(nn.Module):
def __init__(self, in_ch, out_ch, time_emb_dim):
super().__init__()
self.time_mlp = nn.Linear(time_emb_dim, out_ch)
self.gn1 = nn.GroupNorm(8 if in_ch >= 8 else 1, in_ch)
self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=1)
self.gn2 = nn.GroupNorm(8 if out_ch >= 8 else 1, out_ch)
self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1)
if in_ch != out_ch:
self.shortcut = nn.Conv2d(in_ch, out_ch, 1)
else:
self.shortcut = nn.Identity()
def forward(self, x, t_emb):
h = F.silu(self.gn1(x))
h = self.conv1(h)
# Add time conditioning
t_proj = F.silu(self.time_mlp(t_emb))[(..., None, None)]
h = h + t_proj
h = F.silu(self.gn2(h))
h = self.conv2(h)
return h + self.shortcut(x)
class FlowMatchingUNet(nn.Module):
def __init__(self, in_channels=CHANNELS, time_emb_dim=128):
super().__init__()
self.time_mlp = nn.Sequential(
SinusoidalPositionEmbeddings(time_emb_dim),
nn.Linear(time_emb_dim, time_emb_dim),
nn.SiLU(),
nn.Linear(time_emb_dim, time_emb_dim),
)
self.in_conv = nn.Conv2d(in_channels, 64, 3, padding=1)
# Downsampling path
self.down1 = ResBlock(64, 64, time_emb_dim)
self.down2_conv = nn.Conv2d(64, 128, 4, stride=2, padding=1)
self.down2 = ResBlock(128, 128, time_emb_dim)
self.down3_conv = nn.Conv2d(128, 256, 4, stride=2, padding=1)
self.down3 = ResBlock(256, 256, time_emb_dim)
# Bottleneck
self.bot1 = ResBlock(256, 256, time_emb_dim)
self.bot2 = ResBlock(256, 256, time_emb_dim)
# Upsampling path
self.up1_conv = nn.ConvTranspose2d(256, 128, 4, stride=2, padding=1)
self.up1 = ResBlock(256, 128, time_emb_dim)
self.up2_conv = nn.ConvTranspose2d(128, 64, 4, stride=2, padding=1)
self.up2 = ResBlock(128, 64, time_emb_dim)
self.out_gn = nn.GroupNorm(8, 64)
self.out_conv = nn.Conv2d(64, in_channels, 3, padding=1)
def forward(self, x, t):
t_emb = self.time_mlp(t)
x_in = self.in_conv(x)
# Downsample
x1 = self.down1(x_in, t_emb) # [B, 64, 64, 64]
x2 = self.down2(self.down2_conv(x1), t_emb) # [B, 128, 32, 32]
x3 = self.down3(self.down3_conv(x2), t_emb) # [B, 256, 16, 16]
# Bottleneck
h = self.bot1(x3, t_emb)
h = self.bot2(h, t_emb)
# Upsample with Skip Connections
h = self.up1_conv(h) # [B, 128, 32, 32]
h = self.up1(torch.cat([h, x2], dim=1), t_emb) # [B, 256, 32, 32] -> [B, 128, 32, 32]
h = self.up2_conv(h) # [B, 64, 64, 64]
h = self.up2(torch.cat([h, x1], dim=1), t_emb) # [B, 128, 64, 64] -> [B, 64, 64, 64]
h = F.silu(self.out_gn(h))
return self.out_conv(h)
# --- 4. Euler ODE Sampler ---
@torch.no_grad()
def sample_images(model, num_samples=16, steps=SAMPLE_STEPS):
model.eval()
x_t = torch.randn(num_samples, CHANNELS, IMAGE_SIZE, IMAGE_SIZE, device=DEVICE)
dt = 1.0 / steps
for step in range(steps):
t_val = step / steps
t_batch = torch.full((num_samples,), t_val, device=DEVICE, dtype=torch.float32)
v_pred = model(x_t, t_batch)
x_t = x_t + v_pred * dt
# Step clamp to prevent trajectory drift
x_t = x_t.clamp(-1.5, 1.5)
return (x_t.clamp(-1.0, 1.0) + 1.0) / 2.0
if __name__ == '__main__':
if DEVICE.type == "cuda":
torch.backends.cudnn.benchmark = True
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs(CHECKPOINT_DIR, exist_ok=True)
transform = transforms.Compose([
transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),
transforms.CenterCrop(IMAGE_SIZE),
transforms.RandomHorizontalFlip(p=0.5),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
dataset = datasets.ImageFolder(root="data", transform=transform)
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=BATCH_SIZE,
shuffle=True,
drop_last=True,
num_workers=4 if os.cpu_count() >= 4 else 2,
pin_memory=True if DEVICE.type == "cuda" else False,
persistent_workers=True,
prefetch_factor=2
)
model = FlowMatchingUNet().to(DEVICE)
ema = EMA(model, decay=0.999)
optimizer = optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4)
scaler = torch.amp.GradScaler('cuda', enabled=(DEVICE.type == "cuda"))
start_epoch = 0
if os.path.exists(LATEST_CHECKPOINT):
print(f"Loading checkpoint from {LATEST_CHECKPOINT}...")
try:
ckpt = torch.load(LATEST_CHECKPOINT, map_location=DEVICE, weights_only=True)
model.load_state_dict(ckpt['model'])
optimizer.load_state_dict(ckpt['optimizer'])
scaler.load_state_dict(ckpt['scaler'])
if 'ema' in ckpt:
ema.shadow = ckpt['ema']
start_epoch = ckpt['epoch'] + 1
print(f"Resumed successfully at Epoch {start_epoch + 1}.")
except (KeyError, RuntimeError) as e:
print(f"Incompatible checkpoint detected ({e}). Starting fresh from Epoch 1...")
start_epoch = 0
print("Starting Flow Matching Training Loop...")
for epoch in range(start_epoch, EPOCHS):
model.train()
total_loss = 0.0
for i, (x1, _) in enumerate(dataloader):
x1 = x1.to(DEVICE, non_blocking=True)
b_size = x1.size(0)
x0 = torch.randn_like(x1)
t = torch.rand(b_size, device=DEVICE)
t_expanded = t[:, None, None, None]
x_t = (1.0 - t_expanded) * x0 + t_expanded * x1
v_target = x1 - x0
optimizer.zero_grad(set_to_none=True)
with torch.amp.autocast('cuda', enabled=(DEVICE.type == "cuda")):
v_pred = model(x_t, t)
loss = F.mse_loss(v_pred, v_target)
scaler.scale(loss).backward()
# Gradient clipping to prevent exploding velocity vectors
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
scaler.step(optimizer)
scaler.update()
# Update EMA weights
ema.update(model)
total_loss += loss.item()
avg_loss = total_loss / len(dataloader)
if (epoch + 1) % SAVE_EVERY == 0 or (epoch + 1) == EPOCHS:
print(f"Epoch [{epoch+1}/{EPOCHS}] | Velocity MSE Loss: {avg_loss:.5f}")
# Sample using EMA weights
ema.apply_shadow(model)
samples = sample_images(model, num_samples=16, steps=SAMPLE_STEPS)
ema.restore(model)
save_image(samples, os.path.join(OUTPUT_DIR, f"epoch_{epoch+1:03d}.png"), nrow=4)
# Save state including EMA
checkpoint_data = {
'epoch': epoch,
'model': model.state_dict(),
'ema': ema.shadow,
'optimizer': optimizer.state_dict(),
'scaler': scaler.state_dict(),
}
torch.save(checkpoint_data, LATEST_CHECKPOINT)
torch.save(model.state_dict(), os.path.join(CHECKPOINT_DIR, f"model_epoch_{epoch+1:03d}.pth"))
print("Training finished.")