File size: 2,305 Bytes
7d90be6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | from __future__ import annotations
import torch
from torch import nn
class ConditionalEnergyNetwork(nn.Module):
def __init__(self) -> None:
super().__init__()
self.features = nn.Sequential(
nn.Linear(64, 128),
nn.SiLU(),
nn.Linear(128, 64),
nn.SiLU(),
)
self.energy_heads = nn.Linear(64, 10)
def all_energies(self, pixels: torch.Tensor) -> torch.Tensor:
return self.energy_heads(self.features(pixels))
def forward(self, pixels: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
energies = self.all_energies(pixels)
return energies.gather(1, labels[:, None]).squeeze(1)
def langevin_sample(
model: ConditionalEnergyNetwork,
pixels: torch.Tensor,
labels: torch.Tensor,
*,
steps: int,
step_size: float,
noise_scale: float,
generator: torch.Generator | None = None,
) -> torch.Tensor:
was_training = model.training
model.eval()
current = pixels.detach().clone()
for _ in range(steps):
current.requires_grad_(True)
energy = model(current, labels).sum()
gradient = torch.autograd.grad(energy, current)[0]
with torch.no_grad():
noise = torch.randn(
current.shape,
generator=generator,
device=current.device,
)
current = current - step_size * gradient + noise_scale * noise
current.clamp_(0, 1)
model.train(was_training)
return current.detach()
class TinyVisionJudge(nn.Module):
def __init__(self) -> None:
super().__init__()
self.features = nn.Sequential(
nn.Conv2d(1, 8, kernel_size=3, padding=1),
nn.GELU(),
nn.Conv2d(8, 8, kernel_size=3, padding=1, groups=8),
nn.GELU(),
nn.Conv2d(8, 12, kernel_size=1),
nn.GELU(),
nn.MaxPool2d(2),
)
self.classifier = nn.Sequential(
nn.Flatten(),
nn.Linear(12 * 4 * 4, 10),
)
def forward(self, pixels: torch.Tensor) -> torch.Tensor:
return self.classifier(self.features(pixels))
def parameter_count(model: nn.Module) -> int:
return sum(parameter.numel() for parameter in model.parameters())
|