File size: 2,049 Bytes
6be6d8e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """PyTorch model definitions for the training debugger.
SimpleCNN (~50K params) and SimpleMLP (~20K params).
"""
from __future__ import annotations
import torch
import torch.nn as nn
class SimpleCNN(nn.Module):
"""3-layer CNN for CIFAR-10 style classification. ~50K params."""
def __init__(self, num_layers: int = 3, hidden_dim: int = 64) -> None:
super().__init__()
self.conv1 = nn.Conv2d(3, 32, 3, padding=1)
self.bn1 = nn.BatchNorm2d(32)
self.conv2 = nn.Conv2d(32, 64, 3, padding=1)
self.bn2 = nn.BatchNorm2d(64)
self.conv3 = nn.Conv2d(64, 64, 3, padding=1)
self.bn3 = nn.BatchNorm2d(64)
self.fc = nn.Linear(64 * 4 * 4, 10)
self.pool = nn.MaxPool2d(2, 2)
self.relu = nn.ReLU()
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = self.pool(self.relu(self.bn1(self.conv1(x))))
x = self.pool(self.relu(self.bn2(self.conv2(x))))
x = self.pool(self.relu(self.bn3(self.conv3(x))))
x = x.view(x.size(0), -1)
x = self.fc(x)
return x
class SimpleMLP(nn.Module):
"""3-layer MLP for CIFAR-10 style classification. ~20K params."""
def __init__(
self,
input_dim: int = 3072,
hidden_dim: int = 128,
num_classes: int = 10,
) -> None:
super().__init__()
self.flatten = nn.Flatten()
self.fc1 = nn.Linear(input_dim, hidden_dim)
self.bn1 = nn.BatchNorm1d(hidden_dim)
self.fc2 = nn.Linear(hidden_dim, hidden_dim)
self.bn2 = nn.BatchNorm1d(hidden_dim)
self.fc3 = nn.Linear(hidden_dim, num_classes)
self.relu = nn.ReLU()
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = self.flatten(x)
x = self.relu(self.bn1(self.fc1(x)))
x = self.relu(self.bn2(self.fc2(x)))
x = self.fc3(x)
return x
def create_model(model_type: str) -> nn.Module:
"""Create a model by type string."""
if model_type == "mlp":
return SimpleMLP()
return SimpleCNN()
|