File size: 3,700 Bytes
562abfe
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
import argparse
import json
import time
from pathlib import Path

import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms

from model import CNN, count_params


def get_loaders(data_dir, batch_size):
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,)),
    ])
    train_set = datasets.MNIST(data_dir, train=True, download=True, transform=transform)
    val_set = datasets.MNIST(data_dir, train=False, download=True, transform=transform)
    train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=2)
    val_loader = DataLoader(val_set, batch_size=512, shuffle=False, num_workers=2)
    return train_loader, val_loader


@torch.no_grad()
def evaluate(model, loader, device):
    model.eval()
    correct = total = 0
    for x, y in loader:
        x, y = x.to(device), y.to(device)
        correct += (model(x).argmax(1) == y).sum().item()
        total += y.size(0)
    model.train()
    return correct / total


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--depth", type=int, required=True, help="number of blocks (each = 2 conv layers)")
    parser.add_argument("--residual", action="store_true")
    parser.add_argument("--no-bn", action="store_true")
    parser.add_argument("--channels", type=int, default=64)
    parser.add_argument("--epochs", type=int, default=15)
    parser.add_argument("--batch-size", type=int, default=128)
    parser.add_argument("--lr", type=float, default=1e-3)
    parser.add_argument("--data-dir", default="data")
    parser.add_argument("--out-dir", default=None)
    args = parser.parse_args()

    device = "cuda" if torch.cuda.is_available() else "cpu"
    suffix = ("_res" if args.residual else "") + ("_nobn" if args.no_bn else "")
    out_dir = Path(args.out_dir or f"runs/depth{args.depth}{suffix}")
    out_dir.mkdir(parents=True, exist_ok=True)

    model = CNN(args.depth, channels=args.channels,
                use_residual=args.residual, bn=not args.no_bn).to(device)
    train_loader, val_loader = get_loaders(args.data_dir, args.batch_size)

    opt = torch.optim.Adam(model.parameters(), lr=args.lr)
    sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=args.epochs)
    criterion = nn.CrossEntropyLoss()

    print(f"depth={args.depth} residual={args.residual} bn={not args.no_bn} params={count_params(model):,} device={device}")

    history = []
    for epoch in range(1, args.epochs + 1):
        model.train()
        t0 = time.time()
        running_loss, running_correct, n = 0.0, 0, 0
        for x, y in train_loader:
            x, y = x.to(device), y.to(device)
            opt.zero_grad(set_to_none=True)
            logits = model(x)
            loss = criterion(logits, y)
            loss.backward()
            opt.step()
            running_loss += loss.item() * y.size(0)
            running_correct += (logits.argmax(1) == y).sum().item()
            n += y.size(0)
        sched.step()

        train_acc = running_correct / n
        val_acc = evaluate(model, val_loader, device)
        train_loss = running_loss / n
        history.append({"epoch": epoch, "train_loss": train_loss,
                        "train_acc": train_acc, "val_acc": val_acc})
        print(f"epoch {epoch:2d}/{args.epochs}  loss={train_loss:.4f}  "
              f"train_acc={train_acc:.4f}  val_acc={val_acc:.4f}  ({time.time()-t0:.1f}s)")

    torch.save(model.state_dict(), out_dir / "model.pt")
    (out_dir / "history.json").write_text(json.dumps(history, indent=2))
    print(f"saved to {out_dir}")


if __name__ == "__main__":
    main()