train.py

import sys, numpy as np, pandas as pd, torch, torch.nn as nn, torch.nn.functional as F

dev = "cuda" if torch.cuda.is_available() else "cpu"
torch.manual_seed(0)
np.random.seed(0)
csv, L, H = ("/workspace/data/electricity.csv", 512, 96)
smoke = "--smoke" in sys.argv
save = (
    sys.argv[sys.argv.index("--save") + 1]
    if "--save" in sys.argv
    else "/workspace/ts_demo/fela_ts_electricity.pt"
)
epochs = int(sys.argv[sys.argv.index("--epochs") + 1]) if "--epochs" in sys.argv else 30


class RevIN(nn.Module):
    def __init__(s, C):
        super().__init__()
        s.g = nn.Parameter(torch.ones(C))
        s.b = nn.Parameter(torch.zeros(C))

    def norm(s, x):
        s.m = x.mean(1, keepdim=True)
        s.s = x.std(1, keepdim=True) + 1e-05
        return (x - s.m) / s.s * s.g + s.b

    def denorm(s, x):
        return (x - s.b) / s.g * s.s + s.m


class FNO1D(nn.Module):
    def __init__(s, D, m):
        super().__init__()
        s.m = m
        s.w = nn.Parameter(1 / (D * D) * torch.rand(m, D, D, dtype=torch.cfloat))

    def forward(s, x):
        P = x.shape[1]
        xf = torch.fft.rfft(x, dim=1)
        mm = min(s.m, xf.shape[1])
        o = torch.zeros_like(xf)
        o[:, :mm] = torch.einsum("bpd,pde->bpe", xf[:, :mm], s.w[:mm])
        return torch.fft.irfft(o, n=P, dim=1)


class Block(nn.Module):
    def __init__(s, D, m, ff=2, drop=0.2):
        super().__init__()
        s.n1 = nn.LayerNorm(D)
        s.fno = FNO1D(D, m)
        s.d1 = nn.Dropout(drop)
        s.n2 = nn.LayerNorm(D)
        s.ff = nn.Sequential(
            nn.Linear(D, D * ff), nn.GELU(), nn.Dropout(drop), nn.Linear(D * ff, D)
        )

    def forward(s, x):
        x = x + s.d1(s.fno(s.n1(x)))
        return x + s.ff(s.n2(x))


class FELA_TS(nn.Module):
    def __init__(s, C, L, H, patch=16, stride=8, D=128, modes=16, nblk=3):
        super().__init__()
        s.C, s.L, s.H, s.patch, s.stride = (C, L, H, patch, stride)
        s.revin = RevIN(C)
        s.np_ = (L - patch) // stride + 1
        s.embed = nn.Linear(patch, D)
        s.blocks = nn.ModuleList([Block(D, modes) for _ in range(nblk)])
        s.head = nn.Linear(s.np_ * D, H)

    def forward(s, x):
        x = s.revin.norm(x)
        x = x.permute(0, 2, 1).reshape(-1, s.L)
        x = x.unfold(1, s.patch, s.stride)
        h = s.embed(x)
        for b in s.blocks:
            h = b(h)
        y = s.head(h.flatten(1)).reshape(-1, s.C, s.H).permute(0, 2, 1)
        return s.revin.denorm(y)


df = pd.read_csv(csv)
cols = [c for c in df.columns if c != "date"]
data = df[cols].values.astype(np.float32)
n = len(data)
ntr, nva = (int(n * 0.7), int(n * 0.1))
mu = data[:ntr].mean(0)
sd = data[:ntr].std(0) + 1e-08
data = (data - mu) / sd


def win(a):
    xs, ys = ([], [])
    for i in range(0, len(a) - L - H + 1, 1):
        xs.append(a[i : i + L])
        ys.append(a[i + L : i + L + H])
    return (torch.tensor(np.array(xs)), torch.tensor(np.array(ys)))


Xtr, Ytr = win(data[:ntr])
Xte, Yte = win(data[ntr + nva :])
C = data.shape[1]
assert len(Xtr) == 17805
if smoke:
    print(
        f"Electricity C={C} n={n} ntr={ntr} nva={nva} train {len(Xtr)} test {len(Xte)}"
    )
    sys.exit()
m = FELA_TS(C, L, H).to(dev)
opt = torch.optim.Adam(m.parameters(), lr=0.001)
sch = torch.optim.lr_scheduler.CosineAnnealingLR(opt, epochs)
bs = 64
print(f"[Ts] electricity C={C} train {len(Xtr)} test {len(Xte)}")
for ep in range(epochs):
    m.train()
    p = torch.randperm(len(Xtr))
    for i in range(0, len(Xtr) - bs, bs):
        idx = p[i : i + bs]
        loss = F.l1_loss(m(Xtr[idx].to(dev)), Ytr[idx].to(dev))
        opt.zero_grad()
        loss.backward()
        opt.step()
    sch.step()
m.eval()
se = ae = cnt = 0
with torch.no_grad():
    for i in range(0, len(Xte), 256):
        pr = m(Xte[i : i + 256].to(dev))
        y = Yte[i : i + 256].to(dev)
        se += F.mse_loss(pr, y, reduction="sum").item()
        ae += (pr - y).abs().sum().item()
        cnt += y.numel()
mse, mae = (se / cnt, ae / cnt)
print(f"[Ts] electricity/96 TEST MSE {mse:.4f} MAE {mae:.4f}")
torch.save(m.state_dict(), save)
print(f"SAVED {save}")