File size: 4,145 Bytes
39d8cda
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
"""
WVY-Compose: a tiny text-to-image model written from scratch in numpy.

prompt -> word tokens -> (word embedding + its position in the sentence)
       -> 2 hidden layers -> 48x48 RGB image

Forward pass, backprop and the Adam optimizer are all written by hand below.
Nothing here is pretrained; every weight starts random.
"""
import numpy as np, json, time
from data import build, S

L = 12          # max prompt length in tokens
D = 24          # embedding size per token
H1, H2 = 512, 1024
OUT = S * S * 3


def tokenize(caps):
    vocab = ["<pad>"] + sorted({w for c in caps for w in c.split()})
    w2i = {w: i for i, w in enumerate(vocab)}
    T = np.zeros((len(caps), L), dtype=np.int64)
    for i, c in enumerate(caps):
        ids = [w2i[w] for w in c.split()]
        T[i, :len(ids)] = ids
    return T, vocab, w2i


def init(V, rng):
    def he(a, b): return (rng.standard_normal((a, b)) * np.sqrt(2.0 / a)).astype(np.float32)
    return {
        "E": (rng.standard_normal((V, D)) * 0.3).astype(np.float32),   # word embeddings
        "Pos": (rng.standard_normal((L, D)) * 0.3).astype(np.float32), # position embeddings
        "W1": he(L * D, H1), "b1": np.zeros(H1, np.float32),
        "W2": he(H1, H2), "b2": np.zeros(H2, np.float32),
        "W3": (rng.standard_normal((H2, OUT)) * 0.01).astype(np.float32), "b3": np.zeros(OUT, np.float32),
    }


def forward(p, T):
    e = p["E"][T] + p["Pos"][None]                    # (B, L, D)
    x = e.reshape(len(T), -1)
    z1 = x @ p["W1"] + p["b1"]; h1 = np.maximum(z1, 0)
    z2 = h1 @ p["W2"] + p["b2"]; h2 = np.maximum(z2, 0)
    y = 1 / (1 + np.exp(-(h2 @ p["W3"] + p["b3"])))  # pixels in [0,1]
    return y, (T, x, z1, h1, z2, h2)


def backward(p, cache, y, target):
    T, x, z1, h1, z2, h2 = cache
    B = len(T)
    dy = 2 * (y - target) / (B * OUT)
    dz3 = dy * y * (1 - y)
    g = {"W3": h2.T @ dz3, "b3": dz3.sum(0)}
    dh2 = dz3 @ p["W3"].T; dz2 = dh2 * (z2 > 0)
    g["W2"] = h1.T @ dz2; g["b2"] = dz2.sum(0)
    dh1 = dz2 @ p["W2"].T; dz1 = dh1 * (z1 > 0)
    g["W1"] = x.T @ dz1; g["b1"] = dz1.sum(0)
    de = (dz1 @ p["W1"].T).reshape(B, L, D)
    g["Pos"] = de.sum(0)
    gE = np.zeros_like(p["E"]); np.add.at(gE, T, de); g["E"] = gE
    return g


class Adam:
    def __init__(self, p, lr):
        self.lr, self.t = lr, 0
        self.m = {k: np.zeros_like(v) for k, v in p.items()}
        self.v = {k: np.zeros_like(v) for k, v in p.items()}

    def step(self, p, g, lr):
        self.t += 1
        for k in p:
            self.m[k] = 0.9 * self.m[k] + 0.1 * g[k]
            self.v[k] = 0.999 * self.v[k] + 0.001 * g[k] ** 2
            mh = self.m[k] / (1 - 0.9 ** self.t); vh = self.v[k] / (1 - 0.999 ** self.t)
            p[k] -= lr * mh / (np.sqrt(vh) + 1e-8)


def mse(p, T, X):
    return float(np.mean((forward(p, T)[0] - X.reshape(len(X), -1)) ** 2))


if __name__ == "__main__":
    rng = np.random.default_rng(0)
    combos, caps, X, split = build()
    T, vocab, w2i = tokenize(caps)
    lab = np.array([split[c] for c in combos])
    tr, uc, up = (np.where(lab == s)[0] for s in ("train", "unseen_combo", "unseen_pair"))
    Xf = X.reshape(len(X), -1)
    p = init(len(vocab), rng)
    nparams = sum(v.size for v in p.values())
    print(f"vocab={len(vocab)} params={nparams:,} train={len(tr)} unseen_combo={len(uc)} unseen_pair={len(up)}")
    opt = Adam(p, 2e-3)
    STEPS, BS, LR = 9000, 32, 2e-3
    log = []
    t0 = time.time()
    for s in range(STEPS + 1):
        if s % 500 == 0:
            row = {"step": s, "train": mse(p, T[tr], X[tr]), "unseen_combo": mse(p, T[uc], X[uc]),
                   "unseen_pair": mse(p, T[up], X[up]), "sec": round(time.time() - t0)}
            log.append(row); print(row, flush=True)
        if s == STEPS: break
        b = rng.choice(tr, BS, replace=False)
        y, cache = forward(p, T[b])
        g = backward(p, cache, y, Xf[b])
        lr = LR * 0.5 * (1 + np.cos(np.pi * s / STEPS)) + 1e-5
        opt.step(p, g, lr)
    np.savez_compressed("wvy_compose_weights.npz", **p)
    json.dump({"vocab": vocab, "log": log, "params": int(nparams)}, open("train_log.json", "w"), indent=1)