StarpowerTechnology's picture
Upload folder using huggingface_hub
39d8cda verified
Raw History Blame Contribute Delete
4.15 kB
"""
WVY-Compose: a tiny text-to-image model written from scratch in numpy.
prompt -> word tokens -> (word embedding + its position in the sentence)
-> 2 hidden layers -> 48x48 RGB image
Forward pass, backprop and the Adam optimizer are all written by hand below.
Nothing here is pretrained; every weight starts random.
"""
import numpy as np, json, time
from data import build, S
L = 12 # max prompt length in tokens
D = 24 # embedding size per token
H1, H2 = 512, 1024
OUT = S * S * 3
def tokenize(caps):
vocab = ["<pad>"] + sorted({w for c in caps for w in c.split()})
w2i = {w: i for i, w in enumerate(vocab)}
T = np.zeros((len(caps), L), dtype=np.int64)
for i, c in enumerate(caps):
ids = [w2i[w] for w in c.split()]
T[i, :len(ids)] = ids
return T, vocab, w2i
def init(V, rng):
def he(a, b): return (rng.standard_normal((a, b)) * np.sqrt(2.0 / a)).astype(np.float32)
return {
"E": (rng.standard_normal((V, D)) * 0.3).astype(np.float32), # word embeddings
"Pos": (rng.standard_normal((L, D)) * 0.3).astype(np.float32), # position embeddings
"W1": he(L * D, H1), "b1": np.zeros(H1, np.float32),
"W2": he(H1, H2), "b2": np.zeros(H2, np.float32),
"W3": (rng.standard_normal((H2, OUT)) * 0.01).astype(np.float32), "b3": np.zeros(OUT, np.float32),
}
def forward(p, T):
e = p["E"][T] + p["Pos"][None] # (B, L, D)
x = e.reshape(len(T), -1)
z1 = x @ p["W1"] + p["b1"]; h1 = np.maximum(z1, 0)
z2 = h1 @ p["W2"] + p["b2"]; h2 = np.maximum(z2, 0)
y = 1 / (1 + np.exp(-(h2 @ p["W3"] + p["b3"]))) # pixels in [0,1]
return y, (T, x, z1, h1, z2, h2)
def backward(p, cache, y, target):
T, x, z1, h1, z2, h2 = cache
B = len(T)
dy = 2 * (y - target) / (B * OUT)
dz3 = dy * y * (1 - y)
g = {"W3": h2.T @ dz3, "b3": dz3.sum(0)}
dh2 = dz3 @ p["W3"].T; dz2 = dh2 * (z2 > 0)
g["W2"] = h1.T @ dz2; g["b2"] = dz2.sum(0)
dh1 = dz2 @ p["W2"].T; dz1 = dh1 * (z1 > 0)
g["W1"] = x.T @ dz1; g["b1"] = dz1.sum(0)
de = (dz1 @ p["W1"].T).reshape(B, L, D)
g["Pos"] = de.sum(0)
gE = np.zeros_like(p["E"]); np.add.at(gE, T, de); g["E"] = gE
return g
class Adam:
def __init__(self, p, lr):
self.lr, self.t = lr, 0
self.m = {k: np.zeros_like(v) for k, v in p.items()}
self.v = {k: np.zeros_like(v) for k, v in p.items()}
def step(self, p, g, lr):
self.t += 1
for k in p:
self.m[k] = 0.9 * self.m[k] + 0.1 * g[k]
self.v[k] = 0.999 * self.v[k] + 0.001 * g[k] ** 2
mh = self.m[k] / (1 - 0.9 ** self.t); vh = self.v[k] / (1 - 0.999 ** self.t)
p[k] -= lr * mh / (np.sqrt(vh) + 1e-8)
def mse(p, T, X):
return float(np.mean((forward(p, T)[0] - X.reshape(len(X), -1)) ** 2))
if __name__ == "__main__":
rng = np.random.default_rng(0)
combos, caps, X, split = build()
T, vocab, w2i = tokenize(caps)
lab = np.array([split[c] for c in combos])
tr, uc, up = (np.where(lab == s)[0] for s in ("train", "unseen_combo", "unseen_pair"))
Xf = X.reshape(len(X), -1)
p = init(len(vocab), rng)
nparams = sum(v.size for v in p.values())
print(f"vocab={len(vocab)} params={nparams:,} train={len(tr)} unseen_combo={len(uc)} unseen_pair={len(up)}")
opt = Adam(p, 2e-3)
STEPS, BS, LR = 9000, 32, 2e-3
log = []
t0 = time.time()
for s in range(STEPS + 1):
if s % 500 == 0:
row = {"step": s, "train": mse(p, T[tr], X[tr]), "unseen_combo": mse(p, T[uc], X[uc]),
"unseen_pair": mse(p, T[up], X[up]), "sec": round(time.time() - t0)}
log.append(row); print(row, flush=True)
if s == STEPS: break
b = rng.choice(tr, BS, replace=False)
y, cache = forward(p, T[b])
g = backward(p, cache, y, Xf[b])
lr = LR * 0.5 * (1 + np.cos(np.pi * s / STEPS)) + 1e-5
opt.step(p, g, lr)
np.savez_compressed("wvy_compose_weights.npz", **p)
json.dump({"vocab": vocab, "log": log, "params": int(nparams)}, open("train_log.json", "w"), indent=1)