""" WVY-Compose: a tiny text-to-image model written from scratch in numpy. prompt -> word tokens -> (word embedding + its position in the sentence) -> 2 hidden layers -> 48x48 RGB image Forward pass, backprop and the Adam optimizer are all written by hand below. Nothing here is pretrained; every weight starts random. """ import numpy as np, json, time from data import build, S L = 12 # max prompt length in tokens D = 24 # embedding size per token H1, H2 = 512, 1024 OUT = S * S * 3 def tokenize(caps): vocab = [""] + sorted({w for c in caps for w in c.split()}) w2i = {w: i for i, w in enumerate(vocab)} T = np.zeros((len(caps), L), dtype=np.int64) for i, c in enumerate(caps): ids = [w2i[w] for w in c.split()] T[i, :len(ids)] = ids return T, vocab, w2i def init(V, rng): def he(a, b): return (rng.standard_normal((a, b)) * np.sqrt(2.0 / a)).astype(np.float32) return { "E": (rng.standard_normal((V, D)) * 0.3).astype(np.float32), # word embeddings "Pos": (rng.standard_normal((L, D)) * 0.3).astype(np.float32), # position embeddings "W1": he(L * D, H1), "b1": np.zeros(H1, np.float32), "W2": he(H1, H2), "b2": np.zeros(H2, np.float32), "W3": (rng.standard_normal((H2, OUT)) * 0.01).astype(np.float32), "b3": np.zeros(OUT, np.float32), } def forward(p, T): e = p["E"][T] + p["Pos"][None] # (B, L, D) x = e.reshape(len(T), -1) z1 = x @ p["W1"] + p["b1"]; h1 = np.maximum(z1, 0) z2 = h1 @ p["W2"] + p["b2"]; h2 = np.maximum(z2, 0) y = 1 / (1 + np.exp(-(h2 @ p["W3"] + p["b3"]))) # pixels in [0,1] return y, (T, x, z1, h1, z2, h2) def backward(p, cache, y, target): T, x, z1, h1, z2, h2 = cache B = len(T) dy = 2 * (y - target) / (B * OUT) dz3 = dy * y * (1 - y) g = {"W3": h2.T @ dz3, "b3": dz3.sum(0)} dh2 = dz3 @ p["W3"].T; dz2 = dh2 * (z2 > 0) g["W2"] = h1.T @ dz2; g["b2"] = dz2.sum(0) dh1 = dz2 @ p["W2"].T; dz1 = dh1 * (z1 > 0) g["W1"] = x.T @ dz1; g["b1"] = dz1.sum(0) de = (dz1 @ p["W1"].T).reshape(B, L, D) g["Pos"] = de.sum(0) gE = np.zeros_like(p["E"]); np.add.at(gE, T, de); g["E"] = gE return g class Adam: def __init__(self, p, lr): self.lr, self.t = lr, 0 self.m = {k: np.zeros_like(v) for k, v in p.items()} self.v = {k: np.zeros_like(v) for k, v in p.items()} def step(self, p, g, lr): self.t += 1 for k in p: self.m[k] = 0.9 * self.m[k] + 0.1 * g[k] self.v[k] = 0.999 * self.v[k] + 0.001 * g[k] ** 2 mh = self.m[k] / (1 - 0.9 ** self.t); vh = self.v[k] / (1 - 0.999 ** self.t) p[k] -= lr * mh / (np.sqrt(vh) + 1e-8) def mse(p, T, X): return float(np.mean((forward(p, T)[0] - X.reshape(len(X), -1)) ** 2)) if __name__ == "__main__": rng = np.random.default_rng(0) combos, caps, X, split = build() T, vocab, w2i = tokenize(caps) lab = np.array([split[c] for c in combos]) tr, uc, up = (np.where(lab == s)[0] for s in ("train", "unseen_combo", "unseen_pair")) Xf = X.reshape(len(X), -1) p = init(len(vocab), rng) nparams = sum(v.size for v in p.values()) print(f"vocab={len(vocab)} params={nparams:,} train={len(tr)} unseen_combo={len(uc)} unseen_pair={len(up)}") opt = Adam(p, 2e-3) STEPS, BS, LR = 9000, 32, 2e-3 log = [] t0 = time.time() for s in range(STEPS + 1): if s % 500 == 0: row = {"step": s, "train": mse(p, T[tr], X[tr]), "unseen_combo": mse(p, T[uc], X[uc]), "unseen_pair": mse(p, T[up], X[up]), "sec": round(time.time() - t0)} log.append(row); print(row, flush=True) if s == STEPS: break b = rng.choice(tr, BS, replace=False) y, cache = forward(p, T[b]) g = backward(p, cache, y, Xf[b]) lr = LR * 0.5 * (1 + np.cos(np.pi * s / STEPS)) + 1e-5 opt.step(p, g, lr) np.savez_compressed("wvy_compose_weights.npz", **p) json.dump({"vocab": vocab, "log": log, "params": int(nparams)}, open("train_log.json", "w"), indent=1)