"""Evaluate WVY-Compose: can it draw scenes it never saw, or is it replaying memory?""" import numpy as np, json from PIL import Image, ImageDraw, ImageFont import model as M from data import build, S, HELD_PAIRS combos, caps, X, split = build() T, vocab, w2i = M.tokenize(caps) p = dict(np.load("wvy_compose_weights.npz")) Y = M.forward(p, T)[0].reshape(X.shape) lab = np.array([split[c] for c in combos]) tr = np.where(lab == "train")[0] Xf, Yf = X.reshape(len(X), -1), Y.reshape(len(Y), -1) # distance from every generation to every ground-truth scene Yd, Xd = Yf.astype(np.float64), Xf.astype(np.float64) d_all = ((Yd ** 2).sum(1)[:, None] + (Xd ** 2).sum(1)[None] - 2 * Yd @ Xd.T) / Xf.shape[1] res = {} for s in ("train", "unseen_combo", "unseen_pair"): idx = np.where(lab == s)[0] # 1) "Which scene is this?": is the generation closest to its OWN correct scene among all 480? hit = np.mean(d_all[idx].argmin(1) == idx) # 2) Memorization check: is the generation closer to its correct (unseen) target # than to the nearest image the model actually trained on? own = d_all[idx, idx] d_tr = d_all[idx][:, tr].copy() if s == "train": # exclude itself for train rows for r, i in enumerate(idx): d_tr[r, np.where(tr == i)[0]] = np.inf near_tr = d_tr.min(1) res[s] = {"n": int(len(idx)), "mse": float(own.mean()), "scene_match_acc": float(hit), "closer_to_target_than_any_train": float(np.mean(own < near_tr))} print(json.dumps(res, indent=1)) json.dump(res, open("eval.json", "w"), indent=1) # ---------- figures ---------- font = ImageFont.truetype("/usr/share/fonts/opentype/inter/InterDisplay-Medium.otf", 13) if False else None try: import glob f = glob.glob("/usr/share/fonts/**/Inter*Regular*.otf", recursive=True) or glob.glob("/usr/share/fonts/**/OpenSans*.ttf", recursive=True) font = ImageFont.truetype(f[0], 13) except Exception: font = ImageFont.load_default() Z = 3 # upscale factor def tile(a): return Image.fromarray((np.clip(a, 0, 1) * 255).astype("uint8")).resize((S * Z, S * Z), Image.NEAREST) def nearest_train(i): d = d_all[i, tr].copy() return tr[d.argmin()] def figure(rows, path, cols=("generated", "ground truth", "closest training image")): W, pad, txt = S * Z, 8, 34 im = Image.new("RGB", (len(cols) * (W + pad) + pad, len(rows) * (W + txt + pad) + pad + 18), (255, 255, 255)) dr = ImageDraw.Draw(im) for j, c in enumerate(cols): dr.text((pad + j * (W + pad), 4), c, fill=(80, 80, 80), font=font) for r, i in enumerate(rows): y0 = 22 + r * (W + txt + pad) dr.text((pad, y0), '"' + caps[i] + '"', fill=(20, 20, 20), font=font) imgs = [Y[i], X[i], X[nearest_train(i)]][:len(cols)] for j, a in enumerate(imgs): im.paste(tile(a), (pad + j * (W + pad), y0 + 18)) im.save(path) # Hardest test: arrangements never seen, shown with their seen reverse up_rows = [] for a, b in HELD_PAIRS: for c in ("green", "white"): up_rows.append(combos.index((a, b, "top", c))) figure(up_rows, "fig_unseen_pairs.png") # Reverse pairs that WERE seen in training, for side-by-side comparison rev_rows = [combos.index((b, a, "top", "green")) for a, b in HELD_PAIRS] figure(rev_rows, "fig_seen_reverse.png", cols=("generated", "ground truth")) uc = np.where(lab == "unseen_combo")[0] rng = np.random.default_rng(3) figure(rng.choice(uc, 8, replace=False), "fig_unseen_combos.png") # Big grid: one arrangement across all 8 colors, generated def strip(a, b, l, path): ids = [combos.index((a, b, l, c)) for c in ["white", "black", "red", "orange", "yellow", "green", "blue", "purple"]] g = np.concatenate([Y[i] for i in ids], 1) Image.fromarray((np.clip(g, 0, 1) * 255).astype("uint8")).resize((g.shape[1] * Z, g.shape[0] * Z), Image.NEAREST).save(path) return [lab[i] for i in ids] print("cat-on-dog strip splits:", strip("cat", "dog", "top", "fig_strip_cat_on_dog.png")) # Training curve log = json.load(open("train_log.json"))["log"] import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt fig, ax = plt.subplots(figsize=(6, 3.2), dpi=130) for k, c in (("train", "#2a6fdb"), ("unseen_combo", "#e08a1e"), ("unseen_pair", "#c0392b")): ax.plot([r["step"] for r in log], [r[k] for r in log], label=k.replace("_", " "), color=c, lw=2) ax.set_yscale("log"); ax.set_xlabel("training step"); ax.set_ylabel("pixel error (MSE)") ax.spines[["top", "right"]].set_visible(False); ax.legend(frameon=False); fig.tight_layout() fig.savefig("fig_curve.png")