StarpowerTechnology's picture
Upload folder using huggingface_hub
39d8cda verified
Raw History Blame Contribute Delete
4.58 kB
"""Evaluate WVY-Compose: can it draw scenes it never saw, or is it replaying memory?"""
import numpy as np, json
from PIL import Image, ImageDraw, ImageFont
import model as M
from data import build, S, HELD_PAIRS
combos, caps, X, split = build()
T, vocab, w2i = M.tokenize(caps)
p = dict(np.load("wvy_compose_weights.npz"))
Y = M.forward(p, T)[0].reshape(X.shape)
lab = np.array([split[c] for c in combos])
tr = np.where(lab == "train")[0]
Xf, Yf = X.reshape(len(X), -1), Y.reshape(len(Y), -1)
# distance from every generation to every ground-truth scene
Yd, Xd = Yf.astype(np.float64), Xf.astype(np.float64)
d_all = ((Yd ** 2).sum(1)[:, None] + (Xd ** 2).sum(1)[None] - 2 * Yd @ Xd.T) / Xf.shape[1]
res = {}
for s in ("train", "unseen_combo", "unseen_pair"):
idx = np.where(lab == s)[0]
# 1) "Which scene is this?": is the generation closest to its OWN correct scene among all 480?
hit = np.mean(d_all[idx].argmin(1) == idx)
# 2) Memorization check: is the generation closer to its correct (unseen) target
# than to the nearest image the model actually trained on?
own = d_all[idx, idx]
d_tr = d_all[idx][:, tr].copy()
if s == "train": # exclude itself for train rows
for r, i in enumerate(idx): d_tr[r, np.where(tr == i)[0]] = np.inf
near_tr = d_tr.min(1)
res[s] = {"n": int(len(idx)), "mse": float(own.mean()), "scene_match_acc": float(hit),
"closer_to_target_than_any_train": float(np.mean(own < near_tr))}
print(json.dumps(res, indent=1))
json.dump(res, open("eval.json", "w"), indent=1)
# ---------- figures ----------
font = ImageFont.truetype("/usr/share/fonts/opentype/inter/InterDisplay-Medium.otf", 13) if False else None
try:
import glob
f = glob.glob("/usr/share/fonts/**/Inter*Regular*.otf", recursive=True) or glob.glob("/usr/share/fonts/**/OpenSans*.ttf", recursive=True)
font = ImageFont.truetype(f[0], 13)
except Exception:
font = ImageFont.load_default()
Z = 3 # upscale factor
def tile(a):
return Image.fromarray((np.clip(a, 0, 1) * 255).astype("uint8")).resize((S * Z, S * Z), Image.NEAREST)
def nearest_train(i):
d = d_all[i, tr].copy()
return tr[d.argmin()]
def figure(rows, path, cols=("generated", "ground truth", "closest training image")):
W, pad, txt = S * Z, 8, 34
im = Image.new("RGB", (len(cols) * (W + pad) + pad, len(rows) * (W + txt + pad) + pad + 18), (255, 255, 255))
dr = ImageDraw.Draw(im)
for j, c in enumerate(cols): dr.text((pad + j * (W + pad), 4), c, fill=(80, 80, 80), font=font)
for r, i in enumerate(rows):
y0 = 22 + r * (W + txt + pad)
dr.text((pad, y0), '"' + caps[i] + '"', fill=(20, 20, 20), font=font)
imgs = [Y[i], X[i], X[nearest_train(i)]][:len(cols)]
for j, a in enumerate(imgs): im.paste(tile(a), (pad + j * (W + pad), y0 + 18))
im.save(path)
# Hardest test: arrangements never seen, shown with their seen reverse
up_rows = []
for a, b in HELD_PAIRS:
for c in ("green", "white"):
up_rows.append(combos.index((a, b, "top", c)))
figure(up_rows, "fig_unseen_pairs.png")
# Reverse pairs that WERE seen in training, for side-by-side comparison
rev_rows = [combos.index((b, a, "top", "green")) for a, b in HELD_PAIRS]
figure(rev_rows, "fig_seen_reverse.png", cols=("generated", "ground truth"))
uc = np.where(lab == "unseen_combo")[0]
rng = np.random.default_rng(3)
figure(rng.choice(uc, 8, replace=False), "fig_unseen_combos.png")
# Big grid: one arrangement across all 8 colors, generated
def strip(a, b, l, path):
ids = [combos.index((a, b, l, c)) for c in ["white", "black", "red", "orange", "yellow", "green", "blue", "purple"]]
g = np.concatenate([Y[i] for i in ids], 1)
Image.fromarray((np.clip(g, 0, 1) * 255).astype("uint8")).resize((g.shape[1] * Z, g.shape[0] * Z), Image.NEAREST).save(path)
return [lab[i] for i in ids]
print("cat-on-dog strip splits:", strip("cat", "dog", "top", "fig_strip_cat_on_dog.png"))
# Training curve
log = json.load(open("train_log.json"))["log"]
import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(6, 3.2), dpi=130)
for k, c in (("train", "#2a6fdb"), ("unseen_combo", "#e08a1e"), ("unseen_pair", "#c0392b")):
ax.plot([r["step"] for r in log], [r[k] for r in log], label=k.replace("_", " "), color=c, lw=2)
ax.set_yscale("log"); ax.set_xlabel("training step"); ax.set_ylabel("pixel error (MSE)")
ax.spines[["top", "right"]].set_visible(False); ax.legend(frameon=False); fig.tight_layout()
fig.savefig("fig_curve.png")