Download evaluate.py from StarpowerTechnology/WVY-Image-Composition-101: direct link, hf CLI and curl.
- Browser
- Download file 4.58 kB
-
https://huggingface.co/StarpowerTechnology/WVY-Image-Composition-101/resolve/main/evaluate.py
- Command line
-
hf download hf://StarpowerTechnology/WVY-Image-Composition-101/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/StarpowerTechnology/WVY-Image-Composition-101/resolve/main/evaluate.py
4.58 kB
| """Evaluate WVY-Compose: can it draw scenes it never saw, or is it replaying memory?""" | |
| import numpy as np, json | |
| from PIL import Image, ImageDraw, ImageFont | |
| import model as M | |
| from data import build, S, HELD_PAIRS | |
| combos, caps, X, split = build() | |
| T, vocab, w2i = M.tokenize(caps) | |
| p = dict(np.load("wvy_compose_weights.npz")) | |
| Y = M.forward(p, T)[0].reshape(X.shape) | |
| lab = np.array([split[c] for c in combos]) | |
| tr = np.where(lab == "train")[0] | |
| Xf, Yf = X.reshape(len(X), -1), Y.reshape(len(Y), -1) | |
| # distance from every generation to every ground-truth scene | |
| Yd, Xd = Yf.astype(np.float64), Xf.astype(np.float64) | |
| d_all = ((Yd ** 2).sum(1)[:, None] + (Xd ** 2).sum(1)[None] - 2 * Yd @ Xd.T) / Xf.shape[1] | |
| res = {} | |
| for s in ("train", "unseen_combo", "unseen_pair"): | |
| idx = np.where(lab == s)[0] | |
| # 1) "Which scene is this?": is the generation closest to its OWN correct scene among all 480? | |
| hit = np.mean(d_all[idx].argmin(1) == idx) | |
| # 2) Memorization check: is the generation closer to its correct (unseen) target | |
| # than to the nearest image the model actually trained on? | |
| own = d_all[idx, idx] | |
| d_tr = d_all[idx][:, tr].copy() | |
| if s == "train": # exclude itself for train rows | |
| for r, i in enumerate(idx): d_tr[r, np.where(tr == i)[0]] = np.inf | |
| near_tr = d_tr.min(1) | |
| res[s] = {"n": int(len(idx)), "mse": float(own.mean()), "scene_match_acc": float(hit), | |
| "closer_to_target_than_any_train": float(np.mean(own < near_tr))} | |
| print(json.dumps(res, indent=1)) | |
| json.dump(res, open("eval.json", "w"), indent=1) | |
| # ---------- figures ---------- | |
| font = ImageFont.truetype("/usr/share/fonts/opentype/inter/InterDisplay-Medium.otf", 13) if False else None | |
| try: | |
| import glob | |
| f = glob.glob("/usr/share/fonts/**/Inter*Regular*.otf", recursive=True) or glob.glob("/usr/share/fonts/**/OpenSans*.ttf", recursive=True) | |
| font = ImageFont.truetype(f[0], 13) | |
| except Exception: | |
| font = ImageFont.load_default() | |
| Z = 3 # upscale factor | |
| def tile(a): | |
| return Image.fromarray((np.clip(a, 0, 1) * 255).astype("uint8")).resize((S * Z, S * Z), Image.NEAREST) | |
| def nearest_train(i): | |
| d = d_all[i, tr].copy() | |
| return tr[d.argmin()] | |
| def figure(rows, path, cols=("generated", "ground truth", "closest training image")): | |
| W, pad, txt = S * Z, 8, 34 | |
| im = Image.new("RGB", (len(cols) * (W + pad) + pad, len(rows) * (W + txt + pad) + pad + 18), (255, 255, 255)) | |
| dr = ImageDraw.Draw(im) | |
| for j, c in enumerate(cols): dr.text((pad + j * (W + pad), 4), c, fill=(80, 80, 80), font=font) | |
| for r, i in enumerate(rows): | |
| y0 = 22 + r * (W + txt + pad) | |
| dr.text((pad, y0), '"' + caps[i] + '"', fill=(20, 20, 20), font=font) | |
| imgs = [Y[i], X[i], X[nearest_train(i)]][:len(cols)] | |
| for j, a in enumerate(imgs): im.paste(tile(a), (pad + j * (W + pad), y0 + 18)) | |
| im.save(path) | |
| # Hardest test: arrangements never seen, shown with their seen reverse | |
| up_rows = [] | |
| for a, b in HELD_PAIRS: | |
| for c in ("green", "white"): | |
| up_rows.append(combos.index((a, b, "top", c))) | |
| figure(up_rows, "fig_unseen_pairs.png") | |
| # Reverse pairs that WERE seen in training, for side-by-side comparison | |
| rev_rows = [combos.index((b, a, "top", "green")) for a, b in HELD_PAIRS] | |
| figure(rev_rows, "fig_seen_reverse.png", cols=("generated", "ground truth")) | |
| uc = np.where(lab == "unseen_combo")[0] | |
| rng = np.random.default_rng(3) | |
| figure(rng.choice(uc, 8, replace=False), "fig_unseen_combos.png") | |
| # Big grid: one arrangement across all 8 colors, generated | |
| def strip(a, b, l, path): | |
| ids = [combos.index((a, b, l, c)) for c in ["white", "black", "red", "orange", "yellow", "green", "blue", "purple"]] | |
| g = np.concatenate([Y[i] for i in ids], 1) | |
| Image.fromarray((np.clip(g, 0, 1) * 255).astype("uint8")).resize((g.shape[1] * Z, g.shape[0] * Z), Image.NEAREST).save(path) | |
| return [lab[i] for i in ids] | |
| print("cat-on-dog strip splits:", strip("cat", "dog", "top", "fig_strip_cat_on_dog.png")) | |
| # Training curve | |
| log = json.load(open("train_log.json"))["log"] | |
| import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt | |
| fig, ax = plt.subplots(figsize=(6, 3.2), dpi=130) | |
| for k, c in (("train", "#2a6fdb"), ("unseen_combo", "#e08a1e"), ("unseen_pair", "#c0392b")): | |
| ax.plot([r["step"] for r in log], [r[k] for r in log], label=k.replace("_", " "), color=c, lw=2) | |
| ax.set_yscale("log"); ax.set_xlabel("training step"); ax.set_ylabel("pixel error (MSE)") | |
| ax.spines[["top", "right"]].set_visible(False); ax.legend(frameon=False); fig.tight_layout() | |
| fig.savefig("fig_curve.png") | |