File size: 4,580 Bytes
39d8cda
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
"""Evaluate WVY-Compose: can it draw scenes it never saw, or is it replaying memory?"""
import numpy as np, json
from PIL import Image, ImageDraw, ImageFont
import model as M
from data import build, S, HELD_PAIRS

combos, caps, X, split = build()
T, vocab, w2i = M.tokenize(caps)
p = dict(np.load("wvy_compose_weights.npz"))
Y = M.forward(p, T)[0].reshape(X.shape)
lab = np.array([split[c] for c in combos])
tr = np.where(lab == "train")[0]

Xf, Yf = X.reshape(len(X), -1), Y.reshape(len(Y), -1)
# distance from every generation to every ground-truth scene
Yd, Xd = Yf.astype(np.float64), Xf.astype(np.float64)
d_all = ((Yd ** 2).sum(1)[:, None] + (Xd ** 2).sum(1)[None] - 2 * Yd @ Xd.T) / Xf.shape[1]

res = {}
for s in ("train", "unseen_combo", "unseen_pair"):
    idx = np.where(lab == s)[0]
    # 1) "Which scene is this?": is the generation closest to its OWN correct scene among all 480?
    hit = np.mean(d_all[idx].argmin(1) == idx)
    # 2) Memorization check: is the generation closer to its correct (unseen) target
    #    than to the nearest image the model actually trained on?
    own = d_all[idx, idx]
    d_tr = d_all[idx][:, tr].copy()
    if s == "train":                      # exclude itself for train rows
        for r, i in enumerate(idx): d_tr[r, np.where(tr == i)[0]] = np.inf
    near_tr = d_tr.min(1)
    res[s] = {"n": int(len(idx)), "mse": float(own.mean()), "scene_match_acc": float(hit),
              "closer_to_target_than_any_train": float(np.mean(own < near_tr))}
print(json.dumps(res, indent=1))
json.dump(res, open("eval.json", "w"), indent=1)

# ---------- figures ----------
font = ImageFont.truetype("/usr/share/fonts/opentype/inter/InterDisplay-Medium.otf", 13) if False else None
try:
    import glob
    f = glob.glob("/usr/share/fonts/**/Inter*Regular*.otf", recursive=True) or glob.glob("/usr/share/fonts/**/OpenSans*.ttf", recursive=True)
    font = ImageFont.truetype(f[0], 13)
except Exception:
    font = ImageFont.load_default()

Z = 3  # upscale factor


def tile(a):
    return Image.fromarray((np.clip(a, 0, 1) * 255).astype("uint8")).resize((S * Z, S * Z), Image.NEAREST)


def nearest_train(i):
    d = d_all[i, tr].copy()
    return tr[d.argmin()]


def figure(rows, path, cols=("generated", "ground truth", "closest training image")):
    W, pad, txt = S * Z, 8, 34
    im = Image.new("RGB", (len(cols) * (W + pad) + pad, len(rows) * (W + txt + pad) + pad + 18), (255, 255, 255))
    dr = ImageDraw.Draw(im)
    for j, c in enumerate(cols): dr.text((pad + j * (W + pad), 4), c, fill=(80, 80, 80), font=font)
    for r, i in enumerate(rows):
        y0 = 22 + r * (W + txt + pad)
        dr.text((pad, y0), '"' + caps[i] + '"', fill=(20, 20, 20), font=font)
        imgs = [Y[i], X[i], X[nearest_train(i)]][:len(cols)]
        for j, a in enumerate(imgs): im.paste(tile(a), (pad + j * (W + pad), y0 + 18))
    im.save(path)


# Hardest test: arrangements never seen, shown with their seen reverse
up_rows = []
for a, b in HELD_PAIRS:
    for c in ("green", "white"):
        up_rows.append(combos.index((a, b, "top", c)))
figure(up_rows, "fig_unseen_pairs.png")

# Reverse pairs that WERE seen in training, for side-by-side comparison
rev_rows = [combos.index((b, a, "top", "green")) for a, b in HELD_PAIRS]
figure(rev_rows, "fig_seen_reverse.png", cols=("generated", "ground truth"))

uc = np.where(lab == "unseen_combo")[0]
rng = np.random.default_rng(3)
figure(rng.choice(uc, 8, replace=False), "fig_unseen_combos.png")

# Big grid: one arrangement across all 8 colors, generated
def strip(a, b, l, path):
    ids = [combos.index((a, b, l, c)) for c in ["white", "black", "red", "orange", "yellow", "green", "blue", "purple"]]
    g = np.concatenate([Y[i] for i in ids], 1)
    Image.fromarray((np.clip(g, 0, 1) * 255).astype("uint8")).resize((g.shape[1] * Z, g.shape[0] * Z), Image.NEAREST).save(path)
    return [lab[i] for i in ids]
print("cat-on-dog strip splits:", strip("cat", "dog", "top", "fig_strip_cat_on_dog.png"))

# Training curve
log = json.load(open("train_log.json"))["log"]
import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(6, 3.2), dpi=130)
for k, c in (("train", "#2a6fdb"), ("unseen_combo", "#e08a1e"), ("unseen_pair", "#c0392b")):
    ax.plot([r["step"] for r in log], [r[k] for r in log], label=k.replace("_", " "), color=c, lw=2)
ax.set_yscale("log"); ax.set_xlabel("training step"); ax.set_ylabel("pixel error (MSE)")
ax.spines[["top", "right"]].set_visible(False); ax.legend(frameon=False); fig.tight_layout()
fig.savefig("fig_curve.png")