File size: 6,171 Bytes
39d8cda | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | """
WVY-Compose dataset builder.
The idea: a tiny set of hand-made ingredients (6 transparent animal sprites +
8 background colors) gets composited into scenes with exact text captions.
Some combinations are deliberately held out so we can test whether the model
generates scenes it never saw (generalization) or only replays what it saw
(memorization).
"""
import numpy as np
from PIL import Image, ImageDraw
import itertools, json, os
S = 48 # final image size (pixels)
HI = 4 # draw sprites at 4x resolution, then downsample for clean edges
SPRITE = 22 # sprite size in final pixels
ANIMALS = ["dog", "cat", "bird", "fish", "pig", "rabbit"]
COLORS = {
"white": (240, 240, 240), "black": (25, 25, 25), "red": (200, 40, 40),
"orange": (240, 140, 30), "yellow": (240, 220, 60), "green": (50, 160, 70),
"blue": (50, 90, 200), "purple": (130, 60, 170),
}
LAYOUTS = ["top", "next"] # "A on top of B" / "A next to B"
def _sprite(name):
"""Draw one transparent animal sprite. Each is a single 'ingredient image'."""
n = SPRITE * HI
im = Image.new("RGBA", (n, n), (0, 0, 0, 0))
d = ImageDraw.Draw(im)
o = (15, 15, 15, 255) # dark outline so sprites read on any background
w = 2 * HI // 2 + 2
u = n / 22.0 # drawing unit
def E(box, fill): # ellipse in drawing units
d.ellipse([b * u for b in box], fill=fill, outline=o, width=w)
def P(pts, fill):
d.polygon([(x * u, y * u) for x, y in pts], fill=fill, outline=o, width=w)
if name == "dog":
c = (150, 95, 45, 255)
for x in (5, 8, 13, 16): d.rectangle([x*u, 14*u, (x+2)*u, 20*u], fill=c, outline=o, width=w)
E((3, 9, 19, 16), c) # body
P([(18, 9), (21, 5), (20, 10)], c) # tail up
E((0, 3, 9, 11), c) # head
E((0, 3, 3, 10), (90, 55, 25, 255)) # floppy ear
d.ellipse([2*u, 6*u, 3.3*u, 7.3*u], fill=o)
elif name == "cat":
c = (140, 140, 150, 255)
P([(17, 15), (21, 6), (19, 15)], c) # tail
E((4, 10, 18, 20), c) # body
P([(4, 5), (5, 0), (8, 4)], c); P([(10, 4), (13, 0), (14, 5)], c) # pointy ears
E((3, 2, 15, 12), c) # head
d.ellipse([6*u, 6*u, 7.5*u, 7.5*u], fill=(60, 200, 90, 255))
d.ellipse([10.5*u, 6*u, 12*u, 7.5*u], fill=(60, 200, 90, 255))
elif name == "bird":
c = (250, 215, 40, 255)
E((3, 4, 19, 19), c) # round body
P([(18, 9), (22, 11), (18, 13)], (240, 120, 20, 255)) # beak
E((5, 10, 12, 16), (220, 170, 20, 255)) # wing
d.ellipse([14*u, 8*u, 15.5*u, 9.5*u], fill=o)
d.line([9*u, 19*u, 9*u, 22*u], fill=o, width=w); d.line([13*u, 19*u, 13*u, 22*u], fill=o, width=w)
elif name == "fish":
c = (60, 180, 230, 255)
P([(1, 4), (7, 11), (1, 18)], (30, 120, 200, 255)) # tail fin
E((5, 5, 21, 17), c)
P([(11, 6), (14, 2), (16, 6)], (30, 120, 200, 255)) # dorsal fin
d.ellipse([16*u, 9*u, 17.8*u, 10.8*u], fill=o)
elif name == "pig":
c = (245, 160, 185, 255)
for x in (5, 14): d.rectangle([x*u, 15*u, (x+3)*u, 20*u], fill=c, outline=o, width=w)
E((2, 6, 20, 17), c)
E((15, 9, 21, 14), (230, 120, 150, 255)) # snout
P([(12, 6), (14, 2), (16, 6)], c) # ear
d.ellipse([13*u, 8*u, 14.3*u, 9.3*u], fill=o)
elif name == "rabbit":
c = (235, 235, 225, 255)
E((6, 0, 9, 10), c); E((11, 0, 14, 10), c) # long ears
E((4, 12, 18, 22), c) # body
E((5, 6, 15, 15), c) # head
d.ellipse([7.5*u, 9*u, 8.8*u, 10.3*u], fill=o); d.ellipse([11.2*u, 9*u, 12.5*u, 10.3*u], fill=o)
E((16, 15, 20, 19), (255, 255, 255, 255)) # tail puff
return im
def build_sprites(outdir="sprites"):
os.makedirs(outdir, exist_ok=True)
sp = {}
for a in ANIMALS:
im = _sprite(a)
im.save(f"{outdir}/{a}.png")
sp[a] = im
return sp
def compose(sprites, a, b, layout, color):
"""Paste transparent sprite A and B onto a solid background (the NFT-style layer stack)."""
n = S * HI
bg = Image.new("RGBA", (n, n), COLORS[color] + (255,))
k = SPRITE * HI
if layout == "top": # B on the ground, A standing on top of it
bg.alpha_composite(sprites[b], ((n - k) // 2, n - k - 2 * HI))
bg.alpha_composite(sprites[a], ((n - k) // 2, n - 2 * k + 2 * HI))
else: # A on the left, B on the right
y = n - k - 6 * HI
bg.alpha_composite(sprites[a], (1 * HI, y))
bg.alpha_composite(sprites[b], (n - k - 1 * HI, y))
im = bg.convert("RGB").resize((S, S), Image.LANCZOS)
return np.asarray(im, dtype=np.float32) / 255.0
def caption(a, b, layout, color):
mid = "on top of" if layout == "top" else "next to"
return f"a {a} {mid} a {b} on a {color} background"
# Ordered "A on top of B" pairs that are NEVER shown in any color during training.
# Their reverse IS shown (e.g. "dog on top of cat" is seen, "cat on top of dog" never is).
HELD_PAIRS = [("cat", "dog"), ("bird", "pig"), ("rabbit", "fish"), ("pig", "cat")]
def build(seed=0):
rng = np.random.default_rng(seed)
sprites = build_sprites()
combos = [(a, b, l, c) for a, b in itertools.permutations(ANIMALS, 2)
for l in LAYOUTS for c in COLORS]
split = {}
for cb in combos:
a, b, l, c = cb
if l == "top" and (a, b) in HELD_PAIRS:
split[cb] = "unseen_pair" # hardest test: arrangement never seen
elif rng.random() < 0.2:
split[cb] = "unseen_combo" # pieces + arrangement seen, this exact scene not
else:
split[cb] = "train"
X = np.stack([compose(sprites, *cb) for cb in combos])
caps = [caption(*cb) for cb in combos]
return combos, caps, X, split
if __name__ == "__main__":
combos, caps, X, split = build()
from collections import Counter
print(len(combos), Counter(split.values()))
print(caps[0])
|