StarpowerTechnology's picture
Upload folder using huggingface_hub
39d8cda verified
Raw History Blame Contribute Delete
6.17 kB
"""
WVY-Compose dataset builder.
The idea: a tiny set of hand-made ingredients (6 transparent animal sprites +
8 background colors) gets composited into scenes with exact text captions.
Some combinations are deliberately held out so we can test whether the model
generates scenes it never saw (generalization) or only replays what it saw
(memorization).
"""
import numpy as np
from PIL import Image, ImageDraw
import itertools, json, os
S = 48 # final image size (pixels)
HI = 4 # draw sprites at 4x resolution, then downsample for clean edges
SPRITE = 22 # sprite size in final pixels
ANIMALS = ["dog", "cat", "bird", "fish", "pig", "rabbit"]
COLORS = {
"white": (240, 240, 240), "black": (25, 25, 25), "red": (200, 40, 40),
"orange": (240, 140, 30), "yellow": (240, 220, 60), "green": (50, 160, 70),
"blue": (50, 90, 200), "purple": (130, 60, 170),
}
LAYOUTS = ["top", "next"] # "A on top of B" / "A next to B"
def _sprite(name):
"""Draw one transparent animal sprite. Each is a single 'ingredient image'."""
n = SPRITE * HI
im = Image.new("RGBA", (n, n), (0, 0, 0, 0))
d = ImageDraw.Draw(im)
o = (15, 15, 15, 255) # dark outline so sprites read on any background
w = 2 * HI // 2 + 2
u = n / 22.0 # drawing unit
def E(box, fill): # ellipse in drawing units
d.ellipse([b * u for b in box], fill=fill, outline=o, width=w)
def P(pts, fill):
d.polygon([(x * u, y * u) for x, y in pts], fill=fill, outline=o, width=w)
if name == "dog":
c = (150, 95, 45, 255)
for x in (5, 8, 13, 16): d.rectangle([x*u, 14*u, (x+2)*u, 20*u], fill=c, outline=o, width=w)
E((3, 9, 19, 16), c) # body
P([(18, 9), (21, 5), (20, 10)], c) # tail up
E((0, 3, 9, 11), c) # head
E((0, 3, 3, 10), (90, 55, 25, 255)) # floppy ear
d.ellipse([2*u, 6*u, 3.3*u, 7.3*u], fill=o)
elif name == "cat":
c = (140, 140, 150, 255)
P([(17, 15), (21, 6), (19, 15)], c) # tail
E((4, 10, 18, 20), c) # body
P([(4, 5), (5, 0), (8, 4)], c); P([(10, 4), (13, 0), (14, 5)], c) # pointy ears
E((3, 2, 15, 12), c) # head
d.ellipse([6*u, 6*u, 7.5*u, 7.5*u], fill=(60, 200, 90, 255))
d.ellipse([10.5*u, 6*u, 12*u, 7.5*u], fill=(60, 200, 90, 255))
elif name == "bird":
c = (250, 215, 40, 255)
E((3, 4, 19, 19), c) # round body
P([(18, 9), (22, 11), (18, 13)], (240, 120, 20, 255)) # beak
E((5, 10, 12, 16), (220, 170, 20, 255)) # wing
d.ellipse([14*u, 8*u, 15.5*u, 9.5*u], fill=o)
d.line([9*u, 19*u, 9*u, 22*u], fill=o, width=w); d.line([13*u, 19*u, 13*u, 22*u], fill=o, width=w)
elif name == "fish":
c = (60, 180, 230, 255)
P([(1, 4), (7, 11), (1, 18)], (30, 120, 200, 255)) # tail fin
E((5, 5, 21, 17), c)
P([(11, 6), (14, 2), (16, 6)], (30, 120, 200, 255)) # dorsal fin
d.ellipse([16*u, 9*u, 17.8*u, 10.8*u], fill=o)
elif name == "pig":
c = (245, 160, 185, 255)
for x in (5, 14): d.rectangle([x*u, 15*u, (x+3)*u, 20*u], fill=c, outline=o, width=w)
E((2, 6, 20, 17), c)
E((15, 9, 21, 14), (230, 120, 150, 255)) # snout
P([(12, 6), (14, 2), (16, 6)], c) # ear
d.ellipse([13*u, 8*u, 14.3*u, 9.3*u], fill=o)
elif name == "rabbit":
c = (235, 235, 225, 255)
E((6, 0, 9, 10), c); E((11, 0, 14, 10), c) # long ears
E((4, 12, 18, 22), c) # body
E((5, 6, 15, 15), c) # head
d.ellipse([7.5*u, 9*u, 8.8*u, 10.3*u], fill=o); d.ellipse([11.2*u, 9*u, 12.5*u, 10.3*u], fill=o)
E((16, 15, 20, 19), (255, 255, 255, 255)) # tail puff
return im
def build_sprites(outdir="sprites"):
os.makedirs(outdir, exist_ok=True)
sp = {}
for a in ANIMALS:
im = _sprite(a)
im.save(f"{outdir}/{a}.png")
sp[a] = im
return sp
def compose(sprites, a, b, layout, color):
"""Paste transparent sprite A and B onto a solid background (the NFT-style layer stack)."""
n = S * HI
bg = Image.new("RGBA", (n, n), COLORS[color] + (255,))
k = SPRITE * HI
if layout == "top": # B on the ground, A standing on top of it
bg.alpha_composite(sprites[b], ((n - k) // 2, n - k - 2 * HI))
bg.alpha_composite(sprites[a], ((n - k) // 2, n - 2 * k + 2 * HI))
else: # A on the left, B on the right
y = n - k - 6 * HI
bg.alpha_composite(sprites[a], (1 * HI, y))
bg.alpha_composite(sprites[b], (n - k - 1 * HI, y))
im = bg.convert("RGB").resize((S, S), Image.LANCZOS)
return np.asarray(im, dtype=np.float32) / 255.0
def caption(a, b, layout, color):
mid = "on top of" if layout == "top" else "next to"
return f"a {a} {mid} a {b} on a {color} background"
# Ordered "A on top of B" pairs that are NEVER shown in any color during training.
# Their reverse IS shown (e.g. "dog on top of cat" is seen, "cat on top of dog" never is).
HELD_PAIRS = [("cat", "dog"), ("bird", "pig"), ("rabbit", "fish"), ("pig", "cat")]
def build(seed=0):
rng = np.random.default_rng(seed)
sprites = build_sprites()
combos = [(a, b, l, c) for a, b in itertools.permutations(ANIMALS, 2)
for l in LAYOUTS for c in COLORS]
split = {}
for cb in combos:
a, b, l, c = cb
if l == "top" and (a, b) in HELD_PAIRS:
split[cb] = "unseen_pair" # hardest test: arrangement never seen
elif rng.random() < 0.2:
split[cb] = "unseen_combo" # pieces + arrangement seen, this exact scene not
else:
split[cb] = "train"
X = np.stack([compose(sprites, *cb) for cb in combos])
caps = [caption(*cb) for cb in combos]
return combos, caps, X, split
if __name__ == "__main__":
combos, caps, X, split = build()
from collections import Counter
print(len(combos), Counter(split.values()))
print(caps[0])