File size: 6,171 Bytes
39d8cda
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
"""
WVY-Compose dataset builder.

The idea: a tiny set of hand-made ingredients (6 transparent animal sprites +
8 background colors) gets composited into scenes with exact text captions.
Some combinations are deliberately held out so we can test whether the model
generates scenes it never saw (generalization) or only replays what it saw
(memorization).
"""
import numpy as np
from PIL import Image, ImageDraw
import itertools, json, os

S = 48          # final image size (pixels)
HI = 4          # draw sprites at 4x resolution, then downsample for clean edges
SPRITE = 22     # sprite size in final pixels

ANIMALS = ["dog", "cat", "bird", "fish", "pig", "rabbit"]
COLORS = {
    "white": (240, 240, 240), "black": (25, 25, 25), "red": (200, 40, 40),
    "orange": (240, 140, 30), "yellow": (240, 220, 60), "green": (50, 160, 70),
    "blue": (50, 90, 200), "purple": (130, 60, 170),
}
LAYOUTS = ["top", "next"]   # "A on top of B"  /  "A next to B"


def _sprite(name):
    """Draw one transparent animal sprite. Each is a single 'ingredient image'."""
    n = SPRITE * HI
    im = Image.new("RGBA", (n, n), (0, 0, 0, 0))
    d = ImageDraw.Draw(im)
    o = (15, 15, 15, 255)          # dark outline so sprites read on any background
    w = 2 * HI // 2 + 2
    u = n / 22.0                    # drawing unit

    def E(box, fill):  # ellipse in drawing units
        d.ellipse([b * u for b in box], fill=fill, outline=o, width=w)

    def P(pts, fill):
        d.polygon([(x * u, y * u) for x, y in pts], fill=fill, outline=o, width=w)

    if name == "dog":
        c = (150, 95, 45, 255)
        for x in (5, 8, 13, 16): d.rectangle([x*u, 14*u, (x+2)*u, 20*u], fill=c, outline=o, width=w)
        E((3, 9, 19, 16), c)                     # body
        P([(18, 9), (21, 5), (20, 10)], c)        # tail up
        E((0, 3, 9, 11), c)                       # head
        E((0, 3, 3, 10), (90, 55, 25, 255))       # floppy ear
        d.ellipse([2*u, 6*u, 3.3*u, 7.3*u], fill=o)
    elif name == "cat":
        c = (140, 140, 150, 255)
        P([(17, 15), (21, 6), (19, 15)], c)       # tail
        E((4, 10, 18, 20), c)                     # body
        P([(4, 5), (5, 0), (8, 4)], c); P([(10, 4), (13, 0), (14, 5)], c)  # pointy ears
        E((3, 2, 15, 12), c)                      # head
        d.ellipse([6*u, 6*u, 7.5*u, 7.5*u], fill=(60, 200, 90, 255))
        d.ellipse([10.5*u, 6*u, 12*u, 7.5*u], fill=(60, 200, 90, 255))
    elif name == "bird":
        c = (250, 215, 40, 255)
        E((3, 4, 19, 19), c)                      # round body
        P([(18, 9), (22, 11), (18, 13)], (240, 120, 20, 255))  # beak
        E((5, 10, 12, 16), (220, 170, 20, 255))   # wing
        d.ellipse([14*u, 8*u, 15.5*u, 9.5*u], fill=o)
        d.line([9*u, 19*u, 9*u, 22*u], fill=o, width=w); d.line([13*u, 19*u, 13*u, 22*u], fill=o, width=w)
    elif name == "fish":
        c = (60, 180, 230, 255)
        P([(1, 4), (7, 11), (1, 18)], (30, 120, 200, 255))   # tail fin
        E((5, 5, 21, 17), c)
        P([(11, 6), (14, 2), (16, 6)], (30, 120, 200, 255))  # dorsal fin
        d.ellipse([16*u, 9*u, 17.8*u, 10.8*u], fill=o)
    elif name == "pig":
        c = (245, 160, 185, 255)
        for x in (5, 14): d.rectangle([x*u, 15*u, (x+3)*u, 20*u], fill=c, outline=o, width=w)
        E((2, 6, 20, 17), c)
        E((15, 9, 21, 14), (230, 120, 150, 255))  # snout
        P([(12, 6), (14, 2), (16, 6)], c)          # ear
        d.ellipse([13*u, 8*u, 14.3*u, 9.3*u], fill=o)
    elif name == "rabbit":
        c = (235, 235, 225, 255)
        E((6, 0, 9, 10), c); E((11, 0, 14, 10), c)  # long ears
        E((4, 12, 18, 22), c)                        # body
        E((5, 6, 15, 15), c)                          # head
        d.ellipse([7.5*u, 9*u, 8.8*u, 10.3*u], fill=o); d.ellipse([11.2*u, 9*u, 12.5*u, 10.3*u], fill=o)
        E((16, 15, 20, 19), (255, 255, 255, 255))   # tail puff
    return im


def build_sprites(outdir="sprites"):
    os.makedirs(outdir, exist_ok=True)
    sp = {}
    for a in ANIMALS:
        im = _sprite(a)
        im.save(f"{outdir}/{a}.png")
        sp[a] = im
    return sp


def compose(sprites, a, b, layout, color):
    """Paste transparent sprite A and B onto a solid background (the NFT-style layer stack)."""
    n = S * HI
    bg = Image.new("RGBA", (n, n), COLORS[color] + (255,))
    k = SPRITE * HI
    if layout == "top":    # B on the ground, A standing on top of it
        bg.alpha_composite(sprites[b], ((n - k) // 2, n - k - 2 * HI))
        bg.alpha_composite(sprites[a], ((n - k) // 2, n - 2 * k + 2 * HI))
    else:                  # A on the left, B on the right
        y = n - k - 6 * HI
        bg.alpha_composite(sprites[a], (1 * HI, y))
        bg.alpha_composite(sprites[b], (n - k - 1 * HI, y))
    im = bg.convert("RGB").resize((S, S), Image.LANCZOS)
    return np.asarray(im, dtype=np.float32) / 255.0


def caption(a, b, layout, color):
    mid = "on top of" if layout == "top" else "next to"
    return f"a {a} {mid} a {b} on a {color} background"


# Ordered "A on top of B" pairs that are NEVER shown in any color during training.
# Their reverse IS shown (e.g. "dog on top of cat" is seen, "cat on top of dog" never is).
HELD_PAIRS = [("cat", "dog"), ("bird", "pig"), ("rabbit", "fish"), ("pig", "cat")]


def build(seed=0):
    rng = np.random.default_rng(seed)
    sprites = build_sprites()
    combos = [(a, b, l, c) for a, b in itertools.permutations(ANIMALS, 2)
              for l in LAYOUTS for c in COLORS]
    split = {}
    for cb in combos:
        a, b, l, c = cb
        if l == "top" and (a, b) in HELD_PAIRS:
            split[cb] = "unseen_pair"          # hardest test: arrangement never seen
        elif rng.random() < 0.2:
            split[cb] = "unseen_combo"         # pieces + arrangement seen, this exact scene not
        else:
            split[cb] = "train"
    X = np.stack([compose(sprites, *cb) for cb in combos])
    caps = [caption(*cb) for cb in combos]
    return combos, caps, X, split


if __name__ == "__main__":
    combos, caps, X, split = build()
    from collections import Counter
    print(len(combos), Counter(split.values()))
    print(caps[0])