| """How many classes can PixelGen absorb before something gives? |
| |
| Synthesises N extra classes (random valid 16x8 templates + 3 unique nonsense |
| words each), rebuilds the model at each N, and measures: |
| |
| params / buffers / bytes - the capacity cost |
| build time - cost of "compiling" the weights |
| generate latency (1 prompt) - inference cost |
| isolation - the real 21 classes still bit-identical? |
| dense-layer share - how much of the model is the word->class matrix |
| |
| Nothing is trained. The synthetic classes exist only in this process. |
| """ |
|
|
| import random |
| import sys |
| import time |
|
|
| import numpy as np |
| import torch |
|
|
| import pixelgen.model as M |
| import pixelgen.templates as T |
| import pixelgen.tokenizer as K |
|
|
| REAL_TEMPLATES = dict(T.TEMPLATES) |
| REAL_SYN = dict(K.CLASS_SYNONYMS) |
| REAL_CLASSES = list(REAL_TEMPLATES) |
|
|
|
|
| def random_template(rng): |
| rows = ["." * 8, "." * 8] |
| for _ in range(12): |
| row = ".." + "".join(rng.choice("##.?") for _ in range(6)) |
| rows.append(row) |
| rows += ["." * 8, "." * 8] |
| on = sum(r.count("#") for r in rows); free = sum(r.count("?") for r in rows) |
| if on < 8 or on + free > 60: |
| return random_template(rng) |
| return rows |
|
|
|
|
| def build(n_extra, seed=0): |
| rng = random.Random(seed) |
| tmpl = dict(REAL_TEMPLATES); syn = dict(REAL_SYN) |
| for i in range(n_extra): |
| name = f"syn{i:05d}" |
| tmpl[name] = random_template(rng) |
| syn[name] = [name, f"{name}a", f"{name}b"] |
| T.TEMPLATES = tmpl; T.CLASSES = list(tmpl) |
| K.CLASS_SYNONYMS = syn; K.VOCAB = K.build_vocab() |
| M.TEMPLATES, M.CLASSES, M.CLASS_SYNONYMS, M.VOCAB = tmpl, T.CLASSES, syn, K.VOCAB |
| t0 = time.perf_counter(); m = M.PixelGen(); dt = time.perf_counter() - t0 |
| return m, dt |
|
|
|
|
| |
| ref_model, _ = build(0) |
| probe = REAL_CLASSES + ["red potion", "blue invader", "dragon", "", "golden key", "bone"] |
| ref_out = {p: ref_model.generate([p] * 2, [0, 1], scale=1) for p in probe} |
|
|
| Ns = [int(a) for a in sys.argv[1:]] or [0, 10, 30, 100, 300, 1000, 3000] |
| print(f"{'classes':>8} {'vocab':>6} {'params':>11} {'buffers':>9} {'MB':>7} {'dense%':>6} {'build s':>8} {'gen ms':>7} {'isolated':>8}") |
| for n in Ns: |
| m, dt = build(n) |
| P = sum(p.numel() for p in m.parameters()); B = sum(b.numel() for b in m.buffers()) |
| dense = m.word_to_class.weight.numel() |
| t0 = time.perf_counter() |
| for _ in range(5): m.generate(["red potion"], [0], scale=1) |
| gen = (time.perf_counter() - t0) / 5 * 1000 |
| ok = all(np.array_equal(m.generate([p] * 2, [0, 1], scale=1), ref_out[p]) for p in probe) |
| |
| if n: |
| nm = f"syn{n-1:05d}" |
| got = m.generate([f"{nm}b"], [0], scale=1)[0].sum(-1) > 0 |
| want = m.generate([nm], [0], scale=1)[0].sum(-1) > 0 |
| ok = ok and np.array_equal(got, want) |
| print(f"{len(m.classes):>8} {len(m.vocab):>6} {P:>11,} {B:>9,} {(P+B)*4/1e6:>7.2f} {100*dense/(P+B):>5.1f}% {dt:>8.2f} {gen:>7.2f} {str(ok):>8}") |
|
|
| print("\nparam model: word_to_class = V*C where V≈3C -> O(C^2); banks = 256*C -> O(C)") |
| for C in (21, 100, 300, 1000, 3000, 10000): |
| V = 59 - 0 + 3 * (C - 21) if C >= 21 else 59 |
| print(f" C={C:>6}: dense {V*C:>12,} banks {256*C:>10,} dense/banks = {V*C/(256*C):.1f}x") |
|
|
