""" WHAT KIND OF BODY SUPPORTS A MEMBER ADDED LATER? A member injected into a frozen body reached 96.7% of its own ceiling on Fashion and 27% on digits, and disturbed the existing members by exactly zero in both. Sweeping the body's storage over a thirty-two-fold range did not close the gap: the ceiling rose with capacity and the injected member did not. More room does not stay free, because gradient descent spends whatever is available on the task in front of it. Values allocated while training on one set of classes become specialised TO those classes. So a body cannot be provisioned by leaving space. Three constructions might work instead, and they answer different cases. BROAD train the body on a wider distribution than any member needs. This is what a pretrained backbone is, and it fits the case where a provider ships a base and a user's member is new content INSIDE a space the base already covers. It cannot help with anything genuinely outside that space, because one cannot pretrain on what has not happened. RESERVED give part of the body its OWN value set, unreachable from the family's loss, and let an injected member read it. This is not spare capacity -- the sweep showed there is no such thing -- but capacity that is structurally isolated. Two versions: left at initialisation, which makes it random features, and trained on RECONSTRUCTION, which makes it features of the input distribution rather than of anyone's task, and needs no labels. Masking the reserve out of the family's heads would not isolate it. The body's values are shared across the whole matrix, so a gradient anywhere moves values everywhere; the reserve needs a separate path with its own values, which is what is built here. Each body faces two members. One covers classes the body has seen, which is the provider case. One covers classes it never saw in any form, which is the case no amount of broadening reaches. The interesting cell is out-of-domain against a reserved body, because nothing else covers it. """ import numpy as np import time import json from itertools import combinations try: import cupy as _cp _GPU = _cp.cuda.runtime.getDeviceCount() > 0 except Exception: _GPU = False xp = _cp if _GPU else np DT = np.float32 def to_dev(a, dtype=DT): a = np.asarray(a, dtype=dtype) return xp.asarray(a) if _GPU else a def to_host(a): return _cp.asnumpy(a) if _GPU and isinstance(a, _cp.ndarray) else np.asarray(a) _MAPS = {} def tie(shape, K, tag): key = (tuple(shape), K, tag) if key not in _MAPS: import hashlib h = int(hashlib.md5(str(key).encode()).hexdigest()[:8], 16) rg = np.random.default_rng(h) idx = rg.integers(0, K, int(np.prod(shape))) _MAPS[key] = to_dev(idx, np.int64) if _GPU else idx.astype(np.int64) return _MAPS[key] def fold(W, idx, K): flat = W.reshape(-1) s = xp.zeros(K, DT); c = xp.zeros(K, DT) if _GPU: import cupyx cupyx.scatter_add(s, idx, flat); cupyx.scatter_add(c, idx, xp.ones_like(flat)) else: np.add.at(s, idx, flat); np.add.at(c, idx, np.ones_like(flat)) return (s / xp.maximum(c, 1.0))[idx].reshape(W.shape) def finit(rg, fan, shape, K): """Draw so the weights carry the intended scale AFTER folding.""" return rg.normal(0, np.sqrt(2.0/fan)*np.sqrt(max(1.0, np.prod(shape)/K)), shape) class Body: """A family path and, optionally, a RESERVED path with its own values. The reserve is not spare capacity. It is a separate set of values the family's loss cannot reach, so nothing the family learns can specialise it, and an injected member is the first thing that ever writes to it.""" def __init__(self, D, W, K, seed, reserve=0, K_res=None, tag="b"): rg = np.random.default_rng(seed) self.D, self.W, self.K, self.R = D, W, K, reserve self.i0 = tie((D, W), K, tag+"0") self.W0 = to_dev(finit(rg, D, (D, W), K)) self.W0 = fold(self.W0, self.i0, K) self.b0 = xp.zeros(W, DT) if reserve: self.Kr = K_res or K self.ir = tie((D, reserve), self.Kr, tag+"r") self.Wr = to_dev(finit(rg, D, (D, reserve), self.Kr)) self.Wr = fold(self.Wr, self.ir, self.Kr) self.br = xp.zeros(reserve, DT) def out(self, X, with_reserve): a = xp.maximum(X @ self.W0 + self.b0, 0) if with_reserve and self.R: ar = xp.maximum(X @ self.Wr + self.br, 0) return xp.concatenate([a, ar], 1) return a def width(self, with_reserve): return self.W + (self.R if with_reserve and self.R else 0) def stored(self): return self.K + (self.Kr if self.R else 0) def fit_heads(body, X, Y, masks, epochs, batch, lr, seed, with_reserve=False, train_body=False): """Fit one head per member. train_body moves the family path too; an injection leaves it frozen and writes only the head.""" rg = np.random.default_rng(seed) S, n = masks.shape[0], X.shape[0] w = body.width(with_reserve) H = to_dev(rg.normal(0, np.sqrt(2.0/w), (S, w, 10))) B = xp.zeros((S, 10), DT) P = [H, B] + ([body.W0, body.b0] if train_body else []) M = [xp.zeros_like(p) for p in P]; V = [xp.zeros_like(p) for p in P] t = 0 for ep in range(epochs): perm = rg.permutation(n) for st in range(0, n, batch): b = perm[st:st+batch] bi = to_dev(b, np.int64) if _GPU else b x = X[bi]; y = Y[bi]; mk = masks[:, bi] z0 = x @ body.W0 + body.b0 a = xp.maximum(z0, 0) if with_reserve and body.R: ar = xp.maximum(x @ body.Wr + body.br, 0) a = xp.concatenate([a, ar], 1) lg = xp.einsum('ni,sio->sno', a, H) + B[:, None, :] e = xp.exp(lg - lg.max(-1, keepdims=True)) d = (e/e.sum(-1, keepdims=True) - y[None])*mk[:, :, None] d = d/max(1, len(b)) G = [xp.einsum('ni,sno->sio', a, d), d.sum(1)] if train_body: da = xp.einsum('sno,sio->ni', d, H) if with_reserve and body.R: da = da[:, :body.W] dz = da*(z0 > 0) G += [x.T @ dz, dz.sum(0)] t += 1 for i, (p_, gr) in enumerate(zip(P, G)): M[i] = 0.9*M[i] + 0.1*gr V[i] = 0.999*V[i] + 0.001*gr*gr P[i] = p_ - lr*(M[i]/(1-0.9**t))/(xp.sqrt(V[i]/(1-0.999**t))+1e-8) H, B = P[0], P[1] if train_body: body.W0, body.b0 = P[2], P[3] body.W0 = fold(body.W0, body.i0, body.K) P[2] = body.W0 return H, B def fit_reserve_recon(body, X, epochs, batch, lr, seed): """Train the reserved path to RECONSTRUCT the input. No labels, no task: the features it learns are of the input distribution rather than of anyone's classes, which is the point. A provider can do this on whatever unlabelled data exists.""" rg = np.random.default_rng(seed) n, D = X.shape Dec = to_dev(rg.normal(0, np.sqrt(2.0/body.R), (body.R, D))) P = [body.Wr, body.br, Dec] M = [xp.zeros_like(p) for p in P]; V = [xp.zeros_like(p) for p in P] t = 0 for ep in range(epochs): perm = rg.permutation(n) for st in range(0, n, batch): b = perm[st:st+batch] bi = to_dev(b, np.int64) if _GPU else b x = X[bi] z = x @ body.Wr + body.br; a = xp.maximum(z, 0) rec = a @ Dec d = 2*(rec - x)/max(1, len(b)) gDec = a.T @ d dz = (d @ Dec.T)*(z > 0) G = [x.T @ dz, dz.sum(0), gDec] t += 1 for i, (p_, gr) in enumerate(zip(P, G)): M[i] = 0.9*M[i] + 0.1*gr V[i] = 0.999*V[i] + 0.001*gr*gr P[i] = p_ - lr*(M[i]/(1-0.9**t))/(xp.sqrt(V[i]/(1-0.999**t))+1e-8) body.Wr, body.br, Dec = P body.Wr = fold(body.Wr, body.ir, body.Kr) P[0] = body.Wr return body def acc(body, H, B, X, y, mask, with_reserve): a = body.out(X, with_reserve) lg = to_host(xp.einsum('ni,sio->sno', a, H) + B[:, None, :]) m = to_host(mask).astype(bool) return float((lg[0][m[0]].argmax(1) == y[m[0]]).mean()) def load(name): if name == "fashion": from tensorflow import keras (a, b), (c, d) = keras.datasets.fashion_mnist.load_data() X = np.concatenate([a, c]).reshape(-1, 784).astype(np.float32)/255.0 y = np.concatenate([b, d]).ravel().astype(np.int64) else: from sklearn.datasets import load_digits dd = load_digits() X = dd.data.astype(np.float32)/16.0; y = dd.target.astype(np.int64) return X, y # STORAGE IS MATCHED. A first run gave the reserved bodies 768 values # against the others' 512, so "reserved beats narrow by +0.007" was partly # just half again as much capacity. Here every body holds TOTAL_K, and a # reserved one splits it: the reserve is capacity taken FROM the family # path, not added to it, which is the honest question — is isolated # capacity worth more than the same capacity given to the family? CFG = dict(dataset="fashion", width=64, reserve=32, TOTAL_K=512, reserve_frac=0.25, n_train=8000, epochs=40, batch=128, lr=1e-3, seeds=(0, 1, 2), family_classes=(0, 1, 2, 3, 4, 5, 6), unseen_classes=(7, 8, 9)) def one_seed(X, y, seed, cfg): rg = np.random.default_rng(seed) fam = list(cfg["family_classes"]); uns = list(cfg["unseen_classes"]) D = X.shape[1] subs = [tuple(sorted(c)) for c in combinations(fam, 4)] subs = [subs[i] for i in rg.choice(len(subs), 6, replace=False)] in_dom = tuple(sorted(rg.choice(fam, 4, replace=False))) # seen classes out_dom = tuple(sorted(uns)) # never seen def split(classes, n): i = np.where(np.isin(y, classes))[0]; rg.shuffle(i) return i[:n], i[n:n+2000] ftr, fte = split(fam, cfg["n_train"]) itr, ite = split(in_dom, cfg["n_train"]//3) otr, ote = split(uns, cfg["n_train"]//3) dev = lambda i: to_dev(X[i]) Y = lambda i: to_dev(np.eye(10, dtype=np.float32)[y[i]]) mk = lambda i, ss: to_dev(np.stack([np.isin(y[i], s) for s in ss]).astype(np.float32)) allX = to_dev(X[np.concatenate([ftr, otr])]) # for the BROAD body out = {} for name in ("narrow", "broad", "reserved-random", "reserved-recon"): res = cfg["reserve"] if name.startswith("reserved") else 0 if res: kr = int(cfg["TOTAL_K"]*cfg["reserve_frac"]) kf = cfg["TOTAL_K"] - kr else: kf, kr = cfg["TOTAL_K"], None body = Body(D, cfg["width"], kf, seed, res, kr, tag=name) # the family path: narrow sees family classes, broad sees everything if name == "broad": bx = np.concatenate([ftr, otr]) fit_heads(body, dev(bx), Y(bx), mk(bx, subs + [out_dom]), cfg["epochs"], cfg["batch"], cfg["lr"], seed, train_body=True) else: fit_heads(body, dev(ftr), Y(ftr), mk(ftr, subs), cfg["epochs"], cfg["batch"], cfg["lr"], seed, train_body=True) if name == "reserved-recon": fit_reserve_recon(body, dev(ftr), cfg["epochs"], cfg["batch"], cfg["lr"], seed+5) wr = bool(res) for tag, tr, te, sub in (("in", itr, ite, in_dom), ("out", otr, ote, out_dom)): H, B = fit_heads(body, dev(tr), Y(tr), mk(tr, [sub]), cfg["epochs"], cfg["batch"], cfg["lr"], seed+1, with_reserve=wr, train_body=False) out[f"{name}/{tag}"] = acc(body, H, B, dev(te), y[te], mk(te, [sub]), wr) out[f"{name}/stored"] = body.stored() # ceilings: each member with a body of its own for tag, tr, te, sub in (("in", itr, ite, in_dom), ("out", otr, ote, out_dom)): b2 = Body(D, cfg["width"], cfg["TOTAL_K"], seed+9, tag="own"+tag) H, B = fit_heads(b2, dev(tr), Y(tr), mk(tr, [sub]), cfg["epochs"], cfg["batch"], cfg["lr"], seed+9, train_body=True) out[f"ceiling/{tag}"] = acc(b2, H, B, dev(te), y[te], mk(te, [sub]), False) return out def main(**over): CFG.update(over) t0 = time.time() print("=" * 78) print("WHAT KIND OF BODY SUPPORTS A MEMBER ADDED LATER?") print("=" * 78) print(f" backend: {'cupy (GPU)' if _GPU else 'numpy (CPU)'}") for k, v in CFG.items(): print(f" {k:16s} = {v}") print(f"\n IN-DOMAIN a member covering classes the body has seen") print(f" OUT-DOMAIN a member covering {CFG['unseen_classes']}, which") print(f" only the BROAD body has ever seen") print("=" * 78, flush=True) X, y = load(CFG["dataset"]) runs = [] for s in CFG["seeds"]: r = one_seed(X, y, s, CFG) runs.append(r) print(f" seed {s} done [{time.time()-t0:.0f}s]", flush=True) json.dump(runs, open("bodies.json", "w"), indent=2) keys = list(runs[0]) m = {k: float(np.mean([r[k] for r in runs])) for k in keys} sd = {k: float(np.std([r[k] for r in runs])) for k in keys} print("\n per seed, because the differences are small enough that a") print(" mean without a spread says nothing:\n") print(f" {'arm':>22s} " + " ".join(f"{'seed '+str(s):>9s}" for s in CFG["seeds"])) for name in ("narrow", "broad", "reserved-random", "reserved-recon"): for tag in ("in", "out"): k = f"{name}/{tag}" print(f" {k:>22s} " + " ".join(f"{r[k]:9.4f}" for r in runs)) print("\n" + "=" * 78) print(" READOUT") print("=" * 78) ci, co = m["ceiling/in"], m["ceiling/out"] print(f" ceilings: in-domain {ci:.4f}, out-of-domain {co:.4f}") print(f" (each member with a body built for it alone)\n") print(f" {'body':>16s} {'stored':>8s} {'in-domain':>10s} {'sd':>7s}" f" {'out-domain':>11s} {'sd':>7s}") for name in ("narrow", "broad", "reserved-random", "reserved-recon"): i_, o_ = m[f"{name}/in"], m[f"{name}/out"] print(f" {name:>16s} {int(m[name+'/stored']):8,} {i_:10.4f} " f"{sd[name+'/in']:7.4f} {o_:11.4f} {sd[name+'/out']:7.4f}") print(f"\n against a narrow body, PAIRED (the arms share a seed, so an") print(f" unpaired spread overstates the noise):\n") print(f" {'body':>16s} {'in-domain':>18s} {'out-of-domain':>20s}") for name in ("broad", "reserved-random", "reserved-recon"): for tag, w in (("in", 18), ("out", 20)): d = np.array([r[f"{name}/{tag}"] - r[f"narrow/{tag}"] for r in runs]) cell = (f"{d.mean():+.4f} +- {d.std():.4f}") if tag == "in": line = f" {name:>16s} {cell:>18s}" else: print(line + f" {cell:>20s}") print() best_out = max(("narrow", "broad", "reserved-random", "reserved-recon"), key=lambda n: m[f"{n}/out"]) nar = m["narrow/out"] print(f" the out-of-domain member is best served by {best_out} " f"({m[best_out+'/out']:.4f}),") print(f" against {nar:.4f} for a narrow body — " f"{m[best_out+'/out']-nar:+.4f}") for n in ("reserved-random", "reserved-recon"): print(f" {n:>16s} {m[n+'/out']-nar:+.4f} over narrow") print() if best_out == "broad": print(" BREADTH IS WHAT MATTERS, and the reserve adds nothing. A") print(" provider should train the base on as wide a distribution as") print(" it can and not try to hold capacity back — which also means") print(" a member genuinely outside that distribution has no remedy") print(" here.") elif best_out.startswith("reserved"): print(" ISOLATED CAPACITY WORKS. A path the family's loss cannot") print(" reach is available to a member added later, and it helps") print(" where breadth cannot — with something the base never saw in") print(" any form, which is the case a pretrained backbone has no") print(" answer to.") else: print(" NEITHER CONSTRUCTION HELPS. A member outside the body's") print(" experience is not rescued by breadth or by isolation, so") print(" what it needs is its own body and injection has a domain") print(" boundary rather than a capacity one.") print(f"\n total {time.time()-t0:.0f}s; wrote bodies.json") if __name__ == "__main__": main()