| """ |
| WHAT KIND OF BODY SUPPORTS A MEMBER ADDED LATER? |
| |
| A member injected into a frozen body reached 96.7% of its own ceiling on |
| Fashion and 27% on digits, and disturbed the existing members by exactly |
| zero in both. Sweeping the body's storage over a thirty-two-fold range did |
| not close the gap: the ceiling rose with capacity and the injected member |
| did not. More room does not stay free, because gradient descent spends |
| whatever is available on the task in front of it. Values allocated while |
| training on one set of classes become specialised TO those classes. |
| |
| So a body cannot be provisioned by leaving space. Three constructions might |
| work instead, and they answer different cases. |
| |
| BROAD train the body on a wider distribution than any member |
| needs. This is what a pretrained backbone is, and it fits |
| the case where a provider ships a base and a user's member |
| is new content INSIDE a space the base already covers. It |
| cannot help with anything genuinely outside that space, |
| because one cannot pretrain on what has not happened. |
| |
| RESERVED give part of the body its OWN value set, unreachable from |
| the family's loss, and let an injected member read it. This |
| is not spare capacity -- the sweep showed there is no such |
| thing -- but capacity that is structurally isolated. Two |
| versions: left at initialisation, which makes it random |
| features, and trained on RECONSTRUCTION, which makes it |
| features of the input distribution rather than of anyone's |
| task, and needs no labels. |
| |
| Masking the reserve out of the family's heads would not isolate it. The |
| body's values are shared across the whole matrix, so a gradient anywhere |
| moves values everywhere; the reserve needs a separate path with its own |
| values, which is what is built here. |
| |
| Each body faces two members. One covers classes the body has seen, which is |
| the provider case. One covers classes it never saw in any form, which is |
| the case no amount of broadening reaches. The interesting cell is |
| out-of-domain against a reserved body, because nothing else covers it. |
| """ |
|
|
| import numpy as np |
| import time |
| import json |
| from itertools import combinations |
|
|
| try: |
| import cupy as _cp |
| _GPU = _cp.cuda.runtime.getDeviceCount() > 0 |
| except Exception: |
| _GPU = False |
| xp = _cp if _GPU else np |
| DT = np.float32 |
|
|
|
|
| def to_dev(a, dtype=DT): |
| a = np.asarray(a, dtype=dtype) |
| return xp.asarray(a) if _GPU else a |
|
|
|
|
| def to_host(a): |
| return _cp.asnumpy(a) if _GPU and isinstance(a, _cp.ndarray) else np.asarray(a) |
|
|
|
|
| _MAPS = {} |
|
|
|
|
| def tie(shape, K, tag): |
| key = (tuple(shape), K, tag) |
| if key not in _MAPS: |
| import hashlib |
| h = int(hashlib.md5(str(key).encode()).hexdigest()[:8], 16) |
| rg = np.random.default_rng(h) |
| idx = rg.integers(0, K, int(np.prod(shape))) |
| _MAPS[key] = to_dev(idx, np.int64) if _GPU else idx.astype(np.int64) |
| return _MAPS[key] |
|
|
|
|
| def fold(W, idx, K): |
| flat = W.reshape(-1) |
| s = xp.zeros(K, DT); c = xp.zeros(K, DT) |
| if _GPU: |
| import cupyx |
| cupyx.scatter_add(s, idx, flat); cupyx.scatter_add(c, idx, xp.ones_like(flat)) |
| else: |
| np.add.at(s, idx, flat); np.add.at(c, idx, np.ones_like(flat)) |
| return (s / xp.maximum(c, 1.0))[idx].reshape(W.shape) |
|
|
|
|
| def finit(rg, fan, shape, K): |
| """Draw so the weights carry the intended scale AFTER folding.""" |
| return rg.normal(0, np.sqrt(2.0/fan)*np.sqrt(max(1.0, np.prod(shape)/K)), |
| shape) |
|
|
|
|
| class Body: |
| """A family path and, optionally, a RESERVED path with its own values. |
| |
| The reserve is not spare capacity. It is a separate set of values the |
| family's loss cannot reach, so nothing the family learns can specialise |
| it, and an injected member is the first thing that ever writes to it.""" |
|
|
| def __init__(self, D, W, K, seed, reserve=0, K_res=None, tag="b"): |
| rg = np.random.default_rng(seed) |
| self.D, self.W, self.K, self.R = D, W, K, reserve |
| self.i0 = tie((D, W), K, tag+"0") |
| self.W0 = to_dev(finit(rg, D, (D, W), K)) |
| self.W0 = fold(self.W0, self.i0, K) |
| self.b0 = xp.zeros(W, DT) |
| if reserve: |
| self.Kr = K_res or K |
| self.ir = tie((D, reserve), self.Kr, tag+"r") |
| self.Wr = to_dev(finit(rg, D, (D, reserve), self.Kr)) |
| self.Wr = fold(self.Wr, self.ir, self.Kr) |
| self.br = xp.zeros(reserve, DT) |
|
|
| def out(self, X, with_reserve): |
| a = xp.maximum(X @ self.W0 + self.b0, 0) |
| if with_reserve and self.R: |
| ar = xp.maximum(X @ self.Wr + self.br, 0) |
| return xp.concatenate([a, ar], 1) |
| return a |
|
|
| def width(self, with_reserve): |
| return self.W + (self.R if with_reserve and self.R else 0) |
|
|
| def stored(self): |
| return self.K + (self.Kr if self.R else 0) |
|
|
|
|
| def fit_heads(body, X, Y, masks, epochs, batch, lr, seed, with_reserve=False, |
| train_body=False): |
| """Fit one head per member. train_body moves the family path too; an |
| injection leaves it frozen and writes only the head.""" |
| rg = np.random.default_rng(seed) |
| S, n = masks.shape[0], X.shape[0] |
| w = body.width(with_reserve) |
| H = to_dev(rg.normal(0, np.sqrt(2.0/w), (S, w, 10))) |
| B = xp.zeros((S, 10), DT) |
| P = [H, B] + ([body.W0, body.b0] if train_body else []) |
| M = [xp.zeros_like(p) for p in P]; V = [xp.zeros_like(p) for p in P] |
| t = 0 |
| for ep in range(epochs): |
| perm = rg.permutation(n) |
| for st in range(0, n, batch): |
| b = perm[st:st+batch] |
| bi = to_dev(b, np.int64) if _GPU else b |
| x = X[bi]; y = Y[bi]; mk = masks[:, bi] |
| z0 = x @ body.W0 + body.b0 |
| a = xp.maximum(z0, 0) |
| if with_reserve and body.R: |
| ar = xp.maximum(x @ body.Wr + body.br, 0) |
| a = xp.concatenate([a, ar], 1) |
| lg = xp.einsum('ni,sio->sno', a, H) + B[:, None, :] |
| e = xp.exp(lg - lg.max(-1, keepdims=True)) |
| d = (e/e.sum(-1, keepdims=True) - y[None])*mk[:, :, None] |
| d = d/max(1, len(b)) |
| G = [xp.einsum('ni,sno->sio', a, d), d.sum(1)] |
| if train_body: |
| da = xp.einsum('sno,sio->ni', d, H) |
| if with_reserve and body.R: |
| da = da[:, :body.W] |
| dz = da*(z0 > 0) |
| G += [x.T @ dz, dz.sum(0)] |
| t += 1 |
| for i, (p_, gr) in enumerate(zip(P, G)): |
| M[i] = 0.9*M[i] + 0.1*gr |
| V[i] = 0.999*V[i] + 0.001*gr*gr |
| P[i] = p_ - lr*(M[i]/(1-0.9**t))/(xp.sqrt(V[i]/(1-0.999**t))+1e-8) |
| H, B = P[0], P[1] |
| if train_body: |
| body.W0, body.b0 = P[2], P[3] |
| body.W0 = fold(body.W0, body.i0, body.K) |
| P[2] = body.W0 |
| return H, B |
|
|
|
|
| def fit_reserve_recon(body, X, epochs, batch, lr, seed): |
| """Train the reserved path to RECONSTRUCT the input. |
| |
| No labels, no task: the features it learns are of the input |
| distribution rather than of anyone's classes, which is the point. A |
| provider can do this on whatever unlabelled data exists.""" |
| rg = np.random.default_rng(seed) |
| n, D = X.shape |
| Dec = to_dev(rg.normal(0, np.sqrt(2.0/body.R), (body.R, D))) |
| P = [body.Wr, body.br, Dec] |
| M = [xp.zeros_like(p) for p in P]; V = [xp.zeros_like(p) for p in P] |
| t = 0 |
| for ep in range(epochs): |
| perm = rg.permutation(n) |
| for st in range(0, n, batch): |
| b = perm[st:st+batch] |
| bi = to_dev(b, np.int64) if _GPU else b |
| x = X[bi] |
| z = x @ body.Wr + body.br; a = xp.maximum(z, 0) |
| rec = a @ Dec |
| d = 2*(rec - x)/max(1, len(b)) |
| gDec = a.T @ d |
| dz = (d @ Dec.T)*(z > 0) |
| G = [x.T @ dz, dz.sum(0), gDec] |
| t += 1 |
| for i, (p_, gr) in enumerate(zip(P, G)): |
| M[i] = 0.9*M[i] + 0.1*gr |
| V[i] = 0.999*V[i] + 0.001*gr*gr |
| P[i] = p_ - lr*(M[i]/(1-0.9**t))/(xp.sqrt(V[i]/(1-0.999**t))+1e-8) |
| body.Wr, body.br, Dec = P |
| body.Wr = fold(body.Wr, body.ir, body.Kr) |
| P[0] = body.Wr |
| return body |
|
|
|
|
| def acc(body, H, B, X, y, mask, with_reserve): |
| a = body.out(X, with_reserve) |
| lg = to_host(xp.einsum('ni,sio->sno', a, H) + B[:, None, :]) |
| m = to_host(mask).astype(bool) |
| return float((lg[0][m[0]].argmax(1) == y[m[0]]).mean()) |
|
|
|
|
| def load(name): |
| if name == "fashion": |
| from tensorflow import keras |
| (a, b), (c, d) = keras.datasets.fashion_mnist.load_data() |
| X = np.concatenate([a, c]).reshape(-1, 784).astype(np.float32)/255.0 |
| y = np.concatenate([b, d]).ravel().astype(np.int64) |
| else: |
| from sklearn.datasets import load_digits |
| dd = load_digits() |
| X = dd.data.astype(np.float32)/16.0; y = dd.target.astype(np.int64) |
| return X, y |
|
|
|
|
| |
| |
| |
| |
| |
| |
| CFG = dict(dataset="fashion", width=64, reserve=32, TOTAL_K=512, |
| reserve_frac=0.25, |
| n_train=8000, epochs=40, batch=128, lr=1e-3, seeds=(0, 1, 2), |
| family_classes=(0, 1, 2, 3, 4, 5, 6), |
| unseen_classes=(7, 8, 9)) |
|
|
|
|
| def one_seed(X, y, seed, cfg): |
| rg = np.random.default_rng(seed) |
| fam = list(cfg["family_classes"]); uns = list(cfg["unseen_classes"]) |
| D = X.shape[1] |
| subs = [tuple(sorted(c)) for c in combinations(fam, 4)] |
| subs = [subs[i] for i in rg.choice(len(subs), 6, replace=False)] |
| in_dom = tuple(sorted(rg.choice(fam, 4, replace=False))) |
| out_dom = tuple(sorted(uns)) |
|
|
| def split(classes, n): |
| i = np.where(np.isin(y, classes))[0]; rg.shuffle(i) |
| return i[:n], i[n:n+2000] |
| ftr, fte = split(fam, cfg["n_train"]) |
| itr, ite = split(in_dom, cfg["n_train"]//3) |
| otr, ote = split(uns, cfg["n_train"]//3) |
| dev = lambda i: to_dev(X[i]) |
| Y = lambda i: to_dev(np.eye(10, dtype=np.float32)[y[i]]) |
| mk = lambda i, ss: to_dev(np.stack([np.isin(y[i], s) |
| for s in ss]).astype(np.float32)) |
| allX = to_dev(X[np.concatenate([ftr, otr])]) |
|
|
| out = {} |
| for name in ("narrow", "broad", "reserved-random", "reserved-recon"): |
| res = cfg["reserve"] if name.startswith("reserved") else 0 |
| if res: |
| kr = int(cfg["TOTAL_K"]*cfg["reserve_frac"]) |
| kf = cfg["TOTAL_K"] - kr |
| else: |
| kf, kr = cfg["TOTAL_K"], None |
| body = Body(D, cfg["width"], kf, seed, res, kr, tag=name) |
| |
| if name == "broad": |
| bx = np.concatenate([ftr, otr]) |
| fit_heads(body, dev(bx), Y(bx), mk(bx, subs + [out_dom]), |
| cfg["epochs"], cfg["batch"], cfg["lr"], seed, |
| train_body=True) |
| else: |
| fit_heads(body, dev(ftr), Y(ftr), mk(ftr, subs), cfg["epochs"], |
| cfg["batch"], cfg["lr"], seed, train_body=True) |
| if name == "reserved-recon": |
| fit_reserve_recon(body, dev(ftr), cfg["epochs"], cfg["batch"], |
| cfg["lr"], seed+5) |
| wr = bool(res) |
| for tag, tr, te, sub in (("in", itr, ite, in_dom), |
| ("out", otr, ote, out_dom)): |
| H, B = fit_heads(body, dev(tr), Y(tr), mk(tr, [sub]), |
| cfg["epochs"], cfg["batch"], cfg["lr"], seed+1, |
| with_reserve=wr, train_body=False) |
| out[f"{name}/{tag}"] = acc(body, H, B, dev(te), y[te], |
| mk(te, [sub]), wr) |
| out[f"{name}/stored"] = body.stored() |
|
|
| |
| for tag, tr, te, sub in (("in", itr, ite, in_dom), |
| ("out", otr, ote, out_dom)): |
| b2 = Body(D, cfg["width"], cfg["TOTAL_K"], seed+9, tag="own"+tag) |
| H, B = fit_heads(b2, dev(tr), Y(tr), mk(tr, [sub]), cfg["epochs"], |
| cfg["batch"], cfg["lr"], seed+9, train_body=True) |
| out[f"ceiling/{tag}"] = acc(b2, H, B, dev(te), y[te], |
| mk(te, [sub]), False) |
| return out |
|
|
|
|
| def main(**over): |
| CFG.update(over) |
| t0 = time.time() |
| print("=" * 78) |
| print("WHAT KIND OF BODY SUPPORTS A MEMBER ADDED LATER?") |
| print("=" * 78) |
| print(f" backend: {'cupy (GPU)' if _GPU else 'numpy (CPU)'}") |
| for k, v in CFG.items(): |
| print(f" {k:16s} = {v}") |
| print(f"\n IN-DOMAIN a member covering classes the body has seen") |
| print(f" OUT-DOMAIN a member covering {CFG['unseen_classes']}, which") |
| print(f" only the BROAD body has ever seen") |
| print("=" * 78, flush=True) |
|
|
| X, y = load(CFG["dataset"]) |
| runs = [] |
| for s in CFG["seeds"]: |
| r = one_seed(X, y, s, CFG) |
| runs.append(r) |
| print(f" seed {s} done [{time.time()-t0:.0f}s]", flush=True) |
| json.dump(runs, open("bodies.json", "w"), indent=2) |
| keys = list(runs[0]) |
| m = {k: float(np.mean([r[k] for r in runs])) for k in keys} |
| sd = {k: float(np.std([r[k] for r in runs])) for k in keys} |
| print("\n per seed, because the differences are small enough that a") |
| print(" mean without a spread says nothing:\n") |
| print(f" {'arm':>22s} " + " ".join(f"{'seed '+str(s):>9s}" |
| for s in CFG["seeds"])) |
| for name in ("narrow", "broad", "reserved-random", "reserved-recon"): |
| for tag in ("in", "out"): |
| k = f"{name}/{tag}" |
| print(f" {k:>22s} " + " ".join(f"{r[k]:9.4f}" for r in runs)) |
|
|
| print("\n" + "=" * 78) |
| print(" READOUT") |
| print("=" * 78) |
| ci, co = m["ceiling/in"], m["ceiling/out"] |
| print(f" ceilings: in-domain {ci:.4f}, out-of-domain {co:.4f}") |
| print(f" (each member with a body built for it alone)\n") |
| print(f" {'body':>16s} {'stored':>8s} {'in-domain':>10s} {'sd':>7s}" |
| f" {'out-domain':>11s} {'sd':>7s}") |
| for name in ("narrow", "broad", "reserved-random", "reserved-recon"): |
| i_, o_ = m[f"{name}/in"], m[f"{name}/out"] |
| print(f" {name:>16s} {int(m[name+'/stored']):8,} {i_:10.4f} " |
| f"{sd[name+'/in']:7.4f} {o_:11.4f} {sd[name+'/out']:7.4f}") |
| print(f"\n against a narrow body, PAIRED (the arms share a seed, so an") |
| print(f" unpaired spread overstates the noise):\n") |
| print(f" {'body':>16s} {'in-domain':>18s} {'out-of-domain':>20s}") |
| for name in ("broad", "reserved-random", "reserved-recon"): |
| for tag, w in (("in", 18), ("out", 20)): |
| d = np.array([r[f"{name}/{tag}"] - r[f"narrow/{tag}"] |
| for r in runs]) |
| cell = (f"{d.mean():+.4f} +- {d.std():.4f}") |
| if tag == "in": |
| line = f" {name:>16s} {cell:>18s}" |
| else: |
| print(line + f" {cell:>20s}") |
| print() |
| best_out = max(("narrow", "broad", "reserved-random", "reserved-recon"), |
| key=lambda n: m[f"{n}/out"]) |
| nar = m["narrow/out"] |
| print(f" the out-of-domain member is best served by {best_out} " |
| f"({m[best_out+'/out']:.4f}),") |
| print(f" against {nar:.4f} for a narrow body — " |
| f"{m[best_out+'/out']-nar:+.4f}") |
| for n in ("reserved-random", "reserved-recon"): |
| print(f" {n:>16s} {m[n+'/out']-nar:+.4f} over narrow") |
| print() |
| if best_out == "broad": |
| print(" BREADTH IS WHAT MATTERS, and the reserve adds nothing. A") |
| print(" provider should train the base on as wide a distribution as") |
| print(" it can and not try to hold capacity back — which also means") |
| print(" a member genuinely outside that distribution has no remedy") |
| print(" here.") |
| elif best_out.startswith("reserved"): |
| print(" ISOLATED CAPACITY WORKS. A path the family's loss cannot") |
| print(" reach is available to a member added later, and it helps") |
| print(" where breadth cannot — with something the base never saw in") |
| print(" any form, which is the case a pretrained backbone has no") |
| print(" answer to.") |
| else: |
| print(" NEITHER CONSTRUCTION HELPS. A member outside the body's") |
| print(" experience is not rescued by breadth or by isolation, so") |
| print(" what it needs is its own body and injection has a domain") |
| print(" boundary rather than a capacity one.") |
| print(f"\n total {time.time()-t0:.0f}s; wrote bodies.json") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
