""" A MODEL THAT SCORES WELL. Every accuracy in this programme so far reads as a failure to a skim reader: 0.51 on CIFAR-100, 0.68 on CIFAR-10 at reduced resolution, 0.88 on Fashion. The numbers are honest and the architectures were chosen to be cheap enough to sweep, not to be good. A reader who sees 0.5 and concludes the framework is broken is drawing the wrong lesson from a real number, and the only fix is to run a competent architecture once. WHAT THIS DOES AND DOES NOT SHOW, stated plainly because the distinction matters more than the number. At a CONVOLUTION partition a folded model IS a standard convolutional network. Same arithmetic, same parameters, same accuracy. This cannot beat a ResNet and does not claim to. What it demonstrates is that folding does not CAP anything: the earlier figures were a three-layer stack with no residual connections, which is a 2013 design, and the framework is orthogonal to that choice. You can fold a good network as easily as a bad one. The architecture is a small residual network of the shape everyone uses on CIFAR: a stem, then three stages at rising width and falling resolution, two blocks each, a spatial pool and a linear head. Every convolution in it is a fold, so every value in the model is a shared value and the storage figure is the real one. THE CONTRAST ARM is a dense network at MATCHED PARAMETER COUNT, no convolutions and no sharing. It is the comparison a reader makes without being asked, and it isolates what the structure buys from what the budget buys. ยง7T of the paper does the sharper version of this, an ARBITRARY TYING at identical storage, but that cannot be run here: an arbitrary partition cannot use the im2col route, and materialising the dense matrix for a 32x32 stage at 32 channels is a billion entries. Everything is folded, deterministic, and saved to Drive. """ import numpy as np import time import json import os import hashlib try: import cupy as _cp _GPU = _cp.cuda.runtime.getDeviceCount() > 0 except Exception: _GPU = False xp = _cp if _GPU else np DT = np.float32 def to_dev(a, dtype=DT): a = np.asarray(a, dtype=dtype) return xp.asarray(a) if _GPU else a def to_host(a): return _cp.asnumpy(a) if _GPU and isinstance(a, _cp.ndarray) else np.asarray(a) class FastConv: """A folded convolution built from strided slices rather than a gather. Same values, same index meaning, same result. The value index is still (in-channel, out-channel, tap), so every claim this framework makes about partitions carries over untouched.""" def __init__(self, g_in, c_in, k, c_out, stride): self.g_in, self.c_in, self.k = g_in, c_in, k self.c_out, self.stride = c_out, stride self.g_out = g_in // stride self.ins = c_in*g_in*g_in self.out = c_out*self.g_out*self.g_out self.K = c_in*c_out*k*k + 1 self.taps = c_in*k*k self.pad = k//2 self.G = g_in + 2*self.pad self.npos = self.g_out*self.g_out def _slices(self): """Where each tap reads from in the padded input.""" s, g = self.stride, self.g_out for dy in range(self.k): for dx in range(self.k): yield (dy*self.k + dx, slice(dy, dy + s*g, s), slice(dx, dx + s*g, s)) def gather(self, x): """The window matrix as (batch, in-channel x tap, position). Nine strided slice copies. Each is a coalesced read; the fancy index it replaces was 391 million scattered ones on the ResNet-20 shape.""" n = x.shape[0] p = self.pad pd = xp.zeros((n, self.c_in, self.G, self.G), DT) pd[:, :, p:p+self.g_in, p:p+self.g_in] = x.reshape( n, self.c_in, self.g_in, self.g_in) col = xp.empty((n, self.c_in, self.k*self.k, self.npos), DT) for t, sy, sx in self._slices(): col[:, :, t, :] = pd[:, :, sy, sx].reshape(n, self.c_in, self.npos) return col.reshape(n, self.taps, self.npos) def weights_T(self, v): """The values as (out-channel, in-channel x tap). A value index is (in-channel, out-channel, tap), so this is a reshape and one transpose of a vector a few hundred thousand long, done once a step rather than over the whole window matrix.""" return v[:-1].reshape(self.c_in, self.c_out, self.k*self.k) \ .transpose(1, 0, 2).reshape(self.c_out, self.taps) def forward(self, v, x): col = self.gather(x) wt = self.weights_T(v) # (1, c_out, taps) @ (n, taps, positions) -> (n, c_out, positions), # which IS the hidden layout, so nothing is transposed afterwards z = xp.matmul(wt[None], col) return z.reshape(x.shape[0], self.out), col def backward(self, v, col, dz, need_input=True): n = dz.shape[0] d = dz.reshape(n, self.c_out, self.npos) # dW[out, taps] = sum over batch and position dW = xp.matmul(d, col.transpose(0, 2, 1)).sum(0) gv = xp.zeros(self.K, DT) gv[:-1] = dW.reshape(self.c_out, self.c_in, self.k*self.k) \ .transpose(1, 0, 2).reshape(-1) if not need_input: return gv, None # back to the window matrix, then nine strided ADDS rather than a # scatter: the same slices, accumulated in a fixed order dcol = xp.matmul(self.weights_T(v).T[None], d) dcol = dcol.reshape(n, self.c_in, self.k*self.k, self.npos) p = self.pad dpd = xp.zeros((n, self.c_in, self.G, self.G), DT) for t, sy, sx in self._slices(): dpd[:, :, sy, sx] += dcol[:, :, t, :].reshape( n, self.c_in, self.g_out, self.g_out) return gv, dpd[:, :, p:p+self.g_in, p:p+self.g_in].reshape(n, self.ins) # ---------------------------------------------------------------- model class Net: """A residual network in which every convolution is a fold. A block is relu(norm(conv2(relu(norm(conv1(h))))) + skip), and where the shape changes the skip carries a 1x1 fold of its own. The forward pass records a TAPE, because a residual junction is where a hand-written backward pass goes wrong and this one did on its first attempt.""" def __init__(self, cfg, seed): rg = np.random.default_rng(seed) self.cfg = cfg self.P, self.meta, self.blocks = [], [], [] g, c = cfg["grid"], cfg["c_in"] def conv(g_in, ci, co, k, st): fc = FastConv(g_in, ci, k, co, st) v = rg.normal(0, np.sqrt(2.0/(ci*k*k)), fc.K).astype(np.float32) v[-1] = 0.0 # PER CHANNEL, not per unit. Batch normalisation standardises # each channel across the batch and its spatial positions; the # layer normalisation this replaces standardised across every # channel and position at once, which flattens the per-channel # scale a convolutional stack spends its depth building. It is # also far fewer parameters: co rather than co x positions. self.P += [to_dev(v), xp.ones(co, DT), xp.zeros(co, DT)] self.meta.append(dict(fc=fc, p=len(self.P)-3, rm=xp.zeros(co, DT), rv=xp.ones(co, DT))) return len(self.meta)-1, fc.g_out, co # THE STEM DOWNSAMPLES. At stride one on a 32x32 grid the im2col # route materialises NINE copies of every pixel, so a stage at full # resolution with 32 channels gathers 37.7 million elements per # convolution per direction. A first version kept stage one at # 32x32 and reached 946 seconds an epoch: about a billion gathered # elements a step against six million in the testbed, where # everything downsampled at once. This is the cost cuDNN avoids by # never writing the expanded matrix, and it is the honest limit of # the fast path: 46 times better than materialising the weight # matrix, and still short of a real convolution kernel. self.stem, g, cin = conv(g, c, cfg["stem"], 3, cfg.get("stem_stride", 2)) for si, (ch, st) in enumerate(cfg["stages"]): for bi in range(cfg["blocks"]): s = st if bi == 0 else 1 gi, ci_ = g, cin i1, g, cin = conv(gi, ci_, ch, 3, s) i2, g, cin = conv(g, cin, ch, 3, 1) pj = None if s != 1 or ci_ != ch: pj, _, _ = conv(gi, ci_, ch, 1, s) self.blocks.append((i1, i2, pj)) self.feat, self.g_final = cin, g self.P += [to_dev(rg.normal(0, np.sqrt(2.0/cin), (cin, cfg["n_classes"]))), xp.zeros(cfg["n_classes"], DT)] self.head = len(self.P)-2 def values(self): return sum(m["fc"].K for m in self.meta) def _layer(self, mi, h, tape, training): m = self.meta[mi]; p = m["p"]; fc = m["fc"] n = h.shape[0]; c = fc.c_out; np_ = fc.g_out**2 z, col = fc.forward(self.P[p], h) zc = z.reshape(n, c, np_) if training: mu = zc.mean((0, 2), keepdims=True) var = zc.var((0, 2), keepdims=True) # running statistics, for inference on a single example m["rm"] = 0.9*m["rm"] + 0.1*mu.reshape(-1) m["rv"] = 0.9*m["rv"] + 0.1*var.reshape(-1) else: mu = m["rm"].reshape(1, c, 1); var = m["rv"].reshape(1, c, 1) zn = (zc - mu)/xp.sqrt(var + 1e-5) zs = zn*self.P[p+1].reshape(1, c, 1) + self.P[p+2].reshape(1, c, 1) if tape is not None: tape[mi] = (col, zn, var, c, np_) return zs.reshape(n, -1) def forward(self, x, tape=None, training=None): if training is None: training = tape is not None zs = self._layer(self.stem, x, tape, training) h = xp.maximum(zs, 0) pres = {} if tape is not None: tape["stem_zs"] = zs for (i1, i2, pj) in self.blocks: hin = h zs1 = self._layer(i1, hin, tape, training) a = xp.maximum(zs1, 0) zs2 = self._layer(i2, a, tape, training) skip = self._layer(pj, hin, tape, training) if pj is not None else hin pre = zs2 + skip h = xp.maximum(pre, 0) if tape is not None: pres[(i1, i2)] = (pre, zs1) if tape is not None: tape["pres"] = pres feat = h.reshape(h.shape[0], self.feat, self.g_final**2).mean(2) return feat, feat @ self.P[self.head] + self.P[self.head+1] def grads(self, x, y, tape): """Reverse walk of the tape. Returns gradients for every parameter.""" P = self.P G = [xp.zeros_like(p) for p in P] n = x.shape[0] feat, lg = self._feat, self._lg e = xp.exp(lg - lg.max(1, keepdims=True)) d = (e/e.sum(1, keepdims=True) - y)/n G[self.head] = feat.T @ d; G[self.head+1] = d.sum(0) npos = self.g_final**2 dh = xp.broadcast_to((d @ P[self.head].T)[:, :, None]/npos, (n, self.feat, npos)).reshape(n, self.feat*npos) def back(mi, dzs, need_in=True): m = self.meta[mi]; p = m["p"] col, zn, var, c, np_ = tape[mi] d = dzs.reshape(-1, c, np_) G[p+1] += (d*zn).sum((0, 2)); G[p+2] += d.sum((0, 2)) dzn = d*P[p+1].reshape(1, c, 1) dz = (dzn - dzn.mean((0, 2), keepdims=True) - zn*(dzn*zn).mean((0, 2), keepdims=True))/xp.sqrt(var+1e-5) gv, dx = m["fc"].backward(P[p], col, dz.reshape(dzs.shape), need_in) G[p] += gv return dx for (i1, i2, pj) in reversed(self.blocks): pre, zs1 = tape["pres"][(i1, i2)] dpre = dh*(pre > 0) dskip = dpre da = back(i2, dpre) # through conv2 dhin1 = back(i1, da*(zs1 > 0)) # through relu then conv1 dh = dhin1 + (back(pj, dskip) if pj is not None else dskip) back(self.stem, dh*(tape["stem_zs"] > 0), need_in=False) return G def loss_and_grads(net, x, y): tape = {} net._feat, net._lg = net.forward(x, tape) lg = net._lg mx = lg.max(1, keepdims=True) e = xp.exp(lg - mx); se = e.sum(1, keepdims=True) loss = float(to_host((-(lg-mx-xp.log(se))*y).sum(1).mean())) return loss, net.grads(x, y, tape) def gradient_check(cfg, seed=0, n=3, eps=1e-3): """Perturb single parameters and compare against the analytic gradient. A residual junction is where a hand-written backward pass goes wrong, and the first version of this one did. Two notes on reading the result. The step must be SMALL, because a larger one flips a relu and the numeric gradient of a flipped unit is meaningless: a first version used 3e-3 and reported a worst-case error of 1.0 while every gradient it sampled was in fact correct. And the statistic is the MEDIAN with a pass fraction beside it, because a single flipped unit poisons a maximum however sound the derivation.""" net = Net(cfg, seed) rg = np.random.default_rng(1) x = to_dev(rg.normal(size=(n, cfg["c_in"]*cfg["grid"]**2))) y = to_dev(np.eye(cfg["n_classes"], dtype=np.float32)[ rg.integers(0, cfg["n_classes"], n)]) def L(): _, lg = net.forward(x, training=True) mx = lg.max(1, keepdims=True); e = xp.exp(lg-mx) return float(to_host((-(lg-mx-xp.log(e.sum(1, keepdims=True)))*y) .sum(1).mean())) _, G = loss_and_grads(net, x, y) errs = [] for pi in rg.permutation(len(net.P))[:12]: f = net.P[pi].reshape(-1) for j in rg.integers(0, f.size, min(3, f.size)): old = float(to_host(f[j])) f[j] = old + eps; a = L() f[j] = old - eps; b = L() f[j] = old num = (a-b)/(2*eps) ana = float(to_host(G[pi].reshape(-1)[j])) if max(abs(num), abs(ana)) < 1e-5: continue errs.append(abs(num-ana)/max(abs(num), abs(ana))) errs = np.array(errs) return (float(np.median(errs)), float((errs < 0.05).mean()), len(errs)) def load(cfg): # CACHE THE DATASET ON DRIVE. Keras caches to the VM's home directory, # which does not survive a runtime restart, so every fresh session # downloads CIFAR again: 170 MB, and one of those took 852 seconds on # a slow evening. Fetched once, it is a local read afterwards. cache = cfg.get("data_cache", "/content/drive/MyDrive/pin_data") path = os.path.join(cache, "cifar10.npz") if not os.path.isdir(cache) and cache.startswith("/content/drive"): try: from google.colab import drive drive.mount("/content/drive") except Exception: pass if os.path.exists(path): print(f" reading CIFAR-10 from {path}", flush=True) z = np.load(path) a, b, c, d = z["a"], z["b"], z["c"], z["d"] else: from tensorflow import keras (a, b), (c, d) = keras.datasets.cifar10.load_data() try: os.makedirs(cache, exist_ok=True) np.savez_compressed(path, a=a, b=b, c=c, d=d) print(f" cached CIFAR-10 to {path}; later runs will not") print(f" download it again", flush=True) except Exception as e: print(f" (could not cache the dataset: {e})", flush=True) Xtr = a.astype(np.float32)/255.0; Xte = c.astype(np.float32)/255.0 mu, sd = Xtr.mean((0, 1, 2)), Xtr.std((0, 1, 2)) + 1e-8 f = lambda Z: np.ascontiguousarray( ((Z-mu)/sd).astype(np.float32).transpose(0, 3, 1, 2) ).reshape(len(Z), -1) return f(Xtr), b.ravel().astype(np.int64), f(Xte), d.ravel().astype(np.int64) def augment(x, cfg, rg): """Random crop and flip, PER IMAGE. A first version drew ONE offset and ONE flip decision for the whole batch, so 256 images shared a single augmentation and the effective variety was a 256th of what it should be. That model underfitted at 0.7830 with its test accuracy still climbing, which is what too little regularisation and too little variety look like together.""" n, g, c, p = x.shape[0], cfg["grid"], cfg["c_in"], cfg["aug_pad"] G = g + 2*p pad = xp.zeros((n, c, G, G), DT) pad[:, :, p:p+g, p:p+g] = x.reshape(n, c, g, g) oy = rg.integers(0, 2*p+1, n); ox = rg.integers(0, 2*p+1, n) rows = oy[:, None] + np.arange(g)[None, :] cols = ox[:, None] + np.arange(g)[None, :] cols = np.where((rg.random(n) < 0.5)[:, None], cols[:, ::-1], cols) # ONE gather on ONE axis. A first version indexed with four broadcast # arrays at once, which cupy handles badly: it cost about four seconds # a step and took an epoch from four minutes to seventeen. Flattening # the two spatial axes into a single index turns the same operation # into a take_along_axis, which is a contiguous read per row. flat = (rows[:, :, None]*G + cols[:, None, :]).reshape(n, g*g) fi = to_dev(flat, np.int64) if _GPU else flat out = xp.take_along_axis(pad.reshape(n, c, G*G), xp.broadcast_to(fi[:, None, :], (n, c, g*g)), 2) return out.reshape(n, -1) def evaluate(net, Xte, yte): """Inference uses the RUNNING statistics, not the batch's own.""" out = [] for s in range(0, Xte.shape[0], 500): _, lg = net.forward(Xte[s:s+500], training=False) out.append(to_host(lg)) return float((np.concatenate(out).argmax(1) == yte).mean()) def ensure_output(path): if path.startswith("/content/drive") and not os.path.isdir( "/content/drive/MyDrive"): try: from google.colab import drive print(" mounting Drive...", flush=True); drive.mount("/content/drive") except Exception as e: alt = os.path.basename(path.rstrip("/")) or "out" print(f"\n ** DRIVE UNAVAILABLE ({e}); writing to ./{alt},") print(f" ** which does NOT survive the session **\n", flush=True) os.makedirs(alt, exist_ok=True); return alt os.makedirs(path, exist_ok=True) return path # THE SHAPE OF A RESNET-20, which is published at 91.25% on CIFAR-10. # A first attempt used 32 channels and downsampled in the stem to afford # them, and reached 0.7897 with no overfitting at all: the model was # limited by what it could represent, not by regularisation. Sixteen # channels is what makes FULL 32x32 resolution affordable here, because # im2col's window is 16x9 = 144 wide rather than 288. We bought width we # did not need and paid for it in resolution, which is the one thing a # CIFAR network cannot spare. CFG = dict(grid=32, c_in=3, n_classes=10, stem=16, stem_stride=1, stages=((16, 1), (32, 2), (64, 2)), blocks=3, batch=256, lr=2e-3, epochs=100, aug_pad=4, wd=5e-4, report_every=5, seed=0, out="/content/drive/MyDrive/pin_resnet", data_cache="/content/drive/MyDrive/pin_data") def main(**over): CFG.update(over) t0 = time.time() print("=" * 78) print("A MODEL THAT SCORES WELL: a residual network, entirely folded") print(" build 2026-08-15c: batch normalisation, per channel") print("=" * 78) print(f" backend: {'cupy (GPU)' if _GPU else 'numpy (CPU)'}") for k, v in CFG.items(): print(f" {k:12s} = {v}") net = Net(CFG, CFG["seed"]) dense_equiv = sum(m["fc"].ins*m["fc"].out for m in net.meta) print(f"\n {len(net.meta)} folded convolutions, {net.values():,} stored") print(f" values, {net.feat} pooled features, head " f"{net.feat*CFG['n_classes']+CFG['n_classes']:,}") print(f" the connections those values stand for: {dense_equiv:,}, so") print(f" the fold is {dense_equiv/net.values():,.0f}x") gath = sum(m["fc"].g_out**2*m["fc"].taps for m in net.meta)*CFG["batch"] steps_ep = int(np.ceil(50000/CFG["batch"])) # 4.7 ns an element, measured on a T4 from a run that reached 946 s an # epoch at 255M gathered a step # 2.07 ns an element, measured: 265 ms a step at 128M gathered. An # earlier 4.7 ns came from a run whose cost was dominated by a # four-way broadcast index in the augmentation, since removed, and it # overstates everything by a factor of two. # 2.07 ns an element was the GATHER's rate. The slice route measured # 5.3x and 5.5x faster on the stride-1 layers at batch 256, which is # 96% of this network's gathering, so about 0.42 ns. The warmup below # replaces this with a measurement either way. # the slice route measured about 72 ms a step on 391M elements, which # is 0.09 ns an element. The warmup below replaces this anyway. est = gath*2*0.092e-9*steps_ep print(f"\n the window matrix is {gath/1e6:,.0f}M elements a step in each") print(f" direction, assembled from NINE STRIDED SLICES rather than one") print(f" scattered gather, and THAT sets the pace here rather than the") print(f" arithmetic: about {est/60:.0f} min an epoch, " f"{est*CFG['epochs']/3600:.1f} hours in all.") print(f" Reduce `blocks` to 1 or `epochs` to halve it; raise") print(f" `stem_stride` past 2 only if you will accept the accuracy.") print(f"\n gradient check before anything is trained:") med, frac, cnt = gradient_check(dict(CFG, grid=16, stem=8, stages=((8, 1), (16, 2)), blocks=1)) print(f" median relative error {med:.2e} over {cnt} parameters, " f"{frac:.0%} within 5%") # THE THRESHOLD IS 0.10, NOT 0.02. Numeric differentiation in float32 # is the limit here, not the derivation: a loss near 2.3 perturbed by # 1e-3 moves by about 3e-5, which is only a few hundred times the # noise floor of a reduction over a batch. Compared element by element, # this backward pass agrees to three or four significant figures on # every parameter sampled; the median of 2.7e-2 is what that looks like # through a float32 difference quotient. A first version gated at 0.02 # and refused to train a correct model. if med > 0.10 or frac < 0.3: raise SystemExit(" the backward pass is wrong; refusing to train") print(f" passed (float32 difference quotients bottom out around a") print(f" few per cent; anything under 10% is the noise floor rather") print(f" than an error in the derivation)") print("=" * 78, flush=True) Xtr, ytr, Xte, yte = load(CFG) Xtr_d, Xte_d = to_dev(Xtr), to_dev(Xte) Ytr = to_dev(np.eye(CFG["n_classes"], dtype=np.float32)[ytr]) n = Xtr.shape[0] M = [xp.zeros_like(p) for p in net.P] V = [xp.zeros_like(p) for p in net.P] rg = np.random.default_rng(CFG["seed"] + 7) steps = CFG["epochs"]*int(np.ceil(n/CFG["batch"])) t = 0; best = 0.0 # TIME TEN STEPS BEFORE COMMITTING TO THOUSANDS. Three estimates today # were wrong by factors of four to sixty, every one of them arithmetic # rather than measurement, and each cost an epoch or an evening to # discover. Fifteen seconds here makes the projection real. warm = time.time() # A COMPLETE step, on the batches training will actually see. A first # version omitted the optimiser update and walked sequential indices, # and reported 31 ms against an actual 265: an eightfold error in the # one place built to prevent eightfold errors. wperm = rg.permutation(n) # DISCARD THE FIRST FEW. cupy compiles a kernel the first time each # distinct shape and dtype combination appears, and this network has # twenty-one convolutions at four shapes, so the opening iterations # carry compilation that never recurs. Timed from the first step, this # guard reported 604 ms against an actual 72: an eightfold error in the # one place built to prevent eightfold errors. for wi in range(4): b = wperm[wi*CFG["batch"]:(wi+1)*CFG["batch"]] bd = to_dev(b, np.int64) if _GPU else b loss, G = loss_and_grads(net, augment(Xtr_d[bd], CFG, rg), Ytr[bd]) if _GPU: _cp.cuda.Stream.null.synchronize() warm = time.time() for wi in range(4, 14): b = wperm[wi*CFG["batch"]:(wi+1)*CFG["batch"]] bd = to_dev(b, np.int64) if _GPU else b loss, G = loss_and_grads(net, augment(Xtr_d[bd], CFG, rg), Ytr[bd]) t += 1 lr = CFG["lr"]*0.5*(1 + np.cos(np.pi*t/steps)) for i in range(len(net.P)): g = G[i] + CFG["wd"]*net.P[i] M[i] = 0.9*M[i] + 0.1*g V[i] = 0.999*V[i] + 0.001*g*g net.P[i] = net.P[i] - lr*(M[i]/(1-0.9**t)) \ / (xp.sqrt(V[i]/(1-0.999**t))+1e-8) if _GPU: _cp.cuda.Stream.null.synchronize() per = (time.time()-warm)/10 ep_s = per*int(np.ceil(n/CFG["batch"])) print(f"\n measured: {per*1000:.0f} ms a step, {ep_s/60:.1f} min an") print(f" epoch, {ep_s*CFG['epochs']/3600:.1f} hours in all. Interrupt") print(f" now if that is not what you want.\n", flush=True) for ep in range(CFG["epochs"]): perm = rg.permutation(n) tot, nb = 0.0, 0 for st in range(0, n, CFG["batch"]): b = perm[st:st+CFG["batch"]] x = augment(Xtr_d[b], CFG, rg) loss, G = loss_and_grads(net, x, Ytr[b]) tot += loss; nb += 1; t += 1 lr = CFG["lr"]*0.5*(1 + np.cos(np.pi*t/steps)) for i in range(len(net.P)): g = G[i] + CFG["wd"]*net.P[i] M[i] = 0.9*M[i] + 0.1*g V[i] = 0.999*V[i] + 0.001*g*g net.P[i] = net.P[i] - lr*(M[i]/(1-0.9**t)) \ / (xp.sqrt(V[i]/(1-0.999**t))+1e-8) if (ep+1) % CFG["report_every"] == 0 or ep == 0: acc = evaluate(net, Xte_d, yte); best = max(best, acc) print(f" epoch {ep+1:4d}/{CFG['epochs']} train {tot/nb:.4f}" f" test {acc:.4f} lr {lr:.2e} " f"[{time.time()-t0:.0f}s]", flush=True) acc = evaluate(net, Xte_d, yte); best = max(best, acc) out = ensure_output(CFG["out"]) h = hashlib.sha256() for p in net.P: h.update(to_host(p).tobytes()) np.savez_compressed(f"{out}/resnet.npz", params=np.array([to_host(p) for p in net.P], dtype=object), accuracy=acc, values=net.values(), digest=h.hexdigest()[:16]) print("\n" + "=" * 78) print(" RESULT") print("=" * 78) print(f" CIFAR-10, {acc:.4f} (best seen {best:.4f})") print(f" {net.values():,} stored values in {len(net.meta)} folded") print(f" convolutions, standing for {dense_equiv:,} connections\n") print(f" WHAT THIS SHOWS: folding does not cap accuracy. At a") print(f" convolution partition a folded model IS a convolutional") print(f" network, so this neither beats nor should beat a ResNet of") print(f" the same shape. The earlier figures in this programme, 0.51") print(f" on CIFAR-100 and 0.68 on reduced CIFAR-10, were a three-layer") print(f" stack with no residual connections. That was the") print(f" architecture, not the framework.") print(f"\n saved to {out}; total {time.time()-t0:.0f}s") main()