rarity-route verified: spanning reproduces 0.46, sits between population & concentration, redundancy RULED OUT (incr rank 0.75m); candidate closed form rejected (over-predicts). Paper #2 §2 = two routes: low-rank existence proof (2a) + measured rarity mechanism (2b)

jobs/rarity_route_verify_job.py

@@ -0,0 +1,152 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#   "torch", "torchvision", "numpy", "pillow", "scikit-learn", "scipy",
+#   "huggingface_hub>=0.34", "dinov3 @ git+https://github.com/facebookresearch/dinov3",
+# ]
+# ///
+"""Verify the RARITY-ROUTE identity for spanning's lesion retention. HF Job (GPU).
+
+Tests whether spanning (farthest-point) selection's lesion retention R_span(rho) is PREDICTED by the
+closed form  R_pred(rho) = 1 - F_les( Q_bg(1-rho) )  on OUTLIER ENERGY sigma = residual beyond the
+background subspace -- the rarity route -- and reproduces the medical gap (~0.46 @ rho=0.25). Also
+rules out the REDUNDANCY explanation by measuring the lesion cluster's incremental rank beyond
+background. Masks analysis-only. Emits RARITY_RESULT.
+"""
+from __future__ import annotations
+import json, os, sys, time
+from pathlib import Path
+import numpy as np, torch
+from PIL import Image
+from sklearn.neighbors import NearestNeighbors
+from huggingface_hub import hf_hub_download
+sys.path.insert(0,"/mnt/processed/covtoken_code")
+from dinov3.models.vision_transformer import vit_base          # noqa: E402
+
+BACKBONE_REPO="ricklisz123/MedDINOv3-ViTB-16-CT-3M"; MNT=Path("/mnt")
+RAW_LIDC=MNT/"raw"/"lidc"; MASK_ROOT=MNT/"processed"/"lidc_v2"; OUT=MNT/"processed"/"covtoken"
+N_PATCH,CLS_OFF=196,5; LAYER=int(os.environ.get("LAYER","2"))
+BANK_SLICES=int(os.environ.get("BANK_SLICES","500")); EVAL_SLICES=int(os.environ.get("EVAL_SLICES","800"))
+RHOS=[0.1,0.25,0.5,0.75]; QS=[32,64,128]
+CT_MEAN=np.array([0.485,0.456,0.406],np.float32); CT_STD=np.array([0.229,0.224,0.225],np.float32)
+def log(m): print(f"[rarity] {m}", flush=True)
+
+
+def load_backbone(device):
+    ck=hf_hub_download(BACKBONE_REPO,"model.pth",token=os.environ.get("HF_TOKEN"))
+    m=vit_base(drop_path_rate=0.0,layerscale_init=1e-5,n_storage_tokens=4,qkv_bias=False,mask_k_bias=True)
+    raw=torch.load(ck,map_location="cpu"); sd=raw.get("teacher",raw)
+    sd={(k[9:] if k.startswith("backbone.") else k):v for k,v in sd.items()}
+    m.load_state_dict(sd,strict=False); m.eval().to(device)
+    for p in m.parameters(): p.requires_grad_(False)
+    feats={}
+    def h(_m,_i,out):
+        while isinstance(out,(list,tuple)): out=out[0]
+        feats[0]=out.detach()
+    m.blocks[LAYER].register_forward_hook(h); return m,feats
+
+
+def to_t(p):
+    im=Image.open(p).convert("RGB").resize((224,224),Image.BILINEAR)
+    return torch.from_numpy(((np.asarray(im,np.float32)/255.0-CT_MEAN)/CT_STD)).permute(2,0,1)
+
+
+@torch.inference_mode()
+def tok(model,feats,imgs,device):
+    model.forward_features(imgs.to(device,torch.float32))
+    return feats[0][:,CLS_OFF:CLS_OFF+N_PATCH,:].float().cpu().numpy()
+
+
+def fps(Z,k,seed_idx):
+    n=len(Z); keep=[seed_idx]; d2=((Z-Z[seed_idx])**2).sum(1)
+    for _ in range(k-1):
+        j=int(np.argmax(d2)); keep.append(j); d2=np.minimum(d2,((Z-Z[j])**2).sum(1)); d2[keep]=-1
+    return np.array(keep)
+
+
+def main():
+    t0=time.time(); device=torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model,feats=load_backbone(device); rng=np.random.default_rng(0)
+    scan_split=json.load(open(hf_hub_download("Chucks90/eryon-data-pipelines","manifests/lidc/splits_v1.0.0.json",repo_type="dataset",token=os.environ.get("HF_TOKEN"))))["splits"]
+    train=[]
+    for b in sorted(RAW_LIDC.glob("batch_*")):
+        for sd in b.iterdir():
+            if sd.is_dir() and scan_split.get(sd.name)=="train": train+=sorted(sd.glob("slice_*.png"))
+    train=[train[i] for i in rng.choice(len(train),min(BANK_SLICES,len(train)),replace=False)]
+    ev=[]
+    for cd in sorted((MASK_ROOT/"test").iterdir()):
+        npz=cd/"patch_masks.npz"
+        if cd.is_dir() and npz.exists():
+            pm=np.load(npz)["patch_masks"]
+            for idx in range(len(pm)):
+                if pm[idx].sum()>0: ev.append((cd/f"slice_{idx:04d}.png", pm[idx].reshape(-1)))
+    ev=[ev[i] for i in rng.choice(len(ev),min(EVAL_SLICES,len(ev)),replace=False)]
+    log(f"device={device.type}; bank={len(train)} eval(lesion+)={len(ev)} layer block {LAYER+1}")
+
+    # background bank: membership kNN + subspace U_B (PCA) for residual outlier energy
+    B=[]
+    for i in range(0,len(train),48): B.append(tok(model,feats,torch.stack([to_t(p) for p in train[i:i+48]]),device).reshape(-1,768))
+    B=np.concatenate(B); muB=B.mean(0,keepdims=True); Bc=B-muB
+    knn=NearestNeighbors(n_neighbors=11).fit(B[rng.choice(len(B),min(40000,len(B)),replace=False)])
+    _,_,Vt=np.linalg.svd(Bc[rng.choice(len(Bc),min(60000,len(Bc)),replace=False)],full_matrices=False)
+    UB={q:Vt[:q] for q in QS}
+    # background outlier-energy distribution per q (for quantiles)
+    bg_sigma={q: (np.linalg.norm(Bc - (Bc@UB[q].T)@UB[q],axis=1)) for q in QS}
+    bg_sigma={q: s[rng.choice(len(s),min(60000,len(s)),replace=False)] for q,s in bg_sigma.items()}
+
+    R_span={r:[] for r in RHOS}; R_conc={r:[] for r in RHOS}; R_out={q:{r:[] for r in RHOS} for q in QS}
+    les_sigma={q:[] for q in QS}; inc_rank=[]   # lesion residual energy + incremental rank beyond bg
+    for i in range(0,len(ev),48):
+        ch=ev[i:i+48]; T=tok(model,feats,torch.stack([to_t(p) for p,_ in ch]),device)
+        for b,(_,m) in enumerate(ch):
+            Z=T[b]; li=set(np.where(m>0)[0].tolist()); ml=len(li)
+            if ml==0: continue
+            mem,_=knn.kneighbors(Z); mem=mem[:,1:].mean(1)                     # membership (concentration score)
+            for q in QS:
+                Zc=Z-muB; sig=np.linalg.norm(Zc-(Zc@UB[q].T)@UB[q],axis=1)     # outlier energy sigma
+                les_sigma[q]+=sig[list(li)].tolist()
+            for r in RHOS:
+                k=max(1,round(r*N_PATCH)); seed=int(np.argmax(mem))
+                R_span[r].append(len(li & set(fps(Z,k,seed).tolist()))/ml)
+                R_conc[r].append(len(li & set(np.argsort(-mem)[:k].tolist()))/ml)
+                for q in QS:
+                    Zc=Z-muB; sig=np.linalg.norm(Zc-(Zc@UB[q].T)@UB[q],axis=1)
+                    R_out[q][r].append(len(li & set(np.argsort(-sig)[:k].tolist()))/ml)
+            # incremental rank of lesion tokens beyond background top-128 subspace (ruling out redundancy)
+            Zc=Z-muB; res=Zc-(Zc@UB[128].T)@UB[128]; resl=res[list(li)]
+            if ml>=4:
+                s=np.linalg.svd(resl-resl.mean(0,keepdims=True),compute_uv=False)
+                p=s/(s.sum()+1e-9); inc_rank.append(float(np.exp(-(p*np.log(p+1e-9)).sum()))/ml)
+
+    res={"backbone":"MedDINOv3","layer_block":LAYER+1,"rhos":RHOS,"qs":QS,"curves":{}}
+    for r in RHOS:
+        res["curves"][str(r)]={"R_span":round(float(np.mean(R_span[r])),4),"R_conc":round(float(np.mean(R_conc[r])),4),
+            "R_population":r,"R_out_by_q":{str(q):round(float(np.mean(R_out[q][r])),4) for q in QS}}
+    # closed-form prediction R_pred(rho)=1-F_les(Q_bg(1-rho))
+    pred={}
+    for q in QS:
+        ls=np.array(les_sigma[q]); pred[str(q)]={}
+        for r in RHOS:
+            tau=np.quantile(bg_sigma[q],1-r); pred[str(q)][str(r)]=round(float((ls>tau).mean()),4)
+    res["R_pred_closedform"]=pred
+    res["lesion_incremental_rank_over_m_beyond_bg128"]=round(float(np.mean(inc_rank)),3) if inc_rank else None
+    # verdict: does any q's closed form track R_span and reproduce ~0.46@0.25?
+    span025=res["curves"]["0.25"]["R_span"]
+    errs={q: float(np.mean([abs(pred[str(q)][str(r)]-res["curves"][str(r)]["R_span"]) for r in RHOS])) for q in QS}
+    bestq=min(errs,key=errs.get)
+    res["verdict"]={
+        "R_span_at_0.25":span025,"reproduces_medical_gap":bool(abs(span025-0.46)<0.07),
+        "best_q":bestq,"mean_abs_pred_error":round(errs[bestq],4),
+        "closedform_predicts_spanning":bool(errs[bestq]<0.06),
+        "outlier_credit_C_at_0.25":round((span025-0.25)/0.75,3),
+        "redundancy_ruled_out":bool((res["lesion_incremental_rank_over_m_beyond_bg128"] or 0)>0.5),
+        "interpretation":("RARITY-ROUTE IDENTITY VERIFIED: closed form 1-F_les(Q_bg(1-rho)) tracks spanning retention; "
+            "spanning sits between population rate and concentration; redundancy ruled out (lesion incremental rank high)."
+            if errs[bestq]<0.06 else
+            "Closed form does NOT cleanly predict spanning; fall back to two-route framing (existence proof + rarity carries data).")}
+    res["elapsed_s"]=round(time.time()-t0,1)
+    OUT.mkdir(parents=True,exist_ok=True); (OUT/"rarity_route_verify.json").write_text(json.dumps(res,indent=2))
+    print("RARITY_RESULT "+json.dumps(res),flush=True)
+
+
+if __name__=="__main__": main()

paper/paper2_rank_objectives_draft.md

@@ -35,7 +35,16 @@ But many high-stakes tasks are the opposite: the signal is *rare and concentrate
 nodule, a microcalcification, an anomalous event. We ask whether rank-style objectives, used to
 gate token pruning or as SSL regularizers, help or hurt such tasks, and derive when.
 
-## 2. The law (closed form, synthetic)
+## 2. The law: two routes to set-spectrum blindness
+
+A set-level rank/coverage objective optimizes the *retained set's* spectrum, which is insensitive to
+a rare critical cluster by **either** of two routes — the cluster is internally **low-rank**, *or* it
+is simply too **few** tokens (rarity) to move set-level coverage. This section gives the closed form
+for the low-rank route as a clean **existence proof**; §4 shows — by direct measurement — that *real
+lesions take the rarity route, not this one*, and §2b states the rarity-route identity that carries
+the data.
+
+### 2a. The low-rank route (closed form, synthetic — existence proof)
 
 **Setup.** Background tokens are isotropic; a signal of effective rank r is injected across m
 tokens (r=1 fully aligned, r=m fully diverse), each high energy in a lesion/anomaly subspace L. At
@@ -44,13 +53,46 @@ farthest-point / effective-rank maximization in L.
 
 **Result (Fig. 1).** spanning retention `= min(r,m)/m`; concentration retention `= 1`; **gap
 `= (m-r)/m`; crossover `r* = m`.** A spanning objective retains a signal only in proportion to its
-rank; it ties concentration only when the signal is fully diverse. (Multi-seed: §5.)
+rank; it ties concentration only when the signal is fully diverse. (Multi-seed: §5.) This isolates
+*one* mechanism; it is deliberately not a claim about real lesion geometry (which §4 measures).
 
 **Alignment surface.** Adding an SNR axis yields `A(r, SNR)`: concentration dominates iff the
 signal is concentrated (r<m) AND distinct (high SNR); at low SNR both lose the signal. This
 predicts when rank/spanning regularization is safe (high task-rank or low-SNR-anyway) vs harmful
 (rare, salient signal).
 
+### 2b. The rarity route (mechanistically verified; the route the real data takes)
+
+Real lesions are *diverse* but **rare** (`π=m/n ≪ 1`). At budget fraction `ρ`, a spanning selection
+spends its budget covering the abundant high-dimensional background, so a rare cluster claims only a
+budget-proportional-plus-outlier share. We verify this directly on the real lesion spectrum (LIDC,
+operating layer; `research_v4/rarity_route_verify.json`), measuring the spanning (farthest-point)
+lesion retention `R_span(ρ)` against population (`R_pop=ρ`) and concentration (`R_conc`) references:
+
+| ρ | R_pop | **R_span** | R_conc |
+|---|---|---|---|
+| 0.10 | 0.10 | 0.152 | 0.435 |
+| 0.25 | 0.25 | **0.460** | 0.795 |
+| 0.50 | 0.50 | 0.812 | 0.974 |
+| 0.75 | 0.75 | 0.980 | 1.000 |
+
+Three facts establish the rarity mechanism. (i) Spanning **reproduces the medical gap**,
+`R_span(0.25)=0.46`. (ii) `R_span` lies strictly **between** the population rate and concentration at
+every budget — a rare cluster gets *partial* outlier credit (`C=(R_span−ρ)/(1−ρ)=0.28` at ρ=0.25),
+not the full retention concentration gives it. (iii) **Redundancy is ruled out**: the lesion
+cluster's incremental effective rank beyond the background subspace is high (0.75·m), so its tokens
+are diverse *even orthogonal to background* — the loss is not within-cluster redundancy but rarity
+(budget spent covering background). This is the technical crux: spanning drops a *diverse* cluster
+because it is *few*, not because it is redundant.
+
+We do **not** claim a closed form here. The natural per-token candidate `R_span(ρ)=1−F_les(Q_bg(1−ρ))`
+on outlier energy *over-predicts* at every budget (mean error 0.15), because a per-token score keeps
+all high-energy lesion tokens (concentration-like) whereas spanning spreads budget; the outlier
+credit is also non-constant in `ρ`. An exact rarity-route identity is a budget-allocation/covering
+(DPP-like) object we leave open. The law therefore stands as: a verified **existence proof** for the
+low-rank route (§2a) plus the **measured, redundancy-excluded rarity mechanism** (§2b) that carries
+the real data — the closed form describes a parallel route, and we say so.
+
 ## 3. The mechanism is SELECTION, not SCALING (a sharpening)
 
 A natural worry is over-generalization. We separate two things "rank" conflates. As a *selection*

research_v4/rarity_route_derivation.md

@@ -0,0 +1,57 @@
+# The rarity-route identity (candidate — VERIFICATION OUTCOME below)
+
+> **OUTCOME (verified `rarity_route_verify.json`):** the rarity *mechanism* is confirmed — spanning
+> reproduces the medical gap (R_span(0.25)=0.46), sits between population rate and concentration
+> (outlier credit C=0.28), and **redundancy is ruled out** (lesion incremental rank/m beyond
+> background = 0.75). BUT the candidate **closed form below over-predicts** (mean error 0.15): a
+> per-token σ top-k is concentration-like and keeps all high-energy lesion tokens, while spanning
+> spreads budget. So the closed form is **rejected**; the law ships as existence proof (§2a) + the
+> measured, redundancy-excluded rarity mechanism (§2b). An exact spanning identity (DPP/covering) is
+> left open.
+
+
+## Two routes to set-spectrum blindness
+A set-level rank/coverage objective `C(S)=effrank(P_L Z_S)` optimizes the *retained set's* spectrum,
+which is insensitive to a rare critical cluster by **either** of two routes:
+
+- **Low-rank route (synthetic, paper-2 §2).** The cluster is internally low-rank (effective rank
+  `r < m`). Closed form: spanning retention `min(r,m)/m`, gap `(m−r)/m`, crossover `r*=m`. This is an
+  **existence proof**; it is *not* the route real lesions take (measured lesion rank 339 > background 307).
+- **Rarity route (real data).** The cluster is internally *diverse* but **rare** (`π=m/n ≪ 1`). The
+  budget `k=ρn` is spread over the abundant high-dimensional background; the cluster claims only a
+  budget-proportional-plus-outlier share.
+
+## Candidate identity (rarity route)
+Order tokens by spanning-relevant **outlier energy** `σ_i = ‖(I−U_B U_Bᵀ)(z_i−μ_B)‖` (residual beyond
+the background subspace `U_B`). A spanning selection retains the top-`k` by `σ`. With `π→0` (rarity),
+the top-`ρ` threshold is set by the **background** quantile, so the lesion retention is
+
+  **R_span(ρ) ≈ 1 − F_les( Q_bg(1−ρ) )**,
+
+where `F_les` is the CDF of lesion outlier energy and `Q_bg(1−ρ)` the `(1−ρ)`-quantile of background
+outlier energy. Limits, matching the "between population rate and full retention" intuition:
+- lesion σ ≫ background σ (strong outlier credit) ⇒ `R_span → 1`;
+- lesion σ ≈ background σ (no credit) ⇒ `R_span → ρ` (population rate);
+- in between, `R_span ∈ (ρ, 1)`, position set by the outlier-energy shift. Outlier credit
+  `C = (R_span−ρ)/(1−ρ)`.
+
+This **predicts the medical gap** (target: `R_span(0.25) ≈ 0.46`, i.e. `C ≈ 0.28`) from a *measured*
+energy distribution, rather than from a parallel low-rank law.
+
+## Redundancy must be ruled out (the technical check)
+The competing explanation for `R_span(0.25)=0.46` is within-cluster **redundancy** (FPS covers a tight
+blob with few points). This is **already refuted by the spectrum**: lesion tokens are *more* spread
+than background (top-10 SV fraction 0.176 < 0.203; RankMe 339 > 307), so they are not mutually
+redundant. The verification job confirms it by also measuring the lesion cluster's *incremental* rank
+beyond the background subspace — if that is large, redundancy cannot be the cap, and rarity (budget
+spent on background) is the operative mechanism.
+
+## Verification plan
+Measure, at the operating layer (LIDC, masks analysis-only), across budgets ρ∈{0.1,0.25,0.5,0.75}:
+1. **R_span(ρ)** — actual farthest-point/spanning lesion retention (ground truth; must reproduce 0.46@0.25).
+2. **R_pred(ρ)** — the closed-form `1 − F_les(Q_bg(1−ρ))` (sweep `U_B` dim `q`).
+3. **R_conc(ρ)**, **R_pop(ρ)=ρ** — concentration upper / population lower references.
+4. lesion **incremental rank beyond background** — to rule out redundancy.
+PASS (strong option): `R_pred ≈ R_span` across ρ and reproduces 0.46 ⇒ paper-2 states a derived,
+verified rarity-route identity. ELSE: fall back to the two-route framing (minimum), existence-proof
+only, rarity carrying the data.

research_v4/rarity_route_verify.json

@@ -0,0 +1,30 @@
+{
+  "study": "Rarity-route verification — does the closed form predict spanning's lesion retention, and is redundancy ruled out?",
+  "backbone": "MedDINOv3", "layer_block": 3, "dataset": "LIDC", "n_eval_lesion_pos": 800,
+  "curves_by_rho": {
+    "0.10": {"R_population": 0.10, "R_span": 0.152, "R_concentration": 0.435},
+    "0.25": {"R_population": 0.25, "R_span": 0.460, "R_concentration": 0.795},
+    "0.50": {"R_population": 0.50, "R_span": 0.812, "R_concentration": 0.974},
+    "0.75": {"R_population": 0.75, "R_span": 0.980, "R_concentration": 1.000}
+  },
+  "closed_form_candidate": "R_span(rho) = 1 - F_les(Q_bg(1-rho))  on outlier energy sigma (residual beyond background)",
+  "closed_form_pred_best_q32": {"0.10": 0.374, "0.25": 0.692, "0.50": 0.946, "0.75": 0.999},
+  "verdict": {
+    "reproduces_medical_gap": true, "R_span_at_0.25": 0.460,
+    "outlier_credit_C_at_0.25": 0.28,
+    "interpolates_between_population_and_concentration": true,
+    "redundancy_ruled_out": true, "lesion_incremental_rank_over_m_beyond_bg": 0.754,
+    "closed_form_predicts_spanning": false, "mean_abs_pred_error": 0.152,
+    "why_closed_form_fails": "A per-token outlier-energy top-k keeps ALL high-energy lesion tokens (concentration-like) and so OVER-predicts FPS retention at every budget (+0.13 to +0.23). Spanning spreads budget across the background; its rare-cluster retention is not a per-token score threshold, so no per-token sigma reproduces the curve. The outlier credit C is also not constant (0.06, 0.28, 0.62, 0.92 across rho) -- ruling out R_span=rho+C(1-rho) too."
+  },
+  "outcome": {
+    "decision": "MINIMUM-PLUS: two-route framing kept; rarity route backed by VERIFIED facts, not a derived identity.",
+    "verified_facts": [
+      "Spanning reproduces the medical gap R_span(0.25)=0.460.",
+      "R_span lies strictly between population rate (rho) and concentration at every budget (outlier credit, growing with rho).",
+      "Redundancy is ruled out: lesion cluster incremental rank/m beyond the background subspace is 0.754 (high) -- the cluster is diverse even orthogonal to background, so the gap is RARITY (budget spent covering background), not within-cluster redundancy."
+    ],
+    "left_open": "An exact closed form for spanning's rare-cluster retention. The natural per-token sigma-quantile candidate over-predicts; the correct object is a budget-allocation/covering (DPP-like) derivation, deferred. The synthetic low-rank law (2a) remains the existence proof; the rarity route is mechanistically verified but not yet reduced to a clean identity."
+  },
+  "human_signoff": null
+}