repro-bundle-v2/repro/ddsvm_v2.py

#!/usr/bin/env python3
"""DDSVM reproduction v2 -- multi-seed, multi-dataset, pre-registered predicates.

Replaces the v1 single-seed toy run (repro_ddsvm.py, seed 42, one easy moons
dataset, ~5 s). v2 design, fixed BEFORE running:

Datasets (per-seed fresh sample / fresh split, hash-verified distinct):
  moons-hard : sklearn make_moons, n=3000 (train 2000 / test 1000), noise 0.30
  rings      : sklearn make_circles, n=3000 (2000/1000), noise 0.20, factor 0.55
  gauss-xor  : 2-D XOR of 4 Gaussians (sigma 0.75) + 8 nuisance dims, 2000/1000
  digits-3v8 : sklearn load_digits classes {3,8}, per-seed stratified split
               train 120 / test ~237 (small-sample regime, 64 features)

Methods (matched 300-epoch budget for all deep models, same backbone d-64-64-16):
  linear-svm : sklearn LinearSVC on standardized raw features (classical baseline)
  rbf-svm    : sklearn SVC(RBF) (classical baseline)
  deep-ce    : backbone + linear head, cross-entropy, Adam 5e-3, 300 epochs
  deep-svm   : backbone + linear SVM head, hinge + 0.01*0.5||w||^2, Adam, 300 ep
  ddsvm      : 15 cycles x (8 ep Phase A feat | 8 ep Phase B head | Phase C push
               along n = w/||w|| on active set gamma<1, eta 0.08, + 4 ep align)
  ddsvm-rand : identical to ddsvm but Phase C pushes along a RANDOM unit vector
               (fresh per cycle) instead of the boundary normal -- the control
               that makes "geometry-aware" falsifiable.

Seeds: 25 per (dataset x method). Every RNG derived from the seed; SHA1 of the
train matrix recorded per seed; run-integrity block asserts all hashes distinct
and per-method accuracy variance > 0 (guards the constructed-but-never-sampled
RandomState bug found in a sibling reproduction).

PRE-STATED PREDICATES (two-sided / falsifiable, fixed before the run):
 Claim 1 (per dataset):
  P1a structural: in EVERY cycle, Phase A changes (w,b) by exactly 0 and Phase B
      changes theta by exactly 0, and Phase C mean feature displacement > 0.
  P1b convergence: mean end-of-cycle train hinge, cycle15/cycle1 ratio: 95% CI
      (over seeds) entirely <= 0.5, AND final mean hinge > 1e-8 (degeneracy guard).
  P1c trend: OLS of log(mean hinge) vs cycle: slope <= -0.05 AND R^2 >= 0.70.
  Verdict: VERIFIED if >=10/12 dataset-predicate checks pass AND P1a passes on
  all 4 datasets; PARTIAL if 6-9; else NOT REPRODUCED.
 Claim 2 (per dataset):
  P2a direction: mean cosine(achieved active-point displacement, y_i * n) over
      cycles, per-seed; dataset 95% CI lower bound >= 0.5.
  P2b per-cycle effect: paired (post - pre) refinement change of active-set mean
      margin, pooled over cycles per seed; dataset mean > 0 with 95% CI excl. 0.
  P2c end-to-end: (final post-refinement all-point train margin) - (cycle-1
      pre-refinement margin) > 0 with 95% CI excl. 0, AND train margin violators
      (gamma<1) decrease cycle1 -> final in >= 80% of seeds.
  P2d geometry-aware ablation: paired (ddsvm - ddsvm-rand) final normalized test
      margin > 0 with 95% CI excl. 0.
  Verdict: VERIFIED if >=13/16 checks pass; PARTIAL if 8-12; else NOT REPRODUCED.
 Claim 3 (per dataset, paired over seeds, vs the two deep baselines):
  P3 VERIFIED iff on >= 2/4 datasets BOTH paired accuracy differences
     (ddsvm - deep-ce) and (ddsvm - deep-svm) are > 0 with 95% CI excluding 0,
     AND no dataset has a 95% CI entirely below 0 vs either baseline.
  NOT REPRODUCED iff zero datasets show a CI-excluding-0 improvement vs either
     deep baseline, OR >= 2 datasets show a CI entirely below 0 (regression).
  Otherwise PARTIAL.

Outputs: results/v2/ddsvm_v2_<dataset>.json (per-seed raw), merged summary via
merge step in analyze_v2.py.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import math
import os
import time

import numpy as np
import torch
import torch.nn as nn

# Single-thread torch: benchmarked 9.4 ms/epoch vs 218 ms/epoch with 2 threads
# on this Windows CPU (tiny full-batch models are per-op-overhead dominated).
torch.set_num_threads(1)

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
OUTDIR = os.path.join(ROOT, "results", "v2")
os.makedirs(OUTDIR, exist_ok=True)

N_SEEDS = 25
N_CYCLES = 15
EP_A, EP_B, EP_C = 8, 8, 4          # 15*(8+8+4) = 300 epoch budget
EPOCHS_JOINT = 300                   # matched budget for deep-ce / deep-svm
ETA_GEOM = 0.08
MARGIN_TARGET = 1.0
LR_FEAT, LR_HEAD, LR_JOINT = 5e-3, 1e-2, 5e-3
WD = 1e-4
REG_W = 0.01

DATASETS = ["moons-hard", "rings", "gauss-xor", "digits-3v8"]


# ----------------------------------------------------------------- datasets --
def make_dataset(name: str, seed: int):
    """Fresh sample / fresh split per seed. Returns standardized tensors."""
    rng = np.random.default_rng(100_000 + seed)
    if name == "moons-hard":
        from sklearn.datasets import make_moons
        X, y = make_moons(n_samples=3000, noise=0.30,
                          random_state=int(rng.integers(2**31 - 1)))
        n_tr = 2000
    elif name == "rings":
        from sklearn.datasets import make_circles
        X, y = make_circles(n_samples=3000, noise=0.20, factor=0.55,
                            random_state=int(rng.integers(2**31 - 1)))
        n_tr = 2000
    elif name == "gauss-xor":
        n, sigma, d_noise = 3000, 0.75, 8
        centers = np.array([[1, 1], [-1, -1], [1, -1], [-1, 1]], dtype=float)
        labels = np.array([1, 1, 0, 0])
        idx = rng.integers(0, 4, size=n)
        X2 = centers[idx] + rng.normal(0, sigma, size=(n, 2))
        Xn = rng.normal(0, 1.0, size=(n, d_noise))
        X = np.hstack([X2, Xn])
        y = labels[idx]
        n_tr = 2000
    elif name == "digits-3v8":
        from sklearn.datasets import load_digits
        D = load_digits()
        mask = np.isin(D.target, [3, 8])
        X, y = D.data[mask], (D.target[mask] == 8).astype(int)
        # stratified per-seed split, train 120
        idx0 = np.where(y == 0)[0]
        idx1 = np.where(y == 1)[0]
        rng.shuffle(idx0)
        rng.shuffle(idx1)
        tr = np.concatenate([idx0[:60], idx1[:60]])
        te = np.concatenate([idx0[60:], idx1[60:]])
        rng.shuffle(tr)
        rng.shuffle(te)
        Xtr, Xte = X[tr], X[te]
        ytr, yte = y[tr], y[te]
        return _standardize_pack(Xtr, ytr, Xte, yte)
    else:
        raise ValueError(name)

    perm = rng.permutation(len(X))
    X, y = X[perm], y[perm]
    return _standardize_pack(X[:n_tr], y[:n_tr], X[n_tr:], y[n_tr:])


def _standardize_pack(Xtr, ytr, Xte, yte):
    mu, sd = Xtr.mean(0), Xtr.std(0) + 1e-8
    Xtr = (Xtr - mu) / sd
    Xte = (Xte - mu) / sd
    h = hashlib.sha1(np.ascontiguousarray(Xtr).tobytes()).hexdigest()[:12]
    to = lambda a: torch.tensor(a, dtype=torch.float32)
    ypm = lambda a: torch.tensor(2.0 * a - 1.0, dtype=torch.float32)  # {0,1}->{-1,+1}
    return dict(Xtr=to(Xtr), ytr=ypm(ytr), Xte=to(Xte), yte=ypm(yte),
                ytr01=torch.tensor(ytr, dtype=torch.long),
                yte01=torch.tensor(yte, dtype=torch.long), hash=h)


# ------------------------------------------------------------------- models --
class Backbone(nn.Module):
    def __init__(self, d_in, hidden=64, d_feat=16):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(d_in, hidden), nn.GELU(),
            nn.Linear(hidden, hidden), nn.GELU(),
            nn.Linear(hidden, d_feat))

    def forward(self, x):
        return self.net(x)


class SVMHead(nn.Module):
    def __init__(self, d_feat=16):
        super().__init__()
        self.w = nn.Parameter(torch.randn(d_feat, 1) * 0.01)
        self.b = nn.Parameter(torch.zeros(1))

    def forward(self, z):
        return (z @ self.w + self.b).squeeze(-1)


def geo_margin(z, y, w, b):
    return y * ((z @ w).squeeze(-1) + b) / (torch.norm(w) + 1e-12)


def norm_margin(z, y, w, b):
    """Scale-invariant margin: geometric margin / RMS feature norm."""
    g = geo_margin(z, y, w, b)
    rms = torch.sqrt((z ** 2).sum(1).mean()) + 1e-12
    return g / rms


def flat(params):
    return torch.cat([p.detach().reshape(-1).clone() for p in params])


# ----------------------------------------------------------- method runners --
def run_deep_ce(data, seed):
    torch.manual_seed(20_000 + seed)
    d_in = data["Xtr"].shape[1]
    feat, head = Backbone(d_in), nn.Linear(16, 2)
    opt = torch.optim.Adam(list(feat.parameters()) + list(head.parameters()),
                           lr=LR_JOINT)
    lossf = nn.CrossEntropyLoss()
    for _ in range(EPOCHS_JOINT):
        opt.zero_grad()
        loss = lossf(head(feat(data["Xtr"])), data["ytr01"])
        loss.backward()
        opt.step()
    with torch.no_grad():
        pred = head(feat(data["Xte"])).argmax(1)
        acc = (pred == data["yte01"]).float().mean().item()
    return dict(acc=acc)


def run_deep_svm(data, seed):
    torch.manual_seed(30_000 + seed)
    d_in = data["Xtr"].shape[1]
    feat, head = Backbone(d_in), SVMHead()
    opt = torch.optim.Adam(list(feat.parameters()) + list(head.parameters()),
                           lr=LR_JOINT, weight_decay=WD)
    for _ in range(EPOCHS_JOINT):
        opt.zero_grad()
        s = head(feat(data["Xtr"]))
        loss = torch.clamp(MARGIN_TARGET - data["ytr"] * s, min=0).mean() \
            + REG_W * 0.5 * torch.norm(head.w) ** 2
        loss.backward()
        opt.step()
    with torch.no_grad():
        zte = feat(data["Xte"])
        acc = (torch.sign(head(zte)) == data["yte"]).float().mean().item()
        nm = norm_margin(zte, data["yte"], head.w, head.b)
    return dict(acc=acc, test_norm_margin_mean=nm.mean().item(),
                test_norm_margin_p10=nm.quantile(0.10).item())


def run_ddsvm(data, seed, random_push=False):
    torch.manual_seed((50_000 if not random_push else 60_000) + seed)
    g_push = torch.Generator().manual_seed(70_000 + seed)
    d_in = data["Xtr"].shape[1]
    feat, head = Backbone(d_in), SVMHead()
    opt_f = torch.optim.Adam(feat.parameters(), lr=LR_FEAT)
    opt_h = torch.optim.Adam(head.parameters(), lr=LR_HEAD, weight_decay=WD)
    Xtr, ytr = data["Xtr"], data["ytr"]
    cyc = []
    for c in range(N_CYCLES):
        # Phase A: representation learning (head frozen)
        head_before = flat(head.parameters())
        feat_before = flat(feat.parameters())
        for _ in range(EP_A):
            opt_f.zero_grad()
            s = head(feat(Xtr))
            hin = torch.clamp(MARGIN_TARGET - ytr * s, min=0).mean()
            hin.backward()
            opt_f.step()
        head_drift_A = torch.norm(flat(head.parameters()) - head_before).item()
        feat_move_A = torch.norm(flat(feat.parameters()) - feat_before).item()
        with torch.no_grad():
            hinge_A = torch.clamp(
                MARGIN_TARGET - ytr * head(feat(Xtr)), min=0).mean().item()

        # Phase B: boundary optimization (features frozen/detached)
        feat_before = flat(feat.parameters())
        head_before = flat(head.parameters())
        with torch.no_grad():
            z_fix = feat(Xtr)
        for _ in range(EP_B):
            opt_h.zero_grad()
            s = head(z_fix)
            loss = torch.clamp(MARGIN_TARGET - ytr * s, min=0).mean() \
                + REG_W * 0.5 * torch.norm(head.w) ** 2
            loss.backward()
            opt_h.step()
        feat_drift_B = torch.norm(flat(feat.parameters()) - feat_before).item()
        head_move_B = torch.norm(flat(head.parameters()) - head_before).item()
        with torch.no_grad():
            hinge_B = torch.clamp(
                MARGIN_TARGET - ytr * head(feat(Xtr)), min=0).mean().item()

        # Phase C: geometry-aware feature refinement
        with torch.no_grad():
            z_in = feat(Xtr)
            n_vec = (head.w / (torch.norm(head.w) + 1e-12)).squeeze(-1)
            if random_push:
                u = torch.randn(n_vec.shape[0], generator=g_push)
                n_used = u / (torch.norm(u) + 1e-12)
            else:
                n_used = n_vec
            m_pre = geo_margin(z_in, ytr, head.w, head.b)
            active = (m_pre < MARGIN_TARGET)
            direction = ytr.unsqueeze(1) * n_used.unsqueeze(0)
            z_target = z_in + ETA_GEOM * direction * active.float().unsqueeze(1)
            viol_pre = int((m_pre < MARGIN_TARGET).sum().item())
        for _ in range(EP_C):
            opt_f.zero_grad()
            loss_g = nn.functional.mse_loss(feat(Xtr), z_target)
            loss_g.backward()
            opt_f.step()
        with torch.no_grad():
            z_out = feat(Xtr)
            m_post = geo_margin(z_out, ytr, head.w, head.b)
            dz = z_out - z_in
            disp = torch.norm(dz, dim=1)
            # cosine of achieved displacement with prescribed direction, active pts
            if active.any():
                cos = nn.functional.cosine_similarity(
                    dz[active], direction[active], dim=1)
                cos_mean = cos.mean().item()
                d_active = (m_post[active] - m_pre[active]).mean().item()
            else:
                cos_mean, d_active = float("nan"), float("nan")
            hinge_C = torch.clamp(
                MARGIN_TARGET - ytr * head(feat(Xtr)), min=0).mean().item()
            viol_post = int((m_post < MARGIN_TARGET).sum().item())
        cyc.append(dict(
            cycle=c + 1, hinge_A=hinge_A, hinge_B=hinge_B, hinge_end=hinge_C,
            head_drift_A=head_drift_A, feat_move_A=feat_move_A,
            feat_drift_B=feat_drift_B, head_move_B=head_move_B,
            mean_disp_C=disp.mean().item(),
            active_frac=active.float().mean().item(),
            cos_active=cos_mean,
            margin_pre_mean=m_pre.mean().item(),
            margin_post_mean=m_post.mean().item(),
            margin_pre_active_mean=(m_pre[active].mean().item()
                                    if active.any() else float("nan")),
            margin_post_active_mean=(m_post[active].mean().item()
                                     if active.any() else float("nan")),
            delta_margin_active=d_active,
            viol_pre=viol_pre, viol_post=viol_post))
    with torch.no_grad():
        zte = feat(data["Xte"])
        acc = (torch.sign(head(zte)) == data["yte"]).float().mean().item()
        nm = norm_margin(zte, data["yte"], head.w, head.b)
    return dict(acc=acc, test_norm_margin_mean=nm.mean().item(),
                test_norm_margin_p10=nm.quantile(0.10).item(), cycles=cyc)


def run_sklearn_svm(data, seed, kind):
    Xtr = data["Xtr"].numpy()
    Xte = data["Xte"].numpy()
    ytr = data["ytr01"].numpy()
    yte = data["yte01"].numpy()
    if kind == "linear":
        from sklearn.svm import LinearSVC
        clf = LinearSVC(C=1.0, max_iter=2000, dual=False, random_state=seed)
    else:
        from sklearn.svm import SVC
        clf = SVC(C=1.0, kernel="rbf", gamma="scale", random_state=seed)
    clf.fit(Xtr, ytr)
    return dict(acc=float(clf.score(Xte, yte)))


# --------------------------------------------------------------------- main --
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--datasets", nargs="+", default=DATASETS)
    ap.add_argument("--seeds", type=int, default=N_SEEDS)
    args = ap.parse_args()

    for ds in args.datasets:
        t0 = time.time()
        rows = []
        for seed in range(args.seeds):
            data = make_dataset(ds, seed)
            r = dict(seed=seed, data_hash=data["hash"],
                     n_train=int(data["Xtr"].shape[0]),
                     n_test=int(data["Xte"].shape[0]),
                     d=int(data["Xtr"].shape[1]))
            r["linear-svm"] = run_sklearn_svm(data, seed, "linear")
            r["rbf-svm"] = run_sklearn_svm(data, seed, "rbf")
            r["deep-ce"] = run_deep_ce(data, seed)
            r["deep-svm"] = run_deep_svm(data, seed)
            r["ddsvm"] = run_ddsvm(data, seed, random_push=False)
            r["ddsvm-rand"] = run_ddsvm(data, seed, random_push=True)
            rows.append(r)
            print(f"[{ds}] seed {seed:2d} hash={data['hash']} "
                  f"acc: lin={r['linear-svm']['acc']:.4f} "
                  f"rbf={r['rbf-svm']['acc']:.4f} "
                  f"ce={r['deep-ce']['acc']:.4f} "
                  f"dsvm={r['deep-svm']['acc']:.4f} "
                  f"ddsvm={r['ddsvm']['acc']:.4f} "
                  f"rand={r['ddsvm-rand']['acc']:.4f}", flush=True)
        wall = time.time() - t0
        hashes = [r["data_hash"] for r in rows]
        integrity = dict(
            n_seeds=args.seeds,
            n_unique_data_hashes=len(set(hashes)),
            all_hashes_distinct=len(set(hashes)) == args.seeds,
            acc_std_by_method={m: float(np.std([r[m]["acc"] for r in rows]))
                               for m in ["linear-svm", "rbf-svm", "deep-ce",
                                         "deep-svm", "ddsvm", "ddsvm-rand"]})
        out = dict(dataset=ds, wall_time_sec=wall, config=dict(
            n_seeds=args.seeds, n_cycles=N_CYCLES, ep_a=EP_A, ep_b=EP_B,
            ep_c=EP_C, epochs_joint=EPOCHS_JOINT, eta_geom=ETA_GEOM,
            lr_feat=LR_FEAT, lr_head=LR_HEAD, lr_joint=LR_JOINT,
            weight_decay=WD, reg_w=REG_W), integrity=integrity, runs=rows)
        path = os.path.join(OUTDIR, f"ddsvm_v2_{ds}.json")
        with open(path, "w") as f:
            json.dump(out, f)
        print(f"[{ds}] DONE in {wall:.1f}s -> {path}")
        print(f"[{ds}] integrity: {integrity['n_unique_data_hashes']}/"
              f"{args.seeds} unique hashes; "
              f"acc std ddsvm={integrity['acc_std_by_method']['ddsvm']:.5f}")


if __name__ == "__main__":
    main()