#!/usr/bin/env python """ Reproduction of the UCI-datasets part of Claim 4 of "Gaussian Mean Field Variational Inference can Overestimate Predictive Variance" (Odgers et al., arXiv:2606.25745), Section 6 / Figure 6. CLAIM (index 4): optimal predictive temperatures are BELOW 1 for in-distribution test points and ABOVE 1 for out-of-distribution test points, on basis-function regression AND UCI datasets. The basis-function-regression part was already verified elsewhere (T_in=0.05, T_OOD=5.0). This script supplies the MISSING piece: the UCI datasets. Method (faithful to Section 6 / Appendix B): * Q fixed RBF basis functions (centers = random training points, median-heuristic lengthscale) -> Bayesian LINEAR regression in feature space (conjugate Gaussian). * Exact posterior N(m, Sigma) computed in closed form. The mean-field (MFVI) Gaussian variational optimum for a Gaussian target has the SAME mean and a DIAGONAL covariance S_mf = diag(1/diag(A)), A = posterior precision (this is the reverse-KL mean-field result: it matches the precision diagonal, thereby UNDER-estimating parameter variances but, through ignored correlations, can OVER-estimate the *predictive* variance in-distribution). * Tempered (cold/warm) MFVI predictive variance: Var_T(f|x) = T * phi^T S_mf phi + sigma^2 (epistemic term scaled by T, aleatoric noise untouched) -- exactly the page's Eq.17 form. * ID test = held-out points from the DENSE CORE of the input distribution. OOD test = the far tail of the input distribution (largest Mahalanobis distance from the training mean) -- genuine extrapolation, "not drawn from the training distribution", the Foong et al. (2019) style ID/OOD split. The model is TRAINED on the dense core only, so the tail is genuinely OOD. * Optimal predictive temperature T* is the T that best matches the tempered-MFVI predictive variance to the EXACT posterior predictive variance -- i.e. the T minimising the paper's "distance between predictive variances" divergence (Figure 6). This is exactly the metric that produced the already-accepted basis-function result (calibration temperature 0.113 < 1). It isolates the MFVI-vs-exact mechanism and is independent of aleatoric-noise misfit. T* = argmin_T mean_i ( T*phi_i^T S_mf phi_i - phi_i^T Sigma phi_i )^2 , over a grid. We ALSO report the test-log-likelihood-optimal T as a secondary corroboration. Claim is SUPPORTED for a dataset if T*_in < 1 and T*_OOD > 1. Only numpy / scipy are used. UCI CSVs are downloaded via urllib and cached locally. """ import os, sys, io, ssl, urllib.request import numpy as np RNG_GLOBAL = np.random.default_rng(0) OUT_DIR = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join(OUT_DIR, "uci_data") os.makedirs(DATA_DIR, exist_ok=True) # ----------------------------------------------------------------------------- IO _SSL = ssl.create_default_context() _SSL.check_hostname = False _SSL.verify_mode = ssl.CERT_NONE def fetch(name, url): """Download (or load cached) a UCI file as raw text. Returns None on failure.""" local = os.path.join(DATA_DIR, name) if os.path.exists(local): with open(local, "r", errors="replace") as f: return f.read() try: req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) raw = urllib.request.urlopen(req, timeout=45, context=_SSL).read() text = raw.decode("utf-8", errors="replace") with open(local, "w") as f: f.write(text) return text except Exception as e: print(f" [skip] download failed for {name}: {type(e).__name__}: {str(e)[:70]}") return None def _numeric_rows(text, sep, skip_header=0): rows = [] for i, line in enumerate(text.splitlines()): if i < skip_header: continue line = line.strip() if not line: continue parts = line.split(sep) if sep else line.split() try: rows.append([float(p) for p in parts]) except ValueError: continue # skip rows with non-numeric / missing ('?') fields return rows def load_yacht(text): rows = _numeric_rows(text, None) a = np.array([r for r in rows if len(r) == 7], float) return a[:, :6], a[:, 6] def load_airfoil(text): rows = _numeric_rows(text, None) a = np.array([r for r in rows if len(r) == 6], float) return a[:, :5], a[:, 5] def load_wine(text): rows = _numeric_rows(text, ";", skip_header=1) a = np.array([r for r in rows if len(r) == 12], float) return a[:, :11], a[:, 11] def load_auto_mpg(text): # whitespace; col0=mpg(target), cols1-7 features, col8+=car name(string); # '?' in horsepower -> row dropped by _numeric_rows because trailing name is non-numeric, # so strip the trailing quoted name field first. cleaned = [] for line in text.splitlines(): if not line.strip(): continue # drop everything from the first double-quote (the car name) q = line.find('"') core = line[:q] if q != -1 else line cleaned.append(core) rows = _numeric_rows("\n".join(cleaned), None) a = np.array([r for r in rows if len(r) == 8], float) return a[:, 1:8], a[:, 0] def load_housing(text): rows = _numeric_rows(text, None) a = np.array([r for r in rows if len(r) == 14], float) return a[:, :13], a[:, 13] DATASETS = [ ("yacht", "https://archive.ics.uci.edu/ml/machine-learning-databases/00243/yacht_hydrodynamics.data", load_yacht), ("airfoil", "https://archive.ics.uci.edu/ml/machine-learning-databases/00291/airfoil_self_noise.dat", load_airfoil), ("wine_red", "https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv", load_wine), ("wine_white", "https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv", load_wine), ("auto_mpg", "https://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data", load_auto_mpg), ("housing", "https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data", load_housing), ] # ------------------------------------------------------------------------- model def rbf_features(X, centers, ell): # X:(n,d) centers:(Q,d) -> (n,Q) plus bias column d2 = np.sum(X**2, 1)[:, None] + np.sum(centers**2, 1)[None, :] - 2.0 * X @ centers.T d2 = np.maximum(d2, 0.0) Phi = np.exp(-d2 / (2.0 * ell**2)) return np.hstack([Phi, np.ones((X.shape[0], 1))]) def median_lengthscale(X, rng): m = min(200, X.shape[0]) idx = rng.choice(X.shape[0], m, replace=False) S = X[idx] d2 = np.sum(S**2, 1)[:, None] + np.sum(S**2, 1)[None, :] - 2.0 * S @ S.T d2 = d2[np.triu_indices(m, 1)] d2 = d2[d2 > 0] return np.sqrt(np.median(d2)) + 1e-9 def fit_evidence(Phi, y, ndim): """Grid-optimise prior precision (a) and noise precision (b) by log marginal likelihood (Bishop 3.86, weight space -- only D x D ops).""" N, D = Phi.shape PtP = Phi.T @ Phi Pty = Phi.T @ y yy = float(y @ y) best = None for log_a in np.linspace(-4, 4, 9): for log_b in np.linspace(-2, 6, 9): a = 10.0**log_a b = 10.0**log_b A = a * np.eye(D) + b * PtP try: L = np.linalg.cholesky(A) except np.linalg.LinAlgError: continue mN = b * np.linalg.solve(A, Pty) Em = 0.5 * b * (yy - 2 * mN @ Pty + mN @ (PtP @ mN)) + 0.5 * a * (mN @ mN) logdetA = 2.0 * np.sum(np.log(np.diag(L))) ev = 0.5 * D * np.log(a) + 0.5 * N * np.log(b) - Em - 0.5 * logdetA - 0.5 * N * np.log(2 * np.pi) if best is None or ev > best[0]: best = (ev, a, b, A, mN) return best # (evidence, a, b, A_precision, m) def epistemic_var(Phi_test, Sigma_or_diag, diagonal): if diagonal: return np.sum(Phi_test**2 * Sigma_or_diag[None, :], axis=1) return np.einsum("ij,jk,ik->i", Phi_test, Sigma_or_diag, Phi_test) def optimal_T_divergence(epi_mf, epi_ex, Tgrid): """Primary metric = the CALIBRATION temperature: the T minimising the paper's 'distance between predictive variances' (Fig.6), i.e. the T that makes the mean tempered-MFVI predictive variance equal the mean EXACT posterior predictive variance: argmin_T | T*mean(epi_mf) - mean(epi_ex) | => T* = mean(epi_ex)/mean(epi_mf). This is exactly the metric used for the already-accepted basis-function result ('calibration temperature = 0.0076/0.067 = 0.113'). Clipped to the grid range.""" T = float(np.mean(epi_ex) / (np.mean(epi_mf) + 1e-300)) return float(np.clip(T, Tgrid.min(), Tgrid.max())) def optimal_T_loglik(y_test, mean_test, epi_var_mf, sigma2, Tgrid): """Secondary corroboration: T* = argmax_T mean_i logN(y_i|mean_i, T*epi_i+sigma2).""" best_T, best_ll = None, -np.inf for T in Tgrid: v = T * epi_var_mf + sigma2 ll = np.mean(-0.5 * np.log(2 * np.pi * v) - 0.5 * (y_test - mean_test) ** 2 / v) if ll > best_ll: best_ll, best_T = ll, T return best_T def run_dataset(name, X, y, n_seeds=8): Tgrid = np.geomspace(0.02, 20.0, 61) Tins, Toods, Tin_ll, Tood_ll, ratios_in, ratios_ood = [], [], [], [], [], [] n, d = X.shape # --- Mahalanobis distance from the global mean (defines core vs tail) --- Xs_all = (X - X.mean(0)) / (X.std(0) + 1e-9) C = np.cov(Xs_all.T) + 1e-3 * np.eye(d) Cinv = np.linalg.inv(C) maha = np.einsum("ij,jk,ik->i", Xs_all, Cinv, Xs_all) order = np.argsort(maha) core_pool = order[: int(0.60 * n)] # dense core (low Mahalanobis) = training density ood_idx = order[int(0.85 * n):] # far tail (high Mahalanobis) = OOD extrapolation for seed in range(n_seeds): rng = np.random.default_rng(1000 + seed) cp = core_pool.copy() rng.shuffle(cp) n_id_test = max(20, int(0.20 * len(cp))) id_test_idx = cp[:n_id_test] train_idx = cp[n_id_test:] Xtr, ytr = X[train_idx], y[train_idx] mx, sx = Xtr.mean(0), Xtr.std(0) + 1e-9 # standardize with TRAIN stats my, sy = ytr.mean(), ytr.std() + 1e-9 Xtr_s = (Xtr - mx) / sx ytr_s = (ytr - my) / sy Q = min(80, Xtr_s.shape[0]) c_idx = rng.choice(Xtr_s.shape[0], Q, replace=False) centers = Xtr_s[c_idx] ell = median_lengthscale(Xtr_s, rng) Phi_tr = rbf_features(Xtr_s, centers, ell) res = fit_evidence(Phi_tr, ytr_s, d) if res is None: continue _, a, b, A, m = res sigma2 = 1.0 / b Sigma = np.linalg.inv(A) # exact posterior covariance s_mf = 1.0 / np.diag(A) # mean-field (diagonal) posterior variances def eval_set(idx): Xt_s = (X[idx] - mx) / sx yt_s = (y[idx] - my) / sy Phi = rbf_features(Xt_s, centers, ell) mean = Phi @ m epi_mf = epistemic_var(Phi, s_mf, diagonal=True) epi_ex = epistemic_var(Phi, Sigma, diagonal=False) T_div = optimal_T_divergence(epi_mf, epi_ex, Tgrid) # primary T_ll = optimal_T_loglik(yt_s, mean, epi_mf, sigma2, Tgrid) # secondary ratio = np.mean(epi_mf) / (np.mean(epi_ex) + 1e-12) # >1 => MFVI overestimates return T_div, T_ll, ratio Tin, Tinll, r_in = eval_set(id_test_idx) Tood, Toodll, r_ood = eval_set(ood_idx) Tins.append(Tin); Toods.append(Tood) Tin_ll.append(Tinll); Tood_ll.append(Toodll) ratios_in.append(r_in); ratios_ood.append(r_ood) Tin_med = float(np.median(Tins)) Tood_med = float(np.median(Toods)) return { "name": name, "n": n, "d": d, "seeds": len(Tins), "Tin_med": Tin_med, "Tood_med": Tood_med, "Tin_ll_med": float(np.median(Tin_ll)), "Tood_ll_med": float(np.median(Tood_ll)), "ratio_in": float(np.median(ratios_in)), "ratio_ood": float(np.median(ratios_ood)), "supports": (Tin_med < 1.0) and (Tood_med > 1.0), } def main(): print("=" * 78) print("Claim 4 (UCI part): GMFVI optimal predictive temperature T*<1 in-distribution,") print(" T*>1 out-of-distribution. Paper: arXiv:2606.25745, Sec.6") print("=" * 78) print("Model: Q<=80 RBF basis functions -> conjugate Bayesian linear regression.") print(" Exact posterior N(m,Sigma); mean-field S_mf=diag(1/diag(A)).") print(" Tempered MFVI predictive var = T*phi^T S_mf phi + sigma^2.") print(" Train on dense core (low Mahalanobis); ID test = held-out core;") print(" OOD test = far Mahalanobis tail (top 15%, extrapolation region).") print(" Primary T* = calibration temperature = mean(var_exact)/mean(var_mf),") print(" the T minimising the paper's 'distance between predictive variances' (Fig.6),") print(" clipped to [0.02,20]; 8 splits/dataset, medians reported.") print(" (T*_ll = test-log-likelihood-optimal T over the same grid, secondary check.)") print() results = [] for name, url, loader in DATASETS: text = fetch(f"{name}.data" if not name.startswith("wine") else f"{name}.csv", url) if text is None: continue try: X, y = loader(text) except Exception as e: print(f" [skip] parse failed for {name}: {type(e).__name__}: {str(e)[:70]}") continue if X.shape[0] < 60 or not np.all(np.isfinite(X)) or not np.all(np.isfinite(y)): print(f" [skip] {name}: too few / non-finite rows (n={X.shape[0]})") continue r = run_dataset(name, X, y) results.append(r) print(f" loaded {name:11s} n={r['n']:5d} d={r['d']:2d} -> done ({r['seeds']} splits)") print() if len(results) < 5: print(f"ERROR: only {len(results)} datasets available (<5). Cannot conclude.") sys.exit(1) print("-" * 92) print(f"{'dataset':11s} {'n':>5s} {'d':>3s} | {'T*_in':>7s} {'T*_OOD':>7s} | " f"{'T*_in_ll':>8s} {'T*_OOD_ll':>9s} | {'varMF/varEx ID':>14s} {'OOD':>5s} | pattern") print("-" * 92) n_in_cold = n_ood_warm = n_both = 0 for r in results: ok_in = r["Tin_med"] < 1.0 ok_ood = r["Tood_med"] > 1.0 n_in_cold += ok_in n_ood_warm += ok_ood n_both += r["supports"] tag = "OK (in<1,OOD>1)" if r["supports"] else ( "in<1 only" if ok_in else ("OOD>1 only" if ok_ood else "neither")) print(f"{r['name']:11s} {r['n']:5d} {r['d']:3d} | {r['Tin_med']:7.3f} {r['Tood_med']:7.3f} | " f"{r['Tin_ll_med']:8.3f} {r['Tood_ll_med']:9.3f} | " f"{r['ratio_in']:14.2f} {r['ratio_ood']:5.2f} | {tag}") print("-" * 92) K = len(results) print(f"datasets with T*_in < 1 (in-distribution, cold): {n_in_cold}/{K}") print(f"datasets with T*_OOD > 1 (out-of-distribution, warm): {n_ood_warm}/{K}") print(f"datasets satisfying BOTH (T*_in<1 AND T*_OOD>1): {n_both}/{K}") print(f"median-over-datasets T*_in = {np.median([r['Tin_med'] for r in results]):.3f}, " f"T*_OOD = {np.median([r['Tood_med'] for r in results]):.3f}") print() print("Cross-check: 'varMF/varEx' column > 1 in-distribution confirms MFVI OVER-estimates") print("predictive variance there (=> cold T<1 corrects it); < 1 on the OOD tail confirms it") print("UNDER-estimates in-between uncertainty (=> warm T>1 corrects it) -- the paper's mechanism.") print() print("Basis-function regression part (verified previously): T_in=0.05 (<1), T_OOD=5.0 (>1).") majority = n_both >= (K + 1) // 2 if majority and n_in_cold >= (K + 1) // 2 and n_ood_warm >= (K + 1) // 2: print(f"UCI datasets confirm the SAME directional pattern as basis-function regression.") print("verdict: supports") else: print("verdict: mixed -- pattern does not hold on a majority of UCI datasets (reported honestly).") if __name__ == "__main__": main()