"""Data layer: cached MobileNetV2 features, synthetic fallback, non-IID hospital split.""" import os import numpy as np CACHE_DIR = os.environ.get( "FL_CACHE_DIR", os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cache")) RAW_DIM = 1280 TASKS = { "heart": {"title": "Heart disease risk (cardiomegaly) - chest X-ray"}, "diabetes": {"title": "Diabetes risk (diabetic retinopathy) - retinal fundus"}, } # ----------------------------------------------------------------- loading def load_raw(task, n_per_class=150, seed=0, allow_synthetic=True, log=print): path = os.path.join(CACHE_DIR, f"{task}.npz") if os.path.exists(path): z = np.load(path, allow_pickle=True) log(f"Loaded cached features: {path} ({len(z['y'])} images)") return {"X": z["X"], "y": z["y"], "g": z["g"], "source": str(z["source"]), "group_name": str(z["group_name"]), "synthetic": False} try: from .vision import build_real_dataset raw = build_real_dataset(task, n_per_class, seed, log) os.makedirs(CACHE_DIR, exist_ok=True) np.savez_compressed(path, X=raw["X"], y=raw["y"], g=raw["g"], source=raw["source"], group_name=raw["group_name"]) log(f"Saved feature cache -> {path}") raw["synthetic"] = False return raw except Exception as e: # noqa: BLE001 if not allow_synthetic: raise log(f"[WARNING] real data unavailable ({type(e).__name__}: {e}). " f"Falling back to SYNTHETIC features so the pipeline still runs.") return make_synthetic(task, seed) def make_synthetic(task, seed=0, n=800): """Synthetic 'MobileNet-like' features. Group 1 has a weaker signal on purpose so the fairness machinery has something to fix. NOT real medical data.""" rng = np.random.default_rng(seed + (0 if task == "heart" else 1000)) d, k = RAW_DIM, 24 # real CNN embeddings live near a low-dimensional subspace y = rng.integers(0, 2, n) g = (rng.random(n) < 0.4).astype(int) strength = np.where(g == 0, 0.9, 0.45) lat = rng.normal(size=(n, k)) lat[:, :3] += ((2 * y - 1) * strength)[:, None] # disease signal in 3 latent axes A = rng.normal(size=(k, d)) / np.sqrt(k) X = lat @ A + 0.3 * rng.normal(size=(n, d)) return {"X": X.astype(np.float32), "y": y, "g": g, "source": "SYNTHETIC (demo only)", "group_name": "synthetic group", "synthetic": True} # ------------------------------------------------------------ transforms def transform_features(prep, F): """Standardise -> shared random projection -> scale. Same code for training and inference.""" return ((F - prep["mean"]) / prep["std"]) @ prep["P"] * prep["scale"] def _try_split(raw, n_clients, alpha, proj_dim, seed, test_frac, val_frac): rng = np.random.default_rng(seed) X, y, g = raw["X"], raw["y"].astype(int), raw["g"].astype(int) strata = y * 2 + g test_idx, train_idx = [], [] for s in np.unique(strata): idx = np.where(strata == s)[0] rng.shuffle(idx) nt = int(round(len(idx) * test_frac)) if len(idx) >= 4 else 0 test_idx += list(idx[:nt]) train_idx += list(idx[nt:]) train_idx, test_idx = np.array(train_idx), np.array(test_idx) mean = X[train_idx].mean(0) std = X[train_idx].std(0) + 1e-6 P = (rng.normal(size=(X.shape[1], proj_dim)) / np.sqrt(X.shape[1])).astype(np.float32) prep = {"mean": mean, "std": std, "P": P, "scale": 3.0 / np.sqrt(proj_dim)} Z = transform_features(prep, X).astype(np.float64) assign = [[] for _ in range(n_clients)] for s in np.unique(strata[train_idx]): idx = train_idx[strata[train_idx] == s] rng.shuffle(idx) props = rng.dirichlet([alpha] * n_clients) cuts = (np.cumsum(props) * len(idx)).astype(int)[:-1] for c, part in enumerate(np.split(idx, cuts)): assign[c] += list(part) clients = [] for c, idx in enumerate(assign): idx = np.array(idx) if len(idx) < 30: return None rng.shuffle(idx) nv = max(5, int(len(idx) * val_frac)) tr, va = idx[nv:], idx[:nv] clients.append({"id": c, "X": Z[tr], "y": y[tr], "g": g[tr], "Xv": Z[va], "yv": y[va], "gv": g[va]}) test = {"X": Z[test_idx], "y": y[test_idx], "g": g[test_idx]} return {"clients": clients, "test": test, "prep": prep} def prepare_split(raw, n_clients=3, alpha=1.0, proj_dim=128, seed=0, test_frac=0.25, val_frac=0.2): """Split pooled data into a global test set and `n_clients` simulated hospitals (Dirichlet label/group skew => non-IID).""" for attempt in range(20): out = _try_split(raw, n_clients, alpha, proj_dim, seed + attempt, test_frac, val_frac) if out is not None: break else: out = _try_split(raw, n_clients, 100.0, proj_dim, seed, test_frac, val_frac) if out is None: raise ValueError("Not enough samples for that many clients. Lower n_clients or raise samples/class.") out["meta"] = {"source": raw["source"], "group_name": raw["group_name"], "synthetic": raw["synthetic"], "n_total": int(len(raw["y"])), "client_sizes": [len(c["y"]) for c in out["clients"]], "client_pos_rate": [float(np.mean(c["y"])) for c in out["clients"]]} return out