rk147's picture
Upload 15 files (#1)
0a32a7f
Raw History Blame Contribute Delete
5.46 kB
"""Data layer: cached MobileNetV2 features, synthetic fallback, non-IID hospital split."""
import os
import numpy as np
CACHE_DIR = os.environ.get(
"FL_CACHE_DIR", os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cache"))
RAW_DIM = 1280
TASKS = {
"heart": {"title": "Heart disease risk (cardiomegaly) - chest X-ray"},
"diabetes": {"title": "Diabetes risk (diabetic retinopathy) - retinal fundus"},
}
# ----------------------------------------------------------------- loading
def load_raw(task, n_per_class=150, seed=0, allow_synthetic=True, log=print):
path = os.path.join(CACHE_DIR, f"{task}.npz")
if os.path.exists(path):
z = np.load(path, allow_pickle=True)
log(f"Loaded cached features: {path} ({len(z['y'])} images)")
return {"X": z["X"], "y": z["y"], "g": z["g"], "source": str(z["source"]),
"group_name": str(z["group_name"]), "synthetic": False}
try:
from .vision import build_real_dataset
raw = build_real_dataset(task, n_per_class, seed, log)
os.makedirs(CACHE_DIR, exist_ok=True)
np.savez_compressed(path, X=raw["X"], y=raw["y"], g=raw["g"],
source=raw["source"], group_name=raw["group_name"])
log(f"Saved feature cache -> {path}")
raw["synthetic"] = False
return raw
except Exception as e: # noqa: BLE001
if not allow_synthetic:
raise
log(f"[WARNING] real data unavailable ({type(e).__name__}: {e}). "
f"Falling back to SYNTHETIC features so the pipeline still runs.")
return make_synthetic(task, seed)
def make_synthetic(task, seed=0, n=800):
"""Synthetic 'MobileNet-like' features. Group 1 has a weaker signal on purpose
so the fairness machinery has something to fix. NOT real medical data."""
rng = np.random.default_rng(seed + (0 if task == "heart" else 1000))
d, k = RAW_DIM, 24 # real CNN embeddings live near a low-dimensional subspace
y = rng.integers(0, 2, n)
g = (rng.random(n) < 0.4).astype(int)
strength = np.where(g == 0, 0.9, 0.45)
lat = rng.normal(size=(n, k))
lat[:, :3] += ((2 * y - 1) * strength)[:, None] # disease signal in 3 latent axes
A = rng.normal(size=(k, d)) / np.sqrt(k)
X = lat @ A + 0.3 * rng.normal(size=(n, d))
return {"X": X.astype(np.float32), "y": y, "g": g, "source": "SYNTHETIC (demo only)",
"group_name": "synthetic group", "synthetic": True}
# ------------------------------------------------------------ transforms
def transform_features(prep, F):
"""Standardise -> shared random projection -> scale. Same code for training and inference."""
return ((F - prep["mean"]) / prep["std"]) @ prep["P"] * prep["scale"]
def _try_split(raw, n_clients, alpha, proj_dim, seed, test_frac, val_frac):
rng = np.random.default_rng(seed)
X, y, g = raw["X"], raw["y"].astype(int), raw["g"].astype(int)
strata = y * 2 + g
test_idx, train_idx = [], []
for s in np.unique(strata):
idx = np.where(strata == s)[0]
rng.shuffle(idx)
nt = int(round(len(idx) * test_frac)) if len(idx) >= 4 else 0
test_idx += list(idx[:nt])
train_idx += list(idx[nt:])
train_idx, test_idx = np.array(train_idx), np.array(test_idx)
mean = X[train_idx].mean(0)
std = X[train_idx].std(0) + 1e-6
P = (rng.normal(size=(X.shape[1], proj_dim)) / np.sqrt(X.shape[1])).astype(np.float32)
prep = {"mean": mean, "std": std, "P": P, "scale": 3.0 / np.sqrt(proj_dim)}
Z = transform_features(prep, X).astype(np.float64)
assign = [[] for _ in range(n_clients)]
for s in np.unique(strata[train_idx]):
idx = train_idx[strata[train_idx] == s]
rng.shuffle(idx)
props = rng.dirichlet([alpha] * n_clients)
cuts = (np.cumsum(props) * len(idx)).astype(int)[:-1]
for c, part in enumerate(np.split(idx, cuts)):
assign[c] += list(part)
clients = []
for c, idx in enumerate(assign):
idx = np.array(idx)
if len(idx) < 30:
return None
rng.shuffle(idx)
nv = max(5, int(len(idx) * val_frac))
tr, va = idx[nv:], idx[:nv]
clients.append({"id": c, "X": Z[tr], "y": y[tr], "g": g[tr],
"Xv": Z[va], "yv": y[va], "gv": g[va]})
test = {"X": Z[test_idx], "y": y[test_idx], "g": g[test_idx]}
return {"clients": clients, "test": test, "prep": prep}
def prepare_split(raw, n_clients=3, alpha=1.0, proj_dim=128, seed=0, test_frac=0.25, val_frac=0.2):
"""Split pooled data into a global test set and `n_clients` simulated hospitals
(Dirichlet label/group skew => non-IID)."""
for attempt in range(20):
out = _try_split(raw, n_clients, alpha, proj_dim, seed + attempt, test_frac, val_frac)
if out is not None:
break
else:
out = _try_split(raw, n_clients, 100.0, proj_dim, seed, test_frac, val_frac)
if out is None:
raise ValueError("Not enough samples for that many clients. Lower n_clients or raise samples/class.")
out["meta"] = {"source": raw["source"], "group_name": raw["group_name"],
"synthetic": raw["synthetic"], "n_total": int(len(raw["y"])),
"client_sizes": [len(c["y"]) for c in out["clients"]],
"client_pos_rate": [float(np.mean(c["y"])) for c in out["clients"]]}
return out