Spaces:
Sleeping
Sleeping
Download fl/data.py from rk147/federated_learning_model: direct link, hf CLI and curl.
- Browser
- Download file 5.46 kB
-
https://huggingface.co/spaces/rk147/federated_learning_model/resolve/main/fl/data.py
- Command line
-
hf download hf://spaces/rk147/federated_learning_model/fl/data.py
-
curl -L -o data.py https://huggingface.co/spaces/rk147/federated_learning_model/resolve/main/fl/data.py
5.46 kB
| """Data layer: cached MobileNetV2 features, synthetic fallback, non-IID hospital split.""" | |
| import os | |
| import numpy as np | |
| CACHE_DIR = os.environ.get( | |
| "FL_CACHE_DIR", os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cache")) | |
| RAW_DIM = 1280 | |
| TASKS = { | |
| "heart": {"title": "Heart disease risk (cardiomegaly) - chest X-ray"}, | |
| "diabetes": {"title": "Diabetes risk (diabetic retinopathy) - retinal fundus"}, | |
| } | |
| # ----------------------------------------------------------------- loading | |
| def load_raw(task, n_per_class=150, seed=0, allow_synthetic=True, log=print): | |
| path = os.path.join(CACHE_DIR, f"{task}.npz") | |
| if os.path.exists(path): | |
| z = np.load(path, allow_pickle=True) | |
| log(f"Loaded cached features: {path} ({len(z['y'])} images)") | |
| return {"X": z["X"], "y": z["y"], "g": z["g"], "source": str(z["source"]), | |
| "group_name": str(z["group_name"]), "synthetic": False} | |
| try: | |
| from .vision import build_real_dataset | |
| raw = build_real_dataset(task, n_per_class, seed, log) | |
| os.makedirs(CACHE_DIR, exist_ok=True) | |
| np.savez_compressed(path, X=raw["X"], y=raw["y"], g=raw["g"], | |
| source=raw["source"], group_name=raw["group_name"]) | |
| log(f"Saved feature cache -> {path}") | |
| raw["synthetic"] = False | |
| return raw | |
| except Exception as e: # noqa: BLE001 | |
| if not allow_synthetic: | |
| raise | |
| log(f"[WARNING] real data unavailable ({type(e).__name__}: {e}). " | |
| f"Falling back to SYNTHETIC features so the pipeline still runs.") | |
| return make_synthetic(task, seed) | |
| def make_synthetic(task, seed=0, n=800): | |
| """Synthetic 'MobileNet-like' features. Group 1 has a weaker signal on purpose | |
| so the fairness machinery has something to fix. NOT real medical data.""" | |
| rng = np.random.default_rng(seed + (0 if task == "heart" else 1000)) | |
| d, k = RAW_DIM, 24 # real CNN embeddings live near a low-dimensional subspace | |
| y = rng.integers(0, 2, n) | |
| g = (rng.random(n) < 0.4).astype(int) | |
| strength = np.where(g == 0, 0.9, 0.45) | |
| lat = rng.normal(size=(n, k)) | |
| lat[:, :3] += ((2 * y - 1) * strength)[:, None] # disease signal in 3 latent axes | |
| A = rng.normal(size=(k, d)) / np.sqrt(k) | |
| X = lat @ A + 0.3 * rng.normal(size=(n, d)) | |
| return {"X": X.astype(np.float32), "y": y, "g": g, "source": "SYNTHETIC (demo only)", | |
| "group_name": "synthetic group", "synthetic": True} | |
| # ------------------------------------------------------------ transforms | |
| def transform_features(prep, F): | |
| """Standardise -> shared random projection -> scale. Same code for training and inference.""" | |
| return ((F - prep["mean"]) / prep["std"]) @ prep["P"] * prep["scale"] | |
| def _try_split(raw, n_clients, alpha, proj_dim, seed, test_frac, val_frac): | |
| rng = np.random.default_rng(seed) | |
| X, y, g = raw["X"], raw["y"].astype(int), raw["g"].astype(int) | |
| strata = y * 2 + g | |
| test_idx, train_idx = [], [] | |
| for s in np.unique(strata): | |
| idx = np.where(strata == s)[0] | |
| rng.shuffle(idx) | |
| nt = int(round(len(idx) * test_frac)) if len(idx) >= 4 else 0 | |
| test_idx += list(idx[:nt]) | |
| train_idx += list(idx[nt:]) | |
| train_idx, test_idx = np.array(train_idx), np.array(test_idx) | |
| mean = X[train_idx].mean(0) | |
| std = X[train_idx].std(0) + 1e-6 | |
| P = (rng.normal(size=(X.shape[1], proj_dim)) / np.sqrt(X.shape[1])).astype(np.float32) | |
| prep = {"mean": mean, "std": std, "P": P, "scale": 3.0 / np.sqrt(proj_dim)} | |
| Z = transform_features(prep, X).astype(np.float64) | |
| assign = [[] for _ in range(n_clients)] | |
| for s in np.unique(strata[train_idx]): | |
| idx = train_idx[strata[train_idx] == s] | |
| rng.shuffle(idx) | |
| props = rng.dirichlet([alpha] * n_clients) | |
| cuts = (np.cumsum(props) * len(idx)).astype(int)[:-1] | |
| for c, part in enumerate(np.split(idx, cuts)): | |
| assign[c] += list(part) | |
| clients = [] | |
| for c, idx in enumerate(assign): | |
| idx = np.array(idx) | |
| if len(idx) < 30: | |
| return None | |
| rng.shuffle(idx) | |
| nv = max(5, int(len(idx) * val_frac)) | |
| tr, va = idx[nv:], idx[:nv] | |
| clients.append({"id": c, "X": Z[tr], "y": y[tr], "g": g[tr], | |
| "Xv": Z[va], "yv": y[va], "gv": g[va]}) | |
| test = {"X": Z[test_idx], "y": y[test_idx], "g": g[test_idx]} | |
| return {"clients": clients, "test": test, "prep": prep} | |
| def prepare_split(raw, n_clients=3, alpha=1.0, proj_dim=128, seed=0, test_frac=0.25, val_frac=0.2): | |
| """Split pooled data into a global test set and `n_clients` simulated hospitals | |
| (Dirichlet label/group skew => non-IID).""" | |
| for attempt in range(20): | |
| out = _try_split(raw, n_clients, alpha, proj_dim, seed + attempt, test_frac, val_frac) | |
| if out is not None: | |
| break | |
| else: | |
| out = _try_split(raw, n_clients, 100.0, proj_dim, seed, test_frac, val_frac) | |
| if out is None: | |
| raise ValueError("Not enough samples for that many clients. Lower n_clients or raise samples/class.") | |
| out["meta"] = {"source": raw["source"], "group_name": raw["group_name"], | |
| "synthetic": raw["synthetic"], "n_total": int(len(raw["y"])), | |
| "client_sizes": [len(c["y"]) for c in out["clients"]], | |
| "client_pos_rate": [float(np.mean(c["y"])) for c in out["clients"]]} | |
| return out | |