File size: 5,464 Bytes
0a32a7f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
"""Data layer: cached MobileNetV2 features, synthetic fallback, non-IID hospital split."""
import os
import numpy as np

CACHE_DIR = os.environ.get(
    "FL_CACHE_DIR", os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cache"))
RAW_DIM = 1280

TASKS = {
    "heart": {"title": "Heart disease risk (cardiomegaly) - chest X-ray"},
    "diabetes": {"title": "Diabetes risk (diabetic retinopathy) - retinal fundus"},
}


# ----------------------------------------------------------------- loading
def load_raw(task, n_per_class=150, seed=0, allow_synthetic=True, log=print):
    path = os.path.join(CACHE_DIR, f"{task}.npz")
    if os.path.exists(path):
        z = np.load(path, allow_pickle=True)
        log(f"Loaded cached features: {path} ({len(z['y'])} images)")
        return {"X": z["X"], "y": z["y"], "g": z["g"], "source": str(z["source"]),
                "group_name": str(z["group_name"]), "synthetic": False}
    try:
        from .vision import build_real_dataset
        raw = build_real_dataset(task, n_per_class, seed, log)
        os.makedirs(CACHE_DIR, exist_ok=True)
        np.savez_compressed(path, X=raw["X"], y=raw["y"], g=raw["g"],
                            source=raw["source"], group_name=raw["group_name"])
        log(f"Saved feature cache -> {path}")
        raw["synthetic"] = False
        return raw
    except Exception as e:  # noqa: BLE001
        if not allow_synthetic:
            raise
        log(f"[WARNING] real data unavailable ({type(e).__name__}: {e}). "
            f"Falling back to SYNTHETIC features so the pipeline still runs.")
        return make_synthetic(task, seed)


def make_synthetic(task, seed=0, n=800):
    """Synthetic 'MobileNet-like' features. Group 1 has a weaker signal on purpose
    so the fairness machinery has something to fix. NOT real medical data."""
    rng = np.random.default_rng(seed + (0 if task == "heart" else 1000))
    d, k = RAW_DIM, 24            # real CNN embeddings live near a low-dimensional subspace
    y = rng.integers(0, 2, n)
    g = (rng.random(n) < 0.4).astype(int)
    strength = np.where(g == 0, 0.9, 0.45)
    lat = rng.normal(size=(n, k))
    lat[:, :3] += ((2 * y - 1) * strength)[:, None]          # disease signal in 3 latent axes
    A = rng.normal(size=(k, d)) / np.sqrt(k)
    X = lat @ A + 0.3 * rng.normal(size=(n, d))
    return {"X": X.astype(np.float32), "y": y, "g": g, "source": "SYNTHETIC (demo only)",
            "group_name": "synthetic group", "synthetic": True}


# ------------------------------------------------------------ transforms
def transform_features(prep, F):
    """Standardise -> shared random projection -> scale. Same code for training and inference."""
    return ((F - prep["mean"]) / prep["std"]) @ prep["P"] * prep["scale"]


def _try_split(raw, n_clients, alpha, proj_dim, seed, test_frac, val_frac):
    rng = np.random.default_rng(seed)
    X, y, g = raw["X"], raw["y"].astype(int), raw["g"].astype(int)
    strata = y * 2 + g
    test_idx, train_idx = [], []
    for s in np.unique(strata):
        idx = np.where(strata == s)[0]
        rng.shuffle(idx)
        nt = int(round(len(idx) * test_frac)) if len(idx) >= 4 else 0
        test_idx += list(idx[:nt])
        train_idx += list(idx[nt:])
    train_idx, test_idx = np.array(train_idx), np.array(test_idx)

    mean = X[train_idx].mean(0)
    std = X[train_idx].std(0) + 1e-6
    P = (rng.normal(size=(X.shape[1], proj_dim)) / np.sqrt(X.shape[1])).astype(np.float32)
    prep = {"mean": mean, "std": std, "P": P, "scale": 3.0 / np.sqrt(proj_dim)}
    Z = transform_features(prep, X).astype(np.float64)

    assign = [[] for _ in range(n_clients)]
    for s in np.unique(strata[train_idx]):
        idx = train_idx[strata[train_idx] == s]
        rng.shuffle(idx)
        props = rng.dirichlet([alpha] * n_clients)
        cuts = (np.cumsum(props) * len(idx)).astype(int)[:-1]
        for c, part in enumerate(np.split(idx, cuts)):
            assign[c] += list(part)

    clients = []
    for c, idx in enumerate(assign):
        idx = np.array(idx)
        if len(idx) < 30:
            return None
        rng.shuffle(idx)
        nv = max(5, int(len(idx) * val_frac))
        tr, va = idx[nv:], idx[:nv]
        clients.append({"id": c, "X": Z[tr], "y": y[tr], "g": g[tr],
                        "Xv": Z[va], "yv": y[va], "gv": g[va]})
    test = {"X": Z[test_idx], "y": y[test_idx], "g": g[test_idx]}
    return {"clients": clients, "test": test, "prep": prep}


def prepare_split(raw, n_clients=3, alpha=1.0, proj_dim=128, seed=0, test_frac=0.25, val_frac=0.2):
    """Split pooled data into a global test set and `n_clients` simulated hospitals
    (Dirichlet label/group skew => non-IID)."""
    for attempt in range(20):
        out = _try_split(raw, n_clients, alpha, proj_dim, seed + attempt, test_frac, val_frac)
        if out is not None:
            break
    else:
        out = _try_split(raw, n_clients, 100.0, proj_dim, seed, test_frac, val_frac)
        if out is None:
            raise ValueError("Not enough samples for that many clients. Lower n_clients or raise samples/class.")
    out["meta"] = {"source": raw["source"], "group_name": raw["group_name"],
                   "synthetic": raw["synthetic"], "n_total": int(len(raw["y"])),
                   "client_sizes": [len(c["y"]) for c in out["clients"]],
                   "client_pos_rate": [float(np.mean(c["y"])) for c in out["clients"]]}
    return out