File size: 10,964 Bytes
d2a9154 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | """
Evaluate checkpoints, baselines and trained runs on every split manifest.
python -m experiments.evaluate --released --meanmap # released checkpoints and input-free baseline
python -m experiments.evaluate --runs # experiments/runs/*/best_checkpoint.pth
Per-image metrics: MSE, MS-SSIM, pixel-wise Pearson r, RME, mean traction stress (Pa) and inference time.
Outputs: experiments/results/eval/per_image.csv and summary.csv.
"""
import argparse
import glob
import json
import os
import time
import numpy as np
import pandas as pd
import torch
from experiments.ablation_models import build_generator, n_params
from experiments.layout import DEFAULT_DATASET_ROOT, RESULTS_DIR, RUNS_DIR, S2F_ROOT, list_manifests
from experiments.losses import PerImageWFMRMELoss
from experiments.manifest_data import make_loader
from models.s2f_model import create_s2f_model, create_settings_channels
from utils.metrics import TorchMetricsWrapper
from utils.substrate_settings import compute_settings_normalization
CONFIG_PATH = os.path.join(S2F_ROOT, "config", "substrate_settings.json")
SWEEP_KEYS = ["PDMS_1kPa_3um", "fibroblasts_PDMS", "PDMS_10kPa_3um"]
def released_models(device):
out = {}
g, _ = create_s2f_model(in_channels=3, model_type="s2f")
g.load_state_dict(torch.load(os.path.join(S2F_ROOT, "ckp/single_cell/ckp_singlecell_GN.pth"), map_location="cpu",
weights_only=False)["generator_state_dict"], strict=True)
out["released_singlecell"] = dict(model=g.to(device).eval(), kind="singlecell", in_channels=3, tanh=True)
g, _ = create_s2f_model(in_channels=1, model_type="s2f")
g.load_state_dict(torch.load(os.path.join(S2F_ROOT, "ckp/spheroid/ckp_spheroid_GN.pth"), map_location="cpu",
weights_only=False)["generator_state_dict"], strict=True)
out["released_spheroid_GN"] = dict(model=g.to(device).eval(), kind="spheroid", in_channels=1, tanh=True)
fn = os.path.join(S2F_ROOT, "ckp/spheroid/ckp_spheroid_FN.pth")
if os.path.exists(fn):
g, _ = create_s2f_model(in_channels=1, model_type="s2f_spheroid")
g.load_state_dict(torch.load(fn, map_location="cpu", weights_only=False)["model_state_dict"], strict=True)
g.set_output_mode(use_tanh=False)
out["supervised_spheroid_FN"] = dict(model=g.to(device).eval(), kind="spheroid", in_channels=1, tanh=False)
return out
def run_models(device, pattern):
out = {}
for p in sorted(glob.glob(pattern)):
h = os.path.join(os.path.dirname(p), "history.json")
ck = torch.load(p, map_location="cpu", weights_only=False)
cfg = ck["config"]
done = os.path.exists(h) and len(json.load(open(h))) >= cfg.get("epochs", 0)
if not done and not os.environ.get("S2F_EVAL_UNFINISHED"):
print(f" skipping unfinished run {os.path.basename(os.path.dirname(p))} (set S2F_EVAL_UNFINISHED=1 to include)")
continue
g = build_generator(**cfg); g.load_state_dict(ck["generator_state_dict"], strict=True)
name = os.path.basename(os.path.dirname(p))
out[name] = dict(model=g.to(device).eval(), kind="singlecell", in_channels=cfg["in_channels"], tanh=True,
substrate_mode=cfg["substrate_mode"], input_mode=cfg["input_mode"],
epoch=ck.get("epoch"), val_loss=ck.get("val_loss"))
return out
def mean_map(manifest, dataset_root):
acc, n = None, 0
for b in make_loader(manifest, dataset_root, batch_size=8, substrate_mode="none"):
acc = b[1].sum(0, keepdim=True) if acc is None else acc + b[1].sum(0, keepdim=True); n += b[1].size(0)
return acc / n
def evaluate_one(name, spec, manifest, dataset_root, device, mw, norm, substrate_mode, batch, mean_pred=None):
input_mode = spec.get("input_mode", "bf")
use_settings = spec.get("in_channels", 1) == 3
sm = substrate_mode if use_settings else "none"
forced = None
if sm and sm.startswith("force:"):
forced, sm = sm.split(":", 1)[1], "reference"
try:
loader = make_loader(manifest, dataset_root, batch_size=batch, substrate_mode=sm, input_mode=input_mode)
except FileNotFoundError as e:
print(f" [skip] {name} on {manifest}: {e}"); return []
rows_meta, rec, out, k = loader.dataset.rows, PerImageWFMRMELoss(), [], 0
with torch.no_grad():
for b in loader:
x, y = b[0].to(device), b[1].to(device)
t0 = time.time()
if mean_pred is not None:
p = mean_pred.to(device).expand_as(y)
else:
if use_settings:
meta = b[4]
if forced:
meta = dict(meta, substrate=[forced] * x.size(0))
x = torch.cat([x, create_settings_channels(meta, norm, device, x.shape, config_path=CONFIG_PATH)], 1)
p = spec["model"](x)
p = (p + 1) / 2 if spec.get("tanh", True) else p
dt = (time.time() - t0) / x.size(0)
for i in range(y.size(0)):
r = rows_meta[k]; k += 1
t, q = y[i:i + 1], p[i:i + 1]
yt = float(r.get("ythreshold_Pa") or (50 if "spheroid" in manifest else 500))
out.append({"model": name, "manifest": manifest, "substrate_mode": substrate_mode if use_settings else "n/a",
"sample_dir": r["sample_dir"], "set": r.get("set", ""), "condition": r.get("condition", ""),
"mse": mw.compute_mse(t, q), "ms_ssim": mw.compute_ms_ssim(t, q),
"pixel_r": float(np.corrcoef(t.flatten().cpu().numpy(), q.flatten().cpu().numpy())[0, 1]),
"rme": float(rec(q, t)), "mean_gt": float(t.mean()), "mean_pred": float(q.mean()),
"mean_gt_Pa": float(t.mean()) * yt, "mean_pred_Pa": float(q.mean()) * yt,
"gt_max": float(t.max()), "gt_frac_saturated": float((t >= 0.98).float().mean()),
"seconds_per_image": dt})
return out
def summarise(df):
df = df.assign(substrate_mode=df["substrate_mode"].fillna("none")) # "n/a" is read back from CSV as NaN
g = df.groupby(["model", "manifest", "substrate_mode"], dropna=False)
s = g.agg(n=("mse", "size"), **{f"{m}_{f}": (m, f) for m in ["mse", "ms_ssim", "pixel_r", "rme", "mean_gt_Pa",
"mean_pred_Pa", "gt_frac_saturated"] for f in ["mean", "std"]},
seconds_per_image=("seconds_per_image", "mean")).reset_index()
s["pred_over_gt"] = s["mean_pred_Pa_mean"] / s["mean_gt_Pa_mean"]
return s
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--released", action="store_true", help="released single-cell + spheroid checkpoints")
ap.add_argument("--sweep", action="store_true", help="released single-cell model with 1 / 6 / 10 kPa settings")
ap.add_argument("--meanmap", action="store_true", help="input-free baseline (mean training map)")
ap.add_argument("--runs", nargs="?", const=os.path.join(RUNS_DIR, "*", "best_checkpoint.pth"), default=None,
help="glob of trained run checkpoints (default experiments/runs/*/best_checkpoint.pth)")
ap.add_argument("--manifests", default="", help="comma-separated manifest names (default: all in experiments/splits)")
ap.add_argument("--dataset-root", default=DEFAULT_DATASET_ROOT)
ap.add_argument("--out", default=os.path.join(RESULTS_DIR, "eval"))
ap.add_argument("--batch", type=int, default=2)
ap.add_argument("--device", default=None)
ap.add_argument("--run-modes", default=None,
help="substrate modes for trained runs with substrate channels, e.g. legacy,nominal (default: run config)")
a = ap.parse_args()
device = torch.device(a.device or ("cuda" if torch.cuda.is_available() else "cpu"))
mans = [m for m in (a.manifests.split(",") if a.manifests else list_manifests()) if m]
sc_mans = [m for m in mans if not m.startswith("spheroid") and m != "alltune_train" and m != "zenodo_v1_train"]
sph_mans = [m for m in mans if m.startswith("spheroid") and m != "spheroid_train"]
print(f"device {device}; single-cell manifests {sc_mans}; spheroid manifests {sph_mans}")
mw, norm = TorchMetricsWrapper(device=device), compute_settings_normalization(config_path=CONFIG_PATH)
models = {}
if a.released:
models.update(released_models(device))
if a.runs:
models.update(run_models(device, a.runs))
rows, params = [], []
for name, spec in models.items():
params.append({"model": name, "n_params_M": n_params(spec["model"]) / 1e6, "epoch": spec.get("epoch"),
"val_loss": spec.get("val_loss")})
targets = sc_mans if spec["kind"] == "singlecell" else sph_mans
modes = [spec.get("substrate_mode", "legacy")]
if a.run_modes and spec.get("in_channels", 1) == 3 and name not in ("released_singlecell",):
modes = [m for m in a.run_modes.split(",") if m]
if name == "released_singlecell":
modes = ["legacy", "nominal"] + ([f"force:{k}" for k in SWEEP_KEYS] if a.sweep else [])
for m in targets:
for sm in modes:
print(f" {name:28s} {m:28s} {sm}", flush=True)
rows += evaluate_one(name, spec, m, a.dataset_root, device, mw, norm, sm, a.batch)
if a.meanmap:
for train_m, targets, name in [("alltune_train", sc_mans, "baseline_mean_map_singlecell"),
("spheroid_train", sph_mans, "baseline_mean_map_spheroid")]:
if train_m not in list_manifests():
continue
M = mean_map(train_m, a.dataset_root)
for m in targets:
print(f" {name:28s} {m}", flush=True)
rows += evaluate_one(name, {"in_channels": 1}, m, a.dataset_root, device, mw, norm, "n/a", a.batch, mean_pred=M)
os.makedirs(a.out, exist_ok=True)
new = pd.DataFrame(rows)
pi = os.path.join(a.out, "per_image.csv")
if os.path.exists(pi) and len(new): # keep rows of models not re-evaluated now
old = pd.read_csv(pi)
new = pd.concat([old[~old.model.isin(new.model.unique())], new], ignore_index=True)
if len(new):
new.to_csv(pi, index=False)
summarise(new).to_csv(os.path.join(a.out, "summary.csv"), index=False)
pp = os.path.join(a.out, "model_params.csv")
if params:
P = pd.DataFrame(params)
if os.path.exists(pp):
old = pd.read_csv(pp); P = pd.concat([old[~old.model.isin(P.model)], P], ignore_index=True)
P.to_csv(pp, index=False)
print(f"wrote {pi}, summary.csv, model_params.csv in {a.out}")
if __name__ == "__main__":
main()
|