File size: 4,013 Bytes
586c646 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | """Produce comprehensive statistics on the current healthy-brain corpus."""
import pandas as pd
from pathlib import Path
import textwrap
ROOT = Path("/home/MRI-DataSet")
df = pd.read_csv(ROOT / "manifest.csv")
out = []
out.append("=" * 72)
out.append("HEALTHY-BRAIN TRAINING CORPUS — FINAL SNAPSHOT")
out.append("=" * 72)
out.append(f"Manifest: {ROOT/'manifest.csv'}")
out.append(f"Golden-0-to-25 (primary): {ROOT/'Golden-0-to-25'}/")
out.append(f"Golden-25plus (fine-tune): {ROOT/'Golden-25plus'}/")
out.append("")
# Overall
total = len(df)
healthy = (df["healthy"] == True).sum()
with_age = df["age_years"].notna().sum()
age025 = ((df["age_years"].notna()) & (df["age_years"] < 25)).sum()
age25p = ((df["age_years"].notna()) & (df["age_years"] >= 25)).sum()
no_age = df["age_years"].isna().sum()
out.append(f"Total scans: {total}")
out.append(f"Labelled healthy: {healthy} ({100*healthy/total:.1f}%)")
out.append(f"With known age: {with_age} ({100*with_age/total:.1f}%)")
out.append(f" age < 25 (primary): {age025}")
out.append(f" age >= 25 (pretrain): {age25p}")
out.append(f"Unknown age (to impute): {no_age}")
out.append("")
# Age stats
if with_age:
ages = df["age_years"].dropna()
out.append("Age distribution (years) over dated scans:")
out.append(f" min={ages.min():.1f} median={ages.median():.1f} max={ages.max():.1f}")
out.append(f" mean={ages.mean():.1f} ± {ages.std():.1f}")
out.append("")
# Split breakdown
if "split" in df.columns:
out.append("-" * 72)
out.append("SPLIT BREAKDOWN")
out.append("-" * 72)
for split, grp in df.groupby("split"):
n = len(grp)
h = (grp["healthy"] == True).sum()
ages = grp["age_years"].dropna()
rng = f"{ages.min():.1f}-{ages.max():.1f}" if len(ages) else "n/a"
out.append(f" {split:<28} {n:>5} scans (healthy={h}) range={rng}")
out.append("")
# Per-dataset breakdown
out.append("-" * 72)
out.append("PER-DATASET BREAKDOWN")
out.append("-" * 72)
out.append(f"{'dataset':<28}{'scans':>7}{'healthy':>9}{'age<25':>8}{'age>=25':>9}"
f"{'age range':>18}")
out.append("-" * 72)
for ds, grp in df.groupby("dataset"):
n = len(grp)
h = (grp["healthy"] == True).sum()
a = ((grp["age_years"].notna()) & (grp["age_years"] < 25)).sum()
b = ((grp["age_years"].notna()) & (grp["age_years"] >= 25)).sum()
ages = grp["age_years"].dropna()
if len(ages):
rng = f"{ages.min():.1f}-{ages.max():.1f}"
else:
rng = "n/a"
out.append(f"{ds:<28}{n:>7}{h:>9}{a:>8}{b:>9}{rng:>18}")
out.append("-" * 72)
out.append("")
# Age histogram in 5-year bins
out.append("AGE HISTOGRAM (5-year bins, across all dated scans)")
ages = df["age_years"].dropna()
if len(ages):
max_age = int(ages.max()) + 5
edges = list(range(0, max_age + 1, 5))
counts, _ = pd.cut(ages, bins=edges, right=False, include_lowest=True).value_counts(sort=False), None
for interval, n in counts.items():
bar = "#" * int(40 * n / counts.max())
out.append(f" [{interval.left:>3}, {interval.right:>3}) {n:>5} {bar}")
out.append("")
# Split targets
out.append("=" * 72)
out.append("TARGET SPLITS FOR TRAINING")
out.append("=" * 72)
out.append("")
out.append(textwrap.dedent(f"""\
Primary training (0-25 years) : {age025:>5} scans
Pretraining pool (25+ years) : {age25p:>5} scans
Grand total labelled : {age025+age25p:>5} scans
Strategy (per user):
1. Train EfficientNet-B3 multi-view on Training_0to25/ (pediatric + young adult)
2. Ablation: pretrain on Training_0to25 + Pretrain_25plus
(full lifespan), then fine-tune on 0-25
3. Compare pediatric MAE between the two strategies in thesis
"""))
text = "\n".join(out)
print(text)
(ROOT / "STATISTICS.md").write_text(text)
(ROOT / "STATISTICS.txt").write_text(text)
print(f"\n-> wrote {ROOT/'STATISTICS.md'} and {ROOT/'STATISTICS.txt'}")
|