scripts/stats.py

#!/usr/bin/env python3
"""Generate statistical overview of all downloaded MRI datasets."""
import os, re, glob, csv, json
from collections import Counter
import pandas as pd

ROOT = "/home/MRI-DataSet"


def banner(s):
    print()
    print("=" * 78)
    print(s)
    print("=" * 78)


def describe_ages(label, ages_years, extra=""):
    if not ages_years:
        print(f"  {label}: no age data")
        return None
    s = pd.Series(ages_years)
    print(f"  {label}: n={len(s)}  mean={s.mean():.2f}y  median={s.median():.2f}y  "
          f"std={s.std():.2f}  range={s.min():.2f}–{s.max():.2f}y {extra}")
    return s


def age_histogram(title, ages_years, bins=None):
    if bins is None:
        bins = [(0, 3), (3, 6), (6, 10), (10, 14), (14, 18), (18, 99)]
    counts = []
    for lo, hi in bins:
        n = sum(1 for a in ages_years if lo <= a < hi)
        counts.append(n)
    print(f"\n  Age histogram ({title}):")
    total = len(ages_years) or 1
    for (lo, hi), n in zip(bins, counts):
        bar = "#" * int(50 * n / total)
        label = f"{lo}–{hi}y" if hi < 99 else f"{lo}+"
        print(f"    {label:>6}  {n:>4}   {bar}")
    return counts


def analyse_bcp():
    banner("DataSet-1 — Baby Connectome Project (BCP)")
    meta_csv = os.path.join(ROOT, "DataSet-1", "meta.csv")
    df = pd.read_csv(meta_csv, sep=";")

    t1_dir = os.path.join(ROOT, "DataSet-1", "T1_only")
    on_disk = {p.rstrip("/").split("/")[-1]
               for p in glob.glob(os.path.join(t1_dir, "s*/"))}

    df["on_disk"] = df["image_id"].isin(on_disk)

    print(f"  Metadata rows: {len(df)}  |  on-disk T1 scans: {df['on_disk'].sum()}  "
          f"|  missing: {len(df) - df['on_disk'].sum()}")
    print(f"  Age unit: months   (converted to years below)")
    print(f"  Group split: {dict(Counter(df['group']))}")
    print(f"  Myelinisation: {dict(Counter(df['myelinisation']))}")

    ages_y = (df.loc[df["on_disk"], "age"] / 12).tolist()
    describe_ages("AGE (on-disk scans)", ages_y)

    # age in months histogram
    ages_m = df.loc[df["on_disk"], "age"].tolist()
    counts = {}
    for lo, hi, label in [(0, 6, "0–6m"), (6, 12, "6–12m"), (12, 18, "12–18m"),
                         (18, 24, "18–24m"), (24, 36, "24–36m"), (36, 999, "36m+")]:
        counts[label] = sum(1 for a in ages_m if lo <= a < hi)
    print("\n  Age histogram (months):")
    total = len(ages_m) or 1
    for k, v in counts.items():
        bar = "#" * int(50 * v / total)
        print(f"    {k:>7}  {v:>4}   {bar}")

    # diagnosis counts
    diag_counts = Counter()
    for d in df.loc[df["on_disk"], "diagnosis"].dropna():
        for tok in re.split(r"[;,]", str(d)):
            tok = tok.strip()
            if tok:
                diag_counts[tok] += 1
    top = diag_counts.most_common(8)
    print(f"\n  Top diagnoses: {top}")
    return ages_y, "0–3y (0–36 months)"


def analyse_calgary():
    banner("DataSet-2 — Calgary Preschool")
    xlsx = os.path.join(ROOT, "DataSet-2", "metadata",
                        "Calgary_Preschool_Dataset_Updated_20200213.xlsx")
    df = pd.read_excel(xlsx)
    age_col = "Age (Years)"
    sex_col = "Biological Sex (Female = 0; Male = 1)"

    # On-disk scan IDs come from directory names:
    t1_dir = os.path.join(ROOT, "DataSet-2", "T1_Dataset")
    on_disk_scans = set()
    for subj in os.listdir(t1_dir):
        sd = os.path.join(t1_dir, subj)
        if os.path.isdir(sd):
            for scan in os.listdir(sd):
                if os.path.isdir(os.path.join(sd, scan)):
                    on_disk_scans.add(scan)

    df["on_disk"] = df["ScanID"].astype(str).isin(on_disk_scans)
    print(f"  Metadata rows: {len(df)}  |  on-disk scans: {df['on_disk'].sum()}  "
          f"|  unique children: {df.loc[df['on_disk'],'PreschoolID'].nunique()}")
    print(f"  Sex split (on-disk):  M={(df.loc[df['on_disk'],sex_col]==1).sum()}   "
          f"F={(df.loc[df['on_disk'],sex_col]==0).sum()}")

    ages = df.loc[df["on_disk"], age_col].dropna().tolist()
    describe_ages("AGE (years, on-disk scans)", ages)
    age_histogram("years", ages)
    return ages, "~2–8y"


def analyse_ds002726():
    banner("DataSet-3 — OpenNeuro ds002726 (Gifted Children Study)")
    # No participants.tsv downloaded - only what's in BIDS description
    sub_dirs = sorted(glob.glob(os.path.join(ROOT, "DataSet-3", "sub-*/")))
    print(f"  On-disk subjects: {len(sub_dirs)}  (01–15 gifted, 16–29 controls per README)")
    print("  Ages: NOT PROVIDED in dataset  (no participants.tsv in this OpenNeuro release)")
    print("  Per PDF reference: middle childhood ~6–10y; paper: ranged 7.29–12.39y (mean ~9.4)")
    return [], "~6–12y (per paper, not verified from files)"


def analyse_ds000248():
    banner("DataSet-4 — OpenNeuro ds000248 (MNE-Sample-Data, NOT ADHD-200)")
    print("  Subjects: 1 (MEG/EEG tutorial sample; adult researcher)")
    print("  Age: not specified (n/a in participants.tsv)")
    print("  PDF expected: 585 healthy controls 7–18y — wrong dataset, see note")
    return [], "n/a"


def analyse_ptbp():
    banner("DataSet-5 — PTBP (Independent Test Set)")
    csvp = os.path.join(ROOT, "DataSet-5", "ptbp_summary_demographics.csv")
    df = pd.read_csv(csvp)
    cols = df.columns.tolist()[:15]
    # On-disk scans: T1_Anatomy/PEDS###/YYYYMMDD/Anatomy/*_t1.nii.gz
    t1_root = os.path.join(ROOT, "DataSet-5", "T1_Anatomy")
    on_disk = set()
    for subj in os.listdir(t1_root):
        sd = os.path.join(t1_root, subj)
        if os.path.isdir(sd):
            for sess in os.listdir(sd):
                if os.path.isdir(os.path.join(sd, sess)):
                    on_disk.add((subj, str(sess)))
    df = df[["SubID", "ScanDate", "AgeAtScan", "Sex", "FullScaleIQ"]].copy()
    df["ScanDateStr"] = df["ScanDate"].astype(str)
    df["on_disk"] = df.apply(lambda r: (r["SubID"], r["ScanDateStr"]) in on_disk, axis=1)

    print(f"  Metadata rows: {len(df)}  |  on-disk T1 scans: {df['on_disk'].sum()}  "
          f"|  unique children: {df.loc[df['on_disk'],'SubID'].nunique()}")
    m = (df.loc[df["on_disk"], "Sex"] == "M").sum()
    f = (df.loc[df["on_disk"], "Sex"] == "F").sum()
    print(f"  Sex split (on-disk):  M={m}   F={f}")

    ages = df.loc[df["on_disk"], "AgeAtScan"].dropna().tolist()
    describe_ages("AGE (years, on-disk scans)", ages)
    age_histogram("years", ages)

    iq = df.loc[df["on_disk"] & df["FullScaleIQ"].notna(), "FullScaleIQ"].tolist()
    if iq:
        s = pd.Series(iq)
        print(f"\n  Full-Scale IQ: n={len(s)}  mean={s.mean():.1f}  std={s.std():.1f}  "
              f"range={s.min()}–{s.max()}")
    return ages, "7–18y"


def combined_summary(by_ds):
    banner("COMBINED AGE COVERAGE (0–18+ yrs)")
    bins = [(0, 3), (3, 6), (6, 10), (10, 14), (14, 18), (18, 99)]
    labels = [f"{lo}–{hi}y" if hi < 99 else f"{lo}+" for lo, hi in bins]

    header = ["Dataset", "N"] + labels
    rows = []
    grand_total = [0] * len(bins)
    for name, (ages, _range) in by_ds.items():
        if not ages:
            row = [name, "0"] + ["–"] * len(bins)
        else:
            counts = [sum(1 for a in ages if lo <= a < hi) for lo, hi in bins]
            row = [name, str(len(ages))] + [str(c) for c in counts]
            for i, c in enumerate(counts):
                grand_total[i] += c
        rows.append(row)
    rows.append(["COMBINED", str(sum(grand_total))] + [str(c) for c in grand_total])

    widths = [max(len(r[i]) for r in [header] + rows) for i in range(len(header))]
    def fmt(r):
        return "  ".join(v.ljust(widths[i]) for i, v in enumerate(r))
    print(fmt(header))
    print(fmt(["-" * w for w in widths]))
    for r in rows:
        print(fmt(r))


def main():
    by_ds = {}
    by_ds["DataSet-1 BCP"] = analyse_bcp()
    by_ds["DataSet-2 Calgary"] = analyse_calgary()
    by_ds["DataSet-3 ds002726"] = analyse_ds002726()
    by_ds["DataSet-4 ds000248"] = analyse_ds000248()
    by_ds["DataSet-5 PTBP"] = analyse_ptbp()
    combined_summary(by_ds)


if __name__ == "__main__":
    main()