#!/usr/bin/env python3 """Generate statistical overview of all downloaded MRI datasets.""" import os, re, glob, csv, json from collections import Counter import pandas as pd ROOT = "/home/MRI-DataSet" def banner(s): print() print("=" * 78) print(s) print("=" * 78) def describe_ages(label, ages_years, extra=""): if not ages_years: print(f" {label}: no age data") return None s = pd.Series(ages_years) print(f" {label}: n={len(s)} mean={s.mean():.2f}y median={s.median():.2f}y " f"std={s.std():.2f} range={s.min():.2f}–{s.max():.2f}y {extra}") return s def age_histogram(title, ages_years, bins=None): if bins is None: bins = [(0, 3), (3, 6), (6, 10), (10, 14), (14, 18), (18, 99)] counts = [] for lo, hi in bins: n = sum(1 for a in ages_years if lo <= a < hi) counts.append(n) print(f"\n Age histogram ({title}):") total = len(ages_years) or 1 for (lo, hi), n in zip(bins, counts): bar = "#" * int(50 * n / total) label = f"{lo}–{hi}y" if hi < 99 else f"{lo}+" print(f" {label:>6} {n:>4} {bar}") return counts def analyse_bcp(): banner("DataSet-1 — Baby Connectome Project (BCP)") meta_csv = os.path.join(ROOT, "DataSet-1", "meta.csv") df = pd.read_csv(meta_csv, sep=";") t1_dir = os.path.join(ROOT, "DataSet-1", "T1_only") on_disk = {p.rstrip("/").split("/")[-1] for p in glob.glob(os.path.join(t1_dir, "s*/"))} df["on_disk"] = df["image_id"].isin(on_disk) print(f" Metadata rows: {len(df)} | on-disk T1 scans: {df['on_disk'].sum()} " f"| missing: {len(df) - df['on_disk'].sum()}") print(f" Age unit: months (converted to years below)") print(f" Group split: {dict(Counter(df['group']))}") print(f" Myelinisation: {dict(Counter(df['myelinisation']))}") ages_y = (df.loc[df["on_disk"], "age"] / 12).tolist() describe_ages("AGE (on-disk scans)", ages_y) # age in months histogram ages_m = df.loc[df["on_disk"], "age"].tolist() counts = {} for lo, hi, label in [(0, 6, "0–6m"), (6, 12, "6–12m"), (12, 18, "12–18m"), (18, 24, "18–24m"), (24, 36, "24–36m"), (36, 999, "36m+")]: counts[label] = sum(1 for a in ages_m if lo <= a < hi) print("\n Age histogram (months):") total = len(ages_m) or 1 for k, v in counts.items(): bar = "#" * int(50 * v / total) print(f" {k:>7} {v:>4} {bar}") # diagnosis counts diag_counts = Counter() for d in df.loc[df["on_disk"], "diagnosis"].dropna(): for tok in re.split(r"[;,]", str(d)): tok = tok.strip() if tok: diag_counts[tok] += 1 top = diag_counts.most_common(8) print(f"\n Top diagnoses: {top}") return ages_y, "0–3y (0–36 months)" def analyse_calgary(): banner("DataSet-2 — Calgary Preschool") xlsx = os.path.join(ROOT, "DataSet-2", "metadata", "Calgary_Preschool_Dataset_Updated_20200213.xlsx") df = pd.read_excel(xlsx) age_col = "Age (Years)" sex_col = "Biological Sex (Female = 0; Male = 1)" # On-disk scan IDs come from directory names: t1_dir = os.path.join(ROOT, "DataSet-2", "T1_Dataset") on_disk_scans = set() for subj in os.listdir(t1_dir): sd = os.path.join(t1_dir, subj) if os.path.isdir(sd): for scan in os.listdir(sd): if os.path.isdir(os.path.join(sd, scan)): on_disk_scans.add(scan) df["on_disk"] = df["ScanID"].astype(str).isin(on_disk_scans) print(f" Metadata rows: {len(df)} | on-disk scans: {df['on_disk'].sum()} " f"| unique children: {df.loc[df['on_disk'],'PreschoolID'].nunique()}") print(f" Sex split (on-disk): M={(df.loc[df['on_disk'],sex_col]==1).sum()} " f"F={(df.loc[df['on_disk'],sex_col]==0).sum()}") ages = df.loc[df["on_disk"], age_col].dropna().tolist() describe_ages("AGE (years, on-disk scans)", ages) age_histogram("years", ages) return ages, "~2–8y" def analyse_ds002726(): banner("DataSet-3 — OpenNeuro ds002726 (Gifted Children Study)") # No participants.tsv downloaded - only what's in BIDS description sub_dirs = sorted(glob.glob(os.path.join(ROOT, "DataSet-3", "sub-*/"))) print(f" On-disk subjects: {len(sub_dirs)} (01–15 gifted, 16–29 controls per README)") print(" Ages: NOT PROVIDED in dataset (no participants.tsv in this OpenNeuro release)") print(" Per PDF reference: middle childhood ~6–10y; paper: ranged 7.29–12.39y (mean ~9.4)") return [], "~6–12y (per paper, not verified from files)" def analyse_ds000248(): banner("DataSet-4 — OpenNeuro ds000248 (MNE-Sample-Data, NOT ADHD-200)") print(" Subjects: 1 (MEG/EEG tutorial sample; adult researcher)") print(" Age: not specified (n/a in participants.tsv)") print(" PDF expected: 585 healthy controls 7–18y — wrong dataset, see note") return [], "n/a" def analyse_ptbp(): banner("DataSet-5 — PTBP (Independent Test Set)") csvp = os.path.join(ROOT, "DataSet-5", "ptbp_summary_demographics.csv") df = pd.read_csv(csvp) cols = df.columns.tolist()[:15] # On-disk scans: T1_Anatomy/PEDS###/YYYYMMDD/Anatomy/*_t1.nii.gz t1_root = os.path.join(ROOT, "DataSet-5", "T1_Anatomy") on_disk = set() for subj in os.listdir(t1_root): sd = os.path.join(t1_root, subj) if os.path.isdir(sd): for sess in os.listdir(sd): if os.path.isdir(os.path.join(sd, sess)): on_disk.add((subj, str(sess))) df = df[["SubID", "ScanDate", "AgeAtScan", "Sex", "FullScaleIQ"]].copy() df["ScanDateStr"] = df["ScanDate"].astype(str) df["on_disk"] = df.apply(lambda r: (r["SubID"], r["ScanDateStr"]) in on_disk, axis=1) print(f" Metadata rows: {len(df)} | on-disk T1 scans: {df['on_disk'].sum()} " f"| unique children: {df.loc[df['on_disk'],'SubID'].nunique()}") m = (df.loc[df["on_disk"], "Sex"] == "M").sum() f = (df.loc[df["on_disk"], "Sex"] == "F").sum() print(f" Sex split (on-disk): M={m} F={f}") ages = df.loc[df["on_disk"], "AgeAtScan"].dropna().tolist() describe_ages("AGE (years, on-disk scans)", ages) age_histogram("years", ages) iq = df.loc[df["on_disk"] & df["FullScaleIQ"].notna(), "FullScaleIQ"].tolist() if iq: s = pd.Series(iq) print(f"\n Full-Scale IQ: n={len(s)} mean={s.mean():.1f} std={s.std():.1f} " f"range={s.min()}–{s.max()}") return ages, "7–18y" def combined_summary(by_ds): banner("COMBINED AGE COVERAGE (0–18+ yrs)") bins = [(0, 3), (3, 6), (6, 10), (10, 14), (14, 18), (18, 99)] labels = [f"{lo}–{hi}y" if hi < 99 else f"{lo}+" for lo, hi in bins] header = ["Dataset", "N"] + labels rows = [] grand_total = [0] * len(bins) for name, (ages, _range) in by_ds.items(): if not ages: row = [name, "0"] + ["–"] * len(bins) else: counts = [sum(1 for a in ages if lo <= a < hi) for lo, hi in bins] row = [name, str(len(ages))] + [str(c) for c in counts] for i, c in enumerate(counts): grand_total[i] += c rows.append(row) rows.append(["COMBINED", str(sum(grand_total))] + [str(c) for c in grand_total]) widths = [max(len(r[i]) for r in [header] + rows) for i in range(len(header))] def fmt(r): return " ".join(v.ljust(widths[i]) for i, v in enumerate(r)) print(fmt(header)) print(fmt(["-" * w for w in widths])) for r in rows: print(fmt(r)) def main(): by_ds = {} by_ds["DataSet-1 BCP"] = analyse_bcp() by_ds["DataSet-2 Calgary"] = analyse_calgary() by_ds["DataSet-3 ds002726"] = analyse_ds002726() by_ds["DataSet-4 ds000248"] = analyse_ds000248() by_ds["DataSet-5 PTBP"] = analyse_ptbp() combined_summary(by_ds) if __name__ == "__main__": main()