#!/usr/bin/env python """Bootstrap A1 baseline with manifest QC and Protocol C split artifacts. This command currently implements: - Phase 1: strict derivatives manifest and integrity checks - Phase 2: canonical-grid symmetric analysis mask construction - Phase 3 (core): ROI-preservation QC for 7 core language regions - Phase 4: annotation harmonization to unified run/condition/speaker event tables - Phase 5: optional frozen feature extraction wrappers and caching - Phase 6: optional TR-level HRF regressor + z-score alignment input caching - Phase 7 prep: Protocol C (cross-subject) split table """ from __future__ import annotations import argparse import json import random from pathlib import Path from typing import Any import nibabel as nib import pandas as pd from a1_pipeline.alignment import build_and_cache_alignment_inputs from a1_pipeline.annotations import build_harmonized_annotation_tables from a1_pipeline.constants import DEFAULT_ALLOWED_RUNS, DEFAULT_MODEL_IDS, DEFAULT_TIME_SCALE_SECONDS from a1_pipeline.features import extract_and_cache_run_level_features, resolve_feature_num_workers from a1_pipeline.io_utils import ensure_directory, write_json from a1_pipeline.manifest import build_derivatives_manifest from a1_pipeline.model_config import load_model_ids_from_config from a1_pipeline.spatial import build_symmetric_analysis_mask from a1_pipeline.targets import evaluate_core_roi_preservation, load_core_roi_masks def _parse_allowed_runs(raw: str) -> tuple[int, ...]: tokens = [token.strip() for token in raw.split(",") if token.strip()] runs = [] for token in tokens: value = int(token) if value <= 0: raise ValueError(f"Run must be positive: {value}") runs.append(value) if not runs: raise ValueError("At least one allowed run is required") return tuple(sorted(set(runs))) def _parse_subject_list(raw: str) -> list[str]: raw = raw.strip().lower() if raw == "all": return [] tokens = [token.strip() for token in raw.split(",") if token.strip()] return sorted(set(tokens)) def _parse_layer_indices(raw: str) -> list[int] | None: raw = raw.strip().lower() if raw == "all": return None tokens = [token.strip() for token in raw.split(",") if token.strip()] if not tokens: raise ValueError("--feature-layer-indices must be 'all' or a comma-separated list") indices = sorted({int(token) for token in tokens}) if any(idx < 0 for idx in indices): raise ValueError("Layer indices must be non-negative") return indices def _build_protocol_c_folds( manifest_df: pd.DataFrame, n_test_subjects: int, n_folds: int, seed: int, ) -> pd.DataFrame: """Build Protocol C folds using repeated 21:3 subject holdout splits.""" canonical_labels = { 1: "single_female", 2: "single_male", 3: "mixed_female", 4: "mixed_male", } subjects = sorted(manifest_df["subject"].astype(str).unique().tolist()) n_subjects = len(subjects) if n_subjects != 24: raise ValueError( "Protocol C 21:3 split requires exactly 24 accepted subjects. " f"Found {n_subjects}." ) if n_test_subjects != 3: raise ValueError( "Protocol C is fixed to 21:3, so --protocol-c-test-subjects must be 3" ) if n_folds <= 0: raise ValueError("--protocol-c-num-folds must be positive") rng = random.Random(int(seed)) shuffled_subjects = list(subjects) rng.shuffle(shuffled_subjects) rows: list[dict[str, Any]] = [] for fold_idx in range(int(n_folds)): start = (fold_idx * n_test_subjects) % n_subjects test_subjects = [ shuffled_subjects[(start + offset) % n_subjects] for offset in range(n_test_subjects) ] test_subject_set = set(test_subjects) train_subjects = [subject for subject in subjects if subject not in test_subject_set] if len(train_subjects) != 21: raise ValueError( "Protocol C expected 21 train subjects per fold, got " f"{len(train_subjects)}" ) for canonical_run, condition_label in canonical_labels.items(): rows.append( { "fold_id": f"cv2_run{canonical_run}_fold{fold_idx + 1}", "canonical_run": int(canonical_run), "condition_label": str(condition_label), "train_subjects": ",".join(train_subjects), "test_subjects": ",".join(test_subjects), } ) return pd.DataFrame(rows).sort_values(["canonical_run", "fold_id"]).reset_index(drop=True) def _build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Bootstrap A1 manifest and split artifacts") parser.add_argument( "--base-dir", type=str, default=".", help="Dataset repository root", ) parser.add_argument( "--derivatives-dir", type=str, default=None, help="Override derivatives directory (default: /derivatives)", ) parser.add_argument( "--data-dir", type=str, default=None, help="Override raw BIDS data directory (default: /data)", ) parser.add_argument( "--annotation-dir", type=str, default=None, help="Override annotation directory (default: /data/annotation)", ) parser.add_argument( "--output-dir", type=str, default=None, help="Output folder for bootstrap artifacts (default: /outputs/a1_bootstrap)", ) parser.add_argument( "--roi-mask-dir", type=str, default=None, help=( "Core ROI mask directory (default resolution: " "/code/assets/roi_masks, " "fallback: /llms_brain_lateralization/roi_masks)" ), ) parser.add_argument( "--allowed-runs", type=str, default=",".join(str(value) for value in DEFAULT_ALLOWED_RUNS), help="Comma-separated run whitelist", ) parser.add_argument( "--exclude-subjects", type=str, default="sub-03,sub-18", help="Comma-separated subject IDs to exclude from all phases", ) parser.add_argument( "--deep-nifti-integrity-check", action="store_true", help="Force-read tail volumes to catch truncated .nii.gz files early (slower)", ) parser.add_argument( "--disable-mixed-fallback", action="store_true", help="Disable mixed-run condition fallback to generic 'mixed' label", ) parser.add_argument( "--protocol-c-test-subjects", type=int, default=3, help="Protocol C test subject count per fold (fixed to 3 for 21:3)", ) parser.add_argument( "--protocol-c-num-folds", type=int, default=2, help="Protocol C repeated holdout fold count", ) parser.add_argument( "--protocol-c-seed", type=int, default=0, help="Random seed for Protocol C fold construction", ) parser.add_argument( "--run-mask-method", type=str, default="first_volume_nonzero", choices=["first_volume_nonzero", "temporal_any_nonzero"], help="Method for deriving per-run spatial mask from 4D BOLD", ) parser.add_argument( "--run-mask-epsilon", type=float, default=1e-6, help="Absolute threshold used when binarizing run mask from BOLD values", ) parser.add_argument( "--min-core-roi-voxels", type=int, default=20, help="Minimum post-mask voxel count required per core ROI and subject-run", ) parser.add_argument( "--allow-core-roi-failures", action="store_true", help="Do not fail command even if core ROI preservation gate fails", ) parser.add_argument( "--time-scale-seconds", type=float, default=DEFAULT_TIME_SCALE_SECONDS, help="Scale converting annotation onset/offset units to seconds", ) parser.add_argument( "--run-feature-extraction", action="store_true", help="Execute Phase 5 feature extraction and cache run-level hidden states", ) parser.add_argument( "--feature-dry-run", action="store_true", help="Use deterministic random features instead of loading real HF models", ) parser.add_argument( "--feature-dry-run-n-layers", type=int, default=3, help="Number of synthetic layers when --feature-dry-run is enabled", ) parser.add_argument( "--feature-dry-run-hidden-dim", type=int, default=128, help="Synthetic hidden dimension when --feature-dry-run is enabled", ) parser.add_argument( "--feature-layer-indices", type=str, default="all", help="Feature layers to cache: 'all' or comma-separated indices", ) parser.add_argument( "--max-words-per-chunk", type=int, default=256, help="Word chunk size used during feature extraction", ) parser.add_argument( "--feature-device", type=str, default="auto", help="Feature extraction device: auto, cpu, cuda, cuda:0, ...", ) parser.add_argument( "--feature-num-workers", type=str, default="auto", help=( "Number of GPU workers for feature extraction. 'auto' uses every " "visible CUDA device (recommended on 4xA10G). Use 1 to force " "single-GPU/serial extraction." ), ) parser.add_argument( "--feature-local-files-only", action="store_true", help="Do not access network when loading HF models/tokenizers", ) parser.add_argument( "--feature-overwrite", action="store_true", help="Overwrite existing cached feature files", ) parser.add_argument( "--run-alignment", action="store_true", help="Execute Phase 6 HRF/z-score alignment input caching", ) parser.add_argument( "--alignment-subjects", type=str, default="all", help="Alignment BOLD subject subset: 'all' or comma-separated subject IDs", ) parser.add_argument( "--alignment-trim-start-tr", type=int, default=0, help="Trim this many TRs from start for both regressors and BOLD", ) parser.add_argument( "--alignment-trim-end-tr", type=int, default=0, help="Trim this many TRs from end for both regressors and BOLD", ) parser.add_argument( "--alignment-hrf-model", type=str, default="glover", help="HRF model passed to nilearn.compute_regressor", ) parser.add_argument( "--alignment-overwrite", action="store_true", help="Overwrite existing cached alignment arrays", ) parser.add_argument( "--model-config", type=str, default=None, help="JSON model profile config (default: /code/config/model_profiles.json)", ) parser.add_argument( "--model-profile", type=str, default=None, help="Profile name in --model-config (default: active_profile from config)", ) return parser def _resolve_paths(args: argparse.Namespace) -> tuple[Path, Path, Path, Path, Path, Path]: base_dir = Path(args.base_dir).resolve() derivatives_dir = Path(args.derivatives_dir).resolve() if args.derivatives_dir else base_dir / "derivatives" data_dir = Path(args.data_dir).resolve() if args.data_dir else base_dir / "data" annotation_dir = Path(args.annotation_dir).resolve() if args.annotation_dir else data_dir / "annotation" output_dir = Path(args.output_dir).resolve() if args.output_dir else base_dir / "outputs" / "a1_bootstrap" if args.roi_mask_dir: roi_mask_dir = Path(args.roi_mask_dir).resolve() else: roi_candidates = [ base_dir / "code" / "assets" / "roi_masks", base_dir / "llms_brain_lateralization" / "roi_masks", ] roi_mask_dir = roi_candidates[0] for candidate in roi_candidates: if candidate.exists(): roi_mask_dir = candidate break return base_dir, derivatives_dir, data_dir, annotation_dir, output_dir, roi_mask_dir def _validate_required_dirs( derivatives_dir: Path, data_dir: Path, annotation_dir: Path, roi_mask_dir: Path, ) -> None: if not derivatives_dir.exists(): raise FileNotFoundError(f"Derivatives directory not found: {derivatives_dir}") if not data_dir.exists(): raise FileNotFoundError(f"Data directory not found: {data_dir}") if not annotation_dir.exists(): raise FileNotFoundError(f"Annotation directory not found: {annotation_dir}") if not roi_mask_dir.exists(): raise FileNotFoundError(f"ROI mask directory not found: {roi_mask_dir}") def _resolve_requested_models(args: argparse.Namespace, base_dir: Path) -> tuple[list[str], dict[str, Any]]: default_model_config_path = (base_dir / "code" / "config" / "model_profiles.json").resolve() model_config_path = Path(args.model_config).resolve() if args.model_config else default_model_config_path if model_config_path.exists(): return load_model_ids_from_config( config_path=model_config_path, profile=args.model_profile, ) if args.model_config: raise FileNotFoundError(f"Model config not found: {model_config_path}") if args.model_profile: raise FileNotFoundError( "--model-profile was provided but no model config file was found. " "Create code/config/model_profiles.json or pass --model-config." ) return list(DEFAULT_MODEL_IDS), { "source": "constants_default", "config_path": None, "profile": None, "description": "Fallback to DEFAULT_MODEL_IDS", } def main() -> None: parser = _build_arg_parser() args = parser.parse_args() allowed_runs = _parse_allowed_runs(args.allowed_runs) excluded_subjects = _parse_subject_list(args.exclude_subjects) alignment_subjects = _parse_subject_list(args.alignment_subjects) feature_layer_indices = _parse_layer_indices(args.feature_layer_indices) mixed_fallback_enabled = not bool(args.disable_mixed_fallback) base_dir, derivatives_dir, data_dir, annotation_dir, output_dir, roi_mask_dir = _resolve_paths(args) requested_models, model_selection = _resolve_requested_models(args=args, base_dir=base_dir) _validate_required_dirs( derivatives_dir=derivatives_dir, data_dir=data_dir, annotation_dir=annotation_dir, roi_mask_dir=roi_mask_dir, ) ensure_directory(output_dir) manifest_df, rejected_df, manifest_qc = build_derivatives_manifest( derivatives_dir=derivatives_dir, data_dir=data_dir, allowed_runs=allowed_runs, enable_mixed_fallback=mixed_fallback_enabled, excluded_subjects=excluded_subjects, deep_nifti_integrity_check=bool(args.deep_nifti_integrity_check), ) analysis_mask_img, run_mask_map, run_mask_qc_df, mask_qc = build_symmetric_analysis_mask( manifest_df=manifest_df, run_mask_method=str(args.run_mask_method), epsilon=float(args.run_mask_epsilon), ) core_roi_masks, core_roi_pre_mask_voxels = load_core_roi_masks( roi_mask_dir=roi_mask_dir, reference_img=analysis_mask_img, ) core_roi_coverage_df, core_roi_failures_df, core_roi_qc = evaluate_core_roi_preservation( manifest_df=manifest_df, run_mask_map=run_mask_map, core_roi_masks=core_roi_masks, reference_affine=analysis_mask_img.affine, min_voxels_required=int(args.min_core_roi_voxels), ) core_roi_qc["core_roi_pre_mask_voxels"] = core_roi_pre_mask_voxels core_roi_gate_passed = len(core_roi_failures_df) == 0 run_events_df, unified_events_df, annotation_source_resolution_df, annotation_qc = build_harmonized_annotation_tables( manifest_df=manifest_df, annotation_dir=annotation_dir, time_scale_seconds=float(args.time_scale_seconds), enable_mixed_fallback=mixed_fallback_enabled, ) accepted_manifest_path = output_dir / "accepted_manifest.csv" rejected_manifest_path = output_dir / "rejected_manifest.csv" protocol_c_path = output_dir / "protocol_c_cross_subject_folds.csv" analysis_mask_path = output_dir / "analysis_mask.nii.gz" run_mask_qc_path = output_dir / "run_mask_qc.csv" analysis_mask_qc_path = output_dir / "analysis_mask_qc.json" core_roi_coverage_path = output_dir / "core_roi_coverage.csv" core_roi_failures_path = output_dir / "core_roi_failures.csv" core_roi_qc_path = output_dir / "core_roi_qc.json" run_events_path = output_dir / "run_event_templates.csv" unified_events_path = output_dir / "unified_word_events.csv" annotation_source_resolution_path = output_dir / "annotation_source_resolution.csv" annotation_qc_path = output_dir / "annotation_qc.json" manifest_qc_path = output_dir / "manifest_qc.json" bootstrap_summary_path = output_dir / "bootstrap_summary.json" feature_output_dir = output_dir / "features" feature_summary_path = output_dir / "feature_extraction_summary.csv" feature_qc_path = output_dir / "feature_extraction_qc.json" alignment_output_dir = output_dir / "alignment_inputs" alignment_bold_summary_path = output_dir / "alignment_bold_summary.csv" alignment_regressor_summary_path = output_dir / "alignment_regressor_summary.csv" alignment_qc_path = output_dir / "alignment_qc.json" manifest_df.to_csv(accepted_manifest_path, index=False) rejected_df.to_csv(rejected_manifest_path, index=False) protocol_c_df = _build_protocol_c_folds( manifest_df=manifest_df, n_test_subjects=int(args.protocol_c_test_subjects), n_folds=int(args.protocol_c_num_folds), seed=int(args.protocol_c_seed), ) protocol_c_df.to_csv(protocol_c_path, index=False) run_mask_qc_df.to_csv(run_mask_qc_path, index=False) core_roi_coverage_df.to_csv(core_roi_coverage_path, index=False) core_roi_failures_df.to_csv(core_roi_failures_path, index=False) run_events_df.to_csv(run_events_path, index=False) unified_events_df.to_csv(unified_events_path, index=False) annotation_source_resolution_df.to_csv(annotation_source_resolution_path, index=False) nib.save(analysis_mask_img, str(analysis_mask_path)) split_summary = { "n_manifest_rows": int(len(manifest_df)), "n_subjects_manifest": int(manifest_df["subject"].nunique()) if not manifest_df.empty else 0, "n_protocol_a_folds": 0, "n_protocol_b_splits": 0, "n_protocol_b_skipped": 0, "n_protocol_c_folds": int(len(protocol_c_df)), } write_json(manifest_qc_path, manifest_qc) write_json(analysis_mask_qc_path, mask_qc) write_json(core_roi_qc_path, core_roi_qc) write_json(annotation_qc_path, annotation_qc) feature_summary_df = None feature_qc: dict[str, Any] | None = None if bool(args.run_feature_extraction): feature_output_dir.mkdir(parents=True, exist_ok=True) resolved_workers = resolve_feature_num_workers( requested=args.feature_num_workers, device=str(args.feature_device), ) if resolved_workers > 1: print( f"[bootstrap] Feature extraction will use {resolved_workers} GPU workers " f"(requested={args.feature_num_workers}, device={args.feature_device}).", flush=True, ) feature_summary_df, feature_qc = extract_and_cache_run_level_features( run_events_df=run_events_df, model_ids=requested_models, output_dir=feature_output_dir, layer_indices=feature_layer_indices, max_words_per_chunk=int(args.max_words_per_chunk), dry_run=bool(args.feature_dry_run), dry_run_n_layers=int(args.feature_dry_run_n_layers), dry_run_hidden_dim=int(args.feature_dry_run_hidden_dim), device=str(args.feature_device), local_files_only=bool(args.feature_local_files_only), overwrite=bool(args.feature_overwrite), num_workers=resolved_workers, ) feature_summary_df.to_csv(feature_summary_path, index=False) if feature_qc is None: feature_qc = {} write_json(feature_qc_path, feature_qc) if feature_summary_df is None and bool(args.run_alignment): if feature_summary_path.exists(): feature_summary_df = pd.read_csv(feature_summary_path) if feature_qc_path.exists(): with feature_qc_path.open("r", encoding="utf-8") as handle: feature_qc = json.load(handle) else: raise RuntimeError( "Alignment requested but no feature summary is available. " "Run with --run-feature-extraction first or provide existing cached summary." ) alignment_bold_df = None alignment_regressor_df = None alignment_qc: dict[str, Any] | None = None if bool(args.run_alignment): assert feature_summary_df is not None alignment_output_dir.mkdir(parents=True, exist_ok=True) alignment_bold_df, alignment_regressor_df, alignment_qc = build_and_cache_alignment_inputs( manifest_df=manifest_df, run_events_df=run_events_df, feature_summary_df=feature_summary_df, analysis_mask_path=analysis_mask_path, output_dir=alignment_output_dir, trim_start_tr=int(args.alignment_trim_start_tr), trim_end_tr=int(args.alignment_trim_end_tr), hrf_model=str(args.alignment_hrf_model), overwrite=bool(args.alignment_overwrite), alignment_subjects=alignment_subjects, ) alignment_bold_df.to_csv(alignment_bold_summary_path, index=False) alignment_regressor_df.to_csv(alignment_regressor_summary_path, index=False) if alignment_qc is None: alignment_qc = {} write_json(alignment_qc_path, alignment_qc) bootstrap_summary: dict[str, Any] = { "base_dir": str(base_dir), "derivatives_dir": str(derivatives_dir), "data_dir": str(data_dir), "annotation_dir": str(annotation_dir), "roi_mask_dir": str(roi_mask_dir), "output_dir": str(output_dir), "allowed_runs": list(allowed_runs), "excluded_subjects": excluded_subjects, "deep_nifti_integrity_check": bool(args.deep_nifti_integrity_check), "mixed_fallback_enabled": mixed_fallback_enabled, "models_locked": requested_models, "model_selection": model_selection, "time_scale_seconds": float(args.time_scale_seconds), "feature_extraction_config": { "run_feature_extraction": bool(args.run_feature_extraction), "dry_run": bool(args.feature_dry_run), "dry_run_n_layers": int(args.feature_dry_run_n_layers), "dry_run_hidden_dim": int(args.feature_dry_run_hidden_dim), "layer_indices": feature_layer_indices, "max_words_per_chunk": int(args.max_words_per_chunk), "device": str(args.feature_device), "local_files_only": bool(args.feature_local_files_only), "overwrite": bool(args.feature_overwrite), }, "alignment_config": { "run_alignment": bool(args.run_alignment), "alignment_subjects": alignment_subjects, "trim_start_tr": int(args.alignment_trim_start_tr), "trim_end_tr": int(args.alignment_trim_end_tr), "hrf_model": str(args.alignment_hrf_model), "overwrite": bool(args.alignment_overwrite), }, "run_mask_config": { "method": str(args.run_mask_method), "epsilon": float(args.run_mask_epsilon), }, "core_roi_config": { "min_voxels_required": int(args.min_core_roi_voxels), "allow_core_roi_failures": bool(args.allow_core_roi_failures), }, "protocol_c_config": { "n_test_subjects": int(args.protocol_c_test_subjects), "n_train_subjects": 21, "n_folds": int(args.protocol_c_num_folds), "seed": int(args.protocol_c_seed), }, "manifest_qc": manifest_qc, "analysis_mask_qc": mask_qc, "core_roi_qc": core_roi_qc, "annotation_qc": annotation_qc, "feature_qc": feature_qc, "alignment_qc": alignment_qc, "core_roi_gate_passed": core_roi_gate_passed, "split_summary": split_summary, "artifacts": { "accepted_manifest": str(accepted_manifest_path), "rejected_manifest": str(rejected_manifest_path), "manifest_qc": str(manifest_qc_path), "analysis_mask": str(analysis_mask_path), "run_mask_qc": str(run_mask_qc_path), "analysis_mask_qc": str(analysis_mask_qc_path), "core_roi_coverage": str(core_roi_coverage_path), "core_roi_failures": str(core_roi_failures_path), "core_roi_qc": str(core_roi_qc_path), "run_event_templates": str(run_events_path), "unified_word_events": str(unified_events_path), "annotation_source_resolution": str(annotation_source_resolution_path), "annotation_qc": str(annotation_qc_path), "feature_output_dir": str(feature_output_dir), "feature_summary": str(feature_summary_path) if feature_summary_path.exists() else None, "feature_qc": str(feature_qc_path) if feature_qc_path.exists() else None, "alignment_output_dir": str(alignment_output_dir) if alignment_output_dir.exists() else None, "alignment_bold_summary": str(alignment_bold_summary_path) if alignment_bold_summary_path.exists() else None, "alignment_regressor_summary": str(alignment_regressor_summary_path) if alignment_regressor_summary_path.exists() else None, "alignment_qc": str(alignment_qc_path) if alignment_qc_path.exists() else None, "protocol_c_cross_subject": str(protocol_c_path), }, } write_json(bootstrap_summary_path, bootstrap_summary) print("=" * 72) print("A1 bootstrap complete") print(f"Accepted manifest rows: {len(manifest_df)}") print(f"Rejected candidate rows: {len(rejected_df)}") print(f"Protocol C folds: {len(protocol_c_df)}") print(f"Core ROI fail rows: {len(core_roi_failures_df)}") print(f"Core ROI gate passed: {core_roi_gate_passed}") print(f"Run-level event rows: {len(run_events_df)}") print(f"Unified event rows: {len(unified_events_df)}") print(f"Model selection source: {model_selection.get('source')}") print(f"Model profile: {model_selection.get('profile')}") print(f"Locked models: {', '.join(requested_models)}") if feature_summary_df is not None: print(f"Feature cache rows: {len(feature_summary_df)}") if alignment_regressor_df is not None and alignment_bold_df is not None: print(f"Alignment regressor rows: {len(alignment_regressor_df)}") print(f"Alignment BOLD rows: {len(alignment_bold_df)}") print(f"Output directory: {output_dir}") print("=" * 72) if not core_roi_gate_passed and not bool(args.allow_core_roi_failures): raise RuntimeError( "Core ROI preservation gate failed. " "Inspect core_roi_failures.csv or rerun with --allow-core-roi-failures." ) if __name__ == "__main__": main()