Auto-cap MAX_STEPS based on dataset size after Cell 12 builds train_ds

4000 steps on 11 samples = 2-hour wasted run. New rule: cap at
max(200, steps_per_epoch * 20) so small/synthetic datasets complete in
~5 min while full datasets (5000+ samples) still run the full MAX_STEPS.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

notebooks/kaggle_master_trainer.ipynb

@@ -142,7 +142,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# -- Cell 12: Build & merge all datasets --------------------------------------\nfrom datasets import Dataset, Audio as HFAudio, concatenate_datasets\nfrom functools import partial\n\ntrain_ds = None   # set here so Cell 12b can detect whether we succeeded\neval_ds  = None\nall_parts = []\n\n# -- Part A: User corrections -------------------------------------------------\nif correction_records:\n    print(f'Part A: {len(correction_records)} corrections x {CORRECTION_REPEAT}')\n    rec_list = correction_records * CORRECTION_REPEAT\n    corr_ds = Dataset.from_dict({\n        'audio':         [r['local_audio']  for r in rec_list],\n        'transcription': [r.get('corrected_text') or r.get('transcription', '') for r in rec_list],\n    })\n    corr_ds = corr_ds.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    corr_ds = corr_ds.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=corr_ds.column_names,\n    )\n    all_parts.append(corr_ds)\n    print(f'  -> {len(corr_ds)} samples')\nelse:\n    print('Part A: no corrections -- skipping')\n\n# -- Part B: WaxalNLP ---------------------------------------------------------\nif waxal_ds is not None:\n    print(f'Part B: materialising up to {MAX_WAXAL_TRAIN} WaxalNLP samples ...')\n    waxal_rows = list(waxal_ds.take(MAX_WAXAL_TRAIN))\n    waxal_local = Dataset.from_dict({\n        'audio':         [r['audio']               for r in waxal_rows],\n        'transcription': [r.get(WAXAL_TEXT_COL, '') for r in waxal_rows],\n    })\n    waxal_local = waxal_local.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    waxal_local = waxal_local.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=waxal_local.column_names,\n    )\n    all_parts.append(waxal_local)\n    print(f'  -> {len(waxal_local)} samples')\nelse:\n    print('Part B: WaxalNLP not available -- skipping')\n\n# -- Part C: External datasets ------------------------------------------------\nfor ext_ds, text_col in external_datasets:\n    ext_rows = list(ext_ds)\n    if not ext_rows:\n        continue\n    ext_local = Dataset.from_dict({\n        'audio':         [r['audio']          for r in ext_rows],\n        'transcription': [r.get(text_col, '') for r in ext_rows],\n    })\n    ext_local = ext_local.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    ext_local = ext_local.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=ext_local.column_names,\n    )\n    all_parts.append(ext_local)\n    print(f'Part C: {len(ext_local)} external samples')\n\n# -- Result -------------------------------------------------------------------\nprint(f'\\nData summary: {len(all_parts)} source(s) loaded')\n\nif not all_parts:\n    print('No real data available for Bambara yet.')\n    print('Cell 12b (below) will build a synthetic dataset so the full')\n    print('training pipeline can be validated. Switch to TRAIN_LANG=\"ful\"')\n    print('in Cell 3 for a real Fula training run using WaxalNLP.')\nelse:\n    combined = concatenate_datasets(all_parts).shuffle(seed=42)\n    n_eval   = max(1, min(int(0.05 * len(combined)), 200))\n    split    = combined.train_test_split(test_size=n_eval)\n    train_ds = split['train']\n    eval_ds  = split['test']\n    print(f'Train: {len(train_ds)}  Eval: {len(eval_ds)}')"
+    "# -- Cell 12: Build & merge all datasets --------------------------------------\nfrom datasets import Dataset, Audio as HFAudio, concatenate_datasets\nfrom functools import partial\n\ntrain_ds = None   # set here so Cell 12b can detect whether we succeeded\neval_ds  = None\nall_parts = []\n\n# -- Part A: User corrections -------------------------------------------------\nif correction_records:\n    print(f'Part A: {len(correction_records)} corrections x {CORRECTION_REPEAT}')\n    rec_list = correction_records * CORRECTION_REPEAT\n    corr_ds = Dataset.from_dict({\n        'audio':         [r['local_audio']  for r in rec_list],\n        'transcription': [r.get('corrected_text') or r.get('transcription', '') for r in rec_list],\n    })\n    corr_ds = corr_ds.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    corr_ds = corr_ds.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=corr_ds.column_names,\n    )\n    all_parts.append(corr_ds)\n    print(f'  -> {len(corr_ds)} samples')\nelse:\n    print('Part A: no corrections -- skipping')\n\n# -- Part B: WaxalNLP ---------------------------------------------------------\nif waxal_ds is not None:\n    print(f'Part B: materialising up to {MAX_WAXAL_TRAIN} WaxalNLP samples ...')\n    waxal_rows = list(waxal_ds.take(MAX_WAXAL_TRAIN))\n    waxal_local = Dataset.from_dict({\n        'audio':         [r['audio']               for r in waxal_rows],\n        'transcription': [r.get(WAXAL_TEXT_COL, '') for r in waxal_rows],\n    })\n    waxal_local = waxal_local.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    waxal_local = waxal_local.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=waxal_local.column_names,\n    )\n    all_parts.append(waxal_local)\n    print(f'  -> {len(waxal_local)} samples')\nelse:\n    print('Part B: WaxalNLP not available -- skipping')\n\n# -- Part C: External datasets ------------------------------------------------\nfor ext_ds, text_col in external_datasets:\n    ext_rows = list(ext_ds)\n    if not ext_rows:\n        continue\n    ext_local = Dataset.from_dict({\n        'audio':         [r['audio']          for r in ext_rows],\n        'transcription': [r.get(text_col, '') for r in ext_rows],\n    })\n    ext_local = ext_local.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    ext_local = ext_local.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=ext_local.column_names,\n    )\n    all_parts.append(ext_local)\n    print(f'Part C: {len(ext_local)} external samples')\n\n# -- Result -------------------------------------------------------------------\nprint(f'\\nData summary: {len(all_parts)} source(s) loaded')\n\nif not all_parts:\n    print('No real data available for Bambara yet.')\n    print('Cell 12b (below) will build a synthetic dataset so the full')\n    print('training pipeline can be validated. Switch to TRAIN_LANG=\"ful\"')\n    print('in Cell 3 for a real Fula training run using WaxalNLP.')\nelse:\n    combined = concatenate_datasets(all_parts).shuffle(seed=42)\n    n_eval   = max(1, min(int(0.05 * len(combined)), 200))\n    split    = combined.train_test_split(test_size=n_eval)\n    train_ds = split['train']\n    eval_ds  = split['test']\n    print(f'Train: {len(train_ds)}  Eval: {len(eval_ds)}')\n# Auto-cap MAX_STEPS: no point running 4000 steps on a tiny dataset.\n# Rule: at least 20 passes through the data, capped at user's MAX_STEPS.\nif train_ds is not None:\n    steps_per_epoch = max(1, len(train_ds) // (BATCH_SIZE * GRAD_ACCUM))\n    _auto_steps = max(200, steps_per_epoch * 20)\n    if _auto_steps < MAX_STEPS:\n        print(f'Auto-capping MAX_STEPS {MAX_STEPS} -> {_auto_steps} (small dataset)')\n        MAX_STEPS = _auto_steps\n    else:\n        print(f'MAX_STEPS={MAX_STEPS} OK for {len(train_ds)} training samples')"
    ]
   },
   {