Cell 12: never raise, let Cell 12b handle no-data case gracefully

notebooks/kaggle_master_trainer.ipynb

@@ -119,7 +119,7 @@
    "id": "cell-merge",
    "metadata": {},
    "outputs": [],
-   "source": "# -- Cell 12: Build & merge all datasets --------------------------------------\nfrom datasets import Dataset, Audio as HFAudio, concatenate_datasets\nfrom functools import partial\n\nall_parts = []\n\n# -- Part A: User corrections from sahel-agri-feedback ----------------------\nif correction_records:\n    print(f'Part A: {len(correction_records)} corrections x {CORRECTION_REPEAT} upsample')\n    rec_list = correction_records * CORRECTION_REPEAT\n    corr_ds = Dataset.from_dict({\n        'audio':         [r['local_audio']  for r in rec_list],\n        'transcription': [r.get('corrected_text') or r.get('transcription', '') for r in rec_list],\n    })\n    corr_ds = corr_ds.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    corr_ds = corr_ds.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=corr_ds.column_names,\n    )\n    all_parts.append(corr_ds)\n    print(f'  -> {len(corr_ds)} samples ready')\nelse:\n    print('Part A: no corrections available -- skipping')\n\n# -- Part B: WaxalNLP --------------------------------------------------------\nif waxal_ds is not None:\n    print(f'Part B: materialising up to {MAX_WAXAL_TRAIN} WaxalNLP samples ...')\n    waxal_rows = list(waxal_ds.take(MAX_WAXAL_TRAIN))\n    print(f'  fetched {len(waxal_rows)} rows')\n    waxal_local = Dataset.from_dict({\n        'audio':         [r['audio']               for r in waxal_rows],\n        'transcription': [r.get(WAXAL_TEXT_COL, '') for r in waxal_rows],\n    })\n    waxal_local = waxal_local.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    waxal_local = waxal_local.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=waxal_local.column_names,\n        num_proc=2,\n    )\n    all_parts.append(waxal_local)\n    print(f'  -> {len(waxal_local)} samples ready')\nelse:\n    print('Part B: WaxalNLP not available -- skipping')\n\n# -- Part C: External datasets -----------------------------------------------\nfor ext_ds, text_col in external_datasets:\n    ext_rows = list(ext_ds)\n    if not ext_rows:\n        print('Part C: external dataset empty -- skipping')\n        continue\n    print(f'Part C: processing {len(ext_rows)} external samples ...')\n    ext_local = Dataset.from_dict({\n        'audio':         [r['audio']            for r in ext_rows],\n        'transcription': [r.get(text_col, '') for r in ext_rows],\n    })\n    ext_local = ext_local.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    ext_local = ext_local.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=ext_local.column_names,\n        num_proc=2,\n    )\n    all_parts.append(ext_local)\n    print(f'  -> {len(ext_local)} samples ready')\n\n# -- Diagnostics before failing ----------------------------------------------\nprint(f'\\nData summary:')\nprint(f'  corrections loaded : {len(correction_records)}')\nprint(f'  waxal_ds available : {waxal_ds is not None}')\nprint(f'  external sources   : {len(external_datasets)}')\nprint(f'  all_parts count    : {len(all_parts)}')\n\nif not all_parts:\n    print('\\nNo data was loaded. Common causes:')\n    print('  1. HF_TOKEN secret not attached (Add-ons -> Secrets -> toggle ON)')\n    print('  2. Common Voice terms not accepted at huggingface.co/datasets/mozilla-foundation/common_voice_17_0')\n    print('  3. TRAIN_LANG has no matching enabled dataset in Cell 4')\n    print('\\nFor a quick test without any external data, run the cell below to')\n    print('generate a small synthetic dataset and continue training.')\n    raise RuntimeError('No data loaded -- see diagnostics above')\n\n# -- Merge + split -----------------------------------------------------------\ncombined = concatenate_datasets(all_parts).shuffle(seed=42)\nsplit    = combined.train_test_split(test_size=min(0.05, 200 / len(combined)))\ntrain_ds = split['train']\neval_ds  = split['test']\n\nprint(f'\\nDataset ready')\nprint(f'  Train : {len(train_ds)}')\nprint(f'  Eval  : {len(eval_ds)}')"
+   "source": "# -- Cell 12: Build & merge all datasets --------------------------------------\nfrom datasets import Dataset, Audio as HFAudio, concatenate_datasets\nfrom functools import partial\n\ntrain_ds = None   # set here so Cell 12b can detect whether we succeeded\neval_ds  = None\nall_parts = []\n\n# -- Part A: User corrections -------------------------------------------------\nif correction_records:\n    print(f'Part A: {len(correction_records)} corrections x {CORRECTION_REPEAT}')\n    rec_list = correction_records * CORRECTION_REPEAT\n    corr_ds = Dataset.from_dict({\n        'audio':         [r['local_audio']  for r in rec_list],\n        'transcription': [r.get('corrected_text') or r.get('transcription', '') for r in rec_list],\n    })\n    corr_ds = corr_ds.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    corr_ds = corr_ds.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=corr_ds.column_names,\n    )\n    all_parts.append(corr_ds)\n    print(f'  -> {len(corr_ds)} samples')\nelse:\n    print('Part A: no corrections -- skipping')\n\n# -- Part B: WaxalNLP ---------------------------------------------------------\nif waxal_ds is not None:\n    print(f'Part B: materialising up to {MAX_WAXAL_TRAIN} WaxalNLP samples ...')\n    waxal_rows = list(waxal_ds.take(MAX_WAXAL_TRAIN))\n    waxal_local = Dataset.from_dict({\n        'audio':         [r['audio']               for r in waxal_rows],\n        'transcription': [r.get(WAXAL_TEXT_COL, '') for r in waxal_rows],\n    })\n    waxal_local = waxal_local.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    waxal_local = waxal_local.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=waxal_local.column_names,\n        num_proc=2,\n    )\n    all_parts.append(waxal_local)\n    print(f'  -> {len(waxal_local)} samples')\nelse:\n    print('Part B: WaxalNLP not available -- skipping')\n\n# -- Part C: External datasets ------------------------------------------------\nfor ext_ds, text_col in external_datasets:\n    ext_rows = list(ext_ds)\n    if not ext_rows:\n        continue\n    ext_local = Dataset.from_dict({\n        'audio':         [r['audio']          for r in ext_rows],\n        'transcription': [r.get(text_col, '') for r in ext_rows],\n    })\n    ext_local = ext_local.cast_column('audio', HFAudio(sampling_rate=TARGET_SR))\n    ext_local = ext_local.map(\n        partial(prepare_dataset, text_col='transcription'),\n        remove_columns=ext_local.column_names,\n        num_proc=2,\n    )\n    all_parts.append(ext_local)\n    print(f'Part C: {len(ext_local)} external samples')\n\n# -- Result -------------------------------------------------------------------\nprint(f'\\nData summary: {len(all_parts)} source(s) loaded')\n\nif not all_parts:\n    print('No real data available for Bambara yet.')\n    print('Cell 12b (below) will build a synthetic dataset so the full')\n    print('training pipeline can be validated. Switch to TRAIN_LANG=\"ful\"')\n    print('in Cell 3 for a real Fula training run using WaxalNLP.')\nelse:\n    combined = concatenate_datasets(all_parts).shuffle(seed=42)\n    split    = combined.train_test_split(test_size=min(0.05, 200 / len(combined)))\n    train_ds = split['train']\n    eval_ds  = split['test']\n    print(f'Train: {len(train_ds)}  Eval: {len(eval_ds)}')"
   },
   {
    "cell_type": "code",