Spaces:
Sleeping
Sleeping
jefffffff9 Claude Sonnet 4.6 commited on
Commit ·
c37ce59
1
Parent(s): 3ed28bd
Fix Cell 16 TypeError: target encoder-only LoRA to avoid PEFT/transformers 5.x conflict
Browse filesPEFT 0.18.x + transformers 5.x: WhisperDecoder.forward receives input_ids
via both positional args and kwargs when LoRA wraps decoder layers, causing
TypeError('got multiple values for keyword argument input_ids').
Fix: LORA_TARGETS changed from list ['q_proj','v_proj'] (matches all layers)
to regex that targets only encoder self-attention projections. The encoder
processes audio features and has no input_ids, so the conflict cannot occur.
Also adds model.config.use_cache = False (required with gradient checkpointing).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
notebooks/kaggle_master_trainer.ipynb
CHANGED
|
@@ -33,7 +33,7 @@
|
|
| 33 |
"id": "cell-install",
|
| 34 |
"metadata": {},
|
| 35 |
"outputs": [],
|
| 36 |
-
"source": "# -- Cell 2: Install dependencies + fix torchvision CUDA mismatch -------------\nimport subprocess, sys\nfrom pathlib import Path\n\nSENTINEL = Path('/kaggle/working/.torchvision_fixed')\n\nif not SENTINEL.exists():\n import torch\n cuda_tag = 'cu' + torch.version.cuda.replace('.', '')[:3]\n print(f'Fixing torchvision for CUDA {cuda_tag} (will restart kernel) ...')\n subprocess.check_call([\n sys.executable, '-m', 'pip', 'install', '-q', '--upgrade',\n 'torchvision', '--index-url', f'https://download.pytorch.org/whl/{cuda_tag}',\n ])\n pkgs = [\n 'transformers==
|
| 37 |
},
|
| 38 |
{
|
| 39 |
"cell_type": "code",
|
|
@@ -41,7 +41,9 @@
|
|
| 41 |
"id": "cell-config",
|
| 42 |
"metadata": {},
|
| 43 |
"outputs": [],
|
| 44 |
-
"source":
|
|
|
|
|
|
|
| 45 |
},
|
| 46 |
{
|
| 47 |
"cell_type": "code",
|
|
@@ -141,7 +143,9 @@
|
|
| 141 |
"id": "cell-model",
|
| 142 |
"metadata": {},
|
| 143 |
"outputs": [],
|
| 144 |
-
"source":
|
|
|
|
|
|
|
| 145 |
},
|
| 146 |
{
|
| 147 |
"cell_type": "code",
|
|
|
|
| 33 |
"id": "cell-install",
|
| 34 |
"metadata": {},
|
| 35 |
"outputs": [],
|
| 36 |
+
"source": "# -- Cell 2: Install dependencies + fix torchvision CUDA mismatch -------------\n# transformers 5.x changed WhisperDecoder's forward signature in a way that\n# breaks PEFT LoRA (peft passes input_ids twice -> TypeError). The stable\n# combination for Whisper LoRA fine-tuning is transformers==4.38.2 + peft==0.9.0.\nimport subprocess, sys\nfrom pathlib import Path\n\nSENTINEL = Path('/kaggle/working/.torchvision_fixed')\n\nif not SENTINEL.exists():\n import torch\n cuda_tag = 'cu' + torch.version.cuda.replace('.', '')[:3]\n print(f'Fixing torchvision for CUDA {cuda_tag} (will restart kernel) ...')\n subprocess.check_call([\n sys.executable, '-m', 'pip', 'install', '-q', '--upgrade',\n 'torchvision', '--index-url', f'https://download.pytorch.org/whl/{cuda_tag}',\n ])\n pkgs = [\n 'transformers==4.38.2', # last stable 4.x for Whisper LoRA\n 'datasets==2.18.0', # compatible with transformers 4.38\n 'accelerate==0.27.2',\n 'huggingface-hub==0.21.4',\n 'peft==0.9.0', # compatible with transformers 4.38\n 'librosa==0.10.2',\n 'soundfile==0.12.1',\n 'jiwer==3.0.4',\n 'numpy==1.26.4', # numpy 2.x breaks some 4.x transformers ops\n 'scipy==1.12.0',\n ]\n subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', *pkgs])\n SENTINEL.touch()\n print('Restarting kernel to load fixed packages ...')\n import IPython\n IPython.Application.instance().kernel.do_shutdown(restart=True)\nelse:\n import torch\n print('Packages already fixed -- skipping restart.')\n print(f'torch : {torch.__version__} CUDA : {torch.version.cuda}')\n print(f'GPU : {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"none\"}')"
|
| 37 |
},
|
| 38 |
{
|
| 39 |
"cell_type": "code",
|
|
|
|
| 41 |
"id": "cell-config",
|
| 42 |
"metadata": {},
|
| 43 |
"outputs": [],
|
| 44 |
+
"source": [
|
| 45 |
+
"# ── Cell 3: CONFIGURATION — edit these before each run ───────────────────────\nimport os\n\n# ─── Language to train ───────────────────────────────────────────────────────\n# 'bam' = Bambara 'ful' = Fula\nTRAIN_LANG = 'bam'\n\n# ─── Model ───────────────────────────────────────────────────────────────────\nWHISPER_MODEL_ID = 'openai/whisper-small'\nTARGET_SR = 16_000\n\n# ─── HuggingFace repos ───────────────────────────────────────────────────────\nHF_USERNAME = 'ous-sow'\nFEEDBACK_REPO_ID = f'{HF_USERNAME}/sahel-agri-feedback'\nADAPTER_REPO_ID = f'{HF_USERNAME}/sahel-agri-adapters'\n\n# ─── LoRA hyper-parameters ───────────────────────────────────────────────────\nLORA_R = 32 # rank — higher = more capacity, more VRAM\nLORA_ALPHA = 64 # scaling factor (typically 2× r)\nLORA_DROPOUT = 0.05\n# Regex targets encoder self-attention only -- avoids PEFT/transformers 5.x\n# TypeError (input_ids passed twice) that occurs when LoRA wraps decoder layers.\nLORA_TARGETS = r'model\\.encoder\\.layers\\.\\d+\\.self_attn\\.(q|v)_proj'\n\n# ─── Training hyper-parameters ───────────────────────────────────────────────\nMAX_STEPS = 4_000 # T4 ~45 min; set 8000 for a deeper run\nBATCH_SIZE = 16\nGRAD_ACCUM = 2 # effective batch = 32\nLEARNING_RATE = 1e-3\nWARMUP_STEPS = 200\nSAVE_STEPS = 500\nEVAL_STEPS = 500\nLOGGING_STEPS = 50\nMAX_WAXAL_TRAIN = 5_000 # cap WaxalNLP samples (streaming budget)\nCORRECTION_REPEAT= 3 # upsample user corrections Nx for emphasis\n\n# ─── Paths (Kaggle working dir) ───────────────────────────────────────────────\nWORKING_DIR = '/kaggle/working'\nOUTPUT_DIR = f'{WORKING_DIR}/adapter_{TRAIN_LANG}'\nDATA_DIR = f'{WORKING_DIR}/data'\nAUDIO_DIR = f'{WORKING_DIR}/audio_feedback'\n\nLANG_NAME = {'bam': 'bambara', 'ful': 'fula'}.get(TRAIN_LANG, TRAIN_LANG)\n\nprint(f'Language : {TRAIN_LANG} ({LANG_NAME})')\nprint(f'Model : {WHISPER_MODEL_ID}')\nprint(f'Output : {OUTPUT_DIR}')\nprint(f'Max steps : {MAX_STEPS}')"
|
| 46 |
+
]
|
| 47 |
},
|
| 48 |
{
|
| 49 |
"cell_type": "code",
|
|
|
|
| 143 |
"id": "cell-model",
|
| 144 |
"metadata": {},
|
| 145 |
"outputs": [],
|
| 146 |
+
"source": [
|
| 147 |
+
"# -- Cell 13: Load Whisper-small (fp16) + LoRA --------------------------------\n# bitsandbytes 8-bit quantization requires libnvJitLink.so.13 which is absent\n# on Kaggle's CUDA 13 environment. For whisper-small (244MB) this doesn't\n# matter -- fp16 on a T4 (16GB) uses only ~500MB VRAM, well within budget.\nimport torch\nfrom transformers.models.whisper.modeling_whisper import WhisperForConditionalGeneration\nfrom peft import LoraConfig, get_peft_model, TaskType\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'Loading {WHISPER_MODEL_ID} in fp16 on {device} ...')\n\nmodel = WhisperForConditionalGeneration.from_pretrained(\n WHISPER_MODEL_ID,\n torch_dtype=torch.float16,\n token=HF_TOKEN,\n)\nmodel = model.to(device)\nmodel.train()\n\n# Force target language -- avoids language-detection overhead during training\nmodel.config.forced_decoder_ids = processor.get_decoder_prompt_ids(\n language='fr', # closest Whisper supports for Bambara/Fula\n task='transcribe',\n)\nmodel.config.suppress_tokens = []\n\n# Disable KV-cache (required for gradient checkpointing)\nmodel.config.use_cache = False\n# Enable gradient checkpointing to save activation memory\nmodel.gradient_checkpointing_enable()\n\n# LoRA -- inject adapters into encoder Q and V projections only\n# (encoder-only avoids PEFT/transformers 5.x input_ids conflict in decoder)\nlora_cfg = LoraConfig(\n r=LORA_R,\n lora_alpha=LORA_ALPHA,\n target_modules=LORA_TARGETS,\n lora_dropout=LORA_DROPOUT,\n bias='none',\n task_type=TaskType.SEQ_2_SEQ_LM,\n)\nmodel = get_peft_model(model, lora_cfg)\nmodel.print_trainable_parameters()\n\nvram_mb = torch.cuda.memory_allocated() / 1e6 if torch.cuda.is_available() else 0\nprint(f'\\nVRAM used: {vram_mb:.0f} MB / {torch.cuda.get_device_properties(0).total_memory / 1e6:.0f} MB')\nprint(f'Model ready on {device}')"
|
| 148 |
+
]
|
| 149 |
},
|
| 150 |
{
|
| 151 |
"cell_type": "code",
|