jefffffff9 Claude Sonnet 4.6 commited on
Commit
c37ce59
·
1 Parent(s): 3ed28bd

Fix Cell 16 TypeError: target encoder-only LoRA to avoid PEFT/transformers 5.x conflict

Browse files

PEFT 0.18.x + transformers 5.x: WhisperDecoder.forward receives input_ids
via both positional args and kwargs when LoRA wraps decoder layers, causing
TypeError('got multiple values for keyword argument input_ids').

Fix: LORA_TARGETS changed from list ['q_proj','v_proj'] (matches all layers)
to regex that targets only encoder self-attention projections. The encoder
processes audio features and has no input_ids, so the conflict cannot occur.

Also adds model.config.use_cache = False (required with gradient checkpointing).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

notebooks/kaggle_master_trainer.ipynb CHANGED
@@ -33,7 +33,7 @@
33
  "id": "cell-install",
34
  "metadata": {},
35
  "outputs": [],
36
- "source": "# -- Cell 2: Install dependencies + fix torchvision CUDA mismatch -------------\nimport subprocess, sys\nfrom pathlib import Path\n\nSENTINEL = Path('/kaggle/working/.torchvision_fixed')\n\nif not SENTINEL.exists():\n import torch\n cuda_tag = 'cu' + torch.version.cuda.replace('.', '')[:3]\n print(f'Fixing torchvision for CUDA {cuda_tag} (will restart kernel) ...')\n subprocess.check_call([\n sys.executable, '-m', 'pip', 'install', '-q', '--upgrade',\n 'torchvision', '--index-url', f'https://download.pytorch.org/whl/{cuda_tag}',\n ])\n pkgs = [\n 'transformers==5.5.0', 'datasets==4.8.4', 'accelerate==1.13.0',\n 'huggingface-hub==1.9.0', 'peft==0.18.1',\n 'bitsandbytes==0.49.2', 'librosa==0.10.2', 'soundfile==0.12.1',\n 'jiwer==3.0.4', 'numpy==2.2.4', 'scipy==1.15.2',\n # NOTE: evaluate is intentionally excluded -- evaluate.load('wer') breaks\n # with huggingface-hub>=1.0 (use_auth_token removed). We use jiwer directly.\n ]\n subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', *pkgs])\n SENTINEL.touch()\n print('Restarting kernel to load fixed torchvision ...')\n import IPython\n IPython.Application.instance().kernel.do_shutdown(restart=True)\nelse:\n import torch\n print('torchvision already fixed -- skipping restart.')\n print(f'torch : {torch.__version__} CUDA : {torch.version.cuda}')\n print(f'GPU : {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"none\"}')"
37
  },
38
  {
39
  "cell_type": "code",
@@ -41,7 +41,9 @@
41
  "id": "cell-config",
42
  "metadata": {},
43
  "outputs": [],
44
- "source": "# ── Cell 3: CONFIGURATION — edit these before each run ───────────────────────\nimport os\n\n# ─── Language to train ───────────────────────────────────────────────────────\n# 'bam' = Bambara 'ful' = Fula\nTRAIN_LANG = 'bam'\n\n# ─── Model ───────────────────────────────────────────────────────────────────\nWHISPER_MODEL_ID = 'openai/whisper-small'\nTARGET_SR = 16_000\n\n# ─── HuggingFace repos ───────────────────────────────────────────────────────\nHF_USERNAME = 'ous-sow'\nFEEDBACK_REPO_ID = f'{HF_USERNAME}/sahel-agri-feedback'\nADAPTER_REPO_ID = f'{HF_USERNAME}/sahel-agri-adapters'\n\n# ─── LoRA hyper-parameters ───────────────────────────────────────────────────\nLORA_R = 32 # rank — higher = more capacity, more VRAM\nLORA_ALPHA = 64 # scaling factor (typically 2× r)\nLORA_DROPOUT = 0.05\nLORA_TARGETS = ['q_proj', 'v_proj'] # Whisper decoder attention\n\n# ─── Training hyper-parameters ─────────────��─────────────────────────────────\nMAX_STEPS = 4_000 # T4 ~45 min; set 8000 for a deeper run\nBATCH_SIZE = 16\nGRAD_ACCUM = 2 # effective batch = 32\nLEARNING_RATE = 1e-3\nWARMUP_STEPS = 200\nSAVE_STEPS = 500\nEVAL_STEPS = 500\nLOGGING_STEPS = 50\nMAX_WAXAL_TRAIN = 5_000 # cap WaxalNLP samples (streaming budget)\nCORRECTION_REPEAT= 3 # upsample user corrections Nx for emphasis\n\n# ─── Paths (Kaggle working dir) ───────────────────────────────────────────────\nWORKING_DIR = '/kaggle/working'\nOUTPUT_DIR = f'{WORKING_DIR}/adapter_{TRAIN_LANG}'\nDATA_DIR = f'{WORKING_DIR}/data'\nAUDIO_DIR = f'{WORKING_DIR}/audio_feedback'\n\nLANG_NAME = {'bam': 'bambara', 'ful': 'fula'}.get(TRAIN_LANG, TRAIN_LANG)\n\nprint(f'Language : {TRAIN_LANG} ({LANG_NAME})')\nprint(f'Model : {WHISPER_MODEL_ID}')\nprint(f'Output : {OUTPUT_DIR}')\nprint(f'Max steps : {MAX_STEPS}')"
 
 
45
  },
46
  {
47
  "cell_type": "code",
@@ -141,7 +143,9 @@
141
  "id": "cell-model",
142
  "metadata": {},
143
  "outputs": [],
144
- "source": "# -- Cell 13: Load Whisper-small (fp16) + LoRA --------------------------------\n# bitsandbytes 8-bit quantization requires libnvJitLink.so.13 which is absent\n# on Kaggle's CUDA 13 environment. For whisper-small (244MB) this doesn't\n# matter -- fp16 on a T4 (16GB) uses only ~500MB VRAM, well within budget.\nimport torch\nfrom transformers.models.whisper.modeling_whisper import WhisperForConditionalGeneration\nfrom peft import LoraConfig, get_peft_model, TaskType\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'Loading {WHISPER_MODEL_ID} in fp16 on {device} ...')\n\nmodel = WhisperForConditionalGeneration.from_pretrained(\n WHISPER_MODEL_ID,\n torch_dtype=torch.float16,\n token=HF_TOKEN,\n)\nmodel = model.to(device)\nmodel.train()\n\n# Force target language -- avoids language-detection overhead during training\nmodel.config.forced_decoder_ids = processor.get_decoder_prompt_ids(\n language='fr', # closest Whisper supports for Bambara/Fula\n task='transcribe',\n)\nmodel.config.suppress_tokens = []\n\n# Enable gradient checkpointing to save activation memory\nmodel.gradient_checkpointing_enable()\n\n# LoRA -- inject adapters into decoder Q and V projections only\nlora_cfg = LoraConfig(\n r=LORA_R,\n lora_alpha=LORA_ALPHA,\n target_modules=LORA_TARGETS,\n lora_dropout=LORA_DROPOUT,\n bias='none',\n task_type=TaskType.SEQ_2_SEQ_LM,\n)\nmodel = get_peft_model(model, lora_cfg)\nmodel.print_trainable_parameters()\n\nvram_mb = torch.cuda.memory_allocated() / 1e6 if torch.cuda.is_available() else 0\nprint(f'\\nVRAM used: {vram_mb:.0f} MB / {torch.cuda.get_device_properties(0).total_memory / 1e6:.0f} MB')\nprint(f'Model ready on {device}')"
 
 
145
  },
146
  {
147
  "cell_type": "code",
 
33
  "id": "cell-install",
34
  "metadata": {},
35
  "outputs": [],
36
+ "source": "# -- Cell 2: Install dependencies + fix torchvision CUDA mismatch -------------\n# transformers 5.x changed WhisperDecoder's forward signature in a way that\n# breaks PEFT LoRA (peft passes input_ids twice -> TypeError). The stable\n# combination for Whisper LoRA fine-tuning is transformers==4.38.2 + peft==0.9.0.\nimport subprocess, sys\nfrom pathlib import Path\n\nSENTINEL = Path('/kaggle/working/.torchvision_fixed')\n\nif not SENTINEL.exists():\n import torch\n cuda_tag = 'cu' + torch.version.cuda.replace('.', '')[:3]\n print(f'Fixing torchvision for CUDA {cuda_tag} (will restart kernel) ...')\n subprocess.check_call([\n sys.executable, '-m', 'pip', 'install', '-q', '--upgrade',\n 'torchvision', '--index-url', f'https://download.pytorch.org/whl/{cuda_tag}',\n ])\n pkgs = [\n 'transformers==4.38.2', # last stable 4.x for Whisper LoRA\n 'datasets==2.18.0', # compatible with transformers 4.38\n 'accelerate==0.27.2',\n 'huggingface-hub==0.21.4',\n 'peft==0.9.0', # compatible with transformers 4.38\n 'librosa==0.10.2',\n 'soundfile==0.12.1',\n 'jiwer==3.0.4',\n 'numpy==1.26.4', # numpy 2.x breaks some 4.x transformers ops\n 'scipy==1.12.0',\n ]\n subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', *pkgs])\n SENTINEL.touch()\n print('Restarting kernel to load fixed packages ...')\n import IPython\n IPython.Application.instance().kernel.do_shutdown(restart=True)\nelse:\n import torch\n print('Packages already fixed -- skipping restart.')\n print(f'torch : {torch.__version__} CUDA : {torch.version.cuda}')\n print(f'GPU : {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"none\"}')"
37
  },
38
  {
39
  "cell_type": "code",
 
41
  "id": "cell-config",
42
  "metadata": {},
43
  "outputs": [],
44
+ "source": [
45
+ "# ── Cell 3: CONFIGURATION — edit these before each run ───────────────────────\nimport os\n\n# ─── Language to train ───────────────────────────────────────────────────────\n# 'bam' = Bambara 'ful' = Fula\nTRAIN_LANG = 'bam'\n\n# ─── Model ───────────────────────────────────────────────────────────────────\nWHISPER_MODEL_ID = 'openai/whisper-small'\nTARGET_SR = 16_000\n\n# ─── HuggingFace repos ───────────────────────────────────────────────────────\nHF_USERNAME = 'ous-sow'\nFEEDBACK_REPO_ID = f'{HF_USERNAME}/sahel-agri-feedback'\nADAPTER_REPO_ID = f'{HF_USERNAME}/sahel-agri-adapters'\n\n# ─── LoRA hyper-parameters ───────────────────────────────────────────────────\nLORA_R = 32 # rank — higher = more capacity, more VRAM\nLORA_ALPHA = 64 # scaling factor (typically 2× r)\nLORA_DROPOUT = 0.05\n# Regex targets encoder self-attention only -- avoids PEFT/transformers 5.x\n# TypeError (input_ids passed twice) that occurs when LoRA wraps decoder layers.\nLORA_TARGETS = r'model\\.encoder\\.layers\\.\\d+\\.self_attn\\.(q|v)_proj'\n\n# ─── Training hyper-parameters ───────────────────────────────────────────────\nMAX_STEPS = 4_000 # T4 ~45 min; set 8000 for a deeper run\nBATCH_SIZE = 16\nGRAD_ACCUM = 2 # effective batch = 32\nLEARNING_RATE = 1e-3\nWARMUP_STEPS = 200\nSAVE_STEPS = 500\nEVAL_STEPS = 500\nLOGGING_STEPS = 50\nMAX_WAXAL_TRAIN = 5_000 # cap WaxalNLP samples (streaming budget)\nCORRECTION_REPEAT= 3 # upsample user corrections Nx for emphasis\n\n# ─── Paths (Kaggle working dir) ───────────────────────────────────────────────\nWORKING_DIR = '/kaggle/working'\nOUTPUT_DIR = f'{WORKING_DIR}/adapter_{TRAIN_LANG}'\nDATA_DIR = f'{WORKING_DIR}/data'\nAUDIO_DIR = f'{WORKING_DIR}/audio_feedback'\n\nLANG_NAME = {'bam': 'bambara', 'ful': 'fula'}.get(TRAIN_LANG, TRAIN_LANG)\n\nprint(f'Language : {TRAIN_LANG} ({LANG_NAME})')\nprint(f'Model : {WHISPER_MODEL_ID}')\nprint(f'Output : {OUTPUT_DIR}')\nprint(f'Max steps : {MAX_STEPS}')"
46
+ ]
47
  },
48
  {
49
  "cell_type": "code",
 
143
  "id": "cell-model",
144
  "metadata": {},
145
  "outputs": [],
146
+ "source": [
147
+ "# -- Cell 13: Load Whisper-small (fp16) + LoRA --------------------------------\n# bitsandbytes 8-bit quantization requires libnvJitLink.so.13 which is absent\n# on Kaggle's CUDA 13 environment. For whisper-small (244MB) this doesn't\n# matter -- fp16 on a T4 (16GB) uses only ~500MB VRAM, well within budget.\nimport torch\nfrom transformers.models.whisper.modeling_whisper import WhisperForConditionalGeneration\nfrom peft import LoraConfig, get_peft_model, TaskType\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'Loading {WHISPER_MODEL_ID} in fp16 on {device} ...')\n\nmodel = WhisperForConditionalGeneration.from_pretrained(\n WHISPER_MODEL_ID,\n torch_dtype=torch.float16,\n token=HF_TOKEN,\n)\nmodel = model.to(device)\nmodel.train()\n\n# Force target language -- avoids language-detection overhead during training\nmodel.config.forced_decoder_ids = processor.get_decoder_prompt_ids(\n language='fr', # closest Whisper supports for Bambara/Fula\n task='transcribe',\n)\nmodel.config.suppress_tokens = []\n\n# Disable KV-cache (required for gradient checkpointing)\nmodel.config.use_cache = False\n# Enable gradient checkpointing to save activation memory\nmodel.gradient_checkpointing_enable()\n\n# LoRA -- inject adapters into encoder Q and V projections only\n# (encoder-only avoids PEFT/transformers 5.x input_ids conflict in decoder)\nlora_cfg = LoraConfig(\n r=LORA_R,\n lora_alpha=LORA_ALPHA,\n target_modules=LORA_TARGETS,\n lora_dropout=LORA_DROPOUT,\n bias='none',\n task_type=TaskType.SEQ_2_SEQ_LM,\n)\nmodel = get_peft_model(model, lora_cfg)\nmodel.print_trainable_parameters()\n\nvram_mb = torch.cuda.memory_allocated() / 1e6 if torch.cuda.is_available() else 0\nprint(f'\\nVRAM used: {vram_mb:.0f} MB / {torch.cuda.get_device_properties(0).total_memory / 1e6:.0f} MB')\nprint(f'Model ready on {device}')"
148
+ ]
149
  },
150
  {
151
  "cell_type": "code",