Align Space app with Kaggle notebook training pipeline

1. Replace PEFT AdapterManager with full-checkpoint loader
- _fine_tuned_models dict replaces _adapter_manager/PEFT
- _reload_adapters_from_hub loads WhisperForConditionalGeneration
from config.json checkpoints (what the notebook actually saves)
- _run_pipeline uses _fine_tuned_models.get(lang, base_model)

2. Tab 3 audio uploads now write to corrections.jsonl + audio/
- Previously saved to training_audio/ which the notebook never read
- Now saved under audio/{lang}_{ts}.wav with corrected_text field
exactly matching what Cell 7 of the notebook expects

3. Phrase pairs now also append to vocabulary.jsonl
- _append_phrases_to_vocabulary_jsonl writes {word, translation, language}
entries so Cell 7 vocab_entries picks them up as synthetic fallback labels

4. Tab 4 updated: correct notebook name, clear data flow docs,
button renamed to "Reload Fine-tuned Models from Hub"

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

app.py

@@ -54,12 +54,11 @@ except ImportError:
         return fn
 
 # ── Module-level model state (CPU-resident between requests) ─────────────────
-_whisper_model     = None   # WhisperForConditionalGeneration (base)
-_whisper_processor = None
-_adapter_manager   = None   # AdapterManager (wraps base model with PEFT if adapters loaded)
-_model_lock        = threading.Lock()
-_model_status      = "not loaded"
-_adapters_loaded   = set()  # set of language codes with loaded adapters, e.g. {"bam", "ful"}
+_whisper_model      = None   # WhisperForConditionalGeneration (base)
+_whisper_processor  = None
+_fine_tuned_models  = {}     # lang_code -> WhisperForConditionalGeneration (full checkpoint)
+_model_lock         = threading.Lock()
+_model_status       = "not loaded"
 
 from src.tts.mms_tts                  import MMSTTSEngine
 from src.iot.intent_parser             import IntentParser
@@ -82,9 +81,8 @@ if HF_TOKEN:
 # ── Model loading ─────────────────────────────────────────────────────────────
 
 def _do_load_whisper():
-    global _whisper_model, _whisper_processor, _adapter_manager, _model_status
+    global _whisper_model, _whisper_processor, _model_status
     import torch
-    from src.engine.adapter_manager import AdapterManager
 
     # Import concrete Whisper classes directly — bypasses transformers __init__.py
     # Auto-class exports differ between transformers 4.x and 5.x; direct paths are stable.
@@ -111,48 +109,11 @@ def _do_load_whisper():
                 token=HF_TOKEN,
             )
         _whisper_model.eval()
-
-        # Create the AdapterManager wrapping the base model
-        _adapter_manager = AdapterManager(base_model=_whisper_model, config={})
-
-        # Try to load adapters from the local adapter repo snapshot (if already downloaded)
-        _try_load_local_adapters()
-
         _model_status = f"ready ({WHISPER_MODEL_ID})"
     except Exception as e:
         _model_status = f"error: {e}"
 
 
-def _try_load_local_adapters() -> None:
-    """Load any adapter snapshots that are already on disk (downloaded previously)."""
-    global _adapters_loaded
-    if _adapter_manager is None:
-        return
-    if not ADAPTER_REPO_ID:
-        return
-    try:
-        from huggingface_hub import try_to_load_from_cache
-        lang_dirs = {"bam": "adapters/bambara", "ful": "adapters/fula"}
-        for lang, subdir in lang_dirs.items():
-            cached = try_to_load_from_cache(
-                repo_id=ADAPTER_REPO_ID,
-                filename=f"{subdir}/adapter_config.json",
-                repo_type="model",
-                token=HF_TOKEN,
-            )
-            if cached:
-                import os
-                adapter_path = str(os.path.dirname(cached))
-                _adapter_manager.register(lang, adapter_path)
-                try:
-                    _adapter_manager.load_adapter(lang)
-                    _adapters_loaded.add(lang)
-                except Exception:
-                    pass
-    except Exception:
-        pass  # Adapters not cached yet — will load after first Hub download
-
-
 def _ensure_whisper_loaded():
     """Load Whisper to CPU in a background thread on first call. Non-blocking."""
     global _model_status
@@ -198,13 +159,9 @@ def _run_pipeline(audio_path: str, language_code: str):
 
     audio_np, _ = librosa.load(audio_path, sr=16000, mono=True)
 
-    # Use adapter-wrapped model if an adapter for this language is loaded;
+    # Use fine-tuned checkpoint for this language if one has been loaded;
     # otherwise fall back to base Whisper.
-    if _adapter_manager is not None and language_code in _adapters_loaded:
-        _adapter_manager.activate(language_code)
-        active_model = _adapter_manager.get_model()
-    else:
-        active_model = _whisper_model
+    active_model = _fine_tuned_models.get(language_code, _whisper_model)
 
     active_model.to(device)
     with _model_lock:
@@ -376,49 +333,56 @@ def _save_feedback_to_hub(
 # ── Adapter reload ────────────────────────────────────────────────────────────
 
 def _reload_adapters_from_hub() -> str:
-    global _adapters_loaded
+    """Download full fine-tuned checkpoints from Hub and hot-swap them into memory."""
+    global _fine_tuned_models
     if _hf_api is None:
-        return "⚠️ HF_TOKEN not set — cannot download adapters."
-    if _adapter_manager is None:
+        return "⚠️ HF_TOKEN not set — cannot download checkpoints."
+    if _whisper_model is None:
         return "⏳ Base model not loaded yet — wait for model to finish loading and try again."
     try:
+        import torch
         from huggingface_hub import snapshot_download
+        try:
+            from transformers.models.whisper.modeling_whisper import WhisperForConditionalGeneration
+        except ImportError:
+            from transformers import WhisperForConditionalGeneration
+
         local_dir = snapshot_download(
             repo_id=ADAPTER_REPO_ID, repo_type="model", token=HF_TOKEN
         )
         results = []
         for lang, subdir in (("bam", "adapters/bambara"), ("ful", "adapters/fula")):
-            adapter_path = Path(local_dir) / subdir
-            if not adapter_path.exists():
-                results.append(f"⚠️ {lang}: `{subdir}` not found in repo")
+            ckpt_path = Path(local_dir) / subdir
+            if not ckpt_path.exists():
+                results.append(f"⚠️ {lang}: `{subdir}` not found — run training notebook first")
                 continue
-            # Check that this looks like a valid PEFT adapter
-            if not (adapter_path / "adapter_config.json").exists():
-                results.append(f"⚠️ {lang}: `{subdir}` missing adapter_config.json — run training first")
+            if not (ckpt_path / "config.json").exists():
+                results.append(f"⚠️ {lang}: `{subdir}/config.json` missing — incomplete checkpoint")
                 continue
             try:
-                _adapter_manager.register(lang, str(adapter_path))
-                _adapter_manager.load_adapter(lang)
-                _adapters_loaded.add(lang)
-                results.append(f"✅ {lang}: adapter loaded from `{subdir}`")
+                m = WhisperForConditionalGeneration.from_pretrained(
+                    str(ckpt_path), torch_dtype=torch.float32
+                )
+                m.eval()
+                _fine_tuned_models[lang] = m
+                results.append(f"✅ {lang}: fine-tuned checkpoint loaded from `{subdir}`")
             except Exception as e:
                 results.append(f"❌ {lang}: load failed — {e}")
 
         summary = "\n".join(results)
-        active = ", ".join(_adapters_loaded) if _adapters_loaded else "none"
-        return f"{summary}\n\n**Active adapters:** {active}\n**Repo:** `{ADAPTER_REPO_ID}`"
+        active = ", ".join(_fine_tuned_models) if _fine_tuned_models else "none"
+        return f"{summary}\n\n**Active fine-tuned models:** {active}\n**Repo:** `{ADAPTER_REPO_ID}`"
     except Exception as e:
-        return f"❌ Adapter reload failed: {e}"
+        return f"❌ Checkpoint reload failed: {e}"
 
 
 def _get_adapter_status() -> str:
     lines = []
 
-    # Show which adapters are currently active in memory
-    if _adapters_loaded:
-        lines.append(f"**Active adapters (in memory):** {', '.join(sorted(_adapters_loaded))}")
+    if _fine_tuned_models:
+        lines.append(f"**Fine-tuned models loaded:** {', '.join(sorted(_fine_tuned_models))}")
     else:
-        lines.append("**Active adapters:** none — using base Whisper")
+        lines.append("**Fine-tuned models:** none — using base Whisper for all languages")
 
     if _hf_api is None:
         lines.append("_HF_TOKEN not set — Hub check skipped._")
@@ -427,15 +391,15 @@ def _get_adapter_status() -> str:
     try:
         from huggingface_hub import list_repo_files
         files = list(list_repo_files(ADAPTER_REPO_ID, repo_type="model", token=HF_TOKEN))
-        bam_ok = any("bambara" in f and "adapter_config" in f for f in files)
-        ful_ok = any("fula" in f and "adapter_config" in f for f in files)
+        bam_ok = any("bambara" in f and "config.json" in f for f in files)
+        ful_ok = any("fula" in f and "config.json" in f for f in files)
         lines += [
             f"\n**Hub repo:** `{ADAPTER_REPO_ID}`",
-            f"- Bambara (bam): {'✅ trained adapter present' if bam_ok else '⚠️ not yet trained — run bootstrap notebook'}",
-            f"- Fula (ful): {'✅ trained adapter present' if ful_ok else '⚠️ not yet trained — run bootstrap notebook'}",
+            f"- Bambara (bam): {'✅ trained checkpoint present' if bam_ok else '⚠️ not yet trained — run Kaggle notebook'}",
+            f"- Fula (ful): {'✅ trained checkpoint present' if ful_ok else '⚠️ not yet trained — run Kaggle notebook'}",
         ]
         if bam_ok or ful_ok:
-            lines.append("\n_Click **Reload Adapters** to activate them._")
+            lines.append("\n_Click **Reload Models** to activate them._")
     except Exception as e:
         lines.append(f"_Could not read Hub repo: {e}_")
 
@@ -445,7 +409,7 @@ def _get_adapter_status() -> str:
 # ── Knowledge Base handlers ───────────────────────────────────────────────────
 
 def _import_phrase_pairs(lang_label: str, pairs_text: str) -> str:
-    """Import pasted phrase pairs into the phrase library."""
+    """Import pasted phrase pairs into the phrase library and append to vocabulary.jsonl."""
     if not pairs_text.strip():
         return "⚠️ Nothing entered. Use the format:  native phrase | english translation"
     lang = SUPPORTED_LANGUAGES.get(lang_label, "bam")
@@ -453,10 +417,56 @@ def _import_phrase_pairs(lang_label: str, pairs_text: str) -> str:
     if count == 0:
         return "⚠️ No valid phrases found. Each line must contain a | separator.\nExample:  I ni ce | Hello, good day"
     _upload_phrase_additions_to_hub(lang)
+    # Also append to vocabulary.jsonl so the Kaggle training notebook picks them up
+    _append_phrases_to_vocabulary_jsonl(lang, pairs_text)
     total = _phrase_matcher.phrase_count(lang)
     return f"✅ Added {count} phrase(s) for {lang_label}. Library now has {total} phrases. Available immediately."
 
 
+def _append_phrases_to_vocabulary_jsonl(lang: str, pairs_text: str) -> None:
+    """Append phrase pairs to vocabulary.jsonl in the feedback repo (training input)."""
+    if _hf_api is None or not FEEDBACK_REPO_ID:
+        return
+    entries = []
+    for line in pairs_text.splitlines():
+        if "|" not in line:
+            continue
+        parts = line.split("|", 1)
+        word = parts[0].strip()
+        translation = parts[1].strip() if len(parts) > 1 else ""
+        if word:
+            entries.append({"word": word, "translation": translation, "language": lang})
+    if not entries:
+        return
+    try:
+        from huggingface_hub import hf_hub_download
+        for attempt in range(2):
+            try:
+                local = hf_hub_download(
+                    repo_id=FEEDBACK_REPO_ID, filename="vocabulary.jsonl",
+                    repo_type="dataset", token=HF_TOKEN,
+                )
+                with open(local, encoding="utf-8") as f:
+                    existing = f.read()
+            except Exception:
+                existing = ""
+            new_lines = "".join(json.dumps(e, ensure_ascii=False) + "\n" for e in entries)
+            updated = existing + new_lines
+            try:
+                _hf_api.upload_file(
+                    path_or_fileobj=io.BytesIO(updated.encode("utf-8")),
+                    path_in_repo="vocabulary.jsonl",
+                    repo_id=FEEDBACK_REPO_ID,
+                    repo_type="dataset",
+                )
+                break
+            except Exception:
+                if attempt == 1:
+                    pass  # Silent — phrase library still updated locally
+    except Exception:
+        pass  # Non-critical — phrase library already saved via _upload_phrase_additions_to_hub
+
+
 def _upload_phrase_additions_to_hub(lang: str) -> None:
     """Persist user phrase additions to HF Hub so they survive Space restarts."""
     if _hf_api is None or not FEEDBACK_REPO_ID:
@@ -500,7 +510,7 @@ threading.Thread(target=_load_phrase_additions_from_hub, daemon=True).start()
 
 
 def _save_audio_for_training(lang_label: str, audio_path: str | None, transcript: str, source_note: str) -> str:
-    """Save an uploaded audio file + transcription as a training sample to HF Hub."""
+    """Save uploaded audio + transcription to corrections.jsonl so the Kaggle notebook picks it up."""
     transcript = transcript.strip()
     if audio_path is None:
         return "⚠️ Please upload an audio file first."
@@ -508,38 +518,65 @@ def _save_audio_for_training(lang_label: str, audio_path: str | None, transcript
         return "⚠️ Please type the transcription — what is said in this audio."
 
     lang = SUPPORTED_LANGUAGES.get(lang_label, "bam")
-    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
-    audio_repo_path = f"training_audio/{lang}/{timestamp}.wav"
-    meta_repo_path  = f"training_audio/{lang}/{timestamp}.txt"
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f")
+    # Store under audio/ — same path structure that corrections.jsonl expects
+    audio_repo_path = f"audio/{lang}_{timestamp}.wav"
+
+    record = {
+        "id": timestamp,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "language": lang,
+        "audio_file": audio_repo_path,
+        "transcription": transcript,          # notebook reads this field
+        "corrected_text": transcript,          # also populate corrected_text for compatibility
+        "source": source_note.strip() or "uploaded",
+        "is_correction": False,
+        "model": WHISPER_MODEL_ID,
+    }
 
     if _hf_api is None or not FEEDBACK_REPO_ID:
-        return "⚠️ HF_TOKEN not set — file saved locally only, not uploaded to Hub."
+        return "⚠️ HF_TOKEN not set — cannot upload to Hub."
 
     try:
-        import io
+        # Upload audio to audio/ (same bucket corrections use)
         _hf_api.upload_file(
             path_or_fileobj=audio_path,
             path_in_repo=audio_repo_path,
             repo_id=FEEDBACK_REPO_ID,
             repo_type="dataset",
         )
-        meta = (
-            f"language: {lang}\n"
-            f"transcription: {transcript}\n"
-            f"source: {source_note.strip() or 'uploaded'}\n"
-            f"timestamp: {timestamp}\n"
-        )
-        _hf_api.upload_file(
-            path_or_fileobj=io.BytesIO(meta.encode()),
-            path_in_repo=meta_repo_path,
-            repo_id=FEEDBACK_REPO_ID,
-            repo_type="dataset",
-        )
+
+        # Append to corrections.jsonl (same file the notebook reads)
+        from huggingface_hub import hf_hub_download
+        for attempt in range(2):
+            try:
+                local_jsonl = hf_hub_download(
+                    repo_id=FEEDBACK_REPO_ID, filename="corrections.jsonl",
+                    repo_type="dataset", token=HF_TOKEN,
+                )
+                with open(local_jsonl, encoding="utf-8") as f:
+                    existing = f.read()
+            except Exception:
+                existing = ""
+            updated = existing + json.dumps(record, ensure_ascii=False) + "\n"
+            try:
+                _hf_api.upload_file(
+                    path_or_fileobj=io.BytesIO(updated.encode("utf-8")),
+                    path_in_repo="corrections.jsonl",
+                    repo_id=FEEDBACK_REPO_ID,
+                    repo_type="dataset",
+                )
+                break
+            except Exception as e:
+                if attempt == 1:
+                    return f"⚠️ Audio uploaded but corrections.jsonl update failed: {e}"
+
+        total = updated.count("\n")
         return (
-            f"✅ Saved to training dataset!\n"
+            f"✅ Saved to training dataset (#{total} total corrections)!\n"
             f"Audio: {audio_repo_path}\n"
             f"Transcription: {transcript[:80]}{'…' if len(transcript) > 80 else ''}\n"
-            f"Run the training notebook on Kaggle to include this in the next model update."
+            f"Run the Kaggle notebook to include this in the next model update."
         )
     except Exception as exc:
         return f"❌ Upload failed: {exc}"
@@ -815,17 +852,21 @@ def build_ui() -> gr.Blocks:
                     "run the training notebook to fine-tune the speech model."
                 )
                 adapter_status_md = gr.Markdown(value=_get_adapter_status())
-                reload_btn = gr.Button("🔄 Reload Adapters from Hub")
+                reload_btn = gr.Button("🔄 Reload Fine-tuned Models from Hub")
                 reload_out = gr.Markdown()
 
                 gr.Markdown("---")
                 gr.Markdown(
                     "**Training notebook**: "
-                    "`notebooks/train_colab.ipynb` — open in Kaggle or Colab, run all cells.\n\n"
+                    "`notebooks/kaggle_master_trainer.ipynb` — import to Kaggle, run all cells.\n\n"
+                    "**What feeds training:**\n"
+                    "- Tab 2 corrections → `corrections.jsonl` in the feedback dataset\n"
+                    "- Tab 3 audio uploads → `corrections.jsonl` (same file)\n"
+                    "- Tab 3 phrase pairs → `vocabulary.jsonl` (used as synthetic fallback labels)\n\n"
                     "**Feedback dataset**: "
                     f"`{FEEDBACK_REPO_ID}` (auto-updated on each save)\n\n"
-                    "**Adapter repo**: "
-                    f"`{ADAPTER_REPO_ID}` (updated after training)"
+                    "**Model checkpoint repo**: "
+                    f"`{ADAPTER_REPO_ID}` (updated after training, reload above to activate)"
                 )
 
                 reload_btn.click(fn=_reload_adapters_from_hub, outputs=[reload_out])