Add debug output to capture training errors

app.py

@@ -2,10 +2,9 @@ import gradio as gr
 import os
 import subprocess
 import threading
-import time
 
 # Global variable to track training status
-training_status = {"status": "idle", "message": "", "progress": 0}
+training_status = {"status": "idle", "message": "", "full_output": ""}
 
 def check_data():
     """Check if data is available"""
@@ -14,8 +13,6 @@ def check_data():
         files.append("✅ Combined dataset: 9,302 examples")
     if os.path.exists("combined_balanced_training_data.csv"):
         files.append("✅ Balanced dataset: 8,304 examples")
-    if os.path.exists("data/clean_training_data.csv"):
-        files.append("✅ Clean manual data: 478 examples")
     
     if not files:
         return "❌ No training data found. Please upload data files."
@@ -46,7 +43,7 @@ def run_training_subprocess(hf_token, model_size, hub_username, num_epochs, use_
             "--data_path", data_path,
             "--model_size", size,
             "--num_epochs", str(num_epochs),
-            "--use_lora"  # Always use LoRA for efficiency
+            "--use_lora"
         ]
         
         if hf_token:
@@ -56,40 +53,24 @@ def run_training_subprocess(hf_token, model_size, hub_username, num_epochs, use_
         
         training_status["status"] = "running"
         training_status["message"] = "Starting training..."
-        training_status["progress"] = 0
+        training_status["full_output"] = f"Command: {' '.join(cmd)}\n\n"
         
-        # Run training
+        # Run training and capture ALL output
         process = subprocess.Popen(
             cmd,
             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True,
-            bufsize=1
+            stderr=subprocess.PIPE,
+            text=True
         )
         
-        # Read output line by line
-        for line in process.stdout:
-            if "loss" in line.lower():
-                training_status["message"] = line.strip()
-            elif "epoch" in line.lower():
-                # Try to extract progress
-                try:
-                    if "/" in line:
-                        parts = line.split("/")
-                        current = float(parts[0].split()[-1])
-                        total = float(parts[1].split()[0])
-                        training_status["progress"] = int((current / total) * 100)
-                except:
-                    pass
-            elif "exact_match" in line.lower():
-                training_status["message"] = f"Evaluation: {line.strip()}"
+        # Capture both stdout and stderr
+        stdout, stderr = process.communicate()
         
-        process.wait()
+        training_status["full_output"] += f"=== STDOUT ===\n{stdout}\n\n=== STDERR ===\n{stderr}"
         
         if process.returncode == 0:
             training_status["status"] = "completed"
-            training_status["message"] = "Training completed successfully! Model pushed to HuggingFace Hub."
-            training_status["progress"] = 100
+            training_status["message"] = "Training completed successfully!"
         else:
             training_status["status"] = "error"
             training_status["message"] = f"Training failed with exit code {process.returncode}"
@@ -97,6 +78,7 @@ def run_training_subprocess(hf_token, model_size, hub_username, num_epochs, use_
     except Exception as e:
         training_status["status"] = "error"
         training_status["message"] = f"Error: {str(e)}"
+        training_status["full_output"] = str(e)
 
 def train_model(hf_token, model_size, hub_username, num_epochs, use_balanced):
     """Start training in background thread"""
@@ -108,6 +90,9 @@ def train_model(hf_token, model_size, hub_username, num_epochs, use_balanced):
     if training_status["status"] == "running":
         return "⚠️ Training already in progress!"
     
+    # Reset status
+    training_status = {"status": "idle", "message": "", "full_output": ""}
+    
     # Start training in background thread
     thread = threading.Thread(
         target=run_training_subprocess,
@@ -115,44 +100,22 @@ def train_model(hf_token, model_size, hub_username, num_epochs, use_balanced):
     )
     thread.start()
     
-    return "🚀 Training started! Check status below..."
+    return "🚀 Training started! Check the Debug Output tab for detailed logs..."
 
 def get_training_status():
     """Get current training status"""
     global training_status
     
-    if training_status["status"] == "idle":
-        return "💤 No training in progress"
-    elif training_status["status"] == "running":
-        return f"""🏃 Training in progress... ({training_status['progress']}%)
-
-{training_status['message']}"""
-    elif training_status["status"] == "completed":
-        return f"""✅ Training completed!
-
-{training_status['message']}
-
-Your model is available at: https://huggingface.co/{training_status.get('hub_username', 'your-username')}/resume-normalizer-flan-t5"""
-    else:
-        return f"""❌ Training failed!
-
-{training_status['message']}"""
+    status_msg = f"""
+Status: {training_status['status']}
+Message: {training_status['message']}
+"""
+    
+    return status_msg, training_status.get('full_output', '')
 
 # Create Gradio interface
 with gr.Blocks(title="Resume Normalizer Trainer") as app:
-    gr.Markdown("""
-    # Resume Normalizer Trainer
-    
-    Train a Flan-T5 model to normalize company names, job titles, and skills from resumes.
-    
-    **Features:**
-    - Company name normalization (e.g., "Google Inc" → "Alphabet Inc.")
-    - Job title standardization (e.g., "SWE" → "Software Engineer")
-    - Skills normalization (e.g., "JS" → "JavaScript")
-    - Binary equivalency detection
-    
-    **Hardware:** Running on 4xL4 GPUs (96GB VRAM)
-    """)
+    gr.Markdown("# Resume Normalizer Trainer - Debug Mode")
     
     with gr.Tab("📊 Check Data"):
         check_btn = gr.Button("Check Available Datasets", variant="primary")
@@ -165,35 +128,30 @@ with gr.Blocks(title="Resume Normalizer Trainer") as app:
                 hf_token = gr.Textbox(
                     label="HuggingFace Token",
                     type="password",
-                    placeholder="hf_...",
-                    info="Required to push model to Hub"
+                    placeholder="hf_..."
                 )
                 hub_username = gr.Textbox(
                     label="HuggingFace Username",
-                    value="aoisfhdugbos",
-                    info="Your HuggingFace username"
+                    value="aoisfhdugbos"
                 )
             
             with gr.Column():
                 model_size = gr.Dropdown(
                     label="Model Size",
                     choices=["T5-Base (250M)", "T5-Large (770M)"],
-                    value="T5-Base (250M)",
-                    info="Larger models are more accurate but slower"
+                    value="T5-Base (250M)"
                 )
                 num_epochs = gr.Slider(
                     label="Training Epochs",
                     minimum=1,
                     maximum=10,
                     value=5,
-                    step=1,
-                    info="More epochs = better quality but longer training"
+                    step=1
                 )
         
         use_balanced = gr.Checkbox(
             label="Use Balanced Dataset (8,304 examples)",
-            value=False,
-            info="Check to use balanced dataset instead of full dataset (9,302 examples)"
+            value=False
         )
         
         train_btn = gr.Button("🚀 Start Training", variant="primary", size="lg")
@@ -205,39 +163,11 @@ with gr.Blocks(title="Resume Normalizer Trainer") as app:
             outputs=train_output
         )
     
-    with gr.Tab("📈 Training Status"):
-        gr.Markdown("Click the button below to refresh training status")
-        status_btn = gr.Button("🔄 Refresh Status", variant="secondary")
-        status_output = gr.Textbox(label="Current Status", lines=10)
-        
-        status_btn.click(get_training_status, outputs=status_output)
-    
-    with gr.Tab("ℹ️ About"):
-        gr.Markdown("""
-        ## Resume Normalizer Model
-        
-        This trainer fine-tunes a Flan-T5 model for resume entity normalization tasks:
-        
-        ### Supported Tasks:
-        1. **Company Normalization**: Handles mergers, acquisitions, rebranding
-        2. **Job Title Standardization**: Recognizes equivalent roles and seniority
-        3. **Skills Normalization**: Standardizes technology names and abbreviations
-        4. **Equivalency Detection**: Binary classification for entity matching
-        
-        ### Model Architecture:
-        - Base Model: Google Flan-T5 (instruction-tuned)
-        - Fine-tuning: LoRA (Low-Rank Adaptation) for efficiency
-        - Multi-task: Uses task prefixes ([COMPANY], [JOB], [SKILLS])
-        
-        ### Training Data:
-        - 478 manually curated high-quality examples
-        - 8,824 synthetic examples generated with GPT-4
-        - Total: 9,302 training examples
+    with gr.Tab("🐛 Debug Output"):
+        refresh_btn = gr.Button("🔄 Refresh Debug Output", variant="secondary")
+        status_output = gr.Textbox(label="Status", lines=5)
+        debug_output = gr.Textbox(label="Full Training Output", lines=30)
         
-        ### Expected Performance:
-        - Inference: <100ms per query
-        - Accuracy: >90% on test set
-        - Model size: 250M-770M parameters
-        """)
+        refresh_btn.click(get_training_status, outputs=[status_output, debug_output])
 
 app.launch()