Upgrade to Llama 3.1 8B-Instruct for better long-form content

- Switch from Mistral-7B to Llama-3.1-8B-Instruct
- 4x larger context window (32K → 128K tokens)
- Better reasoning and question generation quality
- Same speed and memory usage
- Perfect for long-form content interpretation

app.py

@@ -122,8 +122,8 @@ async def load_model():
         try:
             logger.info("Loading model with transformers...")
             
-            # Use Mistral 7B Instruct - 30-40% faster with same quality
-            base_model_name = "mistralai/Mistral-7B-Instruct-v0.2"
+            # Use Llama 3.1 8B Instruct - 4x context window, better reasoning
+            base_model_name = "meta-llama/Llama-3.1-8B-Instruct"
             
             tokenizer, model = await load_model_with_retry(base_model_name, hf_token)
             
@@ -337,7 +337,7 @@ async def generate_questions(request: QuestionGenerationRequest):
             questions.append(f"What is the main point of this statement: '{request.statement[:100]}...'?")
         
         metadata = {
-            "model": "mistralai/Mistral-7B-Instruct-v0.2",
+            "model": "meta-llama/Llama-3.1-8B-Instruct",
             "temperature": request.temperature,
             "difficulty_level": request.difficulty_level,
             "generated_text_length": len(generated_text),

gradio_app.py

@@ -38,8 +38,8 @@ class ModelManager:
             # Get HF token from environment
             hf_token = os.getenv("HF_TOKEN")
             
-            logger.info("Loading Mistral-7B-Instruct-v0.2 model...")
-            base_model_name = "mistralai/Mistral-7B-Instruct-v0.2"
+            logger.info("Loading Llama-3.1-8B-Instruct model...")
+            base_model_name = "meta-llama/Llama-3.1-8B-Instruct"
             
             self.tokenizer = AutoTokenizer.from_pretrained(
                 base_model_name,

upgrade_models.py

@@ -0,0 +1,75 @@
+# Model upgrade options for better long-form content interpretation
+
+# OPTION 1: Llama 3.1 70B (Best Quality - if you have compute)
+LLAMA_70B = {
+    "model_name": "meta-llama/Llama-3.1-70B-Instruct",
+    "context_window": "128K tokens",
+    "quality": "Excellent - best for complex content",
+    "speed": "Moderate (2-4x slower than 7B)",
+    "memory_required": "~35GB VRAM",
+    "fits_on_a100": True,
+    "upgrade_difficulty": "Easy - just change model name"
+}
+
+# OPTION 2: Qwen2.5-32B (Best Balance)
+QWEN_32B = {
+    "model_name": "Qwen/Qwen2.5-32B-Instruct",
+    "context_window": "128K tokens", 
+    "quality": "Excellent - specialized for reasoning",
+    "speed": "Fast (1.5-2x slower than 7B)",
+    "memory_required": "~16GB VRAM",
+    "fits_on_a100": True,
+    "upgrade_difficulty": "Easy - just change model name"
+}
+
+# OPTION 3: Llama 3.1 8B (Easy Upgrade)
+LLAMA_8B = {
+    "model_name": "meta-llama/Llama-3.1-8B-Instruct", 
+    "context_window": "128K tokens",
+    "quality": "Very good - better than Mistral-7B",
+    "speed": "Fast (similar to current)",
+    "memory_required": "~8GB VRAM",
+    "fits_on_a100": True,
+    "upgrade_difficulty": "Trivial - just change model name"
+}
+
+# OPTION 4: Claude 3.5 Sonnet via API (Best Overall)
+CLAUDE_API = {
+    "model_name": "claude-3-5-sonnet-20241022",
+    "context_window": "200K tokens",
+    "quality": "Excellent - best for nuanced questions",
+    "speed": "Very fast via API",
+    "memory_required": "0GB (API-based)",
+    "cost": "$3 per million input tokens",
+    "upgrade_difficulty": "Medium - requires API integration"
+}
+
+def get_recommended_upgrade():
+    """Get the best upgrade based on priorities"""
+    
+    recommendations = {
+        "best_quality": LLAMA_70B,
+        "best_balance": QWEN_32B, 
+        "easiest_upgrade": LLAMA_8B,
+        "best_overall": CLAUDE_API
+    }
+    
+    return recommendations
+
+# Context window comparison
+CONTEXT_COMPARISON = {
+    "Current Mistral-7B": "32K tokens",
+    "Llama 3.1 8B": "128K tokens (4x more)",
+    "Llama 3.1 70B": "128K tokens (4x more)", 
+    "Qwen2.5-32B": "128K tokens (4x more)",
+    "Claude 3.5 Sonnet": "200K tokens (6x more)"
+}
+
+# Performance for long-form content
+LONG_FORM_PERFORMANCE = {
+    "Mistral-7B": "Good for simple questions",
+    "Llama 3.1 8B": "Better reasoning, longer context",
+    "Qwen2.5-32B": "Excellent reasoning, great for complex content",
+    "Llama 3.1 70B": "Superior understanding, best for nuanced questions",
+    "Claude 3.5 Sonnet": "Best overall, excellent at context understanding"
+}