Update app.py

app.py

@@ -19,6 +19,10 @@ from langchain_community.vectorstores import FAISS
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain.schema import Document
 
+# Suppress transformers warnings about generation flags
+import os
+os.environ['TRANSFORMERS_VERBOSITY'] = 'error'
+
 # Setup logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -27,10 +31,22 @@ logger = logging.getLogger(__name__)
 torch.set_num_threads(4)  # Limit threads for better CPU performance
 torch.set_grad_enabled(False)  # Disable gradients (inference only)
 
-# Suppress specific warnings
+# Suppress specific warnings and asyncio issues
 import warnings
 warnings.filterwarnings("ignore", message="MatMul8bitLt")
 warnings.filterwarnings("ignore", message="torch_dtype")
+warnings.filterwarnings("ignore", message="Invalid file descriptor")
+warnings.filterwarnings("ignore", message="generation flags")
+warnings.filterwarnings("ignore", category=UserWarning)
+
+# Fix asyncio file descriptor warnings
+import asyncio
+import sys
+if sys.platform == 'linux':
+    try:
+        asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
+    except:
+        pass
 
 # ============================================================================
 # CONFIGURATION
@@ -666,8 +682,9 @@ def generate_llm_answer(
 
     def call_model(prompt, max_new_tokens, temperature, top_p, repetition_penalty):
         logger.info(f"    → PHI model call (temp={temperature}, max_new_tokens={max_new_tokens})")
+        logger.info(f"    → Prompt length: {len(prompt)} chars")
         try:
-            # Call local PHI model with speed optimizations
+            # Call local PHI model with optimized parameters
             out = llm_client(
                 prompt,
                 max_new_tokens=max_new_tokens,
@@ -678,26 +695,41 @@ def generate_llm_answer(
                 num_return_sequences=1,
                 pad_token_id=llm_client.tokenizer.eos_token_id,
                 eos_token_id=llm_client.tokenizer.eos_token_id,
-                num_beams=1,  # Greedy/sampling is faster than beam search
-                early_stopping=True,  # Stop as soon as EOS is generated
-                use_cache=True  # Use KV cache for speed
+                truncation=True,
+                return_full_text=False  # Only return new generation, not prompt
             )
             
+            logger.info(f"    → Raw output type: {type(out)}")
+            
             # Extract generated text from pipeline output
-            if isinstance(out, list) and out:
-                generated = out[0].get('generated_text', '') if isinstance(out[0], dict) else str(out[0])
+            if isinstance(out, list) and len(out) > 0:
+                first_item = out[0]
+                if isinstance(first_item, dict):
+                    generated = first_item.get('generated_text', '')
+                else:
+                    generated = str(first_item)
             else:
-                generated = str(out)
+                generated = str(out) if out else ''
+            
+            logger.info(f"    → Generated length before cleanup: {len(generated)} chars")
             
-            # PHI models return prompt + completion, extract only new text
-            if prompt in generated:
-                # Remove the prompt from the output
+            # PHI models may still include prompt, remove it
+            if generated and prompt in generated:
+                prompt_end = generated.find(prompt) + len(prompt)
+                generated = generated[prompt_end:].strip()
+            
+            # Additional cleanup: remove any leading prompt fragments
+            if generated and generated.startswith(prompt[:50]):
                 generated = generated[len(prompt):].strip()
             
-            return generated
+            logger.info(f"    → Final generated length: {len(generated)} chars, words: {len(generated.split())}")
+            
+            return generated.strip()
             
         except Exception as e:
             logger.error(f"    ✗ PHI model call error: {e}")
+            import traceback
+            logger.error(f"    ✗ Traceback: {traceback.format_exc()}")
             return ''
 
     # Natural prompt: let the model generate complete, flowing responses
@@ -724,16 +756,20 @@ Answer:"""
         top_p = 0.93
         repetition_penalty = 1.10
 
+    logger.info(f"  → Starting generation with prompt: {base_prompt[:200]}...")
     initial_output = call_model(base_prompt, max_new_tokens, temperature, top_p, repetition_penalty)
     response = (initial_output or '').strip()
 
     # Basic sanity checks
     if not response:
-        logger.warning("  ✗ Empty initial response")
+        logger.warning("  ✗ Empty initial response - model may not be generating")
+        logger.warning(f"  ✗ Prompt was: {base_prompt[:300]}")
         response = ''
 
     words = response.split()
     word_count = len(words)
+    
+    logger.info(f"  → Initial response: {word_count} words")
 
     # Natural mode: accept ANY response length - let model decide
     # No truncation, no artificial limits
@@ -746,6 +782,11 @@ Answer:"""
     if word_count >= 50:
         logger.info(f"  ✅ Accepted natural response ({word_count} words)")
         return response
+    
+    # Very permissive: accept anything with 20+ words
+    if word_count >= 20:
+        logger.info(f"  ⚠️ Short but acceptable response ({word_count} words)")
+        return response
 
     # Otherwise, try iterative continuation to build up to the target
     accumulated = response