Fixing model halucination

src/api/routes.py

@@ -4,11 +4,47 @@ from models.request_models import ProviderNotesRequest
 from models.response_models import CodingResponse, FileUploadResponse
 from services.groq_service import groq_service
 from services.file_service import file_service
+import re
 
 router = APIRouter()
 logger = logging.getLogger(__name__)
 
 
+def is_likely_medical_text(text: str) -> bool:
+    """
+    Pre-validate if text appears to be medical provider notes
+    Returns True if likely medical, False otherwise
+    """
+    text_lower = text.lower()
+    
+    # Medical keywords that should be present
+    medical_keywords = [
+        'patient', 'diagnosis', 'symptom', 'treatment', 'exam', 'history',
+        'complaint', 'pain', 'fever', 'cough', 'prescribed', 'medication',
+        'procedure', 'surgery', 'vital', 'blood', 'pressure', 'heart',
+        'lung', 'breath', 'chronic', 'acute', 'assessment', 'plan',
+        'condition', 'disease', 'injury', 'wound', 'fracture', 'infection'
+    ]
+    
+    # Red flags for non-medical text
+    casual_phrases = [
+        'how are you', 'hello', 'hi there', 'good morning', 'good evening',
+        'test test', 'sample text', 'doing fine', 'nice to meet',
+        'what\'s up', 'how\'s it going'
+    ]
+    
+    # Check for casual phrases (immediate rejection)
+    for phrase in casual_phrases:
+        if phrase in text_lower:
+            return False
+    
+    # Check for medical keywords (need at least 1)
+    medical_keyword_count = sum(1 for keyword in medical_keywords if keyword in text_lower)
+    
+    # Require at least 1 medical keyword and minimum length
+    return medical_keyword_count >= 1 and len(text.split()) >= 10
+
+
 @router.post("/coding", response_model=CodingResponse)
 async def analyze_provider_notes(request: ProviderNotesRequest):
     """
@@ -20,7 +56,7 @@ async def analyze_provider_notes(request: ProviderNotesRequest):
     - Overall encounter summary
     """
     try:
-        logger.info(f"📥 Received coding request (notes length: {len(request.provider_notes)})")
+        logger.info(f"Received coding request (notes length: {len(request.provider_notes)})")
         
         # Validate input
         if not request.provider_notes or len(request.provider_notes.strip()) < 10:
@@ -29,20 +65,29 @@ async def analyze_provider_notes(request: ProviderNotesRequest):
                 detail="Provider notes must be at least 10 characters long"
             )
         
+        # PRE-VALIDATION: Check if text appears to be medical
+        if not is_likely_medical_text(request.provider_notes):
+            logger.warning("Input rejected - does not appear to be medical provider notes")
+            return {
+                "icd_codes": [],
+                "cpt_codes": [],
+                "overall_summary": "No medical coding applicable - input does not appear to contain clinical provider notes. Please provide legitimate medical documentation."
+            }
+        
         # Analyze with Groq
         result = groq_service.analyze_provider_notes(request.provider_notes)
         
-        logger.info(f"✅ Analysis complete: {len(result.get('icd_codes', []))} ICD codes, {len(result.get('cpt_codes', []))} CPT codes")
+        logger.info(f"Analysis complete: {len(result.get('icd_codes', []))} ICD codes, {len(result.get('cpt_codes', []))} CPT codes")
         
         return result
         
     except HTTPException:
         raise
     except ValueError as e:
-        logger.error(f"❌ Validation error: {str(e)}")
+        logger.error(f"Validation error: {str(e)}")
         raise HTTPException(status_code=400, detail=str(e))
     except Exception as e:
-        logger.error(f"❌ Error processing request: {str(e)}", exc_info=True)
+        logger.error(f"Error processing request: {str(e)}", exc_info=True)
         raise HTTPException(status_code=500, detail=f"Error processing request: {str(e)}")
 
 
@@ -58,7 +103,7 @@ async def upload_provider_notes_file(file: UploadFile = File(...)):
     - Overall summary
     """
     try:
-        logger.info(f"📤 Received file upload: {file.filename}")
+        logger.info(f"Received file upload: {file.filename}")
         
         # Validate file type
         if not file.filename.endswith('.txt'):
@@ -71,14 +116,14 @@ async def upload_provider_notes_file(file: UploadFile = File(...)):
         content = await file.read()
         text = content.decode('utf-8')
         
-        logger.info(f"📄 File read successfully (length: {len(text)})")
+        logger.info(f"File read successfully (length: {len(text)})")
         
         # Remove PII
         try:
             cleaned_text, pii_count = file_service.remove_pii(text)
-            logger.info(f"🔒 PII removal complete: {pii_count} entities removed")
+            logger.info(f"PII removal complete: {pii_count} entities removed")
         except Exception as pii_error:
-            logger.error(f"⚠️ PII removal failed: {str(pii_error)}")
+            logger.error(f"WARNING: PII removal failed: {str(pii_error)}")
             # Continue without PII removal if it fails
             cleaned_text = text
             pii_count = 0
@@ -98,15 +143,15 @@ async def upload_provider_notes_file(file: UploadFile = File(...)):
             "overall_summary": result.get("overall_summary", "")
         }
         
-        logger.info(f"✅ File processing complete")
+        logger.info(f"File processing complete")
         
         return response
         
     except HTTPException:
         raise
     except UnicodeDecodeError:
-        logger.error("❌ File encoding error")
+        logger.error("File encoding error")
         raise HTTPException(status_code=400, detail="File must be UTF-8 encoded text")
     except Exception as e:
-        logger.error(f"❌ Error processing uploaded file: {str(e)}", exc_info=True)
+        logger.error(f"Error processing uploaded file: {str(e)}", exc_info=True)
         raise HTTPException(status_code=500, detail=f"Error processing uploaded file: {str(e)}")

src/config/settings.py

@@ -19,8 +19,8 @@ settings = Settings()
 
 # Validate API key on startup
 if not settings.groq_api_key:
-    print("⚠️ WARNING: GROQ_API_KEY is not set! API will not function properly.")
+    print("WARNING: GROQ_API_KEY is not set! API will not function properly.")
     print("Please set GROQ_API_KEY in Hugging Face Space secrets.")
 else:
-    print(f"✅ Groq API Key loaded (length: {len(settings.groq_api_key)})")
-    print(f"✅ Using model: {settings.groq_model}")
+    print(f"Groq API Key loaded (length: {len(settings.groq_api_key)})")
+    print(f"Using model: {settings.groq_model}")

src/main.py

@@ -21,22 +21,22 @@ async def lifespan(app: FastAPI):
     """
     # Startup
     logger.info("=" * 50)
-    logger.info(f"🚀 {settings.api_title} v{settings.api_version}")
+    logger.info(f"Starting {settings.api_title} v{settings.api_version}")
     logger.info("=" * 50)
     
     if settings.groq_api_key:
-        logger.info(f"✅ Groq API Key: Configured (length: {len(settings.groq_api_key)})")
-        logger.info(f"✅ Groq Model: {settings.groq_model}")
+        logger.info(f"Groq API Key: Configured (length: {len(settings.groq_api_key)})")
+        logger.info(f"Groq Model: {settings.groq_model}")
     else:
-        logger.error("❌ GROQ_API_KEY is NOT set!")
-        logger.error("⚠️ API will NOT function without valid API key")
+        logger.error("GROQ_API_KEY is NOT set!")
+        logger.error("WARNING: API will NOT function without valid API key")
     
     logger.info("=" * 50)
     
     yield
     
     # Shutdown
-    logger.info("👋 Shutting down API...")
+    logger.info("Shutting down API...")
 
 
 # Create FastAPI app with lifespan

src/services/file_service.py

@@ -1,6 +1,3 @@
-from fastapi import UploadFile, HTTPException
-import os
-from typing import Dict
 import logging
 from services.regex_pii_remover import RegexPIIRemover
 
@@ -11,7 +8,7 @@ class FileService:
     def __init__(self):
         """Initialize file service with PII remover"""
         self.pii_remover = RegexPIIRemover()
-        logger.info("✅ FileService initialized")
+        logger.info("FileService initialized")
     
     def remove_pii(self, text: str) -> tuple[str, int]:
         """
@@ -24,16 +21,16 @@ class FileService:
             tuple: (cleaned_text, pii_count)
         """
         try:
-            logger.info(f"🔒 Starting PII removal (text length: {len(text)})")
+            logger.info(f"Starting PII removal (text length: {len(text)})")
             
             cleaned_text, pii_count = self.pii_remover.remove_pii(text)
             
-            logger.info(f"✅ PII removal complete: {pii_count} entities removed")
+            logger.info(f"PII removal complete: {pii_count} entities removed")
             
             return cleaned_text, pii_count
             
         except Exception as e:
-            logger.error(f"❌ Error during PII removal: {str(e)}")
+            logger.error(f"Error during PII removal: {str(e)}")
             # Return original text if PII removal fails
             return text, 0
 

src/services/groq_service.py

@@ -8,22 +8,27 @@ logger = logging.getLogger(__name__)
 
 class GroqService:
     def __init__(self):
-        """Initialize Groq client with API key from settings"""
-        if not settings.groq_api_key:
-            logger.error("❌ GROQ_API_KEY is not set!")
-            raise ValueError("GROQ_API_KEY environment variable is required")
+        """Initialize Groq service (client created on first use)"""
+        self._client = None
+        self.model = settings.groq_model
+    
+    @property
+    def client(self):
+        """Lazy initialization of Groq client"""
+        if self._client is None:
+            if not settings.groq_api_key:
+                logger.error("GROQ_API_KEY is not set!")
+                raise ValueError("GROQ_API_KEY environment variable is required")
+            
+            try:
+                logger.info("Initializing Groq client...")
+                self._client = Groq(api_key=settings.groq_api_key)
+                logger.info(f"Groq client initialized successfully")
+            except Exception as e:
+                logger.error(f"Failed to initialize Groq client: {str(e)}")
+                raise
         
-        try:
-            # Simple initialization without extra parameters
-            self.client = Groq(
-                api_key=settings.groq_api_key
-            )
-            self.model = settings.groq_model
-            logger.info(f"✅ Groq client initialized successfully")
-            logger.info(f"✅ Using model: {self.model}")
-        except Exception as e:
-            logger.error(f"❌ Failed to initialize Groq client: {str(e)}")
-            raise
+        return self._client
     
     def analyze_provider_notes(self, provider_notes: str) -> dict:
         """
@@ -31,19 +36,18 @@ class GroqService:
         
         Args:
             provider_notes: Clinical provider notes text
-            response_format: Desired response format (default: "json")
             
         Returns:
             dict: Parsed coding response with ICD-10, CPT codes and summary
         """
         try:
-            logger.info(f"📝 Analyzing provider notes (length: {len(provider_notes)})")
+            logger.info(f"Analyzing provider notes (length: {len(provider_notes)})")
             
             # Get the prompt
             prompt = get_coding_prompt(provider_notes)
             
-            # Call Groq API
-            logger.info(f"🚀 Calling Groq API with model: {self.model}")
+            # Call Groq API (client initialized here if needed)
+            logger.info(f"Calling Groq API with model: {self.model}")
             response = self.client.chat.completions.create(
                 model=self.model,
                 messages=[
@@ -63,8 +67,7 @@ class GroqService:
             
             # Extract response
             raw_response = response.choices[0].message.content
-            logger.info(f"📥 Received response from Groq (length: {len(raw_response)})")
-            logger.debug(f"Raw response: {raw_response[:500]}...")
+            logger.info(f"Received response from Groq (length: {len(raw_response)})")
             
             # Parse JSON
             parsed_response = json.loads(raw_response)
@@ -72,31 +75,20 @@ class GroqService:
             # Validate and structure response
             result = self._structure_response(parsed_response)
             
-            logger.info(f"✅ Successfully analyzed notes: {len(result.get('icd_codes', []))} ICD codes, {len(result.get('cpt_codes', []))} CPT codes")
+            logger.info(f"Successfully analyzed: {len(result.get('icd_codes', []))} ICD codes, {len(result.get('cpt_codes', []))} CPT codes")
             
             return result
             
         except json.JSONDecodeError as e:
-            logger.error(f"❌ JSON parsing error: {str(e)}")
-            logger.error(f"Raw response: {raw_response}")
+            logger.error(f"JSON parsing error: {str(e)}")
             raise ValueError(f"Failed to parse Groq response as JSON: {str(e)}")
         
         except Exception as e:
-            logger.error(f"❌ Error analyzing provider notes: {str(e)}")
+            logger.error(f"Error analyzing provider notes: {str(e)}")
             raise
     
     def _structure_response(self, parsed_response: dict) -> dict:
-        """
-        Structure and validate the response from Groq
-        
-        Args:
-            parsed_response: Raw parsed JSON from Groq
-            response_format: Desired response format (default: "json")
-            
-        Returns:
-            dict: Properly structured response
-        """
-        # Handle different possible response formats
+        """Structure and validate the response from Groq"""
         icd_codes = []
         cpt_codes = []
         overall_summary = parsed_response.get("overall_summary", "")
@@ -129,5 +121,5 @@ class GroqService:
             "overall_summary": overall_summary
         }
 
-# Global instance
+# Global instance (client initialized on first use)
 groq_service = GroqService()

src/services/regex_pii_remover.py

@@ -35,7 +35,7 @@ class RegexPIIRemover:
             'dob': re.compile(r'(DOB|Date of Birth)[:\s]+\d{1,2}[/-]\d{1,2}[/-]\d{2,4}', re.IGNORECASE),
         }
         
-        logger.info(f"✅ RegexPIIRemover initialized with {len(self.patterns)} patterns")
+        logger.info(f"RegexPIIRemover initialized with {len(self.patterns)} patterns")
     
     def remove_pii(self, text: str) -> tuple[str, int]:
         """
@@ -66,6 +66,6 @@ class RegexPIIRemover:
                 else:
                     cleaned_text = pattern.sub('[REDACTED]', cleaned_text)
         
-        logger.info(f"🔒 Removed {total_removed} PII entities")
+        logger.info(f"Removed {total_removed} PII entities")
         
         return cleaned_text, total_removed

src/utils/prompts.py

@@ -2,11 +2,20 @@ SYSTEM_PROMPT = """You are a specialized medical coding assistant AI that analyz
 
 CRITICAL INSTRUCTIONS:
 1. You MUST respond ONLY in valid JSON format as specified below
-2. Do NOT hallucinate or make up codes - only use codes you are confident about
-3. If you are uncertain about any code, do NOT include it in the response
-4. If you cannot find any relevant ICD or CPT codes, return empty arrays for those sections
-5. Always provide clear, evidence-based explanations for each code you assign
-6. Your response must be parseable JSON - do not add any text before or after the JSON object
+2. ONLY process text that appears to be legitimate clinical provider notes
+3. If the input is NOT medical provider notes (casual conversation, greetings, test text, non-medical content), respond with empty arrays
+4. Do NOT hallucinate or make up codes - only use codes you are 100% confident about
+5. If you are uncertain about any code, do NOT include it in the response
+6. If you cannot find any relevant ICD or CPT codes, return empty arrays for those sections
+7. Always provide clear, evidence-based explanations for each code you assign
+8. Your response must be parseable JSON - do not add any text before or after the JSON object
+
+INPUT VALIDATION - REJECT IF:
+- The text is casual conversation (e.g., "how are you", "hello", "testing")
+- The text lacks medical terminology or clinical context
+- The text does not describe a patient encounter, diagnosis, or medical procedure
+- The text is less than 20 words and contains no medical information
+- The text appears to be a test or non-medical query
 
 REQUIRED JSON FORMAT:
 {
@@ -24,27 +33,53 @@ REQUIRED JSON FORMAT:
       "explanation": "Detailed explanation of why this code was selected based on the provider notes"
     }
   ],
-  "overall_summary": "Brief summary of the coding decisions"
+  "overall_summary": "Brief summary of the coding decisions, or 'No medical coding applicable - input does not contain clinical provider notes' if input is not medical"
 }
 
 CODING PRINCIPLES:
+- FIRST verify the input is legitimate medical provider notes
+- If NOT medical notes, return empty arrays with summary explaining why
 - Only assign codes that are clearly supported by documentation in the provider notes
 - Be conservative - if unsure, omit the code rather than guess
 - Prioritize accuracy over quantity
 - Each explanation must reference specific details from the provider notes
-- If no relevant codes can be determined, respond with empty arrays
+- Medical provider notes should include: patient symptoms, diagnoses, treatments, procedures, or clinical observations
+
+EXAMPLES OF VALID MEDICAL INPUT:
+✅ "Patient presents with acute bronchitis, productive cough for 5 days, prescribed azithromycin"
+✅ "45 y/o female with Type 2 diabetes, HbA1c 8.2%, medication adjustment discussed"
+✅ "Laceration repair of right forearm, 3cm wound, simple closure"
+
+EXAMPLES OF INVALID INPUT (return empty arrays):
+❌ "how are you and i doing fine"
+❌ "hello world"
+❌ "test test test"
+❌ "this is a sample text"
+❌ Any non-medical casual conversation
+
+Remember: Return ONLY the JSON object, nothing else. If input is not medical provider notes, return empty arrays with explanatory summary."""
 
-Remember: Return ONLY the JSON object, nothing else."""
 
 def create_user_prompt(provider_notes: str) -> str:
     """Create the user prompt with provider notes"""
-    return f"""Analyze the following provider notes and extract appropriate ICD-10 and CPT codes.
+    return f"""Analyze the following text and determine if it contains legitimate clinical provider notes.
 
-PROVIDER NOTES:
+INPUT TEXT:
 {provider_notes}
 
+INSTRUCTIONS:
+1. First, determine if this is medical provider notes or non-medical text
+2. If it's NOT medical notes (casual conversation, greetings, test text), return:
+   {{
+     "icd_codes": [],
+     "cpt_codes": [],
+     "overall_summary": "No medical coding applicable - input does not contain clinical provider notes"
+   }}
+3. If it IS medical notes, extract appropriate ICD-10 and CPT codes
+
 Respond ONLY with the JSON object following the exact format specified in the system prompt."""
 
+
 def get_coding_prompt(provider_notes: str) -> str:
     """
     Generate prompt for medical coding analysis
@@ -65,6 +100,8 @@ def get_coding_prompt(provider_notes: str) -> str:
 {provider_notes}
 
 **Instructions:**
+- FIRST verify this is legitimate medical provider notes
+- If NOT medical notes (casual text, greetings, tests), return empty arrays
 - Provide accurate ICD-10 and CPT codes based on current coding guidelines
 - Include detailed explanations for each code
 - Provide an overall summary of the patient encounter
@@ -90,5 +127,6 @@ def get_coding_prompt(provider_notes: str) -> str:
 
 **Important:** 
 - Return ONLY valid JSON, no markdown, no code blocks
-- Include at least one ICD-10 code and one CPT code if applicable
-- Be specific and accurate with coding"""
+- If input is NOT medical notes, return empty arrays with explanatory summary
+- Be specific and accurate with coding
+- Do NOT code non-medical text"""