Refactor language utilities to enhance language support and detection accuracy

app.py

@@ -181,31 +181,63 @@ def get_context(message, conversation_id):
         logger.error(f"Error getting context: {str(e)}")
         return ""
 
-def translate_with_llm(text: str, target_lang: str) -> str:
-    """Translate text using the active LLM"""
+def post_process_response(user_message, bot_response):
+    """Enhanced post-processing of bot responses to ensure correct language"""
     try:
-        prompt = (
-            f"Translate the following text to {target_lang}. "
-            f"Provide ONLY the direct translation, no explanations or additional text. "
-            f"Maintain the same tone and style:\n\n{text}"
-        )
+        user_lang = detect_language(user_message)
+        # Convert to closest supported language
+        user_lang = LanguageUtils.get_closest_supported_language(user_lang)
         
-        response = client.chat_completion(
-            messages=[
-                {"role": "system", "content": "You are a professional translator."},
-                {"role": "user", "content": prompt}
-            ],
-            max_tokens=ACTIVE_MODEL['parameters']['max_length'],
-            temperature=0.3,
-            top_p=0.9,
-            stream=False
-        )
+        logger.info(f"User language detected: {user_lang} ({LanguageUtils.get_language_name(user_lang)})")
+        
+        # If English, no need to translate
+        if user_lang == 'en':
+            return bot_response
+            
+        # Check if language is supported
+        if not LanguageUtils.is_supported(user_lang):
+            logger.warning(f"Unsupported language: {user_lang}")
+            apology = ("I apologize, but I cannot respond in your language. "
+                      "I will answer in English instead.\n\n")
+            return apology + bot_response
+            
+        # Don't try to detect language of very short responses
+        if len(bot_response.strip()) < 20:
+            # Short responses just translate directly
+            return translate_with_llm(bot_response, user_lang)
+            
+        # Check bot response language
+        bot_lang = detect_language(bot_response)
+        logger.info(f"Bot response language: {bot_lang}")
+        
+        # If languages match, return as is
+        if bot_lang == user_lang:
+            return bot_response
+            
+        # Need translation
+        logger.warning(f"Language mismatch! User: {user_lang}, Bot: {bot_lang}")
+        
+        translated_response = translate_with_llm(bot_response, user_lang)
         
-        return response.choices[0].message.content.strip()
+        # Verify translation worked by checking a sample (not the whole text)
+        # This is more reliable than checking the entire text
+        sample_size = min(100, len(translated_response) // 2)
+        if sample_size > 20:  # Only verify if we have enough text
+            sample = translated_response[:sample_size]
+            translated_lang = detect_language(sample)
+            
+            if translated_lang != user_lang:
+                logger.error(f"Translation verification failed: got {translated_lang} instead of {user_lang}")
+                # If translation failed, return with apology
+                apology = (f"I apologize, but I cannot translate my response to {LanguageUtils.get_language_name(user_lang)}. "
+                          "Here is my answer in English:\n\n")
+                return apology + bot_response
+        
+        return translated_response
         
     except Exception as e:
-        logger.error(f"Translation failed: {e}")
-        return text
+        logger.error(f"Post-processing error: {e}")
+        return bot_response
 
 def post_process_response(user_message, bot_response):
     """Check if the response language matches the user's language and translate if needed"""
@@ -268,14 +300,39 @@ def load_vector_store():
         return None
 
 def detect_language(text: str) -> str:
-    """Detect language with fallback"""
+    """Enhanced language detection with better handling of edge cases"""
     try:
-        if len(text.strip()) < 5:
+        # If text is too short, don't try to detect
+        if len(text.strip()) < 10:
             logger.debug(f"Text too short for reliable detection: '{text}'")
             return "en"
             
-        return detect(text.strip())
+        # First detection with langdetect
+        from langdetect import detect, LangDetectException
         
+        try:
+            lang_code = detect(text.strip())
+            logger.debug(f"Detected language: {lang_code}")
+            
+            # Verify detection with confidence check by analyzing a larger portion of text
+            if len(text) > 50:
+                from langdetect import DetectorFactory
+                DetectorFactory.seed = 0  # For consistent results
+                
+                detector = DetectorFactory.create()
+                detector.append(text)
+                lang_probabilities = detector.get_probabilities()
+                
+                # If top language has low probability, fallback to English
+                if lang_probabilities and lang_probabilities[0].prob < 0.5:
+                    logger.warning(f"Low confidence detection ({lang_probabilities[0].prob:.2f}) for '{lang_code}', defaulting to English")
+                    return "en"
+            
+            return lang_code
+        except LangDetectException as e:
+            logger.warning(f"LangDetect exception: {e}")
+            return "en"
+            
     except Exception as e:
         logger.error(f"Language detection error: {str(e)} for text: '{text[:50]}...'")
         return "en"
@@ -290,11 +347,12 @@ def respond(
     top_p,
     attempt_fallback=True
 ):
-    """Generate response with proper error handling"""
+    """Generate response with improved language handling"""
     try:
         # Reset and determine user language for new request
         user_lang = detect_language(message)
-        logger.debug(f"Detected user language: {user_lang}")
+        user_lang = LanguageUtils.get_closest_supported_language(user_lang)
+        logger.info(f"User language detected for request: {user_lang} ({LanguageUtils.get_language_name(user_lang)})")
         
         # Create clean history without system messages
         clean_history = [
@@ -302,9 +360,15 @@ def respond(
             if msg["role"] != "system"
         ]
         
-        # Add fresh system message with current language instruction
-        language_instruction = f"\nIMPORTANT: You MUST respond in {user_lang} language ONLY."
-        full_system_message = system_message + language_instruction
+        # Remove language instruction from system message to avoid confusion
+        base_system_message = system_message.split("\nIMPORTANT:")[0] if "\nIMPORTANT:" in system_message else system_message
+        
+        # Always request English response, we'll translate later
+        full_system_message = (
+            f"{base_system_message}\n\n"
+            f"IMPORTANT: Always respond in English, no matter what language the user speaks. "
+            f"Provide a complete and helpful response - we will handle translation separately."
+        )
         
         # --- API Request ---
         response = client.chat_completion(
@@ -321,7 +385,7 @@ def respond(
         
         bot_response = response.choices[0].message.content
         
-        # Post-process response to check language
+        # Post-process response to translate if needed
         processed_response = post_process_response(message, bot_response)
         
         # --- Format Successful Response ---

src/language_utils.py

@@ -10,89 +10,84 @@ DetectorFactory.seed = 0
 logger = logging.getLogger(__name__)
 
 class LanguageUtils:
-    """Centralized class for language processing"""
+    """Utility class for language operations"""
     
-    # Supported languages (can be extended)
-    SUPPORTED_LANGUAGES = ["en", "ru", "uk", "de", "fr", "es", "it", "pt"]
+    SUPPORTED_LANGUAGES = {
+        # Common European languages
+        'en': 'English',
+        'ru': 'Russian',
+        'de': 'German',
+        'fr': 'French',
+        'es': 'Spanish',
+        'it': 'Italian',
+        'pt': 'Portuguese',
+        'nl': 'Dutch',
+        'pl': 'Polish',
+        'sv': 'Swedish',
+        'no': 'Norwegian',
+        'da': 'Danish',
+        'fi': 'Finnish',
+        
+        # Asian languages
+        'zh': 'Chinese',
+        'ja': 'Japanese',
+        'ko': 'Korean',
+        
+        # Other widely used languages
+        'ar': 'Arabic',
+        'hi': 'Hindi',
+        'tr': 'Turkish',
+        'cs': 'Czech',
+        'uk': 'Ukrainian',
+        'bg': 'Bulgarian',
+        'el': 'Greek',
+        'he': 'Hebrew',
+        'th': 'Thai',
+        'vi': 'Vietnamese',
+        'hu': 'Hungarian',
+        'sk': 'Slovak',
+        'ro': 'Romanian',
+        'id': 'Indonesian',
+        'ms': 'Malay',
+    }
     
     @classmethod
-    def detect_language(cls, text: str, default: str = "en") -> str:
-        """
-        Detects text language with enhanced error handling
-        
-        Args:
-            text: Text to analyze
-            default: Default language in case of error
-            
-        Returns:
-            Language code (ISO 639-1)
-        """
-        try:
-            # Minimum length for reliable detection
-            if len(text.strip()) < 15:
-                logger.warning(f"Text too short for reliable detection: '{text}'")
-                return default
-                
-            lang = detect(text)
-            
-            # Check language support
-            if lang not in cls.SUPPORTED_LANGUAGES:
-                logger.warning(f"Unsupported language detected: {lang}. Defaulting to {default}")
-                return default
-                
-            logger.debug(f"Detected language: {lang} for text: '{text[:50]}...'")
-            return lang
-            
-        except Exception as e:
-            logger.error(f"Language detection failed: {str(e)}. Text: '{text[:100]}...'")
-            return default
+    def get_language_name(cls, lang_code: str) -> str:
+        """Get language name from code"""
+        return cls.SUPPORTED_LANGUAGES.get(lang_code, "Unknown")
+    
+    @classmethod
+    def is_supported(cls, lang_code: str) -> bool:
+        """Check if language is supported"""
+        return lang_code in cls.SUPPORTED_LANGUAGES
     
     @classmethod
-    def get_language_instruction(cls, target_lang: str, user_message: str) -> str:
+    def get_closest_supported_language(cls, lang_code: str) -> str:
         """
-        Generates strict response language instructions
+        Get the closest supported language code
         
-        Args:
-            target_lang: Language the bot should respond in
-            user_message: Original user message
-            
-        Returns:
-            String with prompt instructions
+        This helps with similar language detection issues
+        like confusing 'no' (Norwegian) with 'da' (Danish)
         """
-        instructions = {
-            "en": f"CRITICAL: Respond in English only. Never switch languages.\n\nOriginal message: {user_message}",
-            "ru": f"ВАЖНО: Отвечайте только на русском. Не переключайтесь на другие языки.\n\nОригинальное сообщение: {user_message}",
-            "uk": f"ВАЖЛИВО: Відповідайте лише українською. Не змінюйте мову.\n\nОригінальне повідомлення: {user_message}",
-            "de": f"KRITISCH: Antworten Sie nur auf Deutsch. Wechseln Sie nie die Sprache.\n\nOriginalnachricht: {user_message}",
-            "fr": f"CRITIQUE: Répondez uniquement en français. Ne changez jamais de langue.\n\nMessage original: {user_message}",
-            "es": f"CRÍTICO: Responda sólo en español. Nunca cambie de idioma.\n\nMensaje original: {user_message}",
-            "it": f"IMPORTANTE: Rispondere solo in italiano. Non cambiare lingua.\n\nMessaggio originale: {user_message}",
-            "pt": f"CRÍTICO: Responda apenas em português. Nunca mude de idioma.\n\nMensagem original: {user_message}"
+        if lang_code in cls.SUPPORTED_LANGUAGES:
+            return lang_code
+            
+        # Language mapping for commonly confused languages
+        similar_languages = {
+            'nb': 'no',  # Norwegian Bokmål → Norwegian
+            'nn': 'no',  # Norwegian Nynorsk → Norwegian
+            'zh-cn': 'zh',  # Chinese Simplified → Chinese 
+            'zh-tw': 'zh',  # Chinese Traditional → Chinese
+            'hr': 'sr',  # Croatian → Serbian (similar)
+            'bs': 'sr',  # Bosnian → Serbian (similar)
+            'mk': 'bg',  # Macedonian → Bulgarian (similar)
+            'be': 'ru',  # Belarusian → Russian (similar)
+            'ca': 'es',  # Catalan → Spanish (similar)
+            'gl': 'pt',  # Galician → Portuguese (similar)
+            'af': 'nl',  # Afrikaans → Dutch (similar)
         }
         
-        return instructions.get(target_lang, instructions["en"])
+        return similar_languages.get(lang_code, "en")
     
-    @classmethod
-    def validate_response_language(cls, response: str, expected_lang: str) -> bool:
-        """
-        Validates if response language matches expected language
-        
-        Args:
-            response: Bot's response
-            expected_lang: Expected language (ISO 639-1)
-            
-        Returns:
-            True if language matches, False if not
-        """
-        try:
-            detected_lang = cls.detect_language(response)
-            if detected_lang != expected_lang:
-                logger.warning(f"Language mismatch! Expected {expected_lang}, got {detected_lang}")
-                return False
-            return True
-        except Exception as e:
-            logger.error(f"Language validation failed: {str(e)}")
-            return False
-
 # Create instance for convenient import
 language_processor = LanguageUtils()