updated app state

app.py

@@ -1,139 +1,11 @@
-import os
-import gradio as gr
 import langsmith
 from langsmith import traceable
-from src.rag.agent_chain import ExecutiveAgentChain 
-from src.utils.logging import get_logger, init_logging
-
-init_logging(interactive_mode=False)
-logger = get_logger("chatbot_app")
-
+from src.apps.chat.app import ChatbotApplication
+from src.utils.logging import init_logging
 from dotenv import load_dotenv
-load_dotenv()
-
-class ChatbotApplication:
-    def __init__(self, language: str = 'de') -> None:
-        self._app = gr.Blocks()
-     
-        with self._app:
-            # Initial state variables
-            agent_state = gr.State(None)
-            lang_state  = gr.State(language)
-            
-            with gr.Row():
-                lang_selector = gr.Radio(
-                    choices=["Deutsch", "English"],
-                    value="English" if language == 'en' else 'Deutsch',
-                    label="Selected Language",
-                    interactive=True,
-                )
-                reset_button = gr.Button("Reset Conversation")
-            
-            chat = gr.ChatInterface(
-                fn=lambda msg, history, agent: self._chat( 
-                    message=msg,
-                    history=history,
-                    agent=agent,
-                ),
-                additional_inputs=[agent_state],
-                title="Executive Education Adviser",
-                type='messages',
-            )
-
-
-            def clear_chat_immediate():
-                return []
-            
-            def on_lang_change(language):
-                lang_code = 'en' if language == 'English' else 'de'
-                return switch_language(lang_code)
-            
-            def initalize_agent(language):
-                agent = ExecutiveAgentChain(language=language)
-                greeting = agent.generate_greeting()
-                return agent, [{"role": "assistant", "content": greeting}]
-
-            def switch_language(new_language):
-                new_agent, greeting = initalize_agent(new_language)
-                return (
-                    new_agent,
-                    new_language,
-                    greeting,
-                )
-            
-            lang_selector.change(
-                fn=clear_chat_immediate,
-                outputs=[chat.chatbot_value],
-                queue=True,
-            )
-
-            lang_selector.change(
-                fn=on_lang_change,
-                inputs=[lang_selector],
-                outputs=[agent_state, lang_state, chat.chatbot_value],
-                queue=True,
-            )
-
-            reset_button.click(
-                fn=clear_chat_immediate,
-                outputs=[chat.chatbot_value],
-                queue=True,
-            )
-
-            reset_button.click(
-                fn=switch_language,
-                inputs=[lang_state],
-                outputs=[agent_state, lang_state, chat.chatbot_value],
-                queue=True,
-            )
-            
-            # Initialize the agent chain on the app startup
-            self._app.load(
-                fn=lambda: initalize_agent(language),
-                outputs=[agent_state, chat.chatbot_value],
-            )
-
-    @property
-    def app(self) -> gr.Blocks:
-        """Expose underlying Gradio Blocks for external runners (e.g., HF Spaces)."""
-        return self._app
-
-    def _chat(self, message: str, history: list[dict], agent: ExecutiveAgentChain):
-        if agent is None:
-            logger.error("Agent not initialized")
-            return ["I apologize, but the chatbot is not properly initialized. Please refresh the page or contact support."]
-        
-        answers = []
-        try:
-            # Log user input
-            logger.info(f"Processing user query: {message[:100]}...")
-            
-            # Query agent (now includes input handling, scope checking, and formatting)
-            response = agent.query(query=message)
-            
-            logger.info(f"Received and formatted response from agent ({len(response)} chars)")
-            answers.append(response)
-            
-        except Exception as e:
-            logger.error(f"Error processing query: {e}", exc_info=True)
-            
-            # Provide helpful error message instead of empty string
-            error_message = (
-                "I apologize, but I encountered an error processing your request. "
-                "Please try rephrasing your question or contact our admissions team for assistance."
-            )
-            answers.append(error_message)
-        
-        return answers
-
 
-    def run(self):
-        self._app.launch(
-            share=os.getenv("GRADIO_SHARE", "false").lower() == "true",
-            server_name=os.getenv("SERVER_NAME", "0.0.0.0"),
-            server_port=int(os.getenv("PORT", 7860)),
-        )
+if __name__ == "__main__":
+	load_dotenv()
+	init_logging(interactive_mode=False)
+	ChatbotApplication("de").run()
 
-if __name__ == '__main__':
-    app = ChatbotApplication('de')
-    app.run()

config.py

@@ -123,17 +123,21 @@ class LLMProviderConfiguration:
 # Weaviate database settings 
 class WeaviateConfiguration:
     LOCAL_DATABASE = False
-    WEAVIATE_BACKUP_BACKEND = ''
     WEAVIATE_COLLECTION_BASENAME = 'hsg_rag_content'
     
+    # Weaviate backup settings
+    AVAILABLE_BACKUP_METHODS = ['manual', 'filesystem', 's3']
+    BACKUP_METHOD = 'manual'
+    BACKUP_PATH = os.getenv('WEAVIATE_BACKUP_PATH')
+
     # Weaviate Cloud settings
-    CLUSTER_URL = "r2vd9fuvrcjvx7idsvta.c0.europe-west3.gcp.weaviate.cloud"
+    CLUSTER_URL = os.getenv('WEAVIATE_CLUSTER_URL')
     WEAVIATE_API_KEY = os.getenv('WEAVIATE_API_KEY')
     HUGGING_FACE_API_KEY = os.getenv('HUGGING_FACE_API_KEY')
     
     # Custom timeouts for Cloud connection (in seconds)
-    INIT_TIMEOUT   = 10 
-    QUERY_TIMEOUT  = 3 
+    INIT_TIMEOUT   = 90 
+    QUERY_TIMEOUT  = 10 
     INSERT_TIMEOUT = 120
 
     @classmethod 
@@ -167,9 +171,13 @@ MAX_RESPONSE_WORDS_LEAD = 100  # Maximum words for lead agent responses
 MAX_RESPONSE_WORDS_SUBAGENT = 200  # Maximum words for subagent responses
 ENABLE_RESPONSE_CHUNKING = True  # Break long responses into multiple turns
 
+# Evaluation of agent response quality 
+ENABLE_EVALUATE_RESPONSE_QUALITY = True
+
 # Conversation state settings
 TRACK_USER_PROFILE = True  # Track user preferences and avoid repetition
 LOCK_LANGUAGE_AFTER_FIRST_MESSAGE = True  # Don't change language mid-conversation
+MAX_CONVERSATION_TURNS = 15 # End conversation after max turns reached
 
 # Data processing pipeline settings 
 CHUNK_MAX_TOKENS = 8191

src/apps/chat/app.py

@@ -1,19 +1,22 @@
 import os
 import gradio as gr
-from src.rag.agent_chain import ExecutiveAgentChain 
-from src.utils.logging import get_logger, cached_log_handler
+from src.apps.chat.js import JS_LISTENER, JS_CLEAR
+from src.const.agent_response_constants import *
+from src.rag.agent_chain import ExecutiveAgentChain
+from src.rag.utilclasses import LeadAgentQueryResponse
+from src.utils.logging import get_logger
 
 logger = get_logger("chatbot_app")
 
 class ChatbotApplication:
     def __init__(self, language: str = 'de') -> None:
-        self._app = gr.Blocks()
-     
+        self._app = gr.Blocks(js=JS_LISTENER)
+        self._language = language
+
         with self._app:
-            # Initial state variables
             agent_state = gr.State(None)
-            lang_state  = gr.State(language)
-            
+            lang_state = gr.State(language)
+
             with gr.Row():
                 lang_selector = gr.Radio(
                     choices=["Deutsch", "English"],
@@ -22,9 +25,9 @@ class ChatbotApplication:
                     interactive=True,
                 )
                 reset_button = gr.Button("Reset Conversation")
-            
+
             chat = gr.ChatInterface(
-                fn=lambda msg, history, agent: self._chat( 
+                fn=lambda msg, history, agent: self._chat(
                     message=msg,
                     history=history,
                     agent=agent,
@@ -34,14 +37,19 @@ class ChatbotApplication:
                 type='messages',
             )
 
+            iframe_container = gr.HTML(
+                value="",
+                elem_id="consultation-iframe-container",
+                visible=True
+            )
 
             def clear_chat_immediate():
-                return []
-            
+                return [], ""
+
             def on_lang_change(language):
                 lang_code = 'en' if language == 'English' else 'de'
                 return switch_language(lang_code)
-            
+
             def initalize_agent(language):
                 agent = ExecutiveAgentChain(language=language)
                 greeting = agent.generate_greeting()
@@ -53,37 +61,40 @@ class ChatbotApplication:
                     new_agent,
                     new_language,
                     greeting,
+                    ""
                 )
-            
+
             lang_selector.change(
                 fn=clear_chat_immediate,
-                outputs=[chat.chatbot_value],
+                outputs=[chat.chatbot_value, iframe_container],
                 queue=True,
+                js=JS_CLEAR
             )
 
             lang_selector.change(
                 fn=on_lang_change,
                 inputs=[lang_selector],
-                outputs=[agent_state, lang_state, chat.chatbot_value],
+                outputs=[agent_state, lang_state, chat.chatbot_value, iframe_container],
                 queue=True,
             )
 
             reset_button.click(
                 fn=clear_chat_immediate,
-                outputs=[chat.chatbot_value],
+                outputs=[chat.chatbot_value, iframe_container],
                 queue=True,
+                js=JS_CLEAR
             )
 
             reset_button.click(
                 fn=switch_language,
                 inputs=[lang_state],
-                outputs=[agent_state, lang_state, chat.chatbot_value],
+                outputs=[agent_state, lang_state, chat.chatbot_value, iframe_container],
                 queue=True,
             )
-            
+
             # Initialize the agent chain on the app startup
             self._app.load(
-                fn=lambda: initalize_agent(language),
+                fn=lambda: initalize_agent(self._language),
                 outputs=[agent_state, chat.chatbot_value],
             )
 
@@ -95,31 +106,28 @@ class ChatbotApplication:
     def _chat(self, message: str, history: list[dict], agent: ExecutiveAgentChain):
         if agent is None:
             logger.error("Agent not initialized")
-            return ["I apologize, but the chatbot is not properly initialized. Please refresh the page or contact support."]
-        
+            return ["I apologize, but the chatbot is not properly initialized."]
+
         answers = []
         try:
-            # Log user input
             logger.info(f"Processing user query: {message[:100]}...")
-            
-            # Query agent (now includes input handling, scope checking, and formatting)
-            response = agent.query(query=message)
-            
-            logger.info(f"Received and formatted response from agent ({len(response)} chars)")
-            answers.append(response)
-            
+
+            lead_resp: LeadAgentQueryResponse = agent.query(query=message)
+            answers.append(lead_resp.response)
+            self._language = lead_resp.language
+
+            if lead_resp.confidence_fallback or lead_resp.max_turns_reached:
+                answers.extend(APPOINTMENT_LINKS[self._language])
+
         except Exception as e:
             logger.error(f"Error processing query: {e}", exc_info=True)
-            
-            # Provide helpful error message instead of empty string
             error_message = (
                 "I apologize, but I encountered an error processing your request. "
                 "Please try rephrasing your question or contact our admissions team for assistance."
             )
             answers.append(error_message)
-        
-        return answers
 
+        return answers
 
     def run(self):
         self._app.launch(

src/apps/chat/js.py

@@ -0,0 +1,39 @@
+JS_LISTENER = """
+function() {
+    document.addEventListener('click', function(e) {
+        // 1. Use .closest() to find the <a> tag even if user clicks the text/icon inside it
+        const target = e.target.closest('a.appointment-btn');
+
+        if (target) {
+            // 2. Prevent the link from opening in a new tab/window
+            e.preventDefault();
+
+            // 3. Get the URL from the standard href attribute
+            const url = target.getAttribute('href');
+            const container = document.getElementById('consultation-iframe-container');
+
+            if (container) {
+                container.innerHTML = `
+                    <div style="margin-top: 20px; border: 1px solid #e5e7eb; border-radius: 8px; overflow: hidden;">
+                        <div style="background: #f9fafb; padding: 10px; font-weight: bold; border-bottom: 1px solid #e5e7eb; display: flex; justify-content: space-between;">
+                            <span>Appointment Booking</span>
+                            <button onclick="document.getElementById('consultation-iframe-container').innerHTML=''" style="cursor: pointer; color: red;">✕ Close</button>
+                        </div>
+                        <iframe src="${url}" width="100%" height="600px" frameborder="0"></iframe>
+                    </div>
+                `;
+                container.scrollIntoView({ behavior: 'smooth' });
+            }
+        }
+    });
+}
+"""
+
+JS_CLEAR = """
+function() {
+    const el = document.getElementById('consultation-iframe-container');
+    if (el) {
+        el.innerHTML = '';
+    }
+}
+"""

src/const/agent_response_constants.py

@@ -0,0 +1,140 @@
+from gradio import ChatMessage
+
+""" Constants for Gradio app """
+
+GREETING_MESSAGES = {
+        "en": [
+            "Hello and welcome! I’m your Executive Education Advisor for the HSG Executive MBA programs (**IEMBA**, **emba X**, and **EMBA**). How can I best support your MBA planning today?",
+            "Hello and welcome! I’m your Executive Education Advisor for the University of St.Gallen’s Executive MBA programs (**IEMBA**, **emba X**, **EMBA**). How can I support your MBA planning today?",
+            "Hello and welcome! I’m your Executive Education Advisor for the HSG Executive MBA programs (**EMBA**, **IEMBA**, **emba X**). How can I help you with your EMBA journey today?",
+            "Hello and welcome! I’m your Executive Education Advisor for the University of St.Gallen’s EMBA programs, here to help you navigate our **EMBA**, **IEMBA**, and **emba X** options.",
+            "Hello and welcome. I’m your Executive Education Advisor for the University of St.Gallen’s Executive MBA programs, here to help you assess fit and navigate the **EMBA**, **IEMBA**, and **emba X** options.",
+        ],
+        "de": [
+            "Guten Tag! Ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme und unterstütze Sie gerne bei Fragen zu **EMBA**, **IEMBA** und **emba X**.",
+            "Guten Tag, ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme (**EMBA**, **IEMBA**, **emba X**). Ich unterstütze Sie bei Programmwahl, Ablauf und Zulassungsfragen.",
+            "Guten Tag und herzlich willkommen! Ich bin Ihr Executive Education Advisor für die HSG Executive MBA Programme und unterstütze Sie gern bei Fragen zu **EMBA**, **IEMBA** und **emba X**.",
+            "Guten Tag, ich bin Ihr Executive-Education-Berater für die HSG Executive MBA-Programme (**EMBA**, **IEMBA**, **emba X**) und unterstütze Sie gerne bei Programmwahl und Zulassungsfragen.",
+            "Guten Tag! Ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme (**EMBA**, **IEMBA**, **emba X**) und unterstütze Sie gerne bei Programmwahl und Zulassungsfragen.",
+        ]
+}
+
+QUERY_EXCEPTION_MESSAGE = {
+    "en": "I'm sorry, I cannot provide a helpful response right now. Please contact tech support or try again later.",
+    "de": "Es tut mir leid, ich kann im Moment keine hilfreiche Antwort geben. Bitte wenden Sie sich an den technischen Support oder versuchen Sie es später erneut.",
+}
+
+NOT_VALID_QUERY_MESSAGE = {
+    "en": "I didn't quite understand that. Could you please rephrase your question?",
+    "de": "Das habe ich nicht ganz verstanden. Könnten Sie Ihre Frage bitte anders formulieren?",
+}
+
+CONFIDENCE_FALLBACK_MESSAGE = {
+    "en": (
+        "I'm sorry, but I couldn't find any information in my records that matches your request, "
+        "so I can't answer it with confidence. Could you please rephrase your question?\n\n"
+        "Alternatively, you can book an appointment with a student services advisor using the links below."
+    ),
+    "de": (
+        "Es tut mir leid, aber ich konnte in meinen Unterlagen keine Informationen finden, "
+        "die zu Ihrer Anfrage passen, sodass ich sie nicht mit ausreichender Sicherheit beantworten kann. "
+        "Könnten Sie Ihre Frage bitte umformulieren?\n\n"
+        "Alternativ können Sie über die untenstehenden Links einen Termin bei der Studienberatung buchen."
+    ),
+}
+
+LANGUAGE_FALLBACK_MESSAGE = {
+    "en": (
+        "I am sorry, I can only reply in English or German. "
+        "Would you like to continue our conversation in English?"
+    ),
+    "de": (
+        "Es tut mir leid, ich kann nur auf Englisch oder Deutsch antworten. "
+        "Möchten Sie unser Gespräch auf Deutsch fortführen?"
+    ),
+}
+
+CONVERSATION_END_MESSAGE = {
+    "en": (
+        "This conversation has reached its maximum length. "
+        "To make sure you receive the best possible support, "
+        "please continue with a personal consultation.\n\n"
+        "You can book an appointment with a student services advisor using the links below. "
+        "Thank you for your understanding."
+    ),
+    "de": (
+        "Dieses Gespräch hat die maximale Länge erreicht. "
+        "Damit Sie bestmöglich unterstützt werden, bitten wir Sie, "
+        "das Anliegen in einem persönlichen Beratungsgespräch fortzusetzen.\n\n"
+        "Über die untenstehenden Links können Sie einen Termin mit der Studienberatung buchen. "
+        "Vielen Dank für Ihr Verständnis."
+    ),
+}
+
+
+def create_appt_button(url, title, lang_text):
+    return (
+        f'<a href="{url}" class="appointment-btn" '
+        f'style="display: block; background-color: #f3f4f6; border: 1px solid #d1d5db; '
+        f'padding: 8px 16px; border-radius: 6px; cursor: pointer; '
+        f'color: #374151; font-weight: 600; width: 100%; text-align: left; '
+        f'margin-top: 5px; text-decoration: none;">'
+        f'📅 {lang_text}: {title}'
+        f'</a>'
+    )
+
+
+APPOINTMENT_LINKS = {
+    "en": [
+        ChatMessage(
+            role="assistant",
+            content=create_appt_button(
+                "https://calendly.com/cyra-vonmueller/beratungsgespraech-emba-hsg",
+                "Cyra von Müller",
+                "Book Appointment"
+            ),
+        ),
+        ChatMessage(
+            role="assistant",
+            content=create_appt_button(
+                "https://calendly.com/kristin-fuchs-unisg/iemba-online-personal-consultation",
+                "Kristin Fuchs",
+                "Book Appointment"
+            ),
+        ),
+        ChatMessage(
+            role="assistant",
+            content=create_appt_button(
+                "https://calendly.com/teyuna-giger-unisg",
+                "Teyuna Giger",
+                "Book Appointment"
+            ),
+        ),
+    ],
+    "de": [
+        ChatMessage(
+            role="assistant",
+            content=create_appt_button(
+                "https://calendly.com/cyra-vonmueller/beratungsgespraech-emba-hsg",
+                "Cyra von Müller",
+                "Termin buchen"
+            ),
+        ),
+        ChatMessage(
+            role="assistant",
+            content=create_appt_button(
+                "https://calendly.com/kristin-fuchs-unisg/iemba-online-personal-consultation",
+                "Kristin Fuchs",
+                "Termin buchen"
+            ),
+        ),
+        ChatMessage(
+            role="assistant",
+            content=create_appt_button(
+                "https://calendly.com/teyuna-giger-unisg",
+                "Teyuna Giger",
+                "Termin buchen"
+            ),
+        ),
+    ],
+}

src/database/weavservice.py

@@ -99,9 +99,13 @@ class WeaviateService:
                     if wvtconf.is_local():
                         self._client = wvt.connect_to_local()
                         break
+                    
+                    cluster_url = wvtconf.CLUSTER_URL
+                    if not cluster_url.startswith('http'):
+                        cluster_url = f"https://{cluster_url}"
 
                     self._client = wvt.connect_to_weaviate_cloud(
-                        cluster_url=wvtconf.CLUSTER_URL,
+                        cluster_url=cluster_url,
                         auth_credentials=wvtconf.WEAVIATE_API_KEY,
                         additional_config=AdditionalConfig(
                             timeout=Timeout(

src/rag/agent_chain.py

@@ -1,51 +1,64 @@
+from langchain_core.runnables import RunnableConfig
+from langsmith import traceable
 from langchain.tools import tool
 from langchain.agents import create_agent
 from langchain_core.messages import (
-    HumanMessage, 
-    AIMessage, 
-    SystemMessage, 
+    HumanMessage,
+    AIMessage,
+    SystemMessage,
 )
 from langchain.agents.middleware import ModelFallbackMiddleware
+from langchain.agents.structured_output import ProviderStrategy
 
 import uuid
 import json
 import os
 import re
+import random
 from datetime import datetime
 
 from src.database.weavservice import WeaviateService
 
 from src.rag.utilclasses import *
+from src.const.agent_response_constants import *
 from src.rag.middleware import AgentChainMiddleware as chainmdw
 from src.rag.prompts import PromptConfigurator as promptconf
 from src.rag.models import ModelConfigurator as modelconf
 from src.rag.input_handler import InputHandler
 from src.rag.response_formatter import ResponseFormatter
 from src.rag.scope_guardian import ScopeGuardian
+from src.rag.quality_score_handler import QualityEvaluationResult, QualityScoreHandler
+from src.rag.language_detection import LanguageDetector
 
-from src.utils.lang import detect_language, get_language_name
-from src.utils.logging import get_logger 
+from src.utils.logging import get_logger
+from src.utils.lang import get_language_name
 from config import (
     TOP_K_RETRIEVAL,
-    LOCK_LANGUAGE_AFTER_FIRST_MESSAGE,
     TRACK_USER_PROFILE,
-    ENABLE_RESPONSE_CHUNKING
+    ENABLE_RESPONSE_CHUNKING,
+    ENABLE_EVALUATE_RESPONSE_QUALITY, 
+    MAX_CONVERSATION_TURNS,
 )
 
 chain_logger = get_logger('agent_chain')
 
+
 class ExecutiveAgentChain:
     def __init__(self, language: str = 'en') -> None:
-        self._initial_language = language
-        self._language = language
-        self._user_language = None  # Will be locked after first user message
+        self._initial_language  = language
+        self._stored_language = language
         self._dbservice = WeaviateService()
         self._agents, self._config = self._init_agents()
         self._conversation_history = []
         
+        # AI-middlewares
+        if ENABLE_EVALUATE_RESPONSE_QUALITY:
+            self._quality_handler = QualityScoreHandler()
+        self._language_detector = LanguageDetector()
+
         # Generate unique user ID for this session
         self._user_id = str(uuid.uuid4())
-        
+
         # Initialize conversation state with user profile tracking
         self._conversation_state: ConversationState = {
             'user_id': self._user_id,
@@ -62,12 +75,11 @@ class ExecutiveAgentChain:
             'topics_discussed': [],
             'preferences_known': False
         }
-        
+
         # Track scope violations for escalation
         self._scope_violation_count = 0
-        
-        chain_logger.info(f"Initialized new Agent Chain for language '{language}' with user_id: {self._user_id}")
 
+        chain_logger.info(f"Initialized new Agent Chain for language '{language}' with user_id: {self._user_id}")
 
     def _retrieve_context(self, query: str, language: str = None):
         """
@@ -77,18 +89,17 @@ class ExecutiveAgentChain:
             query: Keywords depicting information you want to retrieve in the primary language. 
             language: Optional parameter (either 'en' for English language or 'de' for German language). This parameter selects the language of the database to query from. The input query must be written in the same language as the selected language. Use this parameter only if there's not enough information in your main language.
         """
-        lang = language or self._language
+        lang = language if language in ['en', 'de'] else self._initial_language
         try:
             response, _ = self._dbservice.query(
-                query=query, 
-                lang=lang, 
+                query=query,
+                lang=lang,
                 limit=TOP_K_RETRIEVAL,
             )
             serialized = '\n\n'.join([doc.properties.get('body', '') for doc in response.objects])
             return serialized
         except Exception as e:
             raise e
-   
 
     def _call_emba_agent(self, query: str) -> str:
         """
@@ -98,17 +109,16 @@ class ExecutiveAgentChain:
             query: Query to the EMBA support agent. Provide collected user data in the query if possible.
         """
         try:
-            response = self._query(
-                agent=self._agents['emba'], 
+            structured_response = self._query(
+                agent=self._agents['emba'],
                 messages=[HumanMessage(query)],
                 thread_id=f"emba_{hash(query)}",
             )
-            return response
+            return structured_response.response
         except Exception as e:
             chain_logger.error(f"EMBA Agent error: {e}")
             raise RuntimeError("Unable to retrieve EMBA information at this time.")
 
-
     def _call_iemba_agent(self, query: str) -> str:
         """
         Invokes the IEMBA support agent to retrieve more detailed information about the IEMBA program.
@@ -117,17 +127,16 @@ class ExecutiveAgentChain:
             query: Query to the IEMBA support agent. Provide collected user data in the query if possible.
         """
         try:
-            response = self._query(
-                agent=self._agents['iemba'], 
+            structured_response = self._query(
+                agent=self._agents['iemba'],
                 messages=[HumanMessage(query)],
                 thread_id=f"emba_{hash(query)}",
             )
-            return response
+            return structured_response.response
         except Exception as e:
             chain_logger.error(f"IEMBA Agent error: {e}")
             raise RuntimeError("Unable to retrieve IEMBA information at this time.")
 
-
     def _call_embax_agent(self, query: str) -> str:
         """
         Invokes the emba X support agent to retrieve more detailed information about the emba X program.
@@ -136,17 +145,16 @@ class ExecutiveAgentChain:
             query: Query to the emba X support agent. Provide collected user data in the query if possible.
         """
         try:
-            response = self._query(
-                agent=self._agents['embax'], 
+            structured_response = self._query(
+                agent=self._agents['embax'],
                 messages=[HumanMessage(query)],
                 thread_id=f"emba_{hash(query)}",
             )
-            return response
+            return structured_response.response
         except Exception as e:
             chain_logger.error(f"emba X Agent error: {e}")
             raise RuntimeError("Unable to retrieve emba X information at this time.")
 
-
     def _init_agents(self):
         config: RunnableConfig = {
             'configurable': {'thread_id': 0}
@@ -186,22 +194,25 @@ class ExecutiveAgentChain:
                 model=modelconf.get_main_agent_model(),
                 tools=tools_agent_calling,
                 state_schema=LeadInformationState,
-                system_prompt=promptconf.get_configured_agent_prompt('lead', language=self._language),
+                system_prompt=promptconf.get_configured_agent_prompt('lead', language=self._initial_language),
                 middleware=[
                     chainmdw.get_tool_wrapper(),
                     chainmdw.get_model_wrapper(),
                     fallback_middleware,
                 ],
                 context_schema=AgentContext,
-            ),            
+                response_format=ProviderStrategy(
+                    StructuredAgentResponse
+                ),
+            ),
         }
         for agent in ['emba', 'iemba', 'embax']:
-            agents[agent]=create_agent(
+            agents[agent] = create_agent(
                 name=f"{agent.upper()} Agent",
                 model=modelconf.get_subagent_model(),
                 tools=[tool_retrieve_context],
                 state_schema=LeadInformationState,
-                system_prompt=promptconf.get_configured_agent_prompt(agent, language=self._language),
+                system_prompt=promptconf.get_configured_agent_prompt(agent, language=self._initial_language),
                 middleware=[
                     fallback_middleware,
                     chainmdw.get_tool_wrapper(),
@@ -210,7 +221,7 @@ class ExecutiveAgentChain:
                 context_schema=AgentContext,
             )
         return agents, config
-   
+
     def _extract_experience_years(self, conversation: str) -> int | None:
         """Extract years of professional experience from conversation text."""
         # Look for patterns like "10 years", "5 years experience", etc.
@@ -243,7 +254,7 @@ class ExecutiveAgentChain:
         """Extract professional field/industry from conversation text."""
         # Common fields mentioned in executive education
         fields = [
-            'finance', 'banking', 'technology', 'tech', 'IT', 'healthcare', 
+            'finance', 'banking', 'technology', 'tech', 'IT', 'healthcare',
             'consulting', 'manufacturing', 'retail', 'marketing', 'sales',
             'engineering', 'pharma', 'telecommunications', 'energy',
             'Finanzwesen', 'Technologie', 'Gesundheitswesen', 'Beratung'  # German
@@ -264,8 +275,8 @@ class ExecutiveAgentChain:
             'Strategie', 'Innovation', 'Führung', 'Digitalisierung'  # German
         ]
         conversation_lower = conversation.lower()
-        found_interests = [interest for interest in interests 
-                          if interest.lower() in conversation_lower]
+        found_interests = [interest for interest in interests
+                           if interest.lower() in conversation_lower]
         return ', '.join(found_interests) if found_interests else None
 
     def _extract_name(self, conversation: str) -> str | None:
@@ -301,15 +312,15 @@ class ExecutiveAgentChain:
     def _determine_suggested_program(self) -> str | None:
         """Determine recommended program based on user profile."""
         state = self._conversation_state
-        
+
         # If program interest was explicitly mentioned
         if state['program_interest']:
             return state['program_interest'][0]
-        
+
         # Make recommendation based on profile
         experience = state.get('experience_years', 0) or 0
         leadership = state.get('leadership_years', 0) or 0
-        
+
         # EMBA: 5+ years experience, 2+ years leadership
         if experience >= 5 and leadership >= 2:
             return 'EMBA'
@@ -317,64 +328,64 @@ class ExecutiveAgentChain:
         elif experience >= 3:
             return 'IEMBA'
         # EMBA X: Digital/Innovation focus
-        elif state.get('interest') and any(kw in state.get('interest', '').lower() 
+        elif state.get('interest') and any(kw in state.get('interest', '').lower()
                                            for kw in ['digital', 'innovation', 'technology']):
             return 'EMBA X'
-        
+
         return None
 
     def _update_conversation_state(self, user_query: str, agent_response: str) -> None:
         """Update conversation state by extracting information from the conversation."""
         if not TRACK_USER_PROFILE:
             return
-        
+
         # Combine query and response for analysis
         conversation_text = f"{user_query} {agent_response}"
-        
+
         # Extract profile information
         if not self._conversation_state.get('experience_years'):
             exp_years = self._extract_experience_years(conversation_text)
             if exp_years:
                 self._conversation_state['experience_years'] = exp_years
                 chain_logger.info(f"Extracted experience years: {exp_years}")
-        
+
         if not self._conversation_state.get('leadership_years'):
             lead_years = self._extract_leadership_years(conversation_text)
             if lead_years:
                 self._conversation_state['leadership_years'] = lead_years
                 chain_logger.info(f"Extracted leadership years: {lead_years}")
-        
+
         if not self._conversation_state.get('field'):
             field = self._extract_field(conversation_text)
             if field:
                 self._conversation_state['field'] = field
                 chain_logger.info(f"Extracted field: {field}")
-        
+
         if not self._conversation_state.get('interest'):
             interest = self._extract_interest(conversation_text)
             if interest:
                 self._conversation_state['interest'] = interest
                 chain_logger.info(f"Extracted interest: {interest}")
-        
+
         # Extract name
         if not self._conversation_state.get('user_name'):
             name = self._extract_name(conversation_text)
             if name:
                 self._conversation_state['user_name'] = name
                 chain_logger.info(f"Extracted name: {name}")
-        
+
         # Detect handover request
         if self._detect_handover_request(conversation_text):
             self._conversation_state['handover_requested'] = True
             chain_logger.info("Handover request detected")
-        
+
         # Check for program mentions
         programs = ['EMBA', 'IEMBA', 'EMBA X']
         for program in programs:
             if program.lower() in conversation_text.lower():
                 if program not in self._conversation_state['program_interest']:
                     self._conversation_state['program_interest'].append(program)
-        
+
         # Update suggested program
         suggested = self._determine_suggested_program()
         if suggested and not self._conversation_state.get('suggested_program'):
@@ -385,12 +396,12 @@ class ExecutiveAgentChain:
         """Log user profile to JSON file."""
         if not TRACK_USER_PROFILE:
             return
-        
+
         try:
             # Create logs directory if it doesn't exist
             log_dir = os.path.join('logs', 'user_profiles')
             os.makedirs(log_dir, exist_ok=True)
-            
+
             # Create profile data
             profile_data = {
                 'user_id': self._conversation_state['user_id'],
@@ -405,34 +416,26 @@ class ExecutiveAgentChain:
                 'user_language': self._conversation_state.get('user_language'),
                 'program_interest': self._conversation_state.get('program_interest', []),
             }
-            
+
             # Log file path with timestamp
             timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
             log_file = os.path.join(log_dir, f'profile_{self._user_id}_{timestamp}.json')
-            
+
             # Write to file
             with open(log_file, 'w', encoding='utf-8') as f:
                 json.dump(profile_data, f, indent=2, ensure_ascii=False)
-            
+
             chain_logger.info(f"User profile logged to {log_file}")
-            
+
         except Exception as e:
             chain_logger.error(f"Failed to log user profile: {e}")
 
     def generate_greeting(self) -> str:
-        self._conversation_history.extend([
-            SystemMessage("Generate a short greeting message and introduce yourself. 30 words max."),
-            SystemMessage(f"Respond in {get_language_name(self._language)} language."),
-        ])
-        response = self._query(
-            agent=self._agents['lead'], 
-            messages=self._conversation_history,
-        )
-        self._conversation_history.append(AIMessage(response))
-        return response
-
+        greeting_message = random.choice(GREETING_MESSAGES[self._stored_language])
+        return greeting_message
 
-    def query(self, query: str) -> str:
+    @traceable
+    def query(self, query: str) -> LeadAgentQueryResponse:
         """
         Process user query with input handling, scope checking, and response formatting.
         
@@ -442,44 +445,47 @@ class ExecutiveAgentChain:
         Returns:
             Formatted response
         """
+        # Select fallback language if language was changed by previous query
+        response_language = self._stored_language 
+
+        if len(self._conversation_history) >= MAX_CONVERSATION_TURNS * 2:
+            return LeadAgentQueryResponse(
+                response = CONVERSATION_END_MESSAGE[response_language],
+                language = response_language,
+                max_turns_reached = True,
+            ) 
+
         # Step 1: Process input (handle numeric inputs, validation)
         processed_query, is_valid = InputHandler.process_input(
             query,
             [msg for msg in self._conversation_history if isinstance(msg, (HumanMessage, AIMessage))]
         )
-        
+
         if not is_valid or not processed_query:
             chain_logger.warning(f"Invalid input received: '{query}'")
-            return "I didn't quite understand that. Could you please rephrase your question?"
-        
+            return LeadAgentQueryResponse(
+                response=NOT_VALID_QUERY_MESSAGE[self._stored_language],
+                language=response_language,
+            )
+
         # Log if input was interpreted
         if processed_query != query:
             chain_logger.info(f"Interpreted input '{query}' as '{processed_query}'")
-        
-        # Step 2: Lock language on first user message
-        if LOCK_LANGUAGE_AFTER_FIRST_MESSAGE and self._user_language is None:
-            self._user_language = detect_language(processed_query)
-            self._conversation_state['user_language'] = self._user_language
-            self._language = self._user_language
-            chain_logger.info(f"Locked conversation language to '{self._user_language}'")
-        
-        # Use locked language or current language
-        response_language = self._user_language or self._language
-        
-        # Step 3: Check scope before querying agent
+
+        # Step 2: Check scope before querying agent
         scope_type = ScopeGuardian.check_scope(processed_query, response_language)
-        
+
         if scope_type != 'on_topic':
             chain_logger.info(f"Out-of-scope query detected: {scope_type}")
             self._scope_violation_count += 1
-            
+
             # Check if should escalate
             should_escalate, escalation_type = ScopeGuardian.should_escalate(
                 processed_query,
                 scope_type,
                 self._scope_violation_count
             )
-            
+
             if should_escalate:
                 redirect_msg = ScopeGuardian.get_escalation_message(
                     escalation_type,
@@ -490,71 +496,116 @@ class ExecutiveAgentChain:
                     scope_type,
                     response_language
                 )
-            
+
             # Add to history
             self._conversation_history.append(HumanMessage(processed_query))
             self._conversation_history.append(AIMessage(redirect_msg))
-            
-            return redirect_msg
-        
+
+            return LeadAgentQueryResponse(
+                response=redirect_msg,
+                language=response_language,
+            )
+
         # Reset violation count on valid topic
         self._scope_violation_count = 0
-        
-        # Step 4: Build messages with locked language
+       
+        # Append user query to conversation history before querying 
         self._conversation_history.append(HumanMessage(processed_query))
-        
-        # Add language instruction (use locked language)
-        language_instruction = SystemMessage(
-            f"Respond in {get_language_name(response_language)} language."
-        )
-        
+
+        # Step 3: Detect query language using the language detector
+        detected_language = self._language_detector.detect_language(processed_query)
+        chain_logger.info(f"Detected query language: {detected_language}")
+        self._conversation_state['user_language'] = detected_language
+        
+        # Store the query language if it's valid; return fallback message otherwise
+        if detected_language in ['de', 'en']:
+            self._stored_language = detected_language
+            response_language = detected_language
+        else:
+            chain_logger.info("User query is not in a valid language, switching to fallback message...")
+            fallback_message = LANGUAGE_FALLBACK_MESSAGE[response_language]
+            self._conversation_history.append(AIMessage(fallback_message))
+
+            return LeadAgentQueryResponse(
+                response=fallback_message,
+                language=response_language,
+            )
+
+        # Step 4: Build messages with locked language
+        language_instruction = SystemMessage(f"Respond in {get_language_name(response_language)} language.")
+
         # Step 5: Query agent
-        response = self._query(
+        structured_response = self._query(
             agent=self._agents['lead'],
-            messages=self._conversation_history + [language_instruction],
+            messages=self._conversation_history + [language_instruction], 
         )
-        
+        agent_response = structured_response.response
+
         # Step 6: Format response (remove tables, chunk if needed)
         if ENABLE_RESPONSE_CHUNKING:
             formatted_response = ResponseFormatter.format_response(
-                response,
+                agent_response,
                 agent_type='lead',
                 enable_chunking=True
             )
         else:
-            formatted_response = ResponseFormatter.remove_tables(response)
-        
+            formatted_response = ResponseFormatter.remove_tables(agent_response)
+
         # Clean up response
         formatted_response = ResponseFormatter.clean_response(formatted_response)
-        
+
+        # Step 7: Language fallback mechanisms and response quality evaluation
+        confidence_fallback = False
+        if ENABLE_EVALUATE_RESPONSE_QUALITY:
+            quality_evaluation: QualityEvaluationResult = self._quality_handler. \
+                evaluate_response_quality(query, formatted_response)
+            chain_logger.info(f"Lead agent response recieved quality score of {quality_evaluation.overall_score:1.2f}")
+
+            if quality_evaluation.overall_score < 0.3:
+                confidence_fallback = True
+                formatted_response = CONFIDENCE_FALLBACK_MESSAGE[response_language]
+             
         # Add to history
         self._conversation_history.append(AIMessage(formatted_response))
-        
-        # Step 7: Update conversation state and log profile if tracking is enabled
+
+        # Step 8: Update conversation state and log profile if tracking is enabled
         if TRACK_USER_PROFILE:
             self._update_conversation_state(processed_query, formatted_response)
             # Log profile every 5 messages or when program is suggested
             message_count = len([m for m in self._conversation_history if isinstance(m, HumanMessage)])
-            if (message_count % 5 == 0 or 
-                self._conversation_state.get('suggested_program')):
+            if (message_count % 5 == 0 or self._conversation_state.get('suggested_program')):
                 self._log_user_profile()
-        
-        return formatted_response
 
+        return LeadAgentQueryResponse(
+            response = formatted_response,
+            language = response_language,
+            confidence_fallback = confidence_fallback,
+        )
 
-    def _query(self, agent, messages: list, thread_id: str = None) -> str:
+    def _query(self, agent, messages: list, thread_id: str = None) -> StructuredAgentResponse:
         try:
             config = self._config.copy()
             config['configurable']['thread_id'] = thread_id or 0
-                
+
             result: AIMessage = agent.invoke(
                 {"messages": messages},
                 config=config,
                 context=AgentContext(agent_name=agent.name),
             )
-            response = result['messages'][-1]
-            return response.text
+            response = result.get(
+                'structured_response',
+                StructuredAgentResponse(
+                    response=result['messages'][-1].text,
+                    confidence_score=0.5,
+                    language=self._initial_language,
+                )
+            )
+            return response
         except Exception as e:
             error_msg = e.body['message'] if hasattr(e, 'body') else str(e)
             chain_logger.error(f"Failed to invoke the agent: {error_msg}")
-            return "I'm sorry, I cannot provide a helpful response right now. Please contact tech support or try again later."
+            return StructuredAgentResponse(
+                response=QUERY_EXCEPTION_MESSAGE[self._stored_language],
+                confidence_score=0.0,
+                language=self._initial_language,
+            )

src/rag/language_detection.py

@@ -0,0 +1,28 @@
+from pydantic import BaseModel, Field
+from langchain_core.messages import HumanMessage
+from src.rag.models import ModelConfigurator as modconf
+from src.rag.prompts import PromptConfigurator as promptconf
+
+from src.utils.logging import get_logger
+
+logger = get_logger('lang_detector')
+
+class LanguageDetectionResult(BaseModel):
+    language_code: str = Field(description="ISO language code (e.g., en, de, fa, ru) of the language in which the message is written")
+
+class LanguageDetector:
+    def __init__(self) -> None:
+        self._model = modconf.get_language_detector_model()
+        self._model = self._model.with_structured_output(LanguageDetectionResult)
+
+    def detect_language(self, query: str) -> str:
+        prompt = promptconf.get_language_detector_prompt(query)
+        messages = [HumanMessage(prompt)]
+
+        try:
+            result = self._model.invoke(messages)
+            return result.language_code
+        except Exception as e:
+            logger.error(f"Failed to detect language: {e}")
+            return ""
+

src/rag/models.py

@@ -10,7 +10,50 @@ class ModelConfigurator:
     _subagent_model_instance: BaseChatModel = None
     _fallback_models_instances: list[BaseChatModel] = None
     _summarization_model_instance: BaseChatModel = None
+    _confidence_scoring_model_instance: BaseChatModel = None 
+    _language_detector_model_instance: BaseChatModel = None
     
+    @classmethod 
+    def get_language_detector_model(cls) -> BaseChatModel:
+        if cls._confidence_scoring_model_instance:
+            return cls._confidence_scoring_model_instance
+        try:
+            from langchain_openai import ChatOpenAI
+            cls._language_detector_model_instance = ChatOpenAI(
+                model='gpt-4o-mini',
+                openai_api_key=llmconf.get_api_key(),
+                max_tokens=3072,
+                temperature=0.00,
+                timeout=60,
+                request_timeout=60,
+            )
+            logger.info(f"Initialized language detection model")
+            return cls._language_detector_model_instance
+        except Exception as e:
+            logger.error(f"Failed to initialize language detection model: {e}")
+            raise e
+
+    @classmethod
+    def get_confidence_scoring_model(cls) -> BaseChatModel:
+        if cls._confidence_scoring_model_instance:
+            return cls._confidence_scoring_model_instance
+        
+        try:
+            from langchain_openai import ChatOpenAI
+            cls._confidence_scoring_model_instance = ChatOpenAI(
+                model='gpt-4o-mini',
+                openai_api_key=llmconf.get_api_key(),
+                max_tokens=3072,
+                temperature=0.00,
+                timeout=60,
+                request_timeout=60,
+            )
+            logger.info(f"Initialized confidence scoring model")
+            return cls._confidence_scoring_model_instance
+        except Exception as e:
+            logger.error(f"Failed to initialize confidence scoring model: {e}")
+            raise e
+
 
     @classmethod
     def get_summarization_model(cls) -> BaseChatModel:

src/rag/prompts.py

@@ -73,6 +73,27 @@ Keep to 100 words max."""
     
     _SUMMARY_PREFIX_PROMPT = "Conversation Summary:"
 
+    _QUALITY_SCORING_PROMPT = """You are performing a quick evaluation of an AI response from an Executive Education Advisor agent for HSG EMBA, IEMBA and emba X programs. Rate the response on a scale 0.0-1.0 on these categories: format adherence, context awareness, pricing adherence, scope compliance and general rules. Deduct points for violations of the agent's guidelines.
+
+Rules for categories:
+- Format adherence: short paragraphs or bullet points, no tables, bold keywords, maximum 100 words.
+- Content awareness: focuses on programs listed in user query, single numbers in user query interpreted as years of experience.
+- Pricing adherence: Prices in range CHF 75'000 - 110'000, mentions included services, mentions Early Bird discount if possible, does not provide detailed financial planning, redirects to admissions team for detailed information.
+- Scope compliance: redirects to MBA if user query is off-topic, discusses only program details and admissions process, suggests contacting admissions team if possible.
+- General rules: no competitive MBA programs mentioned, no admission predictions, no marketing language or undefined claims; if Agent is uncertain, it should recommend contacting the admissions team.
+
+User query: {query}
+AI response: {response}"""
+
+    _LANGUAGE_DETECTOR_PROMPT = """Detect the language the user is writing in or explicitly requests to speak in, and return its ISO language code (e.g., en, de, fa, ru) in the language field.
+
+User query: {query}
+"""
+    
+    @classmethod 
+    def get_language_detector_prompt(cls, query):
+        return cls._LANGUAGE_DETECTOR_PROMPT.format(query=query)
+
     @classmethod
     def get_summarization_prompt(cls):
         return cls._SUMMARIZATION_PROMPT
@@ -101,3 +122,7 @@ Keep to 100 words max."""
                     program_name=agent.upper(),
                     selected_language=selected_language,
                 )
+
+    @classmethod
+    def get_quality_scoring_prompt(cls, query: str, response: str) -> str:
+        return cls._QUALITY_SCORING_PROMPT.format(query=query, response=response)

src/rag/quality_score_handler.py

@@ -0,0 +1,57 @@
+from pydantic import BaseModel, Field
+from langchain_core.messages import HumanMessage
+from langsmith import Client
+from src.rag.models import ModelConfigurator as modconf
+from src.rag.prompts import PromptConfigurator as promptconf
+
+from src.utils.logging import get_logger
+
+from time import perf_counter
+
+logger = get_logger('quality_score_handler')
+
+class QualityEvaluationResult(BaseModel):
+    """Result of response quality evaluation."""
+
+    overall_score:           float = Field(description='Overall response rating')
+    format_adherence_score:  float = Field(description='Format adherence score')
+    context_awareness_score: float = Field(description='Context awareness score')
+    pricing_adherence_score: float = Field(description='Pricing guidelines adherence score')
+    scope_compliance_score:  float = Field(description='Scope compliance score')
+    general_rules_score:     float = Field(description='General rules score')
+    comment:                 str   = Field(description='Brief explanation')
+
+
+class QualityScoreHandler:
+    def __init__(self) -> None:
+        self._smith_client = Client()
+        self._model = modconf.get_confidence_scoring_model()
+        self._model = self._model.with_structured_output(QualityEvaluationResult)
+
+
+    def evaluate_response_quality(self, query: str, response: str) -> QualityEvaluationResult:
+        prompt = promptconf.get_quality_scoring_prompt(query, response)
+        messages = [HumanMessage(prompt)]
+        
+        try:
+            time_start = perf_counter()
+            logger.info("Evaluating the response quality...")
+            evaluation: QualityEvaluationResult = self._model.invoke(messages)
+            time_elapsed = perf_counter() - time_start
+            logger.info(f"Finished confidence evaluation in {time_elapsed:1.3} sec")
+
+            evaluation.overall_score = sum([
+                evaluation.format_adherence_score,
+                evaluation.context_awareness_score, 
+                evaluation.pricing_adherence_score,
+                evaluation.scope_compliance_score,
+                evaluation.general_rules_score, 
+            ]) / 5.0
+            
+            logger.info(f"- scoring: {evaluation.overall_score:1.2f}")
+            logger.info(f"- comment: {evaluation.comment}")
+
+            return evaluation
+        except Exception as e:
+            logger.error(f"Failed to evaluate the response's confidence: {e}")
+            return QualityEvaluationResult()

src/rag/utilclasses.py

@@ -1,4 +1,5 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass
+from pydantic import BaseModel, Field
 from typing_extensions import TypedDict
 from langchain.agents import AgentState
 from langchain_core.messages import AnyMessage
@@ -7,6 +8,18 @@ from langchain_core.messages import AnyMessage
 class AgentContext:
     agent_name: str
 
+@dataclass
+class LeadAgentQueryResponse:
+    response: str 
+    language: str 
+    confidence_fallback: bool = False 
+    max_turns_reached: bool = False
+
+class StructuredAgentResponse(BaseModel):
+    response:         str   = Field(description="Main response to the query.")
+    confidence_score: float = Field("Value in range 0.0 to 1.0 that determines how confident the agent is in it's response based on the accumulated information.")
+    
+
 class State(TypedDict):
     messages: list[AnyMessage]
     answer: str