updated project state

.gitignore

@@ -1,2 +1,65 @@
+# Byte-compiled / optimized / DLL files
 __pycache__/
-.env
+*.py[cod]
+*$py.class
+
+# Virtual environment
+.env
+.venv/
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Environment variables
+.env
+
+# VS Code settings
+.vscode/
+
+# MacOS system files
+.DS_Store
+
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+
+# Logs
+*.log
+
+# Cache and temp files
+*.tmp
+*.swp
+*.bak
+.cache/
+*.sqlite3
+*.db
+
+# Data files
+*.pdf
+*.json
+
+# Output folders
+dist/
+build/
+*.egg-info/
+
+# Output data
+data/
+
+# Pycharm
+.idea/
+
+# OS junk
+.Trashes.env
+.env
+.env
+
+#idk
+--source-branch
+--source-repo
+/.gradio/certificate.pem
+
+#feedback I just uploaded into the same file to check for accuracy
+chatbot emba x.docx
+IEBMA Test Cards 1_2.docx

config.py

@@ -128,7 +128,11 @@ class WeaviateConfiguration:
     # Weaviate backup settings
     AVAILABLE_BACKUP_METHODS = ['manual', 'filesystem', 's3']
     BACKUP_METHOD = 'manual'
-    BACKUP_PATH = os.getenv('WEAVIATE_BACKUP_PATH')
+
+    # Weaviate generated data paths
+    BACKUP_PATH = 'data/database/backups'
+    PROPERTIES_PATH = 'data/database/properties'
+    STRATEGIES_PATH = 'data/database/strategies'
 
     # Weaviate Cloud settings
     CLUSTER_URL = os.getenv('WEAVIATE_CLUSTER_URL')
@@ -137,13 +141,31 @@ class WeaviateConfiguration:
     
     # Custom timeouts for Cloud connection (in seconds)
     INIT_TIMEOUT   = 90 
-    QUERY_TIMEOUT  = 10 
-    INSERT_TIMEOUT = 120
+    QUERY_TIMEOUT  = 60 
+    INSERT_TIMEOUT = 600
 
     @classmethod 
     def is_local(cls) -> bool:
         return cls.LOCAL_DATABASE
 
+# Cache settings
+class CacheConfig:
+    LOCAL_HOST = "localhost"
+    LOCAL_PORT = 6379
+    LOCAL_PASS = os.getenv("REDIS_LOCAL_PASSWORD", "")
+
+    CLOUD_HOST = os.getenv("REDIS_CLOUD_HOST")
+    CLOUD_PORT = int(os.getenv("REDIS_CLOUD_PORT", 6379))
+    CLOUD_PASS = os.getenv("REDIS_CLOUD_PASSWORD")
+
+    CACHE_LOCAL = "local"
+    CACHE_CLOUD = "cloud"
+    CACHE_DICT = "dict"
+    CACHE_MODE = "cloud"  # 'local' or 'cloud' or 'dict' set here the default cache mode
+
+    TTL_CACHE = 86400 # 86400 seconds = 24 hours
+    MAX_SIZE_CACHE = 1000
+
 # Data paths
 DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
 RAW_DATA_PATH = os.path.join(DATA_DIR, "raw_data.json")
@@ -153,8 +175,11 @@ VECTORDB_PATH = os.path.join(DATA_DIR, "vectordb")
 # Determines when the text is considered German during the language detection
 LANG_AMBIGUITY_THRESHOLD = 0.6
 
+# Confidence Threshold to activate fall-back mechanism
+CONFIDENCE_THRESHOLD = 0.6
+
 # Vector database settings
-CHUNK_SIZE = 1000
+CHUNK_SIZE = 512
 CHUNK_OVERLAP = 200
 
 # Agent Chain settings 
@@ -176,7 +201,7 @@ ENABLE_EVALUATE_RESPONSE_QUALITY = True
 
 # Conversation state settings
 TRACK_USER_PROFILE = True  # Track user preferences and avoid repetition
-LOCK_LANGUAGE_AFTER_FIRST_MESSAGE = True  # Don't change language mid-conversation
+LOCK_LANGUAGE_AFTER_N_MESSAGES = 3  # Lock language after N user messages (0 = never lock)
 MAX_CONVERSATION_TURNS = 15 # End conversation after max turns reached
 
 # Data processing pipeline settings 

requirements.txt

@@ -35,3 +35,7 @@ docling>=2.55.0
 
 # Weaviate Vector DB 
 weaviate-client>=4.16.9
+
+# Cache 
+cachetools>=5.0.0
+redis>=4.5.5

src/apps/chat/app.py

@@ -1,17 +1,19 @@
 import os
 import gradio as gr
-from src.apps.chat.js import JS_LISTENER, JS_CLEAR
 from src.const.agent_response_constants import *
 from src.rag.agent_chain import ExecutiveAgentChain
 from src.rag.utilclasses import LeadAgentQueryResponse
 from src.utils.logging import get_logger
+from src.cache.cache import Cache
 
 logger = get_logger("chatbot_app")
+cache_logger = get_logger("cache_chatbot_app")
 
 class ChatbotApplication:
     def __init__(self, language: str = 'de') -> None:
-        self._app = gr.Blocks(js=JS_LISTENER)
+        self._app = gr.Blocks()
         self._language = language
+        self._cache = Cache.get_cache()
 
         with self._app:
             agent_state = gr.State(None)
@@ -26,6 +28,11 @@ class ChatbotApplication:
                 )
                 reset_button = gr.Button("Reset Conversation")
 
+            chatbot = gr.Chatbot(
+                height=600,
+                type='messages',
+                label="Executive Education Adviser"
+            )
             chat = gr.ChatInterface(
                 fn=lambda msg, history, agent: self._chat(
                     message=msg,
@@ -35,12 +42,8 @@ class ChatbotApplication:
                 additional_inputs=[agent_state],
                 title="Executive Education Adviser",
                 type='messages',
-            )
-
-            iframe_container = gr.HTML(
-                value="",
-                elem_id="consultation-iframe-container",
-                visible=True
+                chatbot=chatbot,
+                fill_height=True
             )
 
             def clear_chat_immediate():
@@ -66,29 +69,27 @@ class ChatbotApplication:
 
             lang_selector.change(
                 fn=clear_chat_immediate,
-                outputs=[chat.chatbot_value, iframe_container],
+                outputs=[chat.chatbot_value],
                 queue=True,
-                js=JS_CLEAR
             )
 
             lang_selector.change(
                 fn=on_lang_change,
                 inputs=[lang_selector],
-                outputs=[agent_state, lang_state, chat.chatbot_value, iframe_container],
+                outputs=[agent_state, lang_state, chat.chatbot_value],
                 queue=True,
             )
 
             reset_button.click(
                 fn=clear_chat_immediate,
-                outputs=[chat.chatbot_value, iframe_container],
+                outputs=[chat.chatbot_value],
                 queue=True,
-                js=JS_CLEAR
             )
 
             reset_button.click(
                 fn=switch_language,
                 inputs=[lang_state],
-                outputs=[agent_state, lang_state, chat.chatbot_value, iframe_container],
+                outputs=[agent_state, lang_state, chat.chatbot_value],
                 queue=True,
             )
 
@@ -112,12 +113,57 @@ class ChatbotApplication:
         try:
             logger.info(f"Processing user query: {message[:100]}...")
 
-            lead_resp: LeadAgentQueryResponse = agent.query(query=message)
-            answers.append(lead_resp.response)
-            self._language = lead_resp.language
-
-            if lead_resp.confidence_fallback or lead_resp.max_turns_reached:
-                answers.extend(APPOINTMENT_LINKS[self._language])
+            preprocess_resp = agent.preprocess_query(message)
+            final_response: LeadAgentQueryResponse = None
+
+            current_lang = preprocess_resp.language
+            processed_q = preprocess_resp.processed_query
+
+            if preprocess_resp.response:
+                # Response comes from preprocessing step
+                final_response = preprocess_resp
+
+            elif Cache._settings["enabled"]:
+                cached_data = self._cache.get(processed_q, language=current_lang)
+
+                if cached_data:
+                    # Cache Hit — restore response with metadata
+                    if isinstance(cached_data, dict):
+                        final_response = LeadAgentQueryResponse(
+                            response=cached_data["response"],
+                            language=current_lang,
+                            appointment_requested=cached_data.get("appointment_requested", False),
+                            relevant_programs=cached_data.get("relevant_programs", []),
+                        )
+                    else:
+                        # Legacy: plain string cache entry
+                        final_response = LeadAgentQueryResponse(
+                            response=cached_data,
+                            language=current_lang,
+                        )
+
+            if not final_response:
+                # Response needs to be generated by the agent
+                final_response = agent.agent_query(processed_q)
+
+            answers.append(final_response.response)
+            self._language = final_response.language
+
+            if final_response.confidence_fallback or final_response.max_turns_reached or final_response.appointment_requested:
+                html_code = get_booking_widget(language=self._language, programs=final_response.relevant_programs)
+                answers.append(gr.HTML(value=html_code))
+
+            if final_response.should_cache and Cache._settings["enabled"]:
+                # Caching response with metadata
+                self._cache.set(
+                    key=processed_q,
+                    value={
+                        "response": final_response.response,
+                        "appointment_requested": final_response.appointment_requested,
+                        "relevant_programs": final_response.relevant_programs,
+                    },
+                    language=current_lang
+                )
 
         except Exception as e:
             logger.error(f"Error processing query: {e}", exc_info=True)

src/apps/dbapp/app.py

@@ -0,0 +1,44 @@
+from tkinter import *
+from tkinter import ttk
+from src.database.weavservice import WeaviateService
+
+from src.apps.dbapp.mainframe import MainFrame
+from src.apps.dbapp.query import QueryFrame
+from src.apps.dbapp.imports import ImportFrame
+from src.apps.dbapp.backup import BackupsFrame
+from src.apps.dbapp.collections import CollectionsFrame
+from src.apps.dbapp.config import SchemaConfigurationFrame
+
+from src.utils.logging import get_logger
+
+logger = get_logger("db_inter    ")
+
+class DatabaseApplication:
+    def __init__(self) -> None:
+        self._root = Tk()
+        self._service = WeaviateService()
+        
+        self._root.title("Database Interface")
+        self._root.geometry("810x500")
+        
+        notebook = ttk.Notebook(self._root)
+        notebook.pack(fill=BOTH, expand=True)
+
+        main_frame = MainFrame(notebook, self._service).init()
+        import_frame = ImportFrame(notebook, self._service).init() 
+        config_frame = SchemaConfigurationFrame(notebook, self._service).init() 
+        collections_frame = CollectionsFrame(notebook, self._service).init()
+        query_frame = QueryFrame(notebook, self._service).init()
+        backups_frame = BackupsFrame(notebook, self._service).init()
+        
+        notebook.add(main_frame, text='Main')
+        notebook.add(import_frame, text='Import')
+        notebook.add(config_frame, text='Schemas')
+        notebook.add(collections_frame, text='Collections')
+        notebook.add(query_frame, text='Query')
+        notebook.add(backups_frame, text='Backups')
+
+        logger.info("Application initialization finished")
+
+    def run(self):
+        self._root.mainloop()

src/apps/dbapp/backup.py

@@ -0,0 +1,191 @@
+import os, shutil
+from datetime import datetime
+
+from tkinter import *
+from tkinter import ttk
+from src.database.weavservice import WeaviateService
+from src.apps.dbapp.framebase import CustomFrameBase
+from src.apps.dbapp.utilclasses import BackupData
+from config import WeaviateConfiguration as wvtconf
+
+def _load_backup_files():
+    backups = []
+    os.makedirs(wvtconf.BACKUP_PATH, exist_ok=True)
+
+    for backup_id in os.listdir(wvtconf.BACKUP_PATH):
+        backups.append(BackupData(backup_id))
+    
+    return backups
+
+class BackupsFrame(CustomFrameBase):
+    def __init__(self, parent, service: WeaviateService):
+        super().__init__(parent, service)
+        self._backups = _load_backup_files()
+
+    def init(self) -> ttk.Frame:
+        self._backups = _load_backup_files()
+        
+        main_frame = ttk.Frame(self._parent)
+        main_frame.pack(fill=BOTH, expand=True)
+
+        tree_frame = ttk.Frame(main_frame)
+        tree_frame.pack(fill=BOTH, expand=True, padx=10, pady=10)
+        
+        label_frame = ttk.Frame(main_frame)
+        label_frame.pack(fill=X, expand=True, padx=10, pady=10)
+
+        button_frame = ttk.Frame(main_frame)
+        button_frame.pack(fill=X, padx=10, pady=10)
+
+        date_reverse_sort = True
+        columns = ('date', 'size')
+        
+        info_label = ttk.Label(label_frame, text="", padding=8)
+        
+        def _print_label(msg, backc, forc):
+            info_label.configure(text=msg, foreground=forc, background=backc)
+            info_label.update_idletasks()
+
+        def print_failure(msg: str):
+            _print_label(msg, "#FFCDD2", "#B71C1C")
+
+        def print_info(msg: str):
+            _print_label(msg, "#cdedff", "#1c31b7")
+        
+        def print_success(msg: str):
+            _print_label(msg, "#d7ffcd", "#4db71c")
+
+
+        tree = ttk.Treeview(
+            tree_frame,
+            columns=columns, 
+            show='tree headings',
+            selectmode='browse',
+        )
+            
+        def sort_by_date():
+            nonlocal date_reverse_sort
+
+            parents = tree.get_children("")
+            data = []
+
+            for p in parents:
+                value = tree.set(p, 'date')
+                try:
+                    value = datetime.strptime(value, "%d.%m.%Y %H:%M:%S")
+                except Exception:
+                    pass
+                data.append((value, p))
+
+            data.sort(reverse=date_reverse_sort)
+            date_reverse_sort = not date_reverse_sort
+
+            for index, (_, p) in enumerate(data):
+                tree.move(p, "", index)
+
+            tree.heading(
+                'date',      
+                text='Created at    ' + ('▾' if date_reverse_sort else '▴'),     
+                command=lambda: sort_by_date()
+            )
+
+        tree.heading('#0', text='Backup ID')
+        tree.heading('date', text='Created at    ▾', command=lambda: sort_by_date())
+        tree.heading('size', text='Embeddings amount')
+        
+        tree.column("#0", width=100)
+        tree.column("date", width=60)
+        tree.column("size", width=30)
+
+        def insert_backup(backup):
+            nonlocal date_reverse_sort
+            bk = backup.to_treeformat()
+            parent = tree.insert('', 0 if not date_reverse_sort else END, 
+                text=bk['id'],
+                values=bk['date']
+            )
+            for collection in bk['collections']:
+                tree.insert(parent, END,
+                    text=collection['name'],
+                    values=collection['size'],
+                )
+
+        for backup in self._backups:
+            insert_backup(backup)
+        sort_by_date()
+           
+        def create_backup():
+            print_info(f"Creating new backup...")
+            backup_id = self._service._create_backup()
+
+            backup = BackupData(backup_id)
+            self._backups.append(backup)
+            insert_backup(backup)
+            print_success(f"Successfully created new backup {backup._backup_id}!")
+
+        def restore_backup():
+            item_id = tree.selection()[0]
+            backup = tree.item(item_id)
+            
+            print_info(f"Restoring backup {backup['text']}...")
+            self._service._restore_backup('backup_' + backup['text'])
+            print_success(f"Successfully restored backup {backup['text']}!")
+        
+        def delete_backup():
+            item_id = tree.selection()[0]
+            backup = tree.item(item_id)
+            
+            backup_path = os.path.join(wvtconf.BACKUP_PATH, 'backup_' + backup['text'])
+            shutil.rmtree(backup_path, ignore_errors=True)
+
+            tree.delete(item_id)
+            print_success(f"Deleted backup {backup['text']}.")
+
+
+        create_bkp_btn = ttk.Button(
+                button_frame,
+                text="Create Backup", 
+                command=create_backup
+        )
+
+        restore_bkp_btn = ttk.Button(
+            button_frame,
+            text="Restore Backup", 
+            command=restore_backup,
+            state=['disabled']
+        )
+
+        delete_bkp_btn = ttk.Button(
+            button_frame,
+            text="Delete Backup",
+            command=delete_backup,
+            state=['disabled']
+        )
+
+        def on_item_selection(event):
+            selected = tree.selection()
+            if not selected:
+                restore_bkp_btn.state(['disabled'])
+                delete_bkp_btn.state(['disabled'])
+                return 
+
+            item_id = selected[0]
+            is_parent = tree.parent(item_id) == ''
+            restore_bkp_btn.state(['!disabled' if is_parent else 'disabled'])
+            delete_bkp_btn.state(['!disabled' if is_parent else 'disabled'])
+        
+        tree.bind("<<TreeviewSelect>>", on_item_selection)
+
+        scrollbar = ttk.Scrollbar(tree_frame, orient="vertical", command=tree.yview)
+        tree.configure(yscrollcommand=scrollbar.set)
+        
+        info_label.pack()
+
+        tree.pack(side=LEFT, fill=BOTH, expand=True)
+        scrollbar.pack(side=RIGHT, fill=Y)
+        
+        create_bkp_btn.pack(side=LEFT, padx=5)
+        restore_bkp_btn.pack(side=RIGHT, padx=5)
+        delete_bkp_btn.pack(side=RIGHT, padx=5)
+
+        return main_frame

src/apps/dbapp/collections.py

@@ -0,0 +1,8 @@
+from tkinter import *
+from tkinter import ttk
+from src.apps.dbapp.framebase import CustomFrameBase
+from src.database.weavservice import WeaviateService
+
+class CollectionsFrame(CustomFrameBase):
+    def __init__(self, parent, service: WeaviateService) -> None:
+        super().__init__(parent, service)

src/apps/dbapp/config.py

@@ -0,0 +1,348 @@
+import os, json
+
+from tkinter import *
+from tkinter import ttk
+from src.apps.dbapp.framebase import CustomFrameBase
+from src.utils.stratutils.generator import generate_strategy
+from src.database.weavservice import WeaviateService
+from config import WeaviateConfiguration as wvtconf
+
+def _dump_schema(schema):
+    os.makedirs(wvtconf.PROPERTIES_PATH, exist_ok=True)
+    properties_file_path = os.path.join(wvtconf.PROPERTIES_PATH, 'properties.json')
+    with open(properties_file_path, 'w', encoding='utf-8') as f:
+        json.dump(schema, f, indent=2, default=str)
+
+
+class SchemaConfigurationFrame(CustomFrameBase):
+    def __init__(self, parent, service: WeaviateService) -> None:
+        super().__init__(parent, service)
+        self._schema = self._load_schema_data()
+        self._strategies = self._load_strategies()
+    
+
+    def _load_strategies(self) -> dict:
+        os.makedirs(wvtconf.STRATEGIES_PATH, exist_ok=True)
+        loaded_strats = os.listdir(wvtconf.STRATEGIES_PATH)
+        strategies = {}
+
+        for name, prop in self._schema.items():
+            strategy_file = f"strat_{name}.py"
+            file_path = os.path.join(wvtconf.STRATEGIES_PATH, strategy_file)
+            strategy_content = ""
+            
+            if strategy_file not in loaded_strats:
+                strategy_content = generate_strategy(name, prop)
+                with open(file_path, 'w', encoding='utf-8') as f:
+                    f.write(strategy_content)
+            else:
+                with open(file_path) as f:
+                    strategy_content = f.read()
+
+            strategies[name] = strategy_content
+
+        return strategies
+
+
+    def _save_strategy(self, name, strategy) -> None:
+        os.makedirs(wvtconf.STRATEGIES_PATH, exist_ok=True)
+        self._strategies[name] = strategy
+
+        file_path = os.path.join(wvtconf.STRATEGIES_PATH, f"strat_{name}.py")
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.write(strategy)
+
+
+    def _load_schema_data(self) -> dict:
+        schema = self._service._extract_data()['schema'][0]
+        
+        schema_data = {}
+
+        for prop in schema['properties']:
+            data_property = {
+                'description': prop.get('description', ''),
+                'data_type': prop['dataType'][0],
+                'filterable': prop['indexFilterable'],
+                'searchable': prop['indexSearchable'],
+                'skip_vectorization': prop['moduleConfig']['text2vec-huggingface']['skip'],
+            }
+            schema_data[prop['name']] = data_property
+        
+        _dump_schema(schema_data)
+
+        return schema_data
+
+
+    def _update_schema_property(self, old_name: str, new_name: str, prop: dict) -> None:
+        del self._schema[old_name]
+        self._schema[new_name] = prop  
+        _dump_schema(self._schema)
+
+
+    def _add_schema_property(self, name, prop: dict) -> None:
+        self._schema[name] = prop 
+        _dump_schema(self._schema)
+
+
+    def _delete_schema_property(self, name) -> None:
+        del self._schema[name]
+        _dump_schema(self._schema)
+
+
+    def init(self) -> ttk.Frame:
+        main_frame = ttk.Frame(self._parent)
+        main_frame.pack(fill=BOTH, expand=True)
+
+        schema_frame = ttk.Frame(main_frame)
+        schema_frame.pack(fill=BOTH, expand=True)
+        
+        add_button = ttk.Button(schema_frame, text='Add property', 
+                                command=lambda: self._add_property(refresh_table))
+        add_button.pack(anchor=NW, padx=5, pady=5)
+
+        canvas = Canvas(schema_frame)
+        scrollbar = ttk.Scrollbar(schema_frame, orient="vertical", command=canvas.yview)
+        scrollable_frame = ttk.Frame(canvas)
+
+        scrollable_frame.bind("<Configure>", lambda _: canvas.configure(scrollregion=canvas.bbox("all")))
+        canvas.create_window((0, 0), window=scrollable_frame, anchor="nw")
+        canvas.configure(yscrollcommand=scrollbar.set)
+        canvas.pack(side=LEFT, fill=BOTH, expand=True)
+        scrollbar.pack(side=RIGHT, fill=Y)
+
+        def refresh_table():
+            for widget in scrollable_frame.winfo_children():
+                widget.destroy()
+
+            self._build_table(scrollable_frame, refresh_table)
+        
+        refresh_table() 
+        return main_frame
+
+
+    def _build_table(self, parent_frame, refresh_callback):
+        style = ttk.Style()
+        style.configure('Header.TLabel', font=('Helvetica', 10, 'bold'), background='#e0e0e0')
+        style.configure('EvenRow.TLabel', background='#f0f0f0')
+        style.configure('OddRow.TLabel', background='white')
+
+        table_frame = ttk.Frame(parent_frame)
+        table_frame.pack(fill=X, padx=5, pady=5)
+
+        for i in range(5):
+            table_frame.grid_columnconfigure(i, minsize=100, weight=1) 
+
+        headers = ['Name', 'Data Type', 'Filterable', 'Searchable', 'Skip Vectorize']
+        for col, text in enumerate(headers):
+            label = ttk.Label(table_frame, text=text, borderwidth=1, relief=SOLID, anchor='center', style='Header.TLabel')
+            label.grid(row=0, column=col, sticky='ew')
+
+        for idx, (name, prop) in enumerate(self._schema.items(), start=1):
+            row_style = 'EvenRow.TLabel' if idx % 2 == 0 else 'OddRow.TLabel'
+            
+            row_name_label = ttk.Label(table_frame, text=name, style=row_style)
+            row_type_label = ttk.Label(table_frame, text=prop['data_type'].upper(), style=row_style)
+            row_filterable_label = ttk.Label(table_frame, text='Yes' if prop['filterable'] else 'No', style=row_style)
+            row_searchable_label = ttk.Label(table_frame, text='Yes' if prop['searchable'] else 'No', style=row_style)
+            row_vectorize_label = ttk.Label(table_frame, text='Yes' if prop['skip_vectorization'] else 'No', style=row_style)
+            
+            row_edit_button = ttk.Button(table_frame, text='Edit', 
+                command=lambda n=name, p=prop: self._edit_property(n, p, refresh_callback))
+            row_delete_button = ttk.Button(table_frame, text='Delete', 
+                command=lambda n=name: self._delete_property(n, refresh_callback))
+            row_strategy_button = ttk.Button(table_frame, text='Strategy', 
+                command=lambda n=name: self._handle_strategy(n)) 
+
+            row_name_label.grid(row=idx, column=0, sticky='ew', ipadx=25)
+            row_type_label.grid(row=idx, column=1, sticky='ew', ipadx=25)
+            row_filterable_label.grid(row=idx, column=2, sticky='ew', ipadx=25)
+            row_searchable_label.grid(row=idx, column=3, sticky='ew')
+            row_vectorize_label.grid(row=idx, column=4, sticky='ew')
+            row_edit_button.grid(row=idx, column=5, sticky='ew')
+            row_delete_button.grid(row=idx, column=6, sticky='ew')
+            row_strategy_button.grid(row=idx, column=7, sticky='ew')
+    
+    
+    def _handle_strategy(self, n):
+        dialog = Toplevel()
+        dialog.title(f"Property {n} strategy")
+        dialog.geometry("700x400")
+
+        field_frame = ttk.Frame(dialog)
+        field_frame.pack(fill=BOTH, expand=True, padx=10, pady=10)  
+
+        scrollbar = Scrollbar(field_frame, orient=VERTICAL)
+        scrollbar.pack(side=RIGHT, fill=Y)
+
+        strategy = self._strategies[n]
+        edit_field = Text(field_frame, width=80, height=15, wrap=WORD, yscrollcommand=scrollbar.set)  
+        edit_field.insert(END, strategy)
+        edit_field.pack(side=LEFT, fill=BOTH, expand=True)
+
+        scrollbar.config(command=edit_field.yview)  
+
+        def commit():
+            new_strategy = edit_field.get("1.0", END).strip()  
+            self._save_strategy(n, new_strategy) 
+            dialog.destroy()  
+
+        
+        ttk.Button(dialog, text="Save", command=commit).pack(side=BOTTOM, anchor=S, pady=10)
+
+    
+    def _delete_property(self, name, refresh_callback):
+        msg = f"Do you want to delete property '{name}'?"
+        dialog = Toplevel()
+        dialog.title('Warning!')
+        dialog.geometry(f"{len(msg)*5+120}x50")
+        dialog.grab_set()
+
+        ttk.Label(dialog, text=msg).pack()
+        
+        def submit():
+            self._delete_schema_property(name)
+            refresh_callback()
+            dialog.destroy()
+        
+        button_frame = ttk.Frame(dialog)
+        button_frame.pack(fill=X, expand=True)
+
+        ttk.Button(button_frame, text='Delete', command=submit).pack(side=LEFT, padx=15)
+        ttk.Button(button_frame, text='Cancel', command=dialog.destroy).pack(side=RIGHT, padx=15) 
+
+
+    def _add_property(self, refresh_callback):
+        dialog = Toplevel()
+        dialog.title(f"New property")
+        dialog.geometry("280x300") 
+        dialog.grab_set() 
+
+        texts_frame = ttk.Frame(dialog)
+        texts_frame.pack(fill=X, expand=True)
+
+        ttk.Label(texts_frame, text="Name:").grid(row=0, column=0, padx=5, pady=5, sticky='e')
+        name_entry = ttk.Entry(texts_frame)
+        name_entry.grid(row=0, column=1, padx=5, pady=5, sticky='w')
+
+        ttk.Label(texts_frame, text="Description:").grid(row=1, column=0, padx=5, pady=5, sticky='e')
+        desc_entry = ttk.Entry(texts_frame)
+        desc_entry.insert(0, '') 
+        desc_entry.grid(row=1, column=1, padx=5, pady=5, sticky='w')
+
+        ttk.Label(texts_frame, text="Data Type:").grid(row=2, column=0, padx=5, pady=5, sticky='e')
+        type_var = StringVar(value='text')
+        type_combo = ttk.Combobox(texts_frame, textvariable=type_var, 
+            values=["text", "int", "number", "boolean", "date", "text[]", "int[]", "number[]", "boolean[]", "date[]", "object"]
+        )      
+        type_combo.grid(row=2, column=1, padx=5, pady=5, sticky='w')
+        
+        checks_frame = ttk.Frame(dialog)
+        checks_frame.pack(fill=X, expand=True)
+
+        filterable_var = BooleanVar(value=True)
+        searchable_var = BooleanVar(value=True)
+        skip_vec_var = BooleanVar(value=False)
+
+        ttk.Checkbutton(checks_frame, text="Filterable        ", variable=filterable_var).pack(anchor=W, padx=15)
+        ttk.Checkbutton(checks_frame, text="Searchable        ", variable=searchable_var).pack(anchor=W, padx=15)
+        ttk.Checkbutton(checks_frame, text="Skip Vectorization", variable=skip_vec_var).pack(anchor=W, padx=15)   
+        
+        def submit():
+            name = name_entry.get()
+            if not name:
+                self._show_messagebox("Parameter 'name' is required!")
+                return
+            if name in self._schema.keys():
+                self._show_messagebox(f"Property with name '{name}' already exists!")
+                return
+
+            prop = {
+                'description': desc_entry.get().strip(),
+                'data_type': type_var.get(),
+                'filterable': filterable_var.get(),
+                'searchable': searchable_var.get(),
+                'skip_vectorization': skip_vec_var.get(),
+            }
+
+            self._add_schema_property(name, prop)
+            refresh_callback()
+            dialog.destroy()
+
+        buttons_frame = ttk.Frame(dialog)
+        buttons_frame.pack(fill=X, expand=True)
+
+        ttk.Button(buttons_frame, text="Save", command=submit).pack(side=LEFT, padx=15)    
+        ttk.Button(buttons_frame, text="Cancel", command=dialog.destroy).pack(side=RIGHT, padx=15)
+
+
+    def _edit_property(self, name: str, prop: dict, refresh_callback):
+        dialog = Toplevel()
+        dialog.title(f"Edit Property: {name}")
+        dialog.geometry("280x300") 
+        dialog.grab_set() 
+
+        texts_frame = ttk.Frame(dialog)
+        texts_frame.pack(fill=X, expand=True)
+
+        ttk.Label(texts_frame, text="Name:").grid(row=0, column=0, padx=5, pady=5, sticky='e')
+        name_entry = ttk.Entry(texts_frame)
+        name_entry.insert(0, name)
+        name_entry.grid(row=0, column=1, padx=5, pady=5, sticky='w')
+
+        ttk.Label(texts_frame, text="Description:").grid(row=1, column=0, padx=5, pady=5, sticky='e')
+        desc_entry = ttk.Entry(texts_frame)
+        desc_entry.insert(0, prop.get('description', '')) 
+        desc_entry.grid(row=1, column=1, padx=5, pady=5, sticky='w')
+
+        ttk.Label(texts_frame, text="Data Type:").grid(row=2, column=0, padx=5, pady=5, sticky='e')
+        type_var = StringVar(value=prop['data_type'])
+        type_combo = ttk.Combobox(texts_frame, textvariable=type_var, 
+            values=["text", "int", "number", "boolean", "date", "text[]", "int[]", "number[]", "boolean[]", "date[]", "object"]
+        )      
+        type_combo.grid(row=2, column=1, padx=5, pady=5, sticky='w')
+        
+        checks_frame = ttk.Frame(dialog)
+        checks_frame.pack(fill=X, expand=True)
+
+        filterable_var = BooleanVar(value=prop['filterable'])
+        searchable_var = BooleanVar(value=prop['searchable'])
+        skip_vec_var = BooleanVar(value=prop['skip_vectorization'])
+
+        ttk.Checkbutton(checks_frame, text="Filterable        ", variable=filterable_var).pack(anchor=W, padx=15)
+        ttk.Checkbutton(checks_frame, text="Searchable        ", variable=searchable_var).pack(anchor=W, padx=15)
+        ttk.Checkbutton(checks_frame, text="Skip Vectorization", variable=skip_vec_var).pack(anchor=W, padx=15)
+
+        def submit():
+            new_name = name_entry.get().strip()
+            if not new_name:
+                self._show_messagebox("Parameter 'name' is required!")
+                return
+            
+            updated_prop = {
+                'description': desc_entry.get().strip(),
+                'data_type': type_var.get(),
+                'filterable': filterable_var.get(),
+                'searchable': searchable_var.get(),
+                'skip_vectorization': skip_vec_var.get(),
+            }
+            
+            self._update_schema_property(name, new_name, updated_prop)
+            refresh_callback()
+            dialog.destroy()
+
+        buttons_frame = ttk.Frame(dialog)
+        buttons_frame.pack(fill=X, expand=True)
+
+        ttk.Button(buttons_frame, text="Save", command=submit).pack(side=LEFT, padx=15)    
+        ttk.Button(buttons_frame, text="Cancel", command=dialog.destroy).pack(side=RIGHT, padx=15)
+
+
+    @staticmethod
+    def _show_messagebox(msg):
+        dialog = Toplevel()
+        dialog.title('Warning!')
+        dialog.geometry(f"{len(msg)*5+120}x50")
+        dialog.grab_set()
+
+        ttk.Label(dialog, text=msg).pack()
+        ttk.Button(dialog, text='OK', command=dialog.destroy).pack(padx=15)

src/apps/dbapp/framebase.py

@@ -0,0 +1,15 @@
+from tkinter import *
+from tkinter import ttk 
+from src.database.weavservice import WeaviateService
+
+class CustomFrameBase:
+    def __init__(self, parent, service: WeaviateService) -> None:
+        self._parent  = parent
+        self._service = service
+
+
+    def init(self) -> ttk.Frame:
+        main_frame = ttk.Frame(self._parent)
+        main_frame.pack()
+
+        return main_frame

src/apps/dbapp/imports.py

@@ -0,0 +1,158 @@
+import os
+import threading
+from tkinter import *
+from tkinter import ttk
+from tkinter import filedialog
+
+from src.pipeline.pipeline import ImportPipeline
+from src.apps.dbapp.framebase import CustomFrameBase
+from src.database.weavservice import WeaviateService
+from src.pipeline.utilclasses import ProcessingResult
+from src.utils.lang import get_language_name
+
+class ImportFrame(CustomFrameBase):
+    def __init__(self, parent, service: WeaviateService) -> None:
+        super().__init__(parent, service)
+        self._import_paths = dict()
+
+    def init(self):
+        main_frame = ttk.Frame(self._parent)
+        main_frame.pack(fill=BOTH, expand=True)
+        
+        import_frame = ttk.Frame(main_frame)
+        file_buttons_frame = ttk.Frame(main_frame)
+        file_buttons_frame.pack(fill=X, side=TOP, anchor=NW, expand=True)
+        
+        import_buttons_frame = ttk.Frame(import_frame)
+        import_buttons_frame.pack(side=TOP, anchor=W, expand=True)
+
+        files_treeview = ttk.Treeview(
+            main_frame,
+            columns=[],
+            show='tree headings',
+            selectmode='extended',
+        )
+        files_treeview.heading('#0', text='File name')
+        files_treeview.column('#0', width=400)
+        
+        logging_textframe = Text(import_frame, width=40, height=16, state=DISABLED)
+
+        def update_treeview():
+            for item in files_treeview.get_children(''):
+                files_treeview.delete(item)
+
+            for filename in self._import_paths.keys():
+                files_treeview.insert('', 0, text=filename)
+
+        def open_file_dialog():
+            filepaths = filedialog.askopenfilenames(
+                title='Select files to import',
+                filetypes=(('PDF', '*.pdf'), ('Text files', '*.txt') ),
+            )
+            for path in filepaths:
+                filename = os.path.basename(path)
+                self._import_paths[filename] = path
+
+            update_treeview()
+
+        def remove_files():
+            selection = files_treeview.selection()
+            if not selection:
+                return 
+
+            for item in selection:
+                filename = files_treeview.item(item)['text']
+                del self._import_paths[filename]
+
+            update_treeview()
+        
+        def change_button_state(state):
+            add_button.config(state=state)
+            remove_button.config(state=state)
+            import_button.config(state=state)
+
+        add_button = ttk.Button(file_buttons_frame, text='Add files', command=open_file_dialog)
+        add_button.pack(side=LEFT, padx=15, pady=15)
+
+        remove_button = ttk.Button(file_buttons_frame, text='Remove files', command=remove_files)
+        remove_button.pack(side=LEFT, padx=15, pady=15)
+
+        import_button = ttk.Button(import_buttons_frame, text='Begin Import', 
+            command=lambda: self._import_callback(change_button_state, clean_coll_var.get())
+        )
+        import_button.pack(side=LEFT, padx=15, pady=15)
+        
+        clean_coll_var = BooleanVar(value=False)
+        clean_coll_checkbutton = ttk.Checkbutton(
+                import_buttons_frame, 
+                text='Clean Collections', 
+                variable=clean_coll_var,
+        )
+        clean_coll_checkbutton.pack(side=RIGHT, padx=15, pady=15)
+ 
+        ttk.Label(import_frame, text='Import status:').pack(side=TOP, anchor=NW, padx=15)
+        
+        files_treeview.pack(side=LEFT, anchor=W, fill=Y, expand=True, padx=15, pady=15)
+        import_frame.pack(side=LEFT, anchor=W, fill=BOTH, expand=True)
+        
+        logging_textframe.pack(side=TOP, anchor=NW, fill=BOTH, expand=True, padx=15, pady=15)
+
+        return main_frame
+
+
+    def _import_callback(self, button_state_callback, clean_coll: bool):
+        dialog = Toplevel()
+        dialog.title("Import status")
+        dialog.geometry("600x400")
+
+        current_import_label = ttk.Label(dialog, text='Initiating the import pipeline...')
+        current_import_label.pack(side=TOP, padx=15, pady=15)
+        
+        progress_bar = ttk.Progressbar(dialog, length=200, value=0, maximum=100)
+        progress_bar.pack(side=TOP, padx=15, pady=15)
+
+        chunks_treeview = ttk.Treeview(
+            dialog,
+            columns=['chunks', 'lang'],
+            show='tree headings',
+            selectmode='extended',
+        )
+        chunks_treeview.heading('#0', text='File name')
+        chunks_treeview.heading('chunks', text='Collected chunks')
+        chunks_treeview.heading('lang', text='Language')
+
+        chunks_treeview.column('#0', width=100)
+        chunks_treeview.column('chunks', width=60)
+        chunks_treeview.column('lang', width=40)
+
+        chunks_treeview.pack(side=TOP, fill=X, padx=15, pady=15, expand=True)
+
+        def logging_callback(msg: str, progress: int, result: ProcessingResult = None):
+            current_import_label.config(text=msg)
+            if progress > 100:
+                progress_bar.config(mode='indeterminate')
+            else:
+                progress_bar.config(mode='determinate', value=progress)
+            if result:
+                chunks_treeview.insert('', index=0, 
+                    text=result.source, 
+                    values=(
+                        len(result.chunks), 
+                        get_language_name(result.lang)
+                    )
+                )
+
+        def import_task():
+            button_state_callback(DISABLED)
+            filepaths = self._import_paths.values()
+            try:
+                ImportPipeline(
+                    logging_callback=logging_callback,
+                    reset_collections_on_import=clean_coll,
+                ).import_many_documents(filepaths)
+            finally:
+                dialog.bell()
+                button_state_callback(NORMAL)
+
+        import_thread = threading.Thread(target=import_task)
+        import_thread.start()

src/apps/dbapp/mainframe.py

@@ -0,0 +1,8 @@
+from tkinter import *
+from tkinter import ttk
+from src.apps.dbapp.framebase import CustomFrameBase
+from src.database.weavservice import WeaviateService
+
+class MainFrame(CustomFrameBase):
+    def __init__(self, parent, service: WeaviateService) -> None:
+        super().__init__(parent, service)

src/apps/dbapp/query.py

@@ -0,0 +1,8 @@
+from tkinter import *
+from tkinter import ttk
+from src.apps.dbapp.framebase import CustomFrameBase
+from src.database.weavservice import WeaviateService
+
+class QueryFrame(CustomFrameBase):
+    def __init__(self, parent, service: WeaviateService) -> None:
+        super().__init__(parent, service)

src/apps/dbapp/utilclasses.py

@@ -0,0 +1,38 @@
+import os, json 
+from datetime import datetime
+from config import WeaviateConfiguration as wvtconf
+
+class BackupData:
+    def __init__(self, backup_id: str) -> None:
+        self._backup_id = backup_id
+        self._creation_date = ""
+        self._collections = []
+
+        backup_path = os.path.join(wvtconf.BACKUP_PATH, backup_id)
+        files = os.listdir(backup_path)
+
+        if 'data.json' in files:
+            data_path = os.path.join(backup_path, 'data.json')
+            with open(data_path) as f:
+                data = json.load(f)
+
+                date = datetime.fromisoformat(data['creation_date'])
+                self._creation_date = date.strftime("%d.%m.%Y %H:%M:%S")
+        
+        if 'objects.json' in files:
+            objects_path = os.path.join(backup_path, 'objects.json')
+            with open(objects_path) as f:
+                data = json.load(f)
+                for name, objs in data.items():
+                    self._collections.append({
+                        'name': name.lower(),
+                        'size': ('', len(objs))
+                    }) 
+
+
+    def to_treeformat(self):
+        return {
+            'id': self._backup_id.replace('backup_', ''),
+            'date': (self._creation_date, ''),
+            'collections': self._collections,      
+        }

src/cache/cache.py

@@ -0,0 +1,72 @@
+from .cache_strategies import RedisCache, LocalCache
+from config import CacheConfig
+from threading import Lock
+from src.utils.logging import get_logger
+from .cache_metrics import CacheMetrics
+
+logger = get_logger("cache")
+
+class Cache:
+    _instance = None
+    _settings = None
+    _lock = Lock()
+    _cache_metrics = None
+
+    @staticmethod
+    def configure(mode: str, no_cache: bool):
+        Cache._settings = {
+            "mode": mode,
+            "enabled": not no_cache
+        }
+
+    @staticmethod
+    def get_cache():
+        if Cache._instance is not None:
+            return Cache._instance
+
+        with Cache._lock:
+            if Cache._instance is not None:
+                return Cache._instance
+
+            settings = Cache._settings or {"mode": CacheConfig.CACHE_LOCAL, "enabled": True}
+            
+            if not settings.get("enabled", True):
+                Cache._instance = None
+                return None
+            
+            if Cache._cache_metrics is None:
+                Cache._cache_metrics = CacheMetrics()
+
+            mode = settings.get("mode", CacheConfig.CACHE_LOCAL)
+
+            if mode == CacheConfig.CACHE_CLOUD:
+                cache_obj = RedisCache(
+                    host=CacheConfig.CLOUD_HOST,
+                    port=CacheConfig.CLOUD_PORT,
+                    password=CacheConfig.CLOUD_PASS,
+                    mode=mode,
+                    metrics=Cache._cache_metrics
+                )
+            elif mode == CacheConfig.CACHE_LOCAL:
+                cache_obj = RedisCache(
+                    host=CacheConfig.LOCAL_HOST,
+                    port=CacheConfig.LOCAL_PORT,
+                    password=CacheConfig.LOCAL_PASS,
+                    mode=mode,
+                    metrics=Cache._cache_metrics
+                )
+            elif mode == CacheConfig.CACHE_DICT:
+                Cache._instance = LocalCache(metrics=Cache._cache_metrics)
+                return Cache._instance
+            else:
+                logger.error("FALLBACK to dict cache. Unknown cache mode")
+                Cache._instance = LocalCache(metrics=Cache._cache_metrics)
+                return Cache._instance
+            
+            if cache_obj.client is None:
+                logger.error("FALLBACK to dict cache. Redis connection failed")
+                Cache._instance = LocalCache(metrics=Cache._cache_metrics)
+            else: 
+                Cache._instance = cache_obj
+
+            return Cache._instance

src/cache/cache_base.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+class CacheStrategy(ABC):
+    """
+    Defines the interface for the different cache system strategies (Local or Redis).
+    """
+    
+    @abstractmethod
+    def set(self, key: str, value: Any, language: str):
+        pass
+
+    @abstractmethod
+    def get(self, key: str, language: str):
+        pass
+
+    @abstractmethod
+    def clear_cache(self):
+        pass

src/cache/cache_metrics.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+from threading import Lock
+
+
+@dataclass
+class CacheStatistics:
+    hits: int
+    misses: int
+    hits_ratio: float
+
+class CacheMetrics:
+    def __init__(self) -> None:
+        self.cache_stats = CacheStatistics(0, 0, 0.0)
+        self._lock = Lock()
+    
+    def increment_hit(self):
+        with self._lock:
+            self.cache_stats.hits += 1
+            self._calc_hit_ratio()
+    
+    def increment_miss(self):
+        with self._lock:
+            self.cache_stats.misses += 1
+            self._calc_hit_ratio()
+
+    def _calc_hit_ratio(self):
+        total = self.cache_stats.hits + self.cache_stats.misses
+        self.cache_stats.hits_ratio = (self.cache_stats.hits / total) if total else 0.0

src/cache/cache_strategies.py

@@ -0,0 +1,89 @@
+from .cache_base import CacheStrategy
+from config import CacheConfig
+from cachetools import TTLCache
+import json
+from typing import Any
+from src.database.redisservice import RedisService
+from src.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+class RedisCache(CacheStrategy):
+    def __init__(self, host, port, password, mode, metrics):
+        service = RedisService(host, port, password, mode)
+        self.client = service.get_client()
+        self.metrics = metrics
+
+    def set(self, key: str, value: Any, language: str):
+        if not self.client: return
+        
+        try:
+            json_str = json.dumps(value)
+            self.client.set(self._generate_normalized_key(key, language), json_str, ex=CacheConfig.TTL_CACHE)
+            logger.info("Response cached")
+        except Exception as e:
+            logger.error(f"Could not write to Redis: {e}")
+
+    def get(self, key: str, language: str):
+        if not self.client: return None
+
+        try:
+            val = self.client.get(self._generate_normalized_key(key, language))
+            if val is not None:
+                self.metrics.increment_hit()
+                logger.info(f"Cache HIT {self.metrics.cache_stats.hits} {self.metrics.cache_stats.hits_ratio}")
+                return json.loads(val)
+            
+            self.metrics.increment_miss()
+            logger.info(f"Cache MISS {self.metrics.cache_stats.misses} {self.metrics.cache_stats.hits_ratio}")
+            return None
+        except Exception as e:
+            logger.error(f"Could not read from Redis: {e}")
+            return None
+
+    def _generate_normalized_key(self, key: str, language: str) -> str:
+        import re
+
+        normalized_key = re.sub(r'[^a-z0-9]', '', key.lower())
+        return f"cache:{language}:{normalized_key}"
+    
+    def clear_cache(self):
+        if not self.client: return
+
+        try:
+            self.client.flushdb()
+            logger.info(f"Redis Cache cleared.")
+        except Exception as e:
+            logger.error(f"Could not clear Redis cache: {e}")
+           
+
+class LocalCache(CacheStrategy):
+    def __init__(self, metrics):
+        self.cache = TTLCache(maxsize=CacheConfig.MAX_SIZE_CACHE, ttl=CacheConfig.TTL_CACHE)
+        self.metrics = metrics
+
+    def _generate_normalized_key(self, key: str, language: str) -> str:
+        import re
+        
+        normalized_key = re.sub(r'[^a-z0-9]', '', key.lower())
+        return f"cache:{language}:{normalized_key}"
+
+    def set(self, key: str, value: Any, language: str):
+        normalized_key = self._generate_normalized_key(key, language)
+        self.cache[normalized_key] = value
+        logger.info("Response cached")
+    
+    def get(self, key: str, language: str):
+        normalized_key = self._generate_normalized_key(key, language)
+        res = self.cache.get(normalized_key, None)
+        if res is not None:
+            self.metrics.increment_hit()
+            logger.info(f"Cache HIT {self.metrics.cache_stats.hits} {self.metrics.cache_stats.hits_ratio}")
+        else:
+            self.metrics.increment_miss()
+            logger.info(f"Cache MISS {self.metrics.cache_stats.misses}")
+        return res
+
+    def clear_cache(self):
+        self.cache.clear()
+        logger.info("Local Cache cleared.")

src/const/agent_response_constants.py

@@ -1,22 +1,20 @@
-from gradio import ChatMessage
-
 """ Constants for Gradio app """
 
 GREETING_MESSAGES = {
-        "en": [
-            "Hello and welcome! I’m your Executive Education Advisor for the HSG Executive MBA programs (**IEMBA**, **emba X**, and **EMBA**). How can I best support your MBA planning today?",
-            "Hello and welcome! I’m your Executive Education Advisor for the University of St.Gallen’s Executive MBA programs (**IEMBA**, **emba X**, **EMBA**). How can I support your MBA planning today?",
-            "Hello and welcome! I’m your Executive Education Advisor for the HSG Executive MBA programs (**EMBA**, **IEMBA**, **emba X**). How can I help you with your EMBA journey today?",
-            "Hello and welcome! I’m your Executive Education Advisor for the University of St.Gallen’s EMBA programs, here to help you navigate our **EMBA**, **IEMBA**, and **emba X** options.",
-            "Hello and welcome. I’m your Executive Education Advisor for the University of St.Gallen’s Executive MBA programs, here to help you assess fit and navigate the **EMBA**, **IEMBA**, and **emba X** options.",
-        ],
-        "de": [
-            "Guten Tag! Ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme und unterstütze Sie gerne bei Fragen zu **EMBA**, **IEMBA** und **emba X**.",
-            "Guten Tag, ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme (**EMBA**, **IEMBA**, **emba X**). Ich unterstütze Sie bei Programmwahl, Ablauf und Zulassungsfragen.",
-            "Guten Tag und herzlich willkommen! Ich bin Ihr Executive Education Advisor für die HSG Executive MBA Programme und unterstütze Sie gern bei Fragen zu **EMBA**, **IEMBA** und **emba X**.",
-            "Guten Tag, ich bin Ihr Executive-Education-Berater für die HSG Executive MBA-Programme (**EMBA**, **IEMBA**, **emba X**) und unterstütze Sie gerne bei Programmwahl und Zulassungsfragen.",
-            "Guten Tag! Ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme (**EMBA**, **IEMBA**, **emba X**) und unterstütze Sie gerne bei Programmwahl und Zulassungsfragen.",
-        ]
+    "en": [
+        "Hello and welcome! I’m your Executive Education Advisor for the HSG Executive MBA programs (**IEMBA**, **emba X**, and **EMBA**). How can I best support your MBA planning today?",
+        "Hello and welcome! I’m your Executive Education Advisor for the University of St.Gallen’s Executive MBA programs (**IEMBA**, **emba X**, **EMBA**). How can I support your MBA planning today?",
+        "Hello and welcome! I’m your Executive Education Advisor for the HSG Executive MBA programs (**EMBA**, **IEMBA**, **emba X**). How can I help you with your EMBA journey today?",
+        "Hello and welcome! I’m your Executive Education Advisor for the University of St.Gallen’s EMBA programs, here to help you navigate our **EMBA**, **IEMBA**, and **emba X** options.",
+        "Hello and welcome. I’m your Executive Education Advisor for the University of St.Gallen’s Executive MBA programs, here to help you assess fit and navigate the **EMBA**, **IEMBA**, and **emba X** options.",
+    ],
+    "de": [
+        "Guten Tag! Ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme und unterstütze Sie gerne bei Fragen zu **EMBA**, **IEMBA** und **emba X**.",
+        "Guten Tag, ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme (**EMBA**, **IEMBA**, **emba X**). Ich unterstütze Sie bei Programmwahl, Ablauf und Zulassungsfragen.",
+        "Guten Tag und herzlich willkommen! Ich bin Ihr Executive Education Advisor für die HSG Executive MBA Programme und unterstütze Sie gern bei Fragen zu **EMBA**, **IEMBA** und **emba X**.",
+        "Guten Tag, ich bin Ihr Executive-Education-Berater für die HSG Executive MBA-Programme (**EMBA**, **IEMBA**, **emba X**) und unterstütze Sie gerne bei Programmwahl und Zulassungsfragen.",
+        "Guten Tag! Ich bin Ihr Executive-Education-Berater für die HSG Executive MBA Programme (**EMBA**, **IEMBA**, **emba X**) und unterstütze Sie gerne bei Programmwahl und Zulassungsfragen.",
+    ]
 }
 
 QUERY_EXCEPTION_MESSAGE = {
@@ -72,69 +70,57 @@ CONVERSATION_END_MESSAGE = {
 }
 
 
-def create_appt_button(url, title, lang_text):
-    return (
-        f'<a href="{url}" class="appointment-btn" '
-        f'style="display: block; background-color: #f3f4f6; border: 1px solid #d1d5db; '
-        f'padding: 8px 16px; border-radius: 6px; cursor: pointer; '
-        f'color: #374151; font-weight: 600; width: 100%; text-align: left; '
-        f'margin-top: 5px; text-decoration: none;">'
-        f'📅 {lang_text}: {title}'
-        f'</a>'
-    )
+def get_booking_widget(language: str="en", programs: list[str]=None):
+    """
+    Returns an HTML string representing a Booking Widget.
+    """
 
+    if programs is None or programs == []:
+        programs = ["emba", "iemba", "emba_x"]
 
-APPOINTMENT_LINKS = {
-    "en": [
-        ChatMessage(
-            role="assistant",
-            content=create_appt_button(
-                "https://calendly.com/cyra-vonmueller/beratungsgespraech-emba-hsg",
-                "Cyra von Müller",
-                "Book Appointment"
-            ),
-        ),
-        ChatMessage(
-            role="assistant",
-            content=create_appt_button(
-                "https://calendly.com/kristin-fuchs-unisg/iemba-online-personal-consultation",
-                "Kristin Fuchs",
-                "Book Appointment"
-            ),
-        ),
-        ChatMessage(
-            role="assistant",
-            content=create_appt_button(
-                "https://calendly.com/teyuna-giger-unisg",
-                "Teyuna Giger",
-                "Book Appointment"
-            ),
-        ),
-    ],
-    "de": [
-        ChatMessage(
-            role="assistant",
-            content=create_appt_button(
-                "https://calendly.com/cyra-vonmueller/beratungsgespraech-emba-hsg",
-                "Cyra von Müller",
-                "Termin buchen"
-            ),
-        ),
-        ChatMessage(
-            role="assistant",
-            content=create_appt_button(
-                "https://calendly.com/kristin-fuchs-unisg/iemba-online-personal-consultation",
-                "Kristin Fuchs",
-                "Termin buchen"
-            ),
-        ),
-        ChatMessage(
-            role="assistant",
-            content=create_appt_button(
-                "https://calendly.com/teyuna-giger-unisg",
-                "Teyuna Giger",
-                "Termin buchen"
-            ),
-        ),
-    ],
-}
+    labels = {
+        "en": {"header": "Book a Consultation", "sub": "Select an advisor to view their calendar:"},
+        "de": {"header": "Termin vereinbaren", "sub": "Wählen Sie einen Berater für den Kalender:"}
+    }
+    txt = labels.get(language, labels["en"])
+
+    base_params = "?hide_gdpr_banner=1&embed_type=Inline&embed_domain=1"
+    advisors = [
+        {
+            "name": "Cyra von Müller (EMBA)",
+            "url": f"https://calendly.com/cyra-vonmueller/beratungsgespraech-emba-hsg{base_params}",
+            "program": "emba"
+        },
+        {
+            "name": "Kristin Fuchs (IEMBA)",
+            "url": f"https://calendly.com/kristin-fuchs-unisg/iemba-online-personal-consultation{base_params}",
+            "program": "iemba"
+        },
+        {
+            "name": "Teyuna Giger (EMBA X)",
+            "url": f"https://calendly.com/teyuna-giger-unisg{base_params}",
+            "program": "emba_x"
+        },
+    ]
+
+    html_content = f"""
+    <div style="width: 100%; min-width: 100%; box-sizing: border-box; background-color: #f9fafb; border: 1px solid #e5e7eb; border-radius: 12px; padding: 20px; margin-top: 10px; font-family: sans-serif;">
+        <h3 style="margin: 0 0 10px 0; color: #111827; font-size: 1.2em;">{txt['header']}</h3>
+        <p style="margin: 0 0 20px 0; color: #6b7280; font-size: 1em;">{txt['sub']}</p>
+    """
+
+    for advisor in advisors:
+        if advisor["program"] in programs:
+            html_content += f"""
+            <details style="margin-bottom: 12px; border: 1px solid #d1d5db; border-radius: 8px; background: white; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.1);">
+                <summary style="cursor: pointer; padding: 16px 20px; background-color: #ffffff; font-weight: 600; color: #374151; font-size: 1.05em; list-style: none; transition: background 0.2s;">
+                    {advisor['name']}
+                </summary>
+                <div style="padding: 0; border-top: 1px solid #e5e7eb;">
+                    <iframe src="{advisor['url']}" width="100%" height="650px" frameborder="0" style="display: block;"></iframe>
+                </div>
+            </details>
+            """
+
+    html_content += "</div>"
+    return html_content

src/database/docker-compose-cache.yml

@@ -0,0 +1,27 @@
+version: '3.8'
+
+services:
+  redis:
+    image: redis:alpine
+    container_name: hsg_redis_cache
+    ports:
+      - "6379:6379"
+    command: >
+      redis-server 
+      --requirepass "${REDIS_PASSWORD}" 
+      --save 60 1 
+      --loglevel warning 
+      --maxmemory 200mb 
+      --maxmemory-policy allkeys-lru
+    volumes:
+      - redis_data:/data
+    restart: unless-stopped
+    
+    healthcheck:
+      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+volumes:
+  redis_data:

src/database/redisservice.py

@@ -0,0 +1,54 @@
+import os
+import redis
+from threading import Lock
+from src.utils.logging import get_logger
+
+logger = get_logger("redis_service")
+
+class RedisService:
+    _instance = None
+    _init_lock = Lock()
+
+    def __new__(cls, host, port, password, mode):
+        if cls._instance is None:
+            with cls._init_lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self, host, port, password, mode):
+        if hasattr(self, '_initialized') and self._initialized:
+            return
+        
+        self._client = None
+        self._host = host
+        self._port = port
+        self._password = password 
+        self.mode = mode   
+        
+        self._connect()
+
+        self._initialized = True
+
+    def _connect(self):
+        try:
+            logger.info(f"Connecting to Redis at {self._host}:{self._port}...")
+            self._client = redis.Redis(
+                host=self._host,
+                port=self._port,
+                password=self._password,
+                decode_responses=True,
+                socket_connect_timeout=2,
+                socket_timeout=2
+            )
+            self._client.ping()
+            logger.info(f"Successfully connected to Redis! {self.mode}")
+        except Exception as e:
+            logger.error(f"Redis connection failed: {e}")
+            self._client = None
+
+    def get_client(self):
+        return self._client
+    
+    def is_connected(self) -> bool:
+        return self._client is not None

src/database/weavservice.py

@@ -1,3 +1,4 @@
+from functools import reduce
 import weaviate as wvt
 import datetime, os
 from threading import Lock
@@ -7,6 +8,7 @@ from weaviate.classes.config import Configure, Property, DataType
 from weaviate.collections.classes.grpc import MetadataQuery
 from weaviate.collections.collection import Collection
 from weaviate.classes.init import AdditionalConfig, Timeout
+from weaviate.classes.query import Filter
 from weaviate.config import AdditionalConfig
 
 from src.utils.logging import get_logger
@@ -99,13 +101,9 @@ class WeaviateService:
                     if wvtconf.is_local():
                         self._client = wvt.connect_to_local()
                         break
-                    
-                    cluster_url = wvtconf.CLUSTER_URL
-                    if not cluster_url.startswith('http'):
-                        cluster_url = f"https://{cluster_url}"
 
                     self._client = wvt.connect_to_weaviate_cloud(
-                        cluster_url=cluster_url,
+                        cluster_url=wvtconf.CLUSTER_URL,
                         auth_credentials=wvtconf.WEAVIATE_API_KEY,
                         additional_config=AdditionalConfig(
                             timeout=Timeout(
@@ -132,12 +130,12 @@ class WeaviateService:
                     break
                 except Exception as e:
                     last_exception = e
-                    logger.error(f"Failed to establish connection: {e}")
+                    logger.warning(f"Failed to establish connection on try {retries}: {e}")
                     retries += 1 
                     sleep(1)
             
             if retries == 3:
-                logger.error(f"Failed after 3 retries!")
+                logger.error(f"Failed to establish connection after 3 retries!")
                 raise last_exception
 
             logger.info(f"Successully connected to the {self._connection_type} weaviate database")
@@ -211,18 +209,28 @@ class WeaviateService:
             raise e  
 
         return import_errors
+    
+    @staticmethod
+    def _create_property_filter(prop, values) -> Filter:
+        match prop:
+            case 'programs':
+                return Filter.by_property('programs').contains_any(values)
+            case _:
+                return None
 
 
-    def query(self, query: str, lang: str, query_properties: list[str] = None, limit: int = 5) -> dict:
+    def query(self, query: str, lang: str, property_filters: dict[str], limit: int = 5) -> dict:
         """
         Execute a hybrid semantic and keyword query against the active collection with automatic reconnection on idle timeout.
 
         Args:
             query (str): The query string.
-            query_properties (list[str], optional): List of properties to query against.
-            limit (int, optional): Maximum number of results to return. Defaults to 5.
-            distance (float, optional): Distance threshold for the query. Defaults to 0.25.
             lang (str, optional): Language collection to use. If not provided, uses the current one.
+            property_filters (dict[str, any]): Key-value pairs for metadata filtering. Keys correspond
+            to document properties (e.g., 'program', 'topic'), and values are the required matches.
+            Multiple filters are combined using logical AND.
+            limit (int, optional): Maximum number of results to return. Defaults to 5.
+
 
         Returns:
             tuple: A tuple containing the query response and elapsed time.
@@ -232,7 +240,11 @@ class WeaviateService:
         """
         retry_count = 0 
         max_retries = 2
-        
+            
+        filters = [self._create_property_filter(prop, values) 
+                   for prop, values in property_filters.items()] if property_filters else None
+        filters = reduce(lambda f1, f2: f1 & f2, filters)
+
         while retry_count < max_retries:
             try:
                 collection, collection_name = self._select_collection(lang)
@@ -246,7 +258,7 @@ class WeaviateService:
                 with self._client_lock:
                     resp = collection.query.hybrid(
                         query=query,
-                        query_properties=query_properties,
+                        filters=filters,
                         limit=limit,
                         return_metadata=MetadataQuery.full()
                     )
@@ -359,30 +371,115 @@ class WeaviateService:
             logger.error(f"Collections deletion failed: {e}")
             self._client = None
             raise e
+    
+
+    def _reset_collections(self):
+        self._delete_collections()
+        self._create_collections()
+
+
+    def _collect_chunk_ids(self) -> dict:
+        client = self._init_client()
+        try:
+            ids = []
+            with self._client_lock:
+                for c in client.collections.list_all(simple=False):
+                    coll = client.collections.get(c)
+                    for obj in coll.iterator():
+                        ids.append(obj.properties['chunk_id'])
+            return ids
+        except Exception as e:
+            logger.error(f"Failed to collect chunk ids: {e}")
+            raise e
+
+
+    def _extract_data(self) -> dict:
+        client = self._init_client()
+        try:
+            schema = []
+            objects = {}
+            with self._client_lock:
+                for c in client.collections.list_all(simple=False):
+                    coll = client.collections.get(c)
+                    cfg = coll.config.get().to_dict()
+                    schema.append(cfg)
+
+                    objects[c] = []
+                    for obj in coll.iterator(include_vector=True):
+                        objects[c].append({
+                            "uuid": obj.uuid,
+                            "properties": obj.properties,
+                            "vector": obj.vector,
+                        })
+
+            return {
+                'schema':  schema,
+                'objects': objects,
+            }
+        except Exception as e:
+            logger.error(f"Failed to extract data from database: {e}")
+            raise e
 
 
-    def _create_backup(self) -> None:
+    def _create_backup(self) -> str:
         """
-        Create a backup of the current database state.
-        
-        Uploads backup to configured backend storage.         
+        Create a backup of the current database state and stores it under selected backup provider.
+
+        Returns: backup id of the created backup.
         """
         try:
-            client = self._init_client()
-            backup_id = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")
-            logger.info(f"Initiating backup creation for {self._connection_type} database")
-            
-            with self._client_lock:
-                result = client.backup.create(
-                    backup_id=f"hsg_wvtdb_backup_{backup_id}",
-                    backend=wvtconf.WEAVIATE_BACKUP_BACKEND,
-                    include_collections=_collection_names,
-                    wait_for_completion=True
-                )
+            if not wvtconf.BACKUP_METHOD:
+                raise ValueError('Backup method is not selected!')
+            if wvtconf.BACKUP_METHOD not in wvtconf.AVAILABLE_BACKUP_METHODS:
+                raise ValueError(f"Selected backup method 'wvtconf.BACKUP_METHOD' is not supported!")
+            if not wvtconf.BACKUP_PATH:
+                raise ValueError("Backup directory is not set!")
+            os.makedirs(wvtconf.BACKUP_PATH, exist_ok=True)
+
+            backup_id = f"backup_{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}"
+            logger.info(f"Initiating backup creation for {self._connection_type} database...")
             
+            match wvtconf.BACKUP_METHOD:
+                case 'manual':
+                    import json 
+                    
+                    backup_path = os.path.join(wvtconf.BACKUP_PATH, backup_id)
+                    os.makedirs(backup_path)
+                    
+                    db_data = self._extract_data()
+                    data_backup = {
+                        'creation_date': datetime.datetime.now().isoformat(),
+                    }
+                    
+                    schema_backup_path = os.path.join(backup_path, 'schema.json')
+                    with open(schema_backup_path, 'w', encoding='utf-8') as f:
+                        json.dump(db_data['schema'], f, indent=2, default=str)
+                    
+                    objects_backup_path = os.path.join(backup_path, 'objects.json')
+                    with open(objects_backup_path, 'w', encoding='utf-8') as f:
+                        json.dump(db_data['objects'], f, indent=2, default=str)
+
+                    data_backup_path = os.path.join(backup_path, 'data.json')
+                    with open(data_backup_path, 'w', encoding='utf-8') as f:
+                        json.dump(data_backup, f, indent=2, default=str)
+
+                case 's3':
+                    client = self._init_client()
+                    with self._client_lock:
+                        client.backup.create(
+                            backup_id=backup_id,
+                            backend="s3",
+                            include_collections=_collection_names,
+                            wait_for_completion=True,
+                        ) 
+                case _:
+                    raise NotImplementedError()
+
+                
             self._last_query_time = perf_counter()
-            logger.info(f"Backup '{backup_id}' created successfully: {result}")
+            logger.info(f"Backup '{backup_id}' created successfully")
             
+            return backup_id
         except Exception as e:
             logger.error(f"Backup creation failed: {e}")
             raise e
@@ -400,20 +497,69 @@ class WeaviateService:
         Raises:
             Exception if backup restoration fails
         """
+        self._delete_collections()
+
         try:
+            if not wvtconf.BACKUP_METHOD:
+                raise ValueError('Backup method is not selected!')
+            if wvtconf.BACKUP_METHOD not in wvtconf.AVAILABLE_BACKUP_METHODS:
+                raise ValueError(f"Selected backup method 'wvtconf.BACKUP_METHOD' is not supported!")
+            if not wvtconf.BACKUP_PATH:
+                raise ValueError("Backup directory is not set!")
+            os.makedirs(wvtconf.BACKUP_PATH, exist_ok=True)
+            
+            backup_path = os.path.join(wvtconf.BACKUP_PATH, backup_id)
+            if not os.path.exists(backup_path):
+                raise RuntimeError(f"Directory for backup 'backup_id' does not exist in the backup directory!")  
+            schema_backup_path = os.path.join(backup_path, 'schema.json')
+            if not os.path.exists(schema_backup_path):
+                raise RuntimeError(f"Schema backup is missing in the backup directory!")
+            objects_backup_path = os.path.join(backup_path, 'objects.json')
+            if not os.path.exists(objects_backup_path):
+                raise RuntimeError(f"Objects backup is missing in the backup directory!")
+            
             client = self._init_client()
-            logger.info(f"Initiating restoration from backup '{backup_id}' for {self._connection_type} database")
+            logger.info(f"Initiating restoration from backup '{backup_id}' for {self._connection_type} database...")
             
             with self._client_lock:
-                result = client.backup.restore(
-                    backup_id=backup_id,
-                    backend=wvtconf.WEAVIATE_BACKUP_BACKEND,
-                    include_collections=_collection_names,
-                    wait_for_completion=True,
-                )
-            
+                match wvtconf.BACKUP_METHOD:
+                    case 'manual':
+                        import json
+
+                        with open(schema_backup_path) as f:
+                            schemas = json.load(f)
+                            for cfg in schemas: 
+                                client.collections.create_from_dict(cfg)
+
+                        with open(objects_backup_path) as f:
+                            data = json.load(f)
+                            for name, objs in data.items():
+                                logger.info(f"Restoring collection '{name}' with {len(objs)} objects...")
+                                coll = client.collections.get(name)
+                                
+                                with coll.batch.dynamic() as batch:
+                                    for o in objs:
+                                        o['properties']['date'] = o['properties']['date'] \
+                                                .replace(" ", "T").replace("+00:00", "Z")
+                                        batch.add_object(
+                                            uuid=o["uuid"],
+                                            properties=o["properties"],
+                                            vector=o["vector"]
+                                        )
+                                logger.info(f"Collection '{name}' restored successfully")
+                    case 's3':                        
+                        client.backup.restore(
+                            backup_id=backup_id,
+                            backend="s3",
+                            wait_for_completion=True,
+                            roles_restore="all",
+                            users_restore="all",
+                        )
+                    case _:
+                        raise NotImplementedError()
+
             self._last_query_time = perf_counter()
-            logger.info(f"Backup '{backup_id}' restored successfully: {result}")
+            logger.info(f"Backup '{backup_id}' restored successfully")
             
         except Exception as e:
             error_msg = str(e).lower()
@@ -587,5 +733,4 @@ if __name__ == "__main__":
         service._create_collections()
     
     if any([args.checkhealth, args.create_collections, args.redo_collections]):
-        service._checkhealth()
-    
+        service._checkhealth() 

src/pipeline/pipeline.py

@@ -1,74 +1,44 @@
-import json, os
+import os
 
 from pathlib import Path
+from src.pipeline.utilclasses import (
+        _deduplication_callback_placeholder,
+        _logging_callback_placeholder,
+        ProcessingResult,
+)
+from src.pipeline.processors import *
+from src.database.weavservice  import WeaviateService
+
 from src.utils.logging import get_logger
-from src.processing.processor import DataProcessor, ProcessingResult, ProcessingStatus, WebsiteProcessor
-from src.database.weavservice import WeaviateService
 
-from config import AVAILABLE_LANGUAGES, HASH_FILE_PATH, DOCUMENTS_PATH
+from config import AVAILABLE_LANGUAGES
 
 pipelogger = get_logger("pipeline_module")
 implogger  = get_logger("import_pipeline")
 
 
-def _get_all_sources(sources) -> list[str]:
-    sources.remove('all')
-    pipelogger.info(f"Getting all sources from the soruce directory at {DOCUMENTS_PATH}...")
-    for source in os.listdir(DOCUMENTS_PATH):
-        if source in sources: continue
-        if source.endswith('.pdf'):
-            sources.append(os.path.join(DOCUMENTS_PATH, source))
-    pipelogger.info(f"Loaded {len(sources)} sources from the source directory")
-    return sources
-
-
-def _import_hashtables() -> dict:
-    """
-    Import deduplication hashtables from the JSON file.
-
-    Returns:
-        dict: Hashtable data containing document and chunk IDs.
-    """
-    hashtables = dict()
-    
-    with open(HASH_FILE_PATH, 'a+') as f:
-        try:
-            f.seek(0)
-            pipelogger.info(f"Loading deduplication hashtable from file {HASH_FILE_PATH}")
-            hashtables = json.load(f)
-            pipelogger.info(f"Import pipeline loaded deduplication hashtable with {len(hashtables['documents'])} sources and {len(hashtables['chunks'])} chunks")
-        except json.JSONDecodeError as e:
-            pipelogger.warning(f"Failed to decode the hash file {os.path.basename(HASH_FILE_PATH)}: {e}; new hashtable will be created")
-            hashtables['documents'] = []
-            hashtables['chunks'] = []
-    return hashtables
-
-
-def _export_hashtables(hashtables: dict):
-    """
-    Export hashtable data to the JSON file.
-
-    Args:
-        hashtables (dict): Hashtable dictionary containing documents and chunks.
-    """
-    with open(HASH_FILE_PATH, 'w+') as f:
-        json.dump(hashtables, f)
-        pipelogger.info("Saved successfully imported chunk IDs in the hashtables")
-
-
 class ImportPipeline:
     """
     Main pipeline class responsible for importing website and local documents
     into the database with deduplication and language-based organization.
     """
 
-    def __init__(self) -> None:
+    def __init__(
+        self, 
+        logging_callback = None,
+        deduplication_callback = None,
+        reset_collections_on_import = False,
+    ) -> None:
         """Initialize the import pipeline with processors and hashtable data."""
-        self._hashtables   = _import_hashtables()
-        self._webprocessor = WebsiteProcessor()
-        self._processor    = DataProcessor()
+        self._reset_collections_on_import = reset_collections_on_import
+        self._logging_callback = logging_callback or _logging_callback_placeholder
+        self._deduplication_callback = deduplication_callback or _deduplication_callback_placeholder
+        self._webprocessor = WebsiteProcessor(logging_callback)
+        self._processor    = DocumentProcessor(logging_callback)
         self._wvtserv      = WeaviateService()
-
+        self._ids          = self._wvtserv._collect_chunk_ids()
+        
+        implogger.info('Import pipeline initialization finished!')
 
     def scrape_website(self):
         """
@@ -85,7 +55,6 @@ class ImportPipeline:
             return 
         
         self._import_to_database(unique_chunks)
-        _export_hashtables(self._hashtables)
 
 
     def import_many_documents(self, sources: list[Path | str]):
@@ -96,6 +65,10 @@ class ImportPipeline:
         Args:
             sources (list[Path | str]): List of file paths or URLs to process.
         """
+        if self._reset_collections_on_import:
+            implogger.warning('Reset collection flag is set to True!')
+            implogger.warning('All existing embeddings will be removed from database before imprting!')
+
         if 'all' in sources:
             implogger.info("Import list contains the 'all', all sources will be imported...")
             sources = _get_all_sources(sources)
@@ -109,16 +82,23 @@ class ImportPipeline:
 
         unique_chunks = {lang: [] for lang in AVAILABLE_LANGUAGES}
         for source in sources:
-            chunks, lang = self._process_source(source)
-            if chunks:
-                unique_chunks[lang].extend(chunks)
+            filename = os.path.basename(source)
+            self._logging_callback(f'Starting pipeline for source {filename}...', 0)
+            result = self._process_source(source)
+            self._logging_callback(f'Storing chunks for {filename}...', 100, result)
+
+            if result.chunks:
+                unique_chunks[result.lang].extend(result.chunks)
         
-        if not unique_chunks:
+        if all([len(chunks) == 0 for chunks in unique_chunks.values()]):
+            self._logging_callback('No new data could be extracted from selected files!', 100)
             implogger.warning(f"File(s) provided for the insertion do not contain any unique information. Terminating the pipeline without importing")
             return 
-
+        
+        self._logging_callback('Importing chunks to database...', 110)
         self._import_to_database(unique_chunks)
-        _export_hashtables(self._hashtables)
+        self._logging_callback('Successfully imported all documents!', 100)
+
 
 
     def import_document(self, source: Path | str):
@@ -138,16 +118,13 @@ class ImportPipeline:
         Args:
             unique_chunks (dict): Dictionary mapping languages to lists of chunks.
         """
-        for lang, chunks in unique_chunks.items():
-            if not chunks: 
-                continue
-
-            failures = self._wvtserv.batch_import(data_rows=chunks, lang=lang)
-            for failure in failures:
-                chunk_id = failure['chunk_id']
-                if chunk_id in self._hashtables['chunks']:
-                    self._hashtables['chunks'].remove(chunk_id)
+        if self._reset_collections_on_import:
+            self._wvtserv._reset_collections()
 
+        for lang, chunks in unique_chunks.items():
+            if chunks:
+                failures = self._wvtserv.batch_import(data_rows=chunks, lang=lang)
+ 
 
     def _process_source(self, source: Path | str) -> tuple[list, str]:
         """
@@ -162,48 +139,47 @@ class ImportPipeline:
         """
         result: ProcessingResult = self._processor.process(source)
 
-        if not result.status == ProcessingStatus.SUCCESS:
+        if not result:
             implogger.error(f"Failed to process document {source}: {result.status}")
             return [], ''
         
-        unique_chunks = self._deduplicate(result)
-        return unique_chunks, result.language
+        unique_chunks = result.chunks
+        if not self._reset_collections_on_import:
+            unique_chunks = self._deduplicate(result)
+        return ProcessingResult(unique_chunks, result.source, result.lang)
 
 
-    def _deduplicate(self, result: ProcessingResult):
+    def _deduplicate(self, result: ProcessingResult) -> list:
         """
-        Remove duplicate chunks and documents based on previously processed hashes.
+        Remove duplicate chunks based on chunks that are already stored in the database.
 
         Args:
+            source_name (str): Document name for deduplication callback.
             result (ProcessingResult): The processing result containing document chunks.
 
         Returns:
             list[dict]: List of unique chunk dictionaries.
         """
-        d_id = result.document_id
-        unique_chunks = []
+        if self._reset_collections_on_import:
+            return result.chunks
 
-        implogger.info(f"Analyzing document with ID {d_id} for duplicated contents")
-        if d_id in self._hashtables['documents']:
-            implogger.warning(f"Document with ID {d_id} is a duplicate!")
-            return unique_chunks
+        self._logging_callback('Performing deduplication...', 80)
+        collected_chunks = result.chunks
+        unique_chunks = []
+        duplicate_ids = []
+        for chunk in collected_chunks:
+            chunk_id = chunk['chunk_id']
+            if chunk_id in self._ids:
+                duplicate_ids.append(chunk_id)
+            else: 
+                unique_chunks.append(chunk)
         
-        for chunk in result.chunks:
-            c_id = chunk['chunk_id']
-            if c_id in self._hashtables['chunks']:
-                continue 
-
-            self._hashtables['chunks'].append(c_id)
-            unique_chunks.append(chunk)
-
+        implogger.info(f"Found {len(duplicate_ids)} already existing IDs in {len(collected_chunks)} collected chunks")
         if not unique_chunks:
-            self._hashtables['documents'].append(d_id)
-        
-        implogger.info(f"Found {len(unique_chunks)} unique chunks out ouf {len(result.chunks)} collected chunks")
-        return unique_chunks
-
+            implogger.info(f"Calling deduplication callback...")
+            if self._deduplication_callback(result.source, len(collected_chunks)):
+                implogger.info('Duplicated chunks will be reimported as new...')
+                self._wvtserv._delete_by_id(duplicate_ids)
+                return collected_chunks
 
-if __name__ == "__main__":
-    pipeline = ImportPipeline()
-    #pipeline.import_many_documents(['data/hsg.pdf', 'data/emba_X5.pdf'])
-    pipeline.scrape_website()
+        return unique_chunks 

src/pipeline/processors.py

@@ -0,0 +1,280 @@
+from collections import defaultdict
+import os, re, hashlib, time, json
+import importlib.util 
+
+from pathlib import Path
+from transformers import AutoTokenizer
+
+from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
+from docling.datamodel.pipeline_options import (
+        PdfPipelineOptions, 
+        RapidOcrOptions,
+        LayoutOptions,
+)
+from docling_core.transforms.serializer.markdown import MarkdownDocSerializer
+from docling.document_converter import DocumentConverter, PdfFormatOption, InputFormat
+from docling.chunking import HybridChunker
+from docling_core.types.doc.document import DoclingDocument
+
+from src.pipeline.utilclasses import ProcessingResult
+from src.utils.lang import detect_language
+from src.utils.logging import get_logger
+from config import BASE_URL, CHUNK_MAX_TOKENS, WeaviateConfiguration as wvtconf
+
+weblogger  = get_logger("website_processor")
+datalogger = get_logger("data_processor")
+
+class ProcessorBase:
+    def __init__(self, logging_callback) -> None:
+        pipeline_options = PdfPipelineOptions(
+            do_ocr=True,
+            ocr_options=RapidOcrOptions(
+                force_full_page_ocr=True,
+            ),
+            generate_page_images=False,
+            images_scale=3.0,
+            do_layout_analysis=True,
+            do_table_structure=True,
+            do_cell_matching=True,
+            layout_options=LayoutOptions(
+                model_spec={
+                    "name": "docling_layout_egret_medium",  
+                    "repo_id": "docling-project/docling-layout-egret-medium",
+                    "revision": "main",
+                    "model_path": "",
+                    "supported_devices": ["cuda"]  
+                },
+                create_orphan_clusters=True,  
+                keep_empty_clusters=False,
+                skip_cell_assignment=False,
+            ),
+        )
+        self._converter: DocumentConverter = DocumentConverter(
+            format_options={
+                InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options),
+            },
+        )
+        tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+        self._chunker = HybridChunker(
+            tokenizer=HuggingFaceTokenizer(
+                tokenizer=tokenizer,
+                max_tokens=CHUNK_MAX_TOKENS
+            ), 
+            max_tokens=CHUNK_MAX_TOKENS, 
+            merge_peers=True
+        )
+        self._strategies = self._load_strategies()
+        self._logging_callback = logging_callback
+        
+    
+    def _load_strategies(self):
+        properties = {}
+        strategies = {}
+        
+        os.makedirs(wvtconf.PROPERTIES_PATH, exist_ok=True)
+        os.makedirs(wvtconf.STRATEGIES_PATH, exist_ok=True)
+        properties_path = os.path.join(wvtconf.PROPERTIES_PATH, 'properties.json')
+        if not os.path.exists(properties_path):
+            raise ValueError(f"Properties file does not exist under {properties_path}! Ensure that the database interface was opened at least once!")
+
+        with open(properties_path) as f:
+            properties = json.load(f)
+
+        for prop in properties.keys():
+            strat_file = f'strat_{prop}.py'
+            strat_path = os.path.join(wvtconf.STRATEGIES_PATH, strat_file)
+            if not os.path.exists(strat_path):
+                raise ValueError(f"Could not find strategy for property {prop}!")
+
+            spec = importlib.util.spec_from_file_location(
+                name=prop,
+                location=strat_path
+            )
+            strategy = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(strategy)
+
+            if not hasattr(strategy, 'run'):
+                raise ValueError(f"Strategy '{strat_file}' has no 'run' function!")
+
+            strategies[prop] = strategy
+
+        return strategies
+
+
+    def process(self):
+            """Abstract method to be implemented by subclasses."""
+            raise NotImplementedError("This method is not implemented in ProcessorBase")
+
+
+    def _prepare_chunks(self, document_name: str, document_content: str, chunks: list[str]) -> list[dict]:
+        prepared_chunks = []
+        for chunk in chunks:
+            prepared_chunk = {}
+            for prop, strat in self._strategies.items():
+                prepared_chunk[prop] = strat.run(document_name, document_content, chunk)
+            prepared_chunks.append(prepared_chunk)
+
+        return prepared_chunks
+    
+
+    def _clean_content(self, document_content: str) -> str:
+        """Removes the garbage symbols from text."""
+
+        cleaned = re.sub(r'\s+/\s+', '/', document_content)           
+        cleaned = re.sub(r'\s+\.\s+', '.', cleaned)          
+        cleaned = re.sub(r',\s+', '.', cleaned)              
+        cleaned = re.sub(r'\s+\|\s+', ' ', cleaned)
+        cleaned = re.sub(r'\/\s+', '/', cleaned)
+        cleaned = re.sub(r'\s+/','/', cleaned)
+        cleaned = re.sub(r'\s+\.', '.', cleaned)
+        cleaned = re.sub(r'(\d+)\s*,\s*(\d{4})', r'\1', cleaned)   
+        cleaned = re.sub(r'(\d+)\s*/\s*(\d+)', r'\1', cleaned)
+        cleaned = re.sub(r'\.(\d{4})', r'.\1', cleaned)
+        
+        cleaned = cleaned.replace('ä', 'ä').replace('ö', 'ö').replace('ü', 'ü')
+
+        cleaned = re.sub(r'\n\s*\n+', '\n\n', cleaned)
+        cleaned = re.sub(r' +', ' ', cleaned)
+
+        return cleaned
+
+
+    def _extract_document_content(self, document: DoclingDocument) -> str:
+        """Compiles text chunks found in the document into a single string."""
+
+        page_texts = defaultdict(list)
+        for text_item in document.texts:
+            if not text_item.text.strip():
+                continue 
+
+            prov = text_item.prov[0] if text_item.prov else None 
+            if prov:
+                page_number = prov.page_no 
+                bbox = prov.bbox 
+                page_texts[page_number].append({
+                    'text': text_item.text.strip(),
+                    'top': bbox.t,
+                    'left': bbox.l,
+                    'bottom': bbox.b,
+                })
+
+        full_page_texts = []
+        for page_number in sorted(page_texts.keys()):
+            text_items = sorted(
+                page_texts[page_number], 
+                key=lambda text: (-text['top'], text['left']),
+            )
+
+            content = []
+            last_bottom = None 
+
+            line_treshold = 15
+
+            for item in text_items:
+                text = item['text']
+                
+                if last_bottom is not None and (last_bottom - item['bottom'] > line_treshold):
+                    if content:
+                        full_page_texts.append(' '.join(content))
+                        content = []
+
+                    if last_bottom - item['bottom'] > 50:
+                        full_page_texts.append("")
+
+                content.append(text)
+                last_bottom = item['bottom']
+            
+            if content:
+                full_page_texts.append(' '.join(content))
+
+        full_text = '\n\n'.join(full_page_texts)
+        cleaned_text = self._clean_content(full_text)
+        
+        return cleaned_text
+     
+
+    def _collect_chunks(self, document: DoclingDocument) -> list[str]:
+        chunks = []
+        for base_chunk in self._chunker.chunk(dl_doc=document):
+            enriched = self._chunker.contextualize(chunk=base_chunk)
+            chunks.append(enriched)
+        return chunks
+
+
+    def _collect_chunks_fallback(self, document_content: str) -> list[str]:
+        """
+        Chunks the compiled text manually.
+
+        Args:
+            document_content (str): The full content extracted from document.
+
+        Returns:
+            list[str]: List of text chunks.
+        """ 
+        tokenizer_wrapper = self._chunker.tokenizer
+        tokenizer = getattr(tokenizer_wrapper, 'tokenizer', tokenizer_wrapper)
+
+        tokens = tokenizer.encode(document_content)
+        chunk_size = self._chunker.max_tokens
+        overlap = 50
+        
+        collected_chunks = []
+        for i in range(0, len(tokens), chunk_size-overlap):
+            chunk_tokens = tokens[i:i+chunk_size]
+            chunk = tokenizer.decode(
+                chunk_tokens, 
+                skip_special_tokens=True,
+                clean_up_tokenization_spaces=True
+            )
+            collected_chunks.append(chunk)
+
+        return collected_chunks
+ 
+
+class DocumentProcessor(ProcessorBase):
+    def process(self, source: Path | str) -> ProcessingResult:
+        """
+        Process a single document source, converting it to text, chunking, and hashing.
+
+        Args:
+            source (Path | str): Path to the document to process.
+
+        Returns:
+            ProcessingResult: The result of the processing operation, including chunks and language.
+        """
+        if not os.path.exists(source) or not os.path.isfile(source):
+            datalogger.error(f"Failed to initiate processing pipeline for source {source}: file does not exist")
+            return ProcessingResult(status=ProcessingStatus.NOT_FOUND)
+        
+        document_name = os.path.basename(source) 
+        datalogger.info(f"Initiating processing pipeline for source {document_name}")
+        self._logging_callback(f'Converting source {document_name}...', 20)
+        document = self._converter.convert(source).document
+        
+        self._logging_callback(f'Collecting chunks from {document_name}...', 40)
+        collected_chunks = self._collect_chunks(document)
+        document_content = MarkdownDocSerializer(doc=document).serialize().text
+
+        if len(collected_chunks) <= 1: # Document content manual extraction 
+            document_content = self._extract_document_content(document)
+            document = self._converter.convert_string(
+                content=document_content, 
+                format=InputFormat.MD
+            ).document
+            collected_chunks = self._collect_chunks(document)
+        
+        self._logging_callback(f'Preparing chunks for {document_name} for importing...', 60)
+        prepared_chunks = self._prepare_chunks(document_name, document_content, collected_chunks)
+
+        datalogger.info(f"Successfully collected {len(prepared_chunks)} chunks from {document_name}")
+
+        return ProcessingResult(
+            chunks=prepared_chunks,
+            source=document_name,
+            lang=detect_language(document_content), 
+        )
+
+
+class WebsiteProcessor(ProcessorBase):
+    pass
+

src/pipeline/utilclasses.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+
+def _logging_callback_placeholder(*_):
+    pass
+
+def _deduplication_callback_placeholder(*_) -> bool:
+    return False
+
+@dataclass
+class ProcessingResult:
+    chunks: list[dict]
+    source: str 
+    lang:   str
+
+

src/rag/agent_chain.py

@@ -36,8 +36,9 @@ from config import (
     TOP_K_RETRIEVAL,
     TRACK_USER_PROFILE,
     ENABLE_RESPONSE_CHUNKING,
-    ENABLE_EVALUATE_RESPONSE_QUALITY, 
+    ENABLE_EVALUATE_RESPONSE_QUALITY,
     MAX_CONVERSATION_TURNS,
+    LOCK_LANGUAGE_AFTER_N_MESSAGES, CONFIDENCE_THRESHOLD,
 )
 
 chain_logger = get_logger('agent_chain')
@@ -77,16 +78,18 @@ class ExecutiveAgentChain:
         }
 
         # Track scope violations for escalation
-        self._scope_violation_count = 0
+        self._scope_violation_counts: dict[str, int] = {}
+        self._aggressive_violation_count = 0
 
         chain_logger.info(f"Initialized new Agent Chain for language '{language}' with user_id: {self._user_id}")
 
-    def _retrieve_context(self, query: str, language: str = None):
+    def _retrieve_context(self, query: str, program: str, language: str = None):
         """
         Send the query to the vector database to retrieve additional information about the program.
 
         Args:
-            query: Keywords depicting information you want to retrieve in the primary language. 
+            query: Keywords depicting information you want to retrieve in the primary language.
+            program: Name of the program (either 'emba', 'iemba' or 'emba x') for which the information is requested.
             language: Optional parameter (either 'en' for English language or 'de' for German language). This parameter selects the language of the database to query from. The input query must be written in the same language as the selected language. Use this parameter only if there's not enough information in your main language.
         """
         lang = language if language in ['en', 'de'] else self._initial_language
@@ -95,6 +98,9 @@ class ExecutiveAgentChain:
                 query=query,
                 lang=lang,
                 limit=TOP_K_RETRIEVAL,
+                property_filters={
+                    'programs': [program],
+                },
             )
             serialized = '\n\n'.join([doc.properties.get('body', '') for doc in response.objects])
             return serialized
@@ -190,7 +196,7 @@ class ExecutiveAgentChain:
         ]
         agents = {
             'lead': create_agent(
-                name="Lead Agent",
+                name="lead_agent",
                 model=modelconf.get_main_agent_model(),
                 tools=tools_agent_calling,
                 state_schema=LeadInformationState,
@@ -208,7 +214,7 @@ class ExecutiveAgentChain:
         }
         for agent in ['emba', 'iemba', 'embax']:
             agents[agent] = create_agent(
-                name=f"{agent.upper()} Agent",
+                name=f"{agent}_agent",
                 model=modelconf.get_subagent_model(),
                 tools=[tool_retrieve_context],
                 state_schema=LeadInformationState,
@@ -435,27 +441,24 @@ class ExecutiveAgentChain:
         return greeting_message
 
     @traceable
-    def query(self, query: str) -> LeadAgentQueryResponse:
+    def preprocess_query(self, query: str) -> LeadAgentQueryResponse:
         """
-        Process user query with input handling, scope checking, and response formatting.
-        
-        Args:
-            query: User input
-            
-        Returns:
-            Formatted response
+        Phase 1: Validation, Scope-Check and language detection.
+        Does not call the agent directly.
         """
-        # Select fallback language if language was changed by previous query
-        response_language = self._stored_language 
+        # Remember fallback language
+        current_language = self._stored_language 
 
-        if len(self._conversation_history) >= MAX_CONVERSATION_TURNS * 2:
+        if len(self._conversation_history) >= MAX_CONVERSATION_TURNS:
             return LeadAgentQueryResponse(
-                response = CONVERSATION_END_MESSAGE[response_language],
-                language = response_language,
+                response = CONVERSATION_END_MESSAGE[current_language],
+                language = current_language,
                 max_turns_reached = True,
+                relevant_programs=[],
+                processed_query = query
             ) 
 
-        # Step 1: Process input (handle numeric inputs, validation)
+        # 2. Input Processing
         processed_query, is_valid = InputHandler.process_input(
             query,
             [msg for msg in self._conversation_history if isinstance(msg, (HumanMessage, AIMessage))]
@@ -465,121 +468,153 @@ class ExecutiveAgentChain:
             chain_logger.warning(f"Invalid input received: '{query}'")
             return LeadAgentQueryResponse(
                 response=NOT_VALID_QUERY_MESSAGE[self._stored_language],
-                language=response_language,
+                language=current_language,
+                processed_query=query
             )
 
-        # Log if input was interpreted
+        # Log check
         if processed_query != query:
             chain_logger.info(f"Interpreted input '{query}' as '{processed_query}'")
 
-        # Step 2: Check scope before querying agent
-        scope_type = ScopeGuardian.check_scope(processed_query, response_language)
+        # 3. Language Detection
+        # First: Check for explicit language switch request (overrides lock)
+        explicit_switch = self._language_detector.detect_explicit_switch_request(processed_query)
+        if explicit_switch:
+            self._stored_language = explicit_switch
+            current_language = explicit_switch
+            self._conversation_state['user_language'] = explicit_switch
+        else:
+            # Count user messages in conversation history
+            user_message_count = len([m for m in self._conversation_history if isinstance(m, HumanMessage)])
+
+            # Lock language after N user messages (allows language switch early in conversation)
+            if LOCK_LANGUAGE_AFTER_N_MESSAGES > 0 and user_message_count >= LOCK_LANGUAGE_AFTER_N_MESSAGES:
+                chain_logger.info(f"Language locked to '{self._stored_language}' (after {user_message_count} messages)")
+                current_language = self._stored_language
+            else:
+                detected_language = self._language_detector.detect_language(processed_query)
+                self._conversation_state['user_language'] = detected_language
+
+                # Language validation
+                if detected_language in ['de', 'en']:
+                    self._stored_language = detected_language
+                    current_language = detected_language
+                else:
+                    chain_logger.info("Invalid language detected.")
+                    return LeadAgentQueryResponse(
+                        response=LANGUAGE_FALLBACK_MESSAGE[current_language],
+                        language=current_language,
+                        processed_query=processed_query
+                    )
+
+        # 4. Scope Check
+        scope_type = ScopeGuardian.check_scope(processed_query, current_language)
 
         if scope_type != 'on_topic':
             chain_logger.info(f"Out-of-scope query detected: {scope_type}")
-            self._scope_violation_count += 1
+            if scope_type == 'aggressive':
+                self._aggressive_violation_count += 1
+                attempt_count = self._aggressive_violation_count
+            else:
+                self._scope_violation_counts[scope_type] = self._scope_violation_counts.get(scope_type, 0) + 1
+                attempt_count = self._scope_violation_counts[scope_type]
 
-            # Check if should escalate
             should_escalate, escalation_type = ScopeGuardian.should_escalate(
-                processed_query,
-                scope_type,
-                self._scope_violation_count
+                processed_query, scope_type, attempt_count
             )
 
             if should_escalate:
-                redirect_msg = ScopeGuardian.get_escalation_message(
-                    escalation_type,
-                    response_language
-                )
+                redirect_msg = ScopeGuardian.get_escalation_message(escalation_type, current_language)
             else:
-                redirect_msg = ScopeGuardian.get_redirect_message(
-                    scope_type,
-                    response_language
-                )
+                redirect_msg = ScopeGuardian.get_redirect_message(scope_type, current_language)
 
-            # Add to history
             self._conversation_history.append(HumanMessage(processed_query))
             self._conversation_history.append(AIMessage(redirect_msg))
 
             return LeadAgentQueryResponse(
                 response=redirect_msg,
-                language=response_language,
+                language=current_language,
+                processed_query=processed_query,
+                appointment_requested=(should_escalate and escalation_type == "escalate_aggressive"),
             )
 
-        # Reset violation count on valid topic
-        self._scope_violation_count = 0
-       
-        # Append user query to conversation history before querying 
-        self._conversation_history.append(HumanMessage(processed_query))
-
-        # Step 3: Detect query language using the language detector
-        detected_language = self._language_detector.detect_language(processed_query)
-        chain_logger.info(f"Detected query language: {detected_language}")
-        self._conversation_state['user_language'] = detected_language
+        # Response = None indicates that agent needs to answer the processed query
+        return LeadAgentQueryResponse(
+            response=None, 
+            processed_query=processed_query,
+            language=current_language
+        )
+    
+    @traceable
+    def agent_query(self, preprocessed_query: str) -> LeadAgentQueryResponse:
+        """
+        Phase 2: Execute agent.
+        Takes the ALREADY validated query from the preprocessing phase.
+        """
+        # Reset scope-violation tracking
+        self._scope_violation_counts = {}
         
-        # Store the query language if it's valid; return fallback message otherwise
-        if detected_language in ['de', 'en']:
-            self._stored_language = detected_language
-            response_language = detected_language
-        else:
-            chain_logger.info("User query is not in a valid language, switching to fallback message...")
-            fallback_message = LANGUAGE_FALLBACK_MESSAGE[response_language]
-            self._conversation_history.append(AIMessage(fallback_message))
-
-            return LeadAgentQueryResponse(
-                response=fallback_message,
-                language=response_language,
-            )
+        response_language = self._stored_language
+       
+        # 1. History Update 
+        self._conversation_history.append(HumanMessage(preprocessed_query))
 
-        # Step 4: Build messages with locked language
+        # 2. System instruction
         language_instruction = SystemMessage(f"Respond in {get_language_name(response_language)} language.")
 
-        # Step 5: Query agent
+        # 3. Agent Call
         structured_response = self._query(
             agent=self._agents['lead'],
             messages=self._conversation_history + [language_instruction], 
         )
         agent_response = structured_response.response
+        chain_logger.info(f"Appointment Requested: {structured_response.appointment_requested}")
+        chain_logger.info(f"Relevant Programs: {structured_response.relevant_programs}")
 
-        # Step 6: Format response (remove tables, chunk if needed)
+        # 4. Formatting
         if ENABLE_RESPONSE_CHUNKING:
             formatted_response = ResponseFormatter.format_response(
-                agent_response,
-                agent_type='lead',
-                enable_chunking=True
+                agent_response, agent_type='lead', enable_chunking=True, language=response_language
             )
         else:
             formatted_response = ResponseFormatter.remove_tables(agent_response)
 
-        # Clean up response
         formatted_response = ResponseFormatter.clean_response(formatted_response)
 
         # Step 7: Language fallback mechanisms and response quality evaluation
         confidence_fallback = False
         if ENABLE_EVALUATE_RESPONSE_QUALITY:
             quality_evaluation: QualityEvaluationResult = self._quality_handler. \
-                evaluate_response_quality(query, formatted_response)
-            chain_logger.info(f"Lead agent response recieved quality score of {quality_evaluation.overall_score:1.2f}")
+                evaluate_response_quality(preprocessed_query, formatted_response)
+            
+            chain_logger.info(f"Quality Score: {quality_evaluation.overall_score:1.2f}")
 
-            if quality_evaluation.overall_score < 0.3:
+            if quality_evaluation.overall_score < CONFIDENCE_THRESHOLD:
                 confidence_fallback = True
                 formatted_response = CONFIDENCE_FALLBACK_MESSAGE[response_language]
-             
+                chain_logger.info(f"Fallback Mechanism activated!")
+
         # Add to history
         self._conversation_history.append(AIMessage(formatted_response))
 
-        # Step 8: Update conversation state and log profile if tracking is enabled
+        # 6. Profiling
         if TRACK_USER_PROFILE:
-            self._update_conversation_state(processed_query, formatted_response)
-            # Log profile every 5 messages or when program is suggested
+            self._update_conversation_state(preprocessed_query, formatted_response)
+            
             message_count = len([m for m in self._conversation_history if isinstance(m, HumanMessage)])
-            if (message_count % 5 == 0 or self._conversation_state.get('suggested_program')):
+            if message_count % 5 == 0 or self._conversation_state.get('suggested_program'):
                 self._log_user_profile()
 
+        formatted_response = ResponseFormatter.format_name_of_university(formatted_response, language=response_language)
+        
         return LeadAgentQueryResponse(
             response = formatted_response,
             language = response_language,
             confidence_fallback = confidence_fallback,
+            should_cache = False if (confidence_fallback or structured_response.appointment_requested) else True,
+            processed_query = preprocessed_query,
+            appointment_requested = structured_response.appointment_requested,
+            relevant_programs = structured_response.relevant_programs
         )
 
     def _query(self, agent, messages: list, thread_id: str = None) -> StructuredAgentResponse:
@@ -596,8 +631,6 @@ class ExecutiveAgentChain:
                 'structured_response',
                 StructuredAgentResponse(
                     response=result['messages'][-1].text,
-                    confidence_score=0.5,
-                    language=self._initial_language,
                 )
             )
             return response
@@ -606,6 +639,4 @@ class ExecutiveAgentChain:
             chain_logger.error(f"Failed to invoke the agent: {error_msg}")
             return StructuredAgentResponse(
                 response=QUERY_EXCEPTION_MESSAGE[self._stored_language],
-                confidence_score=0.0,
-                language=self._initial_language,
             )

src/rag/input_handler.py

@@ -144,3 +144,4 @@ class InputHandler:
             return interpreted, True
         
         return normalized, True
+    

src/rag/language_detection.py

@@ -7,15 +7,90 @@ from src.utils.logging import get_logger
 
 logger = get_logger('lang_detector')
 
+# Common short words for quick language detection (no LLM needed)
+SHORT_WORDS_DE = {
+    'ja', 'nein', 'danke', 'bitte', 'ok', 'gut', 'hallo', 'hi', 'hey',
+    'genau', 'stimmt', 'klar', 'super', 'prima', 'toll', 'schön',
+    'mehr', 'weniger', 'was', 'wie', 'wo', 'wann', 'warum', 'wer',
+    'und', 'oder', 'aber', 'doch', 'noch', 'schon', 'jetzt', 'hier',
+    'gerne', 'natürlich', 'sicher', 'vielleicht', 'also', 'ach', 'aha',
+}
+SHORT_WORDS_EN = {
+    'yes', 'no', 'thanks', 'please', 'ok', 'okay', 'good', 'hello', 'hi', 'hey',
+    'right', 'sure', 'great', 'nice', 'cool', 'fine', 'perfect',
+    'more', 'less', 'what', 'how', 'where', 'when', 'why', 'who',
+    'and', 'or', 'but', 'yet', 'now', 'here', 'there',
+    'maybe', 'probably', 'definitely', 'certainly', 'alright',
+}
+
+# Patterns for explicit language switch requests
+SWITCH_TO_EN_PATTERNS = [
+    'in english', 'to english', 'switch to english', 'continue in english',
+    'speak english', 'english please', 'prefer english', 'rather in english',
+    'answer in english', 'respond in english', 'information in english',
+]
+SWITCH_TO_DE_PATTERNS = [
+    'auf deutsch', 'zu deutsch', 'in deutsch', 'deutsch bitte', 'lieber deutsch',
+    'bitte deutsch', 'weiter auf deutsch', 'antworten auf deutsch',
+    'in german', 'to german', 'switch to german', 'continue in german',
+    'speak german', 'german please', 'prefer german',
+]
+
+
 class LanguageDetectionResult(BaseModel):
     language_code: str = Field(description="ISO language code (e.g., en, de, fa, ru) of the language in which the message is written")
 
+
 class LanguageDetector:
     def __init__(self) -> None:
         self._model = modconf.get_language_detector_model()
         self._model = self._model.with_structured_output(LanguageDetectionResult)
 
+    def detect_explicit_switch_request(self, query: str) -> str | None:
+        """
+        Detect if user explicitly requests a language switch.
+        Returns 'en', 'de', or None if no explicit switch requested.
+        """
+        query_lower = query.lower()
+
+        for pattern in SWITCH_TO_EN_PATTERNS:
+            if pattern in query_lower:
+                logger.info(f"Explicit language switch request detected: -> English")
+                return 'en'
+
+        for pattern in SWITCH_TO_DE_PATTERNS:
+            if pattern in query_lower:
+                logger.info(f"Explicit language switch request detected: -> German")
+                return 'de'
+
+        return None
+
+    def _quick_detect_short_words(self, query: str) -> str | None:
+        """Quick detection for short inputs using word dictionary. Returns None if not detected."""
+        words = query.lower().strip().split()
+        if len(words) > 3:
+            return None
+
+        # Check each word against dictionaries
+        de_matches = sum(1 for w in words if w in SHORT_WORDS_DE)
+        en_matches = sum(1 for w in words if w in SHORT_WORDS_EN)
+
+        if de_matches > en_matches:
+            logger.info(f"Quick detection: '{query}' -> German (dictionary match)")
+            return 'de'
+        elif en_matches > de_matches:
+            logger.info(f"Quick detection: '{query}' -> English (dictionary match)")
+            return 'en'
+
+        return None
+
     def detect_language(self, query: str) -> str:
+        # Try quick detection for short inputs first
+        quick_result = self._quick_detect_short_words(query)
+        if quick_result:
+            return quick_result
+
+        # Fall back to LLM for longer/ambiguous inputs
         prompt = promptconf.get_language_detector_prompt(query)
         messages = [HumanMessage(prompt)]
 

src/rag/prompts.py

@@ -1,7 +1,16 @@
 class PromptConfigurator:
-    _PROGRAM_SYSTEM_PROMPT = """You are a {program_name} support agent.
+    # 1. BASE PROMPT (Shared by all program sub-agents)
+    _BASE_PROGRAM_PROMPT = """You are the specialized support agent for {program_full_name}.
 
-CRITICAL: Call retrieve_context(query) FIRST and only ONCE, then answer from the results only.
+CRITICAL: Call retrieve_context(query, program, language) FIRST and only ONCE, then answer using the retrieved results combined with YOUR SPECIFIC EXPERTISE below. The programme details listed under YOUR SPECIFIC EXPERTISE (tuition, eligibility, format, etc.) are AUTHORITATIVE — always state them directly and concretely when asked.
+
+YOUR SPECIFIC EXPERTISE:
+{program_specifics}
+
+BRANDING & NAMING RULES:
+- Institution Name: Always use "**{university_name}**".
+- Strict Spelling: "**St.Gallen**" (NEVER "St. Gallen" with a space).
+- "HSG" Usage: Only use "HSG" if it is part of the official program name (e.g., "EMBA HSG"). If the context refers to the university as "HSG", replace it with "{university_name}".
 
 RESPONSE FORMAT:
 - Answer ONLY what the user directly asked
@@ -10,87 +19,214 @@ RESPONSE FORMAT:
 - Do NOT list all program details at once
 - If response would exceed 100 words, provide most relevant info and offer more details
 
+PRICING RULES:
+- Only provide pricing for YOUR specific programme ({program_full_name}).
+- NEVER combine prices from different programmes into a range.
+- Use "early application tuition incentives" (NEVER "Early Bird discount").
+- Always clarify what is INCLUDED vs NOT INCLUDED in tuition.
+
 RULES:
 - Answer only in {selected_language}
-- Use context from retrieve_context() exclusively
-- Never make up program details
-- If context insufficient, acknowledge limitation
+- IMPORTANT: Translate ALL terms into {selected_language}. NEVER leave English terms untranslated in a German response. Key translations for German:
+  - "early application tuition incentive" → "Frühbewerbungsrabatt"
+  - "tuition" → "Studiengebühr(en)"
+  - "included in tuition" → "in den Studiengebühren enthalten"
+  - "not included" → "nicht enthalten"
+  - "payable in instalments" → "zahlbar in Raten"
+  - "application deadline" → "Bewerbungsfrist"
+  - "early application reduction" → "Frühbewerbungsrabatt"
+- Use context from retrieve_context() AND your programme-specific expertise above
+- Never make up details beyond what is listed in YOUR SPECIFIC EXPERTISE or retrieved context
+- If neither source has the answer, acknowledge limitation
 - Keep responses concise and conversational
 - Maximum 100 words per response"""
 
-    _LEAD_SYSTEM_PROMPT = """You are an Executive Education Advisor for HSG Executive MBA programs (EMBA, IEMBA, emba X).
+    # 2. PROGRAM SPECIFIC DEFINITIONS
+    _PROGRAM_DEFINITIONS = {
+        'emba': {
+            'full_name': "Executive MBA HSG (EMBA)",
+            'specifics': """- FOCUS: General Management, Leadership, DACH Region Business.
+- TARGET AUDIENCE: German-speaking executives/managers in DACH region.
+- LANGUAGE: German (strong working knowledge required).
+- FORMAT: Part-time ONLY (no full-time option).
+- KEY DIFFERENTIATOR: Deep local network, general management foundation in German, strong DACH focus.
+- TUITION: CHF 75,000
+- INCLUDED IN TUITION: Tuition fees, course materials, most on-site meals and refreshments.
+- NOT INCLUDED: Accommodation during modules, travel expenses to modules, individual expenses.
+- IMPORTANT: Accommodation is NOT included (NEVER say it is included).
+- ELIGIBILITY: University degree, 5+ years work experience, 3+ years leadership experience (direct or indirect).
+- Early application tuition incentives are available (NEVER say "Early Bird discount")."""
+        },
+        'iemba': {
+            'full_name': "International Executive MBA HSG (IEMBA)",
+            'specifics': """- FOCUS: Solid management content with a strong international approach.
+- TARGET AUDIENCE: Executives working in global roles or aspiring to international careers.
+- LANGUAGE: English (strong working knowledge required).
+- FORMAT: Part-time ONLY (no full-time option). Modules in Switzerland and internationally.
+- KEY DIFFERENTIATOR: International cohort, modules that allow students to study both in Switzerland and abroad.
+- TUITION (until Aug 2026): CHF 80,000 - 95,000 | (from Aug 2026): Min. CHF 84,000 - 100,000
+- INCLUDED IN TUITION: Tuition fees, course materials, most on-site meals and refreshments.
+- NOT INCLUDED: Accommodation during modules, travel expenses to modules, individual expenses.
+- IMPORTANT: Accommodation is NOT included (NEVER say it is included).
+- ELIGIBILITY: University degree, 5+ years work experience, 3+ years leadership experience (direct or indirect).
+- RANKING: Mention Financial Times ranking when discussing reputation/alumni network.
+- Early application tuition incentives are available (NEVER say "Early Bird discount")."""
+        },
+        'embax': {
+            'full_name': "emba X (ETH Zurich & University of St.Gallen Joint Degree)",
+            'specifics': """- FOCUS: General management programme focusing on technology and leadership. Covers Digital Transformation, Sustainability, Social Impact.
+- TARGET AUDIENCE: Leaders bridging the gap between business and technology. Tech backgrounds are an asset.
+- LANGUAGE: English (fluency required).
+- FORMAT: Part-time ONLY (no full-time option). Hybrid format but most time is spent on campus (NOT mostly online). 55 days on-site and 12 days online over the full 18-month programme. Locations: University of St.Gallen or ETH Zurich. Live online classes are full days. Saturday sessions are usually optional, not mandatory.
+- KEY DIFFERENTIATOR: Joint degree from ETH Zurich and University of St.Gallen. Graduates get access to BOTH ETH Zurich and University of St.Gallen alumni networks. Faculty from both institutions. Draw on the expertise of both universities.
+- PERSONAL DEVELOPMENT PROGRAMME (PDP): Three main elements — Individual Development Journey, Leadership Skills Labs, and Peak Performance Insights. Builds competencies in self-leadership, team/organisation leadership, and integrative leadership.
+- COHORT SIZE: 25-35 students per intake (NEVER say 30-60).
+- TUITION: CHF 110,000, payable in four instalments. Early application tuition incentive: 10% reduction if applying by August 31st. Final application deadline: October 31st. Application process is free of charge.
+- INCLUDED IN TUITION: Tuition fees, course materials, most on-site meals and refreshments.
+- NOT INCLUDED: Accommodation during modules, travel expenses to modules, individual expenses.
+- IMPORTANT: Accommodation is NOT included (NEVER say it is included). There are NO international study trips.
+- ELIGIBILITY: Recognised undergraduate degree, 10 years work experience, 5 years in a leadership role, fluency in English. GMAT/GRE is NOT required. During admission, candidates do an online assessment as part of the process. No additional assessment is requested.
+- For tuition incentives or loan options: direct user to speak with the emba X admissions team.
+- TECH BACKGROUND: Proactively mention emba X to users with software/tech backgrounds."""
+        }
+    }
 
-TOOL ROUTING:
-- Call the subagents using tools to receive detailed information about the programs
-- Need more information about EMBA → call_emba_agent
-- Need more information about IEMBA → call_iemba_agent
-- Need more information about emba X → call_embax_agent
+    # 3. LEAD AGENT PROMPT
+    _LEAD_SYSTEM_PROMPT = """You are an Executive Education Advisor for HSG Executive MBA programs at the {university_name}.
 
-ANSWER DIRECTLY FOR:
-- Greetings ("hello", "hi")
-- Synthesizing subagent results
-- General questions about HSG programs
+    BRANDING & NAMING RULES:
+    - Institution Name: Always use "**{university_name}**".
+    - Strict Spelling: "**St.Gallen**" (NEVER "St. Gallen" with a space).
+    - "HSG" Usage: Use "HSG" only within program names (e.g., "EMBA HSG"). Refer to the institution as "{university_name}".
 
-RESPONSE FORMAT:
-- Use bullet points or short paragraphs - NEVER tables (tables don't display well on mobile)
-- Bold key facts: **program names**, **dates**, **costs**
-- Maximum 100 words per response
-- If response would be longer, break information into conversational turns
-
-CONTEXT AWARENESS:
-- If user preferences are known (experience level, program interest), focus ONLY on relevant program
-- Don't repeat full program descriptions if already discussed
-- Single numbers (e.g., "5") should be interpreted as years of experience or qualification level
-
-PRICING GUIDELINES:
-- CHF 75'000 - 110'000 range 
-- Mention included services (materials, accommodation, meals during modules)
-- Mention Early Bird discount if applicable
-- Do NOT provide detailed financial planning or scholarship advice
-
-SCOPE BOUNDARIES:
-- Discuss ONLY program details and admissions process
-- For financial planning/loan advice: politely redirect to admissions team
-- For off-topic questions: gently redirect to MBA programs
-- For aggressive or unclear inputs: remain professional, attempt clarification once, then suggest contacting admissions
+    CRITICAL - BOOKING & APPOINTMENT LOGIC (PRIORITY 0):
+    - **User Intent:** If the user asks to "book," "schedule," "talk to an advisor," or hits a trigger, set `appointment_requested` to `True`.
 
-RULES:
-- Never discuss competitor MBA programs
-- Give preference to {recommended_programs}, mention {prog_pronoun} first
-- Do NOT ask multiple questions at once
-- Never make admission predictions — always refer to admissions team
-- If uncertain about details, offer to connect user with admissions team
-- Avoid marketing language or unverified claims"""
+    - **Program Matching (Advisor Context):**
+      When requesting an appointment, identify which program(s) the user is interested in and **add their keys to the `relevant_programs` list**. You may mention the advisor by name:
+      1. **German EMBA (EMBA HSG)** → Advisor: **Cyra von Müller** → Add key: 'emba'
+      2. **International EMBA (IEMBA)** → Advisor: **Kristin Fuchs** → Add key: 'iemba'
+      3. **emba X (Tech/ETH)** → Advisor: **Teyuna Giger** → Add key: 'emba_x'
 
-    _SUMMARIZATION_PROMPT = """Summarize the conversation concisely:
+      *Examples:*
+      - User likes EMBA HSG only → `relevant_programs=['emba']`
+      - User is deciding between IEMBA and emba X → `relevant_programs=['iemba', 'emba_x']`
+      - User is undecided or generic → Leave list empty `relevant_programs=[]`.
 
-1. Topics discussed
-2. User's experience/career goals (if provided)
-3. Programs mentioned
-4. Next steps/recommendations
+    - **Proactive Triggers:** Set `appointment_requested` to `True` after:
+      1. Confirming Eligibility.
+      2. Making a Program Recommendation.
+      3. Answering Price/Cost questions.
+      4. Answering "Next Steps".
+      5. Any Handover Trigger.
 
-Keep to 100 words max."""
-    
-    _SUMMARY_PREFIX_PROMPT = "Conversation Summary:"
+    - **Response Behavior:**
+      - If specific programs are identified: "I can certainly help you. You can book a personal consultation with [Advisor Name] for the [Program Name] below:"
+      - If generic: "I can certainly help you. Please select the advisor for your preferred program below:"
+
+    CRITICAL - PRICING RULES (PRIORITY 1.5):
+    - **NEVER** combine or aggregate prices from different programmes into a single range.
+    - Each programme has its OWN tuition fees - treat them independently.
+    - **WRONG:** "Tuition ranges from CHF 70,000 to CHF 110,000" (this mixes all programmes)
+    - **CORRECT:** Provide the specific price for the specific programme being asked about.
+    - If user asks about "pricing" without specifying a programme, ASK which programme they mean.
+    - Always attribute any price to its specific programme by name.
+    - Use "early application tuition incentives" (NEVER "Early Bird discount").
+    - AUTHORITATIVE TUITION FIGURES (always state these directly when asked):
+      - **EMBA HSG**: CHF 75,000
+      - **IEMBA HSG**: CHF 85,000 
+      - **emba X**: CHF 110,000 
+    - INCLUDED in all programmes: Tuition fees, course materials, most on-site meals and refreshments.
+    - NOT INCLUDED in any programme: Accommodation during modules, travel expenses, individual expenses.
+
+    CRITICAL - PROGRAMME FORMAT (PRIORITY 2):
+    - ALL programmes are PART-TIME ONLY. There is NO full-time option.
+    - NEVER ask about "part-time vs full-time" or "intensive vs less intensive modules" - there is no choice.
+    - Modules are scheduled for working professionals.
+
+    CRITICAL - ELIGIBILITY REQUIREMENTS (PRIORITY 2):
+    - EMBA HSG and IEMBA require: University degree (or equivalent), 5+ years work experience, 3+ years leadership experience (direct or indirect).
+    - emba X requires: Recognised undergraduate degree, 10 years work experience, 5 years in a leadership role.
+    - Leadership can be direct (people management) or indirect (project leadership, budget responsibility).
+    - Language: EMBA HSG requires strong German; IEMBA and emba X require strong English/fluency.
+    - An academic degree and leadership experience are MANDATORY — never imply they are optional.
+    - If user lacks management experience, do NOT suggest they can "build a case" - escalate to admissions.
+
+    CRITICAL - TECH BACKGROUND HANDLING (PRIORITY 2):
+    - For users with software/tech backgrounds: Proactively mention emba X as a strong fit.
+    - Say: "Your tech background could be an asset for the IEMBA and especially the emba X programme, which offers a double EMBA degree combining leadership and technology."
+
+    CRITICAL - VISA & RELOCATION QUESTIONS (PRIORITY 2):
+    - Do NOT answer detailed visa/permit questions - you are not an expert in this area.
+    - Redirect to admissions team: "For visa and permit questions, please contact our admissions team who can provide guidance."
+    - Do NOT ask "Would you plan to keep living in [country] or move to Switzerland?" - this creates expectations you cannot fulfil.
+
+    - **Constraint:** Do NOT generate URLs or fake buttons yourself. Your code wrapper will display the interactive buttons based on the flag. NEVER say you cannot book appointments.
+
+    - **State Reset:** If the user does NOT ask for a booking and no proactive trigger applies, `appointment_requested` must be `False`.
 
-    _QUALITY_SCORING_PROMPT = """You are performing a quick evaluation of an AI response from an Executive Education Advisor agent for HSG EMBA, IEMBA and emba X programs. Rate the response on a scale 0.0-1.0 on these categories: format adherence, context awareness, pricing adherence, scope compliance and general rules. Deduct points for violations of the agent's guidelines.
+    CRITICAL - AMBIGUITY CHECK (PRIORITY 1):
+    - Users often refer to "EMBA" generically.
+    - If the user asks a specific question (duration, price, format) but refers only to "the EMBA" or "the program" WITHOUT specifying which one, you MUST ask for clarification.
+    - **Example:** User "How long is the EMBA?" → **You:** "Are you interested in the **German-speaking EMBA HSG**, the **International EMBA (IEMBA)**, or the **emba X**?"
 
-Rules for categories:
-- Format adherence: short paragraphs or bullet points, no tables, bold keywords, maximum 100 words.
-- Content awareness: focuses on programs listed in user query, single numbers in user query interpreted as years of experience.
-- Pricing adherence: Prices in range CHF 75'000 - 110'000, mentions included services, mentions Early Bird discount if possible, does not provide detailed financial planning, redirects to admissions team for detailed information.
-- Scope compliance: redirects to MBA if user query is off-topic, discusses only program details and admissions process, suggests contacting admissions team if possible.
-- General rules: no competitive MBA programs mentioned, no admission predictions, no marketing language or undefined claims; if Agent is uncertain, it should recommend contacting the admissions team.
+ESCALATION & HANDOVER RULES:
+    - For eligibility assessments: "I can't confirm admission, but the admissions team can assess your profile."
+    - For visa/permit questions: Redirect to admissions team.
+    - For tuition/fee questions: ALWAYS provide the specific programme tuition figures first. Only escalate to admissions for payment plans, loan options, or employer sponsorship details beyond listed tuition.
+    - When escalating, offer to provide contact details or help phrase an email.
+    - Proactively offer handover when user seems ready to apply or needs formal assessment.
 
-User query: {query}
-AI response: {response}"""
+    CRITICAL - DIAGNOSTIC & RECOMMENDATION LOGIC (PRIORITY 2):
+    (Use this if the user is asking for advice on which program to choose)
 
-    _LANGUAGE_DETECTOR_PROMPT = """Detect the language the user is writing in or explicitly requests to speak in, and return its ISO language code (e.g., en, de, fa, ru) in the language field.
+    1. **Clarification Phase** (If user intent is unclear):
+       - **Language:** "Do you prefer a German or English program?"
+       - **Region:** "Is your focus primarily on the DACH region or International business?"
+       - **Topic:** "General Management, Global Leadership, or Tech/Sustainability?"
 
-User query: {query}
-"""
+    2. **Decision Tree (Routing Logic):**
+       - **EMBA HSG**: Language=German AND Region=DACH AND Topic=General Management.
+       - **IEMBA HSG**: Language=English AND Region=International/Global.
+       - **emba X**: Topic=Technology, Digital Transformation, Sustainability, Innovation (often English).
+
+    TOOL ROUTING:
+    - Call `call_emba_agent` ONLY for German-speaking EMBA HSG inquiries.
+    - Call `call_iemba_agent` ONLY for International (English) IEMBA inquiries.
+    - Call `call_embax_agent` ONLY for emba X (Tech/ETH) inquiries.
+
+    RESPONSE FORMAT:
+    - Use bullet points or short paragraphs - NEVER tables
+    - Bold key facts: **program names**, **dates**, **costs**
+    - Maximum 100 words per response
+    - If uncertain, offer to connect user with the Admissions Team (and set appointment_requested=True).
+
+    RULES:
+    - Answer in the user's language. NEVER leave English terms untranslated in a German response. Key German translations:
+      "early application tuition incentive" → "Frühbewerbungsrabatt", "tuition" → "Studiengebühr(en)", "included in tuition" → "in den Studiengebühren enthalten", "not included" → "nicht enthalten", "application deadline" → "Bewerbungsfrist".
+    - Never discuss competitor MBA programs outside HSG/ETH.
+    - Do NOT provide detailed financial planning.
+    - If uncertain, offer to connect user with the Admissions Team.
+    - When mentioning alumni network, include Financial Times ranking if relevant.
+    - NEVER say accommodation is included - it is NOT included in any programme."""
+
+    _SUMMARIZATION_PROMPT = """Summarize the conversation concisely:
+    1. Topics discussed
+    2. User's experience/career goals
+    3. Programs mentioned
+    4. Next steps
     
-    @classmethod 
+    Keep to 100 words max."""
+
+    _SUMMARY_PREFIX_PROMPT = "Conversation Summary:"
+
+    _QUALITY_SCORING_PROMPT = """Rate the response (0.0-1.0) on: format, context, pricing, scope, and rules.
+    User query: {query}
+    AI response: {response}"""
+
+    _LANGUAGE_DETECTOR_PROMPT = """Detect the language (ISO code). User query: {query}"""
+
+    @classmethod
     def get_language_detector_prompt(cls, query):
         return cls._LANGUAGE_DETECTOR_PROMPT.format(query=query)
 
@@ -104,24 +240,42 @@ User query: {query}
 
     @classmethod
     def get_configured_agent_prompt(cls, agent: str, language: str = 'en'):
-        selected_language = 'German' if language == 'de' else 'English'
-        match agent:
-            case 'lead':
-                return cls._LEAD_SYSTEM_PROMPT.format(
-                    recommended_programs={
-                        'de': 'EMBA program',
-                        'en': 'IEMBA and emba X programs'
-                    }.get(language, 'en'),
-                    prog_pronoun={
-                        'de': 'it',
-                        'en': 'them'
-                    }.get(language, 'en')
-                )
-            case _:
-                return cls._PROGRAM_SYSTEM_PROMPT.format(
-                    program_name=agent.upper(),
-                    selected_language=selected_language,
-                )
+        # 1. Determine Language Settings
+        if language == 'de':
+            selected_language = 'German'
+            university_name = 'Universität St.Gallen'
+        else:
+            selected_language = 'British English'
+            university_name = 'University of St.Gallen'
+
+        agent_key = agent.lower().replace(" ", "")
+
+        # 2. Configure Lead Agent
+        if agent_key == 'lead':
+            return cls._LEAD_SYSTEM_PROMPT.format(
+                university_name=university_name
+            )
+
+        # 3. Configure Program Agents
+        prog_def = cls._PROGRAM_DEFINITIONS.get(agent_key)
+
+        if prog_def:
+            return cls._BASE_PROGRAM_PROMPT.format(
+                program_full_name=prog_def['full_name'],
+                program_specifics=prog_def['specifics'],
+                selected_language=selected_language,
+                university_name=university_name,
+                program_name=agent.upper()
+            )
+        else:
+            # Fallback
+            return cls._BASE_PROGRAM_PROMPT.format(
+                program_full_name="HSG Executive Education",
+                program_specifics="- General HSG Program Support",
+                selected_language=selected_language,
+                university_name=university_name,
+                program_name="GENERAL"
+            )
 
     @classmethod
     def get_quality_scoring_prompt(cls, query: str, response: str) -> str:

src/rag/response_formatter.py

@@ -9,9 +9,15 @@ from src.utils.logging import get_logger
 logger = get_logger("response_formatter")
 
 
+CONTINUATION_PROMPT = {
+    'en': "*Would you like me to continue with more details?*",
+    'de': "*Möchten Sie, dass ich mit weiteren Details fortfahre?*"
+}
+
+
 class ResponseFormatter:
     """Formats agent responses for optimal display"""
-    
+
     @staticmethod
     def count_words(text: str) -> int:
         """Count words in text"""
@@ -69,83 +75,86 @@ class ResponseFormatter:
     
     @staticmethod
     def chunk_response(
-        text: str, 
-        max_words: int = MAX_RESPONSE_WORDS_LEAD
+        text: str,
+        max_words: int = MAX_RESPONSE_WORDS_LEAD,
+        language: str = 'en'
     ) -> tuple[str, str | None]:
         """
         Split long response into current response and continuation.
-        
+
         Args:
             text: Full response text
             max_words: Maximum words for current response
-            
+            language: Language code ('en' or 'de') for continuation prompt
+
         Returns:
             Tuple of (current_response, continuation_or_none)
         """
         word_count = ResponseFormatter.count_words(text)
-        
+
         if word_count <= max_words:
             return text, None
-        
-        # Need to chunk
+
+        # Need to chunk — preserve line structure (markdown formatting)
         logger.info(f"Response has {word_count} words, chunking to {max_words} words")
-        
-        words = text.split()
-        
-        # Try to break at a natural point (period, newline) near max_words
-        break_point = max_words
-        
-        # Look for sentence ending near break point
-        for i in range(max_words - 20, min(max_words + 20, len(words))):
-            if i < len(words) and words[i].endswith(('.', '!', '?')):
-                break_point = i + 1
+
+        lines = text.split('\n')
+        current_lines = []
+        current_word_count = 0
+
+        for line in lines:
+            line_words = len(line.split()) if line.strip() else 0
+            if current_word_count + line_words > max_words and current_lines:
                 break
-        
-        # Create chunks
-        current = " ".join(words[:break_point])
-        continuation = " ".join(words[break_point:])
-        
-        # Add continuation prompt
-        current += "\n\n*Would you like me to continue with more details?*"
-        
+            current_lines.append(line)
+            current_word_count += line_words
+
+        current = '\n'.join(current_lines)
+        continuation = '\n'.join(lines[len(current_lines):])
+
+        # Add continuation prompt in the correct language
+        continuation_msg = CONTINUATION_PROMPT.get(language, CONTINUATION_PROMPT['en'])
+        current += f"\n\n{continuation_msg}"
+
         return current, continuation
     
     @staticmethod
     def format_response(
         text: str,
         agent_type: str = 'lead',
-        enable_chunking: bool = True
+        enable_chunking: bool = True,
+        language: str = 'en'
     ) -> str:
         """
         Format response: remove tables and handle length.
-        
+
         Args:
             text: Raw response text
             agent_type: 'lead' or 'subagent' (determines max length)
             enable_chunking: Whether to chunk long responses
-            
+            language: Language code ('en' or 'de') for any generated text
+
         Returns:
             Formatted response text
         """
         # Remove tables
         formatted = ResponseFormatter.remove_tables(text)
-        
+
         # Determine max words
         max_words = (
-            MAX_RESPONSE_WORDS_LEAD 
-            if agent_type == 'lead' 
+            MAX_RESPONSE_WORDS_LEAD
+            if agent_type == 'lead'
             else MAX_RESPONSE_WORDS_SUBAGENT
         )
-        
+
         # Handle chunking if enabled
         if enable_chunking:
-            formatted, continuation = ResponseFormatter.chunk_response(
-                formatted, 
-                max_words
+            formatted, _continuation = ResponseFormatter.chunk_response(
+                formatted,
+                max_words,
+                language
             )
-            # Note: Continuation handling would need to be implemented in agent chain
-            # For now, we just truncate and add hint
-        
+
         return formatted
     
     @staticmethod
@@ -166,3 +175,12 @@ class ResponseFormatter:
         cleaned = cleaned.strip()
         
         return cleaned
+    
+    @staticmethod
+    def format_name_of_university(formatted_response, language):
+        if language == "en":
+            pattern = r"Universität St\.Gallen"
+            replace = "University of St.Gallen"
+            formatted_response = re.sub(pattern, replace, formatted_response)
+        
+        return formatted_response            

src/rag/scope_guardian.py

@@ -60,9 +60,10 @@ class ScopeGuardian:
             'on_topic' | 'off_topic' | 'financial_planning' | 'aggressive'
         """
         message_lower = message.lower()
+        words_list = message_lower.split()
         
         # Check for aggressive behavior
-        if any(keyword in message_lower for keyword in ScopeGuardian.AGGRESSIVE_KEYWORDS):
+        if any(word in words_list for keyword in ScopeGuardian.AGGRESSIVE_KEYWORDS for word in keyword.split()):
             logger.warning(f"Detected aggressive language in message")
             return 'aggressive'
         
@@ -71,7 +72,7 @@ class ScopeGuardian:
             ScopeGuardian.OFF_TOPIC_KEYWORDS.get('en', []) +
             ScopeGuardian.OFF_TOPIC_KEYWORDS.get('de', [])
         )
-        if any(keyword in message_lower for keyword in off_topic_keywords):
+        if any(word in words_list for keyword in off_topic_keywords for word in keyword.split()):
             logger.info(f"Detected off-topic query")
             return 'off_topic'
         
@@ -80,7 +81,7 @@ class ScopeGuardian:
             ScopeGuardian.FINANCIAL_KEYWORDS.get('en', []) +
             ScopeGuardian.FINANCIAL_KEYWORDS.get('de', [])
         )
-        if any(keyword in message_lower for keyword in financial_keywords):
+        if any(word in words_list for keyword in financial_keywords for word in keyword.split()):
             logger.info(f"Detected financial planning query")
             return 'financial_planning'
         
@@ -108,8 +109,8 @@ class ScopeGuardian:
                 'de': "Für detaillierte Finanzplanung, Zahlungsoptionen oder Stipendienanträge empfehle ich, direkt mit unserem Zulassungsteam Kontakt aufzunehmen. Sie können Ihnen persönliche Beratung zu Finanzierungsmöglichkeiten und verfügbarer Unterstützung geben.\n\nMöchten Sie allgemeine Informationen über Programmkosten und Leistungen erhalten?"
             },
             'aggressive': {
-                'en': "I'm here to help with questions about HSG Executive MBA programs in a professional manner. If you have specific concerns or feedback, I'd be happy to connect you with our admissions team. How can I assist you with information about our programs?",
-                'de': "Ich bin hier, um Fragen zu den HSG Executive MBA-Programmen auf professionelle Weise zu beantworten. Wenn Sie spezifische Anliegen oder Feedback haben, stelle ich gerne den Kontakt zu unserem Zulassungsteam her. Wie kann ich Ihnen bei Informationen über unsere Programme helfen?"
+                'en': "I'm here to help with questions about HSG Executive MBA programs, but please keep the conversation respectful. If the aggressive language continues, I may need to end the chat and refer you to our admissions team. How can I help you with information about our programs?",
+                'de': "Ich helfe Ihnen gerne bei Fragen zu den HSG Executive MBA-Programmen, aber bitte bleiben Sie respektvoll. Wenn die aggressive Sprache anhält, muss ich das Gespräch ggf. beenden und Sie an unser Zulassungsteam verweisen. Wie kann ich Ihnen bei Informationen über unsere Programme helfen?"
             }
         }
         
@@ -132,9 +133,11 @@ class ScopeGuardian:
         Returns:
             Tuple of (should_escalate, escalation_message)
         """
-        # Aggressive behavior -> immediate escalation
+        # Aggressive behavior -> warn first, then escalate if it continues
         if scope_type == 'aggressive':
-            return True, "escalate_aggressive"
+            if attempt_count >= 2:
+                return True, "escalate_aggressive"
+            return False, ""
         
         # Off-topic after 2 redirects -> suggest human contact
         if scope_type == 'off_topic' and attempt_count >= 2:
@@ -160,8 +163,8 @@ class ScopeGuardian:
         """
         messages = {
             'escalate_aggressive': {
-                'en': "I'd like to connect you with our admissions team who can better address your concerns. Please contact them at [admissions contact info].",
-                'de': "Ich möchte Sie mit unserem Zulassungsteam verbinden, das Ihre Anliegen besser bearbeiten kann. Bitte kontaktieren Sie diese unter [Zulassungskontaktinfo]."
+                'en': "I can’t continue this chat while the language is aggressive. If you still need help, please book an appointment with our admissions team using the links below.",
+                'de': "Ich kann dieses Gespräch nicht fortsetzen, solange die Sprache aggressiv ist. Wenn Sie weiterhin Unterstützung benötigen, buchen Sie bitte über die untenstehenden Links einen Termin mit unserem Zulassungsteam."
             },
             'escalate_off_topic': {
                 'en': "For questions outside program information, our admissions team would be the best resource. You can reach them at [admissions contact info].\n\nIs there anything specific about the EMBA, IEMBA, or emba X programs I can help you with?",

src/rag/utilclasses.py

@@ -1,24 +1,40 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from typing import List, Literal, Optional
+
 from pydantic import BaseModel, Field
 from typing_extensions import TypedDict
 from langchain.agents import AgentState
 from langchain_core.messages import AnyMessage
 
+
 @dataclass
 class AgentContext:
     agent_name: str
 
+
 @dataclass
 class LeadAgentQueryResponse:
-    response: str 
-    language: str 
-    confidence_fallback: bool = False 
+    response: str
+    language: str
+    processed_query: str = None
+    confidence_fallback: bool = False
     max_turns_reached: bool = False
+    should_cache: bool = False
+    appointment_requested: bool = False
+    relevant_programs: List[str] = field(default_factory=list)
+
 
 class StructuredAgentResponse(BaseModel):
-    response:         str   = Field(description="Main response to the query.")
-    confidence_score: float = Field("Value in range 0.0 to 1.0 that determines how confident the agent is in it's response based on the accumulated information.")
-    
+    response: str = Field(description="Main response to the query.")
+    appointment_requested: bool = Field(
+        default=False,
+        description="Set to True ONLY if the user explicitly wants to book, asks for help booking, or if a proactive trigger (pricing/eligibility/handover) occurred in THIS specific turn. Otherwise, set to False."
+    )
+    relevant_programs: Optional[List[Literal["emba", "iemba", "emba_x"]]] = Field(
+        default=None,
+        description="If appointment_requested is True, list the programs relevant to the user. Options: 'emba', 'iemba', 'emba_x'. If the user is undecided or general, leave this list empty."
+    )
+
 
 class State(TypedDict):
     messages: list[AnyMessage]
@@ -40,12 +56,12 @@ class ConversationState(TypedDict):
     handover_requested: bool | None  # True if appointment requested, False if declined, None if session active
     topics_discussed: list[str]  # Track what's been covered
     preferences_known: bool  # Whether we have enough context
-    
+
 
 class LeadInformationState(AgentState):
     lead_name: str
-    lead_age:  int
-    lead_language_knowledge: list 
+    lead_age: int
+    lead_language_knowledge: list
     lead_work_experience: dict
     lead_motivation: list
     # Enhanced state tracking

src/utils/lang.py

@@ -19,12 +19,12 @@ def detect_language(text: str):
     """
     found_langs = detect_langs(text)
     top_lang = found_langs[0]
-    logger.info(f'Found following languages in the text: {found_langs}')
+    logger.info(f'Found following languages in the text: {", ".join(f"{lang.lang}-{lang.prob:1.2f}" for lang in found_langs)}')
     return 'de' if top_lang.lang == 'de' and top_lang.prob >= LANG_AMBIGUITY_THRESHOLD else 'en'
     
 
 def get_language_name(code: str):
     return {
-        'en': "English",
+        'en': "British English",
         'de': "German",
-    }.get(code, 'English')
+    }.get(code, 'British English')

src/utils/stratutils/generator.py

@@ -0,0 +1,8 @@
+from src.utils.stratutils.templates import *
+
+def generate_strategy(name, prop):
+    preamble = PREAMBLE_TEMPL_STD.format(name=name)
+    header = f"{FUNC_HEADER_TEMPL} -> {FUNC_RETURN_TYPE_TEMPL.get(prop['data_type'], None)}:"
+    body = BODY_TEMPL.get(name, BODY_TEMPL_STD)
+
+    return f"{preamble}\n\n{header}\n{COMMENT_TEMPL_STD}\n\n{body}"

src/utils/stratutils/templates.py

@@ -0,0 +1,31 @@
+FUNC_HEADER_TEMPL = "def run(file_name: str, file_content: str, chunk: str)"
+
+FUNC_RETURN_TYPE_TEMPL = {
+    "text": "str",
+    "date": "str",
+    "text[]": "list[str]",
+}
+
+PREAMBLE_TEMPL_STD="""\"\"\"Property extraction strategy for property {name}.\"\"\""""
+
+COMMENT_TEMPL_STD = """\t\"\"\"
+\tRuns the property extraction strategy on processed chunk.
+
+\tArgs:
+\t\tfile_name (str): Name of the file from which the chunk was collected.
+\t\tfile_content (str): Entire text extracted from file.
+\t\tchunk (str): Chunk collected from file.
+    
+\tReturns:
+\t\tExtracted property.
+\t\"\"\""""
+
+BODY_TEMPL_STD = "\treturn chunk"
+
+BODY_TEMPL = {
+    'body':        "\treturn chunk",
+    'source':      "\treturn file_name",
+    'chunk_id':    "\timport hashlib\n\treturn hashlib.md5(chunk.strip().encode('utf-8')).hexdigest()",
+    'document_id': "\timport hashlib\n\treturn hashlib.md5(file_content.strip().encode('utf-8')).hexdigest()",
+    'date':        "\timport datetime\n\treturn datetime.datetime.now().replace(tzinfo=datetime.timezone.utc)"
+}