Sync from GitHub 7e4c3e8eb6f1afe4620f55cca1a22dfef2c1ee7c

AUDIT_LATENCY_HALLUCINATIONS.md

@@ -88,7 +88,7 @@ Offline (kein User wartet):
 
 ## 4. Offene Punkte (bewusst nicht gemacht)
 
-- **Embeddings** laufen weiter über die HF Inference API (`text2vec_huggingface`). Wechsel auf OpenAI-Embeddings oder lokales Modell erfordert Neu-Erstellung der Weaviate-Collection + Re-Import — lohnt sich zusammen mit Re-Chunking (200 → 512–1024 Tokens). Der BM25-Fallback loggt jetzt sichtbar.
+- **Embeddings:** Die Migration auf app-seitig erzeugte OpenRouter-Embeddings (`openai/text-embedding-3-small`) erfordert weiterhin Neu-Erstellung der Weaviate-Collection + Re-Import. Der BM25-Fallback loggt sichtbar, falls Embedding oder Vektor-Hybrid-Query fehlschlägt.
 - **Merge-Strategie:** lokale Arbeit liegt auf `master`, Remote-Hauptbranch ist `main` — vor dem Push klären (Rename oder PR).
 - Cron läuft auf dem Entwicklungs-Mac nur, wenn er wach ist — auf dem Produktivserver einrichten.
 - `scripts/remove_legacy_code.py` ist ein verbrauchtes Einweg-Skript und kann gelöscht werden.

README.md

@@ -65,9 +65,9 @@ Following variables are required for every mode to run:
 
 ```bash
 OPENAI_API_KEY=...
+OPEN_ROUTER_API_KEY=...
 WEAVIATE_API_KEY=...
 WEAVIATE_CLUSTER_URL=...
-HUGGING_FACE_API_KEY=...
 ```
 
 Optional but commonly useful:
@@ -79,7 +79,6 @@ LANGSMITH_PROJECT=...
 LANGSMITH_ENDPOINT=https://api.smith.langchain.com
 
 GROQ_API_KEY=...
-OPEN_ROUTER_API_KEY=...
 
 REDIS_CLOUD_HOST=...
 REDIS_CLOUD_PORT=...
@@ -171,6 +170,18 @@ python main.py --clear-cache
 python main.py --dbapp
 ```
 
+Embedding model changes require a Weaviate collection rebuild and re-import:
+
+```bash
+python main.py --weaviate redo
+python main.py --scrape
+# plus python main.py --imports ... for any local source files you maintain
+```
+
+The default cloud embedding path uses OpenRouter `openai/text-embedding-3-small`
+and stores app-generated vectors in Weaviate. The existing scraper restoration
+flow is unchanged.
+
 Cache mode can be selected explicitly:
 
 ```bash

TODO_TEAM.md

@@ -20,7 +20,7 @@ Status: 2026-06-11 · Base: branch `master` · Context: `AUDIT_LATENCY_HALLUCINA
 
 ## Backlog (prioritised)
 
-- [ ] **Move embeddings off the HF Inference API** (OpenAI `text-embedding-3-small` or local) + **re-chunking** 200 → 512–1024 tokens; requires rebuilding the Weaviate collection + re-import. Trigger: frequent BM25 fallbacks or weak long-tail answers
+- [ ] **Rebuild retrieval collections after embedding migration** — current code uses OpenRouter `openai/text-embedding-3-small` with app-side vectors and 512-token chunks; run `python main.py --weaviate redo`, then the existing scrape/import jobs, and monitor BM25 fallback warnings
 - [ ] **LLM eval as CI gate** (GitHub Action with `RUN_LLM_EVAL=1` on PRs against `main`; secret for the API key budget)
 - [ ] **Rework or remove the cache** — currently exact query match per session (hit rate ~0); either normalised/semantic keys or delete the code
 - [ ] **Simplify booking logic further** — replace the remaining keyword heuristics in `_query_lead` (explicit booking intent, preference follow-up) with pure structured-output flags

config.py

@@ -36,7 +36,6 @@ MAX_CONVERSATION_TURNS = 20
 # Keep the master branch's latency-oriented defaults.
 MAIN_AGENT_MODEL = ('openai', 'gpt-4.1')
 FALLBACK_MODELS = [('openai', 'gpt-5-mini')]
-SUBAGENT_MODEL = ('openai', 'gpt-5-mini')
 LANGUAGE_DETECTION_MODEL = ('openai', 'gpt-4o-mini')
 CONFIDENCE_SCORING_MODEL = ('openai', 'gpt-4o-mini')
 SUMMARIZATION_MODEL = ('openai', 'gpt-4.1')
@@ -50,17 +49,12 @@ OPENAI_MODEL = 'gpt-4.1'
 
 # ==================================== Weaviate Database Configuration ======================================
 
-# A boolean; either True or False. 
-# Defines whether the database is set as a local instance (via Docker container), 
-# or as a cloud service. More information on https://docs.weaviate.io/weaviate.
-WEAVIATE_IS_LOCAL = False
-
 # A string. Defines the name of the colletions stored in the database.
 # For each available language a new collection will be created
 # with set name <WEAVIATE_COLLECTION_BASENAME>_<LANGUAGE>.
 WEAVIATE_COLLECTION_BASENAME = 'hsg_rag_content'
 
-# A string; either 'manual', 'filesystem' (local instance), 's3' (AWS).
+# A string; either 'manual', 'filesystem', 's3' (AWS).
 # Defines the service for storing the database backups.
 # More information on https://docs.weaviate.io/deploy/configuration/backups.
 WEAVIATE_BACKUP_METHOD = 'manual'
@@ -93,7 +87,9 @@ WEAVIATE_INSERT_TIMEOUT = 600
 WEAVIATE_KEEP_WARM_ENABLED = True
 
 # An integer. Defines how often the keep-warm loop may query Weaviate while idle (in seconds).
-WEAVIATE_KEEP_WARM_INTERVAL = 30
+# 180s keeps the vectorizer warm enough between turns while cutting idle
+# hybrid queries (and HF API usage) to a sixth of the previous 30s setting.
+WEAVIATE_KEEP_WARM_INTERVAL = 180
 
 # An integer. Defines when an idle Weaviate client is considered stale enough to
 # reconnect proactively (in seconds).
@@ -122,9 +118,14 @@ CACHE_LOCAL_PORT = 6379
 
 # ===================================== Data Processing Configuration =======================================
 
-# A string representing the name of an embeding model for embedding generation.
-# The parameter MAX_TOKENS must match this model's maximum token amount.
-EMBEDDING_MODEL = 'sentence-transformers/multi-qa-mpnet-base-dot-v1'
+# Embeddings are generated by the application through OpenRouter and stored as
+# self-provided vectors in Weaviate. Rebuild the collection after changing any
+# embedding provider/model/vector dimension setting.
+EMBEDDING_MODEL = 'openai/text-embedding-3-small'
+EMBEDDING_BASE_URL = 'https://openrouter.ai/api/v1'
+EMBEDDING_DIMENSIONS = 1536
+EMBEDDING_BATCH_SIZE = 32
+EMBEDDING_VECTOR_NAME = 'hsg_rag_embeddings'
 
 # A float in range from 0 to 1. Sets the threshold for english language in the language detector.
 # If the language detection certanty is lower than the threshold, the English language will be returned.

docs/deploy_readiness_checklist.md

@@ -38,8 +38,7 @@ This checklist reflects the GitHub `main` state at commit `c0462c1b7c5074af682ec
 - DNS for `bot.hsg.ch` points to the target server
 - Caddy is installed and configured with `deploy/Caddyfile`
 - Port `7860` is reachable internally on the host
-- Weaviate is available:
-  - local or cloud
+- Weaviate Cloud is available
 - Redis is available:
   - local or cloud
 - Outbound network access exists for required model downloads, or models are pre-cached

docs/weaviate_database_setup.md

@@ -1,12 +1,13 @@
 # Weaviate Database Setup
 
-This project uses a local instance of a Weaviate vector database to store vector embeddings. The chosen embedding model is Sentence Transformers. The model is integrated in the embedding insertion pipeline and will be installed alongside the database.
+This project uses Weaviate Cloud to store retrieval chunks and vectors. The
+application generates embeddings through OpenRouter
+`openai/text-embedding-3-small` and stores them as self-provided vectors.
 
 ## Installation steps
 1. Create a new python virtual environment using `python -m venv venv`, activate the environment via `source venv/bin/activate`, install the needed requirements from the `requirements.txt` file if you haven't done it already.
-2. Follow the installation guide to install [Docker Desktop](https://docs.docker.com/desktop/) on your device.
-3. Navigate to `src/database` and locate the `docker-compose.yml` file. Inside this directory call the command `docker compose up -d` to install and setup the database and embedding model containers. Wait for installation to finish gracefully.
-4. With the python environment activated, call the collection creation script from the `weaviate.py` located in the same directory using `python wvt_service.py --create_collections`. Inspect the logs to check whether the creation of the collections was successfull.
+2. Configure `WEAVIATE_CLUSTER_URL`, `WEAVIATE_API_KEY`, and `OPEN_ROUTER_API_KEY`.
+3. With the python environment activated, initialize the collections with `python main.py --weaviate init`. Inspect the logs to check whether collection creation was successful.
 
 If you've managed to setup the database and create the collections, the installation process is finished and the database is accessible from the other parts of the program.
 
@@ -20,6 +21,16 @@ To manage the state of the database directly, multiple useful scripts were devel
 - `-cb` or `--create_backup`: creates a backup of the current state of the database.
 - `-rb` pr `--restore_backup`: restores the state of the database from the provided backup\_id.
 
+When changing embedding model, tokenizer, or vector dimensions, rebuild the collections and re-import content:
+
+```bash
+python main.py --weaviate redo
+python main.py --scrape
+```
+
+Run `python main.py --imports ...` afterward for any local documents that are
+part of the knowledge base.
+
 ## Data properties
 Embeddings are stored in the corresponding language collection with a set of properties that define chunk metadata:
 
@@ -32,4 +43,4 @@ Embeddings are stored in the corresponding language collection with a set of pro
 
 
 ## WeaviateService
-The WeaviateService class manages the connection and the interaction with the local database.
+The WeaviateService class manages the connection and interaction with Weaviate Cloud.

requirements.txt

@@ -16,8 +16,8 @@ colorama>=0.4.6
 # Language detection
 langdetect>=1.0.9
 
-# Transformers for tokenization
-transformers>=4.34.0
+# Tokenization
+tiktoken>=0.12.0
 
 # Web applications
 gradio==6.14.0

src/apps/dbapp/config.py

@@ -16,6 +16,14 @@ def _dump_schema(schema):
         yaml.safe_dump(schema, f)
 
 
+def _skip_vectorization_from_module_config(prop: dict) -> bool:
+    module_config = prop.get('moduleConfig', {}) or {}
+    for module_data in module_config.values():
+        if isinstance(module_data, dict) and 'skip' in module_data:
+            return module_data['skip']
+    return False
+
+
 class SchemaConfigurationFrame(CustomFrameBase):
     def __init__(self, parent, service: WeaviateService) -> None:
         super().__init__(parent, service)
@@ -71,7 +79,7 @@ class SchemaConfigurationFrame(CustomFrameBase):
                 'data_type': prop['dataType'][0],
                 'filterable': prop['indexFilterable'],
                 'searchable': prop['indexSearchable'],
-                'skip_vectorization': prop['moduleConfig']['text2vec-huggingface']['skip'],
+                'skip_vectorization': _skip_vectorization_from_module_config(prop),
             }
             schema_data[prop['name']] = data_property
         

src/config/configs.py

@@ -77,7 +77,12 @@ class ConversationStateConfig(ConfigBase):
 
 class ProcessingConfig(ConfigBase):
     LANG_AMBIGUITY_THRESHOLD: float = _get('LANG_AMBIGUITY_THRESHOLD')
-    EMBEDDING_MODEL:          float = _get('EMBEDDING_MODEL')
+    EMBEDDING_MODEL:          str   = _get('EMBEDDING_MODEL', 'openai/text-embedding-3-small')
+    EMBEDDING_BASE_URL:       str   = _get('EMBEDDING_BASE_URL', 'https://openrouter.ai/api/v1')
+    EMBEDDING_API_KEY:        str   = _get('EMBEDDING_API_KEY') or _get('OPEN_ROUTER_API_KEY')
+    EMBEDDING_DIMENSIONS:     int   = _get('EMBEDDING_DIMENSIONS', 1536, type_=int)
+    EMBEDDING_BATCH_SIZE:     int   = _get('EMBEDDING_BATCH_SIZE', 32, type_=int)
+    EMBEDDING_VECTOR_NAME:    str   = _get('EMBEDDING_VECTOR_NAME', 'hsg_rag_embeddings')
     MAX_TOKENS:    int = _get('MAX_TOKENS')
     CHUNK_OVERLAP: int = _get('CHUNK_OVERLAP')
 
@@ -118,7 +123,6 @@ class CacheConfig(ConfigBase):
 
 
 class WeaviateConfig(ConfigBase):
-    LOCAL_DATABASE: bool = _get('WEAVIATE_IS_LOCAL')
     WEAVIATE_COLLECTION_BASENAME: str = _get('WEAVIATE_COLLECTION_BASENAME')
     
     BACKUP_METHODS: list[str] = ['manual', 'filesystem', 's3']
@@ -130,13 +134,12 @@ class WeaviateConfig(ConfigBase):
 
     CLUSTER_URL:          str = _get('WEAVIATE_CLUSTER_URL')
     WEAVIATE_API_KEY:     str = _get('WEAVIATE_API_KEY')
-    HUGGING_FACE_API_KEY: str = _get('HUGGING_FACE_API_KEY')
    
     INIT_TIMEOUT:   int  = _get('WEAVIATE_INIT_TIMEOUT', 90) 
     QUERY_TIMEOUT:  int  = _get('WEAVIATE_QUERY_TIMEOUT', 60) 
     INSERT_TIMEOUT: int  = _get('WEAVIATE_INSERT_TIMEOUT', 600)
     KEEP_WARM_ENABLED: bool = _get_bool('WEAVIATE_KEEP_WARM_ENABLED', True)
-    KEEP_WARM_INTERVAL: int = _get('WEAVIATE_KEEP_WARM_INTERVAL', 30, type_=int)
+    KEEP_WARM_INTERVAL: int = _get('WEAVIATE_KEEP_WARM_INTERVAL', 180, type_=int)
     CLIENT_IDLE_TIMEOUT: int = _get('WEAVIATE_CLIENT_IDLE_TIMEOUT', 25 * 60, type_=int)
 
 
@@ -174,7 +177,6 @@ class LLMConfig(ConfigBase):
 
     MAIN_AGENT_MODEL: tuple[str, str] = _get('MAIN_AGENT_MODEL', ('openai', 'gpt-4.1'))
     FALLBACK_MODELS: list[tuple[str, str]] = _get('FALLBACK_MODELS', [('openai', 'gpt-5-mini')])
-    SUBAGENT_MODEL: tuple[str, str] = _get('SUBAGENT_MODEL', ('openai', 'gpt-5-mini'))
     LANGUAGE_DETECTION_MODEL: tuple[str, str] = _get('LANGUAGE_DETECTION_MODEL', ('openai', 'gpt-4o-mini'))
     CONFIDENCE_SCORING_MODEL: tuple[str, str] = _get('CONFIDENCE_SCORING_MODEL', ('openai', 'gpt-4o-mini'))
     SUMMARIZATION_MODEL: tuple[str, str] = _get('SUMMARIZATION_MODEL', ('openai', 'gpt-4.1'))

src/database/docker-compose.yml

@@ -1,29 +0,0 @@
-version: '3.4'
-
-services:
-  weaviate:
-    image: semitechnologies/weaviate:1.33.0
-    restart: on-failure:0
-    ports:
-      - "8080:8080"
-      - "50051:50051"
-    environment:
-      QUERY_DEFAULTS_LIMIT: 25
-      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
-      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
-      ENABLE_API_BASED_MODULES: 'true'
-      ENABLE_MODULES: 'text2vec-transformers'
-      TRANSFORMERS_INFERENCE_API: 'http://t2v-transformers:8080'
-      CLUSTER_HOSTNAME: 'node1'
-    volumes:
-      - weaviate_data:/var/lib/weaviate
-
-  t2v-transformers:
-    image: semitechnologies/transformers-inference:sentence-transformers-all-MiniLM-L6-v2
-    restart: on-failure:0
-    ports:
-      - "8081:8080"
-
-volumes:
-  weaviate_data:
-

src/database/embeddings.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+from time import sleep
+from typing import Iterable
+
+from ..config import config
+
+MAX_EMBEDDING_ATTEMPTS = 3
+
+
+class EmbeddingError(RuntimeError):
+    pass
+
+
+class OpenRouterEmbeddingClient:
+    def __init__(self) -> None:
+        self._client = None
+
+    @property
+    def client(self):
+        if self._client is None:
+            if not config.processing.EMBEDDING_API_KEY:
+                raise EmbeddingError("OPEN_ROUTER_API_KEY is not configured for embeddings")
+
+            from openai import OpenAI
+
+            self._client = OpenAI(
+                api_key=config.processing.EMBEDDING_API_KEY,
+                base_url=config.processing.EMBEDDING_BASE_URL,
+            )
+
+        return self._client
+
+    def embed_query(self, text: str) -> list[float]:
+        return self.embed_documents([text])[0]
+
+    def embed_documents(self, texts: Iterable[str]) -> list[list[float]]:
+        clean_texts = [(text or " ").strip() or " " for text in texts]
+        if not clean_texts:
+            return []
+
+        last_error: Exception | None = None
+        for attempt in range(1, MAX_EMBEDDING_ATTEMPTS + 1):
+            try:
+                response = self.client.embeddings.create(
+                    model=config.processing.EMBEDDING_MODEL,
+                    input=clean_texts,
+                )
+                embeddings = self._extract_embeddings(response)
+                self._validate_embeddings(embeddings, expected_count=len(clean_texts))
+                return embeddings
+            except Exception as exc:
+                last_error = exc
+                if attempt == MAX_EMBEDDING_ATTEMPTS or not self._is_retryable(exc):
+                    break
+                sleep(min(2 ** (attempt - 1), 8))
+
+        raise EmbeddingError(f"Failed to generate OpenRouter embeddings: {last_error}") from last_error
+
+    @staticmethod
+    def _extract_embeddings(response) -> list[list[float]]:
+        data = list(getattr(response, "data", []) or [])
+        data.sort(key=lambda item: getattr(item, "index", 0))
+        return [list(getattr(item, "embedding", []) or []) for item in data]
+
+    def _validate_embeddings(self, embeddings: list[list[float]], expected_count: int) -> None:
+        if len(embeddings) != expected_count:
+            raise EmbeddingError(
+                f"Embedding response count mismatch: expected {expected_count}, got {len(embeddings)}"
+            )
+
+        for idx, embedding in enumerate(embeddings):
+            if len(embedding) != config.processing.EMBEDDING_DIMENSIONS:
+                raise EmbeddingError(
+                    f"Embedding {idx} has dimension {len(embedding)}; "
+                    f"expected {config.processing.EMBEDDING_DIMENSIONS}"
+                )
+
+    @staticmethod
+    def _is_retryable(error: Exception) -> bool:
+        status_code = getattr(error, "status_code", None)
+        if status_code in {408, 409, 425, 429, 500, 502, 503, 504}:
+            return True
+
+        text = str(error).lower()
+        return any(
+            signal in text
+            for signal in [
+                "rate limit",
+                "timeout",
+                "temporarily unavailable",
+                "connection",
+                "server error",
+            ]
+        )

src/database/weavservice.py

@@ -13,6 +13,7 @@ from weaviate.config import AdditionalConfig
 
 from ..utils.logging import get_logger
 from ..config import config
+from .embeddings import EmbeddingError, OpenRouterEmbeddingClient
 
 logger = get_logger("weaviate_service")
 
@@ -54,7 +55,7 @@ class WeaviateService:
         if hasattr(self, '_initialized'):
             return 
 
-        self._connection_type = 'local' if config.weaviate.LOCAL_DATABASE else 'cloud'
+        self._connection_type = 'cloud'
         self._client = None 
         self._client_lock = RLock()
         
@@ -66,6 +67,7 @@ class WeaviateService:
         self._keep_warm_interval = max(1, config.weaviate.KEEP_WARM_INTERVAL)
         self._keep_warm_stop = Event()
         self._keep_warm_thread = None
+        self._embedding_client = OpenRouterEmbeddingClient()
         self._initialized = True
 
         # Initialize the client for the first time 
@@ -114,24 +116,18 @@ class WeaviateService:
             last_exception: Exception = None
             while retries < 3:
                 try:
-                    if config.weaviate.LOCAL_DATABASE:
-                        self._client = wvt.connect_to_local()
-                    else:
-                        self._client = wvt.connect_to_weaviate_cloud(
-                            cluster_url=config.weaviate.CLUSTER_URL,
-                            auth_credentials=config.weaviate.WEAVIATE_API_KEY,
-                            additional_config=AdditionalConfig(
-                                timeout=Timeout(
-                                    init=config.weaviate.INIT_TIMEOUT,
-                                    query=config.weaviate.QUERY_TIMEOUT,
-                                    insert=config.weaviate.INSERT_TIMEOUT,
-                                ),
-                                skip_init_checks=False,
+                    self._client = wvt.connect_to_weaviate_cloud(
+                        cluster_url=config.weaviate.CLUSTER_URL,
+                        auth_credentials=config.weaviate.WEAVIATE_API_KEY,
+                        additional_config=AdditionalConfig(
+                            timeout=Timeout(
+                                init=config.weaviate.INIT_TIMEOUT,
+                                query=config.weaviate.QUERY_TIMEOUT,
+                                insert=config.weaviate.INSERT_TIMEOUT,
                             ),
-                            headers={
-                                "X-HuggingFace-Api-Key": config.weaviate.HUGGING_FACE_API_KEY,
-                            },
-                        )
+                            skip_init_checks=False,
+                        ),
+                    )
 
                     self._warm_up_client(self._client)
                     break
@@ -176,19 +172,39 @@ class WeaviateService:
 
 
     def _keep_warm_loop(self) -> None:
-        while not self._keep_warm_stop.wait(self._keep_warm_interval):
+        # Exponential backoff on consecutive failures: a broken embedding key
+        # would otherwise be hammered every interval, spamming logs and quota.
+        consecutive_failures = 0
+        max_wait = 30 * 60
+        while True:
+            wait = min(self._keep_warm_interval * (2 ** consecutive_failures), max_wait)
+            if self._keep_warm_stop.wait(wait):
+                return
             try:
-                self._keep_warm_once()
+                result = self._keep_warm_once()
+                if result is True:
+                    consecutive_failures = 0
+                elif result is False:
+                    consecutive_failures += 1
+                # result None: skipped (recent real query) — leave backoff as is
             except Exception as e:
+                consecutive_failures += 1
                 logger.warning(f"Weaviate keep-warm tick failed (non-critical): {e}")
+            if consecutive_failures == 3:
+                logger.warning(
+                    "Keep-warm failed 3 times in a row - backing off. If the error is a "
+                    "vector query or embedding 401, check OPEN_ROUTER_API_KEY."
+                )
 
 
-    def _keep_warm_once(self) -> bool:
+    def _keep_warm_once(self) -> bool | None:
         """
         Run one scheduled keep-warm tick when the client has been idle long enough.
 
         Returns:
-            bool: True when a warm-up query was attempted, False when skipped.
+            True when the warm-up query succeeded, False when it was attempted
+            but failed (drives the backoff), None when skipped because a real
+            query happened recently.
         """
         client = self._client
         if client is None:
@@ -196,14 +212,13 @@ class WeaviateService:
 
         time_since_query = perf_counter() - self._last_query_time
         if time_since_query < self._keep_warm_interval:
-            return False
+            return None
 
         logger.debug(
             "Running scheduled Weaviate keep-warm after %3.2f seconds idle",
             time_since_query,
         )
-        self._warm_up_client(client)
-        return True
+        return self._warm_up_client(client)
 
 
     def stop_keep_warm(self) -> None:
@@ -215,9 +230,59 @@ class WeaviateService:
         self._keep_warm_thread = None
 
 
-    def _warm_up_client(self, client: wvt.WeaviateClient) -> None:
+    @staticmethod
+    def _embedding_vector_name() -> str:
+        return config.processing.EMBEDDING_VECTOR_NAME
+
+
+    def _embed_query(self, query: str) -> list[float]:
+        return self._embedding_client.embed_query(query)
+
+
+    def _embed_batch_vectors(self, batch_rows: list[tuple[int, dict]]) -> list[dict]:
+        embeddings = self._embedding_client.embed_documents(
+            data_row.get("body", "") for _, data_row in batch_rows
+        )
+        vector_name = self._embedding_vector_name()
+        return [{vector_name: embedding} for embedding in embeddings]
+
+
+    def _hybrid_query_kwargs(
+        self,
+        query: str,
+        filters=None,
+        limit: int = 5,
+        query_vector: list[float] | None = None,
+    ) -> dict:
+        kwargs = {
+            "query": query,
+            "filters": filters,
+            "limit": limit,
+            "return_metadata": MetadataQuery.full(),
+        }
+
+        kwargs["vector"] = query_vector if query_vector is not None else self._embed_query(query)
+        kwargs["target_vector"] = self._embedding_vector_name()
+
+        return kwargs
+
+
+    @staticmethod
+    def _bm25_query_kwargs(query: str, filters=None, limit: int = 5) -> dict:
+        return {
+            "query": query,
+            "filters": filters,
+            "limit": limit,
+            "return_metadata": MetadataQuery.full(),
+        }
+
+
+    def _warm_up_client(self, client: wvt.WeaviateClient) -> bool:
         """
         Quickly query the client to decrease the response latency of future calls.
+
+        Returns:
+            bool: True when the warm-up query succeeded.
         """
         logger.debug("Running warm-up query to initialize server and vectorizer...")
         try:
@@ -225,18 +290,16 @@ class WeaviateService:
             with self._client_lock:
                 if not client.collections.exists(collection_name):
                     logger.warning("Warm-up skipped because collection %s does not exist", collection_name)
-                    return
+                    return False
 
                 collection = client.collections.get(collection_name)
-                collection.query.hybrid(
-                    query="HSG",
-                    limit=1,
-                    return_metadata=MetadataQuery.full(),
-                )
+                collection.query.hybrid(**self._hybrid_query_kwargs(query="HSG", limit=1))
             self._last_query_time = perf_counter()
-            logger.debug("Warm-up finished - server and vectorizer are ready!")
+            logger.debug("Warm-up finished - server and embeddings are ready!")
+            return True
         except Exception as warmup_err:
             logger.warning(f"Warm-up query failed (non-critical): {warmup_err}")
+            return False
 
 
     def _select_collection(self, lang: str) -> tuple[Collection, str]:
@@ -282,14 +345,15 @@ class WeaviateService:
         import_errors = []
         logger.info(f"Batch importing {len(data_rows)} rows into {collection_name}")
 
-        batch_size = 10
+        batch_size = max(1, config.processing.EMBEDDING_BATCH_SIZE)
         max_attempts = 2
 
         def _import_batch(batch_rows: list[tuple[int, dict]]) -> None:
+            vectors = self._embed_batch_vectors(batch_rows)
             with collection.batch.fixed_size(batch_size=batch_size, concurrent_requests=1) as batch:
-                for idx, data_row in batch_rows:
+                for (idx, data_row), vector in zip(batch_rows, vectors):
                     try:
-                        batch.add_object(properties=data_row)
+                        batch.add_object(properties=data_row, vector=vector)
                     except Exception as e:
                         import_errors.append({'index': idx, 'chunk_id': data_row['chunk_id'], 'error': str(e)})
 
@@ -397,9 +461,7 @@ class WeaviateService:
 
     def ping(self, lang: str) -> dict:
         try:
-            collection, _ = self._select_collection(lang)
-            with self._client_lock:
-                collection.query.hybrid("health check query")
+            self.query("health check query", lang=lang, limit=1)
             return { 'status': 'OK' }
         except Exception as e: 
             return { 'status': 'ERROR', 'error': e } 
@@ -443,28 +505,30 @@ class WeaviateService:
                 logger.info(f"Querying collection {collection_name}")
                 query_start_time = perf_counter()
 
+                try:
+                    query_vector = self._embed_query(query)
+                except EmbeddingError as embed_err:
+                    query_vector = None
+                    logger.warning(
+                        "OpenRouter embedding query failed. "
+                        "Falling back to BM25 keyword retrieval: %s",
+                        embed_err,
+                    )
+
                 with self._client_lock:
-                    try:
-                        resp = collection.query.hybrid(
-                            query=query,
-                            filters=filters,
-                            limit=limit,
-                            return_metadata=MetadataQuery.full()
-                        )
-                    except Exception as hybrid_err:
-                        if not self._should_fallback_to_bm25(hybrid_err):
-                            raise hybrid_err
-                        logger.warning(
-                            "Hybrid query failed during remote vectorization. "
-                            "Falling back to BM25 keyword retrieval: %s",
-                            hybrid_err,
-                        )
-                        resp = collection.query.bm25(
-                            query=query,
-                            filters=filters,
-                            limit=limit,
-                            return_metadata=MetadataQuery.full()
+                    if query_vector is None:
+                        resp = collection.query.bm25(**self._bm25_query_kwargs(query, filters, limit))
+                    else:
+                        try:
+                            resp = collection.query.hybrid(
+                                **self._hybrid_query_kwargs(query, filters, limit, query_vector=query_vector)
                         )
+                        except Exception as hybrid_err:
+                            logger.warning(
+                                "Hybrid vector query failed. Falling back to BM25 keyword retrieval: %s",
+                                hybrid_err,
+                            )
+                            resp = collection.query.bm25(**self._bm25_query_kwargs(query, filters, limit))
                 elapsed = perf_counter() - query_start_time
                 self._last_query_time = perf_counter()
                 logger.info(f"Querying retrieved {len(resp.objects)} objects in {elapsed:3.2f} seconds")
@@ -479,15 +543,6 @@ class WeaviateService:
                         raise e
                 else: # Probably not a server issue
                     raise e
-
-    @staticmethod
-    def _should_fallback_to_bm25(error: Exception) -> bool:
-        error_text = str(error).lower()
-        return (
-            "remote client vectorize" in error_text
-            or "vectorize" in error_text and "401" in error_text
-            or "invalid username or password" in error_text
-        )
     
 
     def _load_properties(self) -> list[Property]:
@@ -539,6 +594,10 @@ class WeaviateService:
         return final_properties
 
 
+    def _vector_config(self):
+        return Configure.Vectors.self_provided(name=self._embedding_vector_name())
+
+
     def _create_collections(self):
         """
         Create and initialize language-specific collections.
@@ -550,14 +609,7 @@ class WeaviateService:
             client = self._init_client()
             logger.info('Attempting collections creation...')
             
-            vector_config = (
-                Configure.Vectors.text2vec_transformers() if config.weaviate.LOCAL_DATABASE
-                else Configure.Vectors.text2vec_huggingface(
-                    name='hsg_rag_embeddings',
-                    source_properties=['body'],
-                    model=config.processing.EMBEDDING_MODEL,
-                )
-            )
+            vector_config = self._vector_config()
             
             successful_creations = 0
             
@@ -849,29 +901,18 @@ class WeaviateService:
                 with self._client_lock:
                     metainfo = client.get_meta()
                 
-                # Format module information
                 modules = metainfo.get('modules', {})
                 modules_list = list(modules.keys()) if isinstance(modules, dict) else modules
                 modules_str = ', '.join(str(m) for m in modules_list) if modules_list else 'None'
                 
-                # Truncate long module strings for logging
                 if len(modules_str) > 50:
                     modules_str = modules_str[:47] + '...'
                 
-                # Log connection details
-                if config.weaviate.LOCAL_DATABASE:
-                    logger.info(
-                        f"Database metadata: "
-                        f"HOSTNAME={metainfo.get('hostname', 'unknown')}, "
-                        f"VERSION={metainfo.get('version', 'unknown')}, "
-                        f"MODULES={modules_str}"
-                    )
-                else:
-                    logger.info(
-                        f"Database metadata: "
-                        f"VERSION={metainfo.get('version', 'unknown')}, "
-                        f"MODULES={modules_str}"
-                    )
+                logger.info(
+                    f"Database metadata: "
+                    f"VERSION={metainfo.get('version', 'unknown')}, "
+                    f"MODULES={modules_str}"
+                )
                 
             except Exception as e:
                 logger.warning(f"Could not retrieve database metadata: {e}")

src/notification/notification_center.py

@@ -22,9 +22,9 @@ class EmailNotifier:
         self.smtp_use_tls = NC.SMTP_USE_TLS
         self.from_email = NC.FROM_EMAIL
         self.to_emails = self._parse_recipients(NC.TO_EMAIL)
-
-        if self.enabled:
-            self._validate()
+        # Validation happens at send time, not at construction: constructing a
+        # NotificationCenter (e.g. inside Scraper.__init__) must not require a
+        # complete SMTP configuration on machines that never send alerts.
 
     @staticmethod
     def _parse_recipients(value: str | None) -> list[str]:
@@ -58,6 +58,8 @@ class EmailNotifier:
         if not self.enabled:
             return
 
+        self._validate()
+
         if isinstance(attachments, str):
             attachments = [attachments]
 
@@ -95,9 +97,7 @@ class SlackNotifier:
     def __init__(self):
         self.enabled = NC.ENABLE_SLACK_ALERTS
         self.webhook_url = NC.SLACK_WEBHOOK_URL
-
-        if self.enabled:
-            self._validate()
+        # Validated at send time (see EmailNotifier note above).
 
     def _validate(self) -> None:
         if not self.webhook_url:
@@ -107,6 +107,8 @@ class SlackNotifier:
         if not self.enabled:
             return
 
+        self._validate()
+
         text = f"*{subject}*\n{body}"
 
         response = requests.post(

src/pipeline/processors.py

@@ -2,9 +2,11 @@ from collections import defaultdict
 import os, re
 
 from pathlib import Path
-from transformers import AutoTokenizer
+from typing import Any
 
-from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
+import tiktoken
+from pydantic import ConfigDict
+from docling_core.transforms.chunker.tokenizer.base import BaseTokenizer
 from docling.datamodel.pipeline_options import PdfPipelineOptions, LayoutOptions
 from docling_core.transforms.serializer.markdown import MarkdownDocSerializer 
 from docling.document_converter import DocumentConverter, PdfFormatOption, InputFormat
@@ -20,6 +22,41 @@ from ..config import config
 weblogger  = get_logger("website_processor")
 datalogger = get_logger("data_processor")
 
+
+class TiktokenTokenizer(BaseTokenizer):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    encoding: Any
+    max_tokens: int
+
+    def count_tokens(self, text: str) -> int:
+        return len(self.encode(text))
+
+    def get_max_tokens(self) -> int:
+        return self.max_tokens
+
+    def get_tokenizer(self) -> Any:
+        return self.encoding
+
+    def encode(self, text: str) -> list[int]:
+        return self.encoding.encode(text, disallowed_special=())
+
+    def decode(self, tokens: list[int], *_, **__) -> str:
+        return self.encoding.decode(tokens)
+
+
+def _embedding_tokenizer_model() -> str:
+    return config.processing.EMBEDDING_MODEL.split('/', 1)[-1]
+
+
+def _embedding_tokenizer() -> TiktokenTokenizer:
+    try:
+        encoding = tiktoken.encoding_for_model(_embedding_tokenizer_model())
+    except KeyError:
+        encoding = tiktoken.get_encoding('cl100k_base')
+    return TiktokenTokenizer(encoding=encoding, max_tokens=config.processing.MAX_TOKENS)
+
+
 class ProcessorBase:
     def __init__(self) -> None:
         """
@@ -50,20 +87,22 @@ class ProcessorBase:
                 InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options),
             },
         )
-        tokenizer = AutoTokenizer.from_pretrained(config.processing.EMBEDDING_MODEL)
-        self._chunker = HybridChunker(
-            tokenizer=HuggingFaceTokenizer(
-                tokenizer=tokenizer,
-                max_tokens=config.processing.MAX_TOKENS
-            ),
-            serializer_provider=EnhansedSerializerProvider(),
-            max_tokens=config.processing.MAX_TOKENS, 
-            merge_peers=True
-        )
+        self._chunker_instance: HybridChunker | None = None
         self.strategies_processor = StrategiesProcessor()
         self._logging_callback = config.dbapp['logging_callback'] or logging_callback_placeholder
     
 
+    @property
+    def _chunker(self) -> HybridChunker:
+        if self._chunker_instance is None:
+            self._chunker_instance = HybridChunker(
+                tokenizer=_embedding_tokenizer(),
+                serializer_provider=EnhansedSerializerProvider(),
+                max_tokens=config.processing.MAX_TOKENS,
+                merge_peers=True
+            )
+        return self._chunker_instance
+
     def process(self):
         """
         Abstract method to be implemented by subclasses for processing sources.
@@ -240,7 +279,7 @@ class ProcessorBase:
 
         tokens = tokenizer.encode(document_content)
         chunk_size = self._chunker.max_tokens
-        overlap = 50
+        overlap = min(config.processing.CHUNK_OVERLAP, max(0, chunk_size - 1))
         
         collected_chunks = []
         for i in range(0, len(tokens), chunk_size-overlap):

src/rag/models.py

@@ -7,7 +7,6 @@ logger = get_logger("model_config")
 
 class ModelConfigurator:
     _main_model_instance: BaseChatModel = None
-    _subagent_model_instance: BaseChatModel = None
     _fallback_models_instances: list[BaseChatModel] = None
     _summarization_model_instance: BaseChatModel = None
     _confidence_scoring_model_instance: BaseChatModel = None 
@@ -66,23 +65,6 @@ class ModelConfigurator:
             raise e
 
 
-    @classmethod
-    def get_subagent_model(cls) -> BaseChatModel:
-        if cls._subagent_model_instance:
-            return cls._subagent_model_instance
-        provider, model = config.llm.SUBAGENT_MODEL
-        try:
-            cls._subagent_model_instance = cls._initialize_model(
-                provider=provider,
-                model=model,
-                role="main",
-            )
-            logger.info(f"Initialized subagent model '{provider}:{model}'")
-            return cls._subagent_model_instance
-        except Exception as e:
-            logger.error(f"Failed to initialize subagent model '{provider}:{model}': {e}")
-            raise e
-
     @classmethod
     def get_main_agent_model(cls) -> BaseChatModel:
         """Initialize the language model based on config."""

src/rag/prompts.py

@@ -226,13 +226,16 @@ GENERAL:
 
         agent_key = agent.lower().replace(" ", "")
 
-        # Verified programme facts (auto-generated from official sources).
-        # Hallucination fix: gives the model an authoritative in-prompt source
-        # for volatile core facts instead of regex-extraction from chunks.
-        facts_block = VerifiedFacts.render_prompt_block(language=language)
-
         # 2. Configure Lead Agent
         if agent_key == 'lead':
+            # Verified programme facts (auto-generated from official sources).
+            # Hallucination fix: gives the model an authoritative in-prompt
+            # source for volatile core facts instead of regex-extraction from
+            # chunks. ONLY the lead prompt gets this block — the (currently
+            # unused) programme-agent prompts keep their retrieval-first
+            # invariant: no volatile facts embedded in the prompt text
+            # (guarded by tests/test_pricing_prompts.py).
+            facts_block = VerifiedFacts.render_prompt_block(language=language)
             return cls._LEAD_SYSTEM_PROMPT.format(
                 university_name=university_name,
                 tool_routing=cls._RETRIEVE_CONTEXT_TOOL_ROUTING,
@@ -248,7 +251,7 @@ GENERAL:
                 selected_language=selected_language,
                 university_name=university_name,
                 program_name=agent.upper()
-            ) + facts_block
+            )
         else:
             # Fallback
             return cls._BASE_PROGRAM_PROMPT.format(

src/scraping/scraper.py

@@ -351,13 +351,13 @@ class Scraper:
 
         raw_chunks     = []
         deleted_chunks = []
-        new_urls = {entry.document.name for entry in tagged_documents}
-        self._active_temp_chunks = {
-            url: chunks
-            for url, chunks in (temp_chunks or {}).items()
-            if url not in new_urls
-        }
         merged_chunks, final_chunks = self._read_temp_chunks(temp_chunks, tagged_documents)
+        # Temp snapshots must only carry (a) URLs newly chunked in this session
+        # (added by _store_temp_chunks) and (b) URLs whose restore failed and
+        # which must survive for the next session. Successfully restored URLs
+        # are finalized now — keeping them in temp would re-restore them on the
+        # next resume and duplicate their chunks.
+        self._active_temp_chunks = dict(getattr(self, '_unrestorable_temp_chunks', {}))
 
         program_counter = self._build_program_counter_from_merged_chunks(merged_chunks)
 
@@ -443,14 +443,18 @@ class Scraper:
                 del loaded_temp_chunks[url]
         
         restored_temp_chunks = []
+        # URLs whose chunks could not be finalized; they must stay in the temp
+        # store so the next session can re-scrape them (read in _collect_chunks).
+        self._unrestorable_temp_chunks: dict[str, list[ChunkMetadata]] = {}
         for url, chunks in loaded_temp_chunks.items():
             url_filename = self._normalizer.url_to_filename(url)
             extracted_text_path = os.path.join(self._path.EXTRACTED_TEXT_OUTPUT, url_filename + '.txt')
             if not os.path.exists(extracted_text_path):
                 incupd_logger.warning(f"Cannot restore chunks for URL {url}: Failed to locate previously extracted contents!")
                 incupd_logger.warning(f"This URL will has to be rescraped in the next session")
+                self._unrestorable_temp_chunks[url] = chunks
                 restored_temp_chunks.extend(chunks)
-                continue 
+                continue
             
             with open(extracted_text_path, 'r') as f:
                 url_text = f.read()

src/scraping/url_normalizer.py

@@ -6,13 +6,21 @@ class UrlNormalizer:
     @staticmethod
     def is_url_blacklisted(url: str) -> bool:
         url_lower = url.lower()
-        path = url_lower.split('://', 1)[-1].split('/', 1)[-1] 
-        
+        path = url_lower.split('://', 1)[-1].split('/', 1)[-1]
+
         for forbidden in PAGE_BLACKLIST:
             if forbidden in path:
                 return True
-                
-        return len(path) > 35
+
+        # Guard against junk/tracking URLs without dropping legitimate content
+        # pages. A plain length cap (previously: len(path) > 35) silently
+        # skipped real pages such as /admissions/ready-to-relearn-the-future/
+        # (39 chars). Use structural signals instead:
+        if '?' in path or '&' in path or '=' in path:
+            return True  # query strings: filters, tracking, form states
+        if path.count('/') > 4:
+            return True  # deeper than any real content page on the targets
+        return len(path) > 100  # extreme guard for runaway slugs
     
 
     @staticmethod

tests/conftest.py

@@ -7,10 +7,10 @@ from pathlib import Path
 TEST_DEPENDENCIES = {
     "tests/consent/test_agent_chain_session.py": {"langchain_core", "langchain", "langsmith"},
     "tests/consent/test_consent_logger.py": {"colorama"},
-    "tests/scraping/test_happy_path.py": {"colorama", "docling", "docling_core", "usp", "fake_useragent"},
-    "tests/scraping/test_page_chunking.py": {"colorama", "docling", "docling_core"},
-    "tests/scraping/test_scraping.py": {"colorama", "docling_core", "usp", "fake_useragent"},
-    "tests/scraping/test_scraping_resume.py": {"colorama", "docling_core", "usp", "fake_useragent"},
+    "tests/scraping/test_happy_path.py": {"colorama", "docling", "docling_core", "usp", "fake_useragent", "tiktoken"},
+    "tests/scraping/test_page_chunking.py": {"colorama", "docling", "docling_core", "tiktoken"},
+    "tests/scraping/test_scraping.py": {"colorama", "docling_core", "usp", "fake_useragent", "tiktoken"},
+    "tests/scraping/test_scraping_resume.py": {"colorama", "docling_core", "usp", "fake_useragent", "tiktoken"},
     "tests/scraping/test_utils.py": {"fake_useragent"},
     "tests/test_cache.py": {"langchain"},
     "tests/test_chatbot_improvements.py": {"langchain_core", "langchain", "langsmith"},

tests/scraping/test_page_chunking.py

@@ -9,20 +9,31 @@ class TestPageChunking:
 
     def test_chunking_pipeline(self):
         init_logging()
-        
-        processor = HTMLProcessor()
-        cleaner   = ContentCleaner(full_scraping=True)
-        raw_texts = []
-        documents = []
-        
+
         html_path = config.paths.RAW_HTML_OUTPUT
-        for raw_html_file_path in [
+        required_fixtures = [
             os.path.join(html_path, 'embax-ch.html'),
             os.path.join(html_path, 'embax-ch_admissions_student-profile.html'),
             # Tests for tables and lists
-            os.path.join(html_path, 'embax-ch_admissions_deadlines-fees.html'), 
+            os.path.join(html_path, 'embax-ch_admissions_deadlines-fees.html'),
             os.path.join(html_path, 'embax-ch_events.html')
-        ]:
+        ]
+        missing = [path for path in required_fixtures if not os.path.exists(path)]
+        if missing:
+            # Integration test over local scrape artifacts (data/* is not in
+            # git). Skip instead of failing on machines without a prior scrape.
+            pytest.skip(
+                "Requires scraped raw HTML in data/raw_html "
+                f"(missing: {', '.join(os.path.basename(p) for p in missing)}). "
+                "Run a scrape of embax.ch first."
+            )
+
+        processor = HTMLProcessor()
+        cleaner   = ContentCleaner(full_scraping=True)
+        raw_texts = []
+        documents = []
+
+        for raw_html_file_path in required_fixtures:
             raw_html = open(raw_html_file_path, 'r', encoding='utf-8').read()
             cleaned_html = cleaner.clean_mobile_content(raw_html)
             document = processor.process(url='https://embax.ch', html_content=cleaned_html)

tests/test_master_transfer_integrations.py

@@ -1,6 +1,10 @@
 from threading import RLock
 from types import SimpleNamespace
 
+import pytest
+
+from src.database.embeddings import EmbeddingError
+from src.database import weavservice
 from src.database.weavservice import WeaviateService
 from src.rag.agent_chain import ExecutiveAgentChain
 from src.rag.models import ModelConfigurator
@@ -38,11 +42,16 @@ def test_retrieve_context_filters_embax_with_canonical_programme_id():
 class FakeQuery:
     def __init__(self):
         self.hybrid_calls = []
+        self.bm25_calls = []
 
     def hybrid(self, **kwargs):
         self.hybrid_calls.append(kwargs)
         return SimpleNamespace(objects=[])
 
+    def bm25(self, **kwargs):
+        self.bm25_calls.append(kwargs)
+        return SimpleNamespace(objects=[])
+
 
 class FakeCollections:
     def __init__(self, collection):
@@ -57,6 +66,26 @@ class FakeCollections:
         return self.collection
 
 
+class FakeEmbeddingClient:
+    def __init__(self, vector=None, fail=False):
+        self.vector = vector or [0.1, 0.2, 0.3]
+        self.fail = fail
+        self.document_inputs = []
+        self.query_inputs = []
+
+    def embed_documents(self, texts):
+        self.document_inputs.append(list(texts))
+        if self.fail:
+            raise EmbeddingError("embedding service unavailable")
+        return [self.vector for _ in self.document_inputs[-1]]
+
+    def embed_query(self, text):
+        self.query_inputs.append(text)
+        if self.fail:
+            raise EmbeddingError("embedding service unavailable")
+        return self.vector
+
+
 def test_weaviate_keep_warm_once_runs_hybrid_warmup():
     collection = SimpleNamespace(query=FakeQuery())
     client = SimpleNamespace(collections=FakeCollections(collection))
@@ -65,10 +94,143 @@ def test_weaviate_keep_warm_once_runs_hybrid_warmup():
     service._client_lock = RLock()
     service._last_query_time = 0
     service._keep_warm_interval = 1
+    service._embedding_client = FakeEmbeddingClient(vector=[0.4, 0.5, 0.6])
 
     assert service._keep_warm_once() is True
     assert collection.query.hybrid_calls[0]["query"] == "HSG"
     assert collection.query.hybrid_calls[0]["limit"] == 1
+    assert collection.query.hybrid_calls[0]["vector"] == [0.4, 0.5, 0.6]
+    assert collection.query.hybrid_calls[0]["target_vector"] == config.processing.EMBEDDING_VECTOR_NAME
+
+
+def test_embedding_config_defaults_to_openrouter_small_model():
+    assert config.processing.EMBEDDING_MODEL == "openai/text-embedding-3-small"
+    assert config.processing.EMBEDDING_BASE_URL == "https://openrouter.ai/api/v1"
+    assert config.processing.EMBEDDING_DIMENSIONS == 1536
+    assert config.processing.EMBEDDING_BATCH_SIZE == 32
+    assert config.processing.MAX_TOKENS == 512
+
+
+def test_processor_uses_embedding_model_tokenizer(monkeypatch):
+    processors = pytest.importorskip("src.pipeline.processors")
+    calls = []
+
+    class FakeEncoding:
+        def encode(self, text, **kwargs):
+            return [1, 2]
+
+        def decode(self, tokens):
+            return "decoded"
+
+    class FakeHybridChunker:
+        def __init__(self, **kwargs):
+            self.kwargs = kwargs
+
+    monkeypatch.setattr(
+        processors.tiktoken,
+        "encoding_for_model",
+        lambda model: calls.append(model) or FakeEncoding(),
+    )
+    monkeypatch.setattr(processors, "HybridChunker", FakeHybridChunker)
+    monkeypatch.setattr(processors, "EnhansedSerializerProvider", lambda: object())
+
+    processor = object.__new__(processors.ProcessorBase)
+    processor._chunker_instance = None
+
+    chunker = processors.ProcessorBase._chunker.fget(processor)
+
+    assert calls == ["text-embedding-3-small"]
+    assert chunker.kwargs["max_tokens"] == config.processing.MAX_TOKENS
+    assert chunker.kwargs["tokenizer"].count_tokens("test") == 2
+
+
+def test_weaviate_vector_config_uses_self_provided_for_openrouter(monkeypatch):
+    monkeypatch.setattr(config.processing, "EMBEDDING_VECTOR_NAME", "test_vectors")
+    monkeypatch.setattr(
+        weavservice.Configure.Vectors,
+        "self_provided",
+        lambda name: ("self_provided", name),
+    )
+
+    service = object.__new__(WeaviateService)
+
+    assert service._vector_config() == ("self_provided", "test_vectors")
+
+
+class FakeBatchContext:
+    def __init__(self):
+        self.added = []
+        self.number_errors = 0
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        return False
+
+    def add_object(self, properties, vector=None):
+        self.added.append({"properties": properties, "vector": vector})
+
+
+class FakeBatchFactory:
+    def __init__(self, context):
+        self.context = context
+
+    def fixed_size(self, **kwargs):
+        self.kwargs = kwargs
+        return self.context
+
+
+def test_batch_import_embeds_rows_and_writes_named_vectors(monkeypatch):
+    monkeypatch.setattr(config.processing, "EMBEDDING_VECTOR_NAME", "test_vectors")
+    batch_context = FakeBatchContext()
+    collection = SimpleNamespace(batch=FakeBatchFactory(batch_context))
+    service = object.__new__(WeaviateService)
+    service._client_lock = RLock()
+    service._last_query_time = 0
+    service._embedding_client = FakeEmbeddingClient(vector=[0.7, 0.8, 0.9])
+    service._select_collection = lambda lang: (collection, "test_collection")
+
+    errors = service.batch_import(
+        data_rows=[{"chunk_id": "c1", "body": "First chunk"}],
+        lang="en",
+    )
+
+    assert errors == []
+    assert service._embedding_client.document_inputs == [["First chunk"]]
+    assert batch_context.added[0]["vector"] == {"test_vectors": [0.7, 0.8, 0.9]}
+
+
+def test_query_embeds_once_and_passes_vector_to_hybrid(monkeypatch):
+    monkeypatch.setattr(config.processing, "EMBEDDING_VECTOR_NAME", "test_vectors")
+    collection = SimpleNamespace(query=FakeQuery())
+    service = object.__new__(WeaviateService)
+    service._client_lock = RLock()
+    service._last_query_time = 0
+    service._embedding_client = FakeEmbeddingClient(vector=[0.2, 0.3, 0.4])
+    service._select_collection = lambda lang: (collection, "test_collection")
+
+    service.query(query="admissions", lang="en", limit=3)
+
+    assert service._embedding_client.query_inputs == ["admissions"]
+    assert collection.query.hybrid_calls[0]["vector"] == [0.2, 0.3, 0.4]
+    assert collection.query.hybrid_calls[0]["target_vector"] == "test_vectors"
+    assert collection.query.hybrid_calls[0]["limit"] == 3
+
+
+def test_query_falls_back_to_bm25_when_embedding_fails(monkeypatch):
+    collection = SimpleNamespace(query=FakeQuery())
+    service = object.__new__(WeaviateService)
+    service._client_lock = RLock()
+    service._last_query_time = 0
+    service._embedding_client = FakeEmbeddingClient(fail=True)
+    service._select_collection = lambda lang: (collection, "test_collection")
+
+    service.query(query="admissions", lang="en", limit=3)
+
+    assert collection.query.hybrid_calls == []
+    assert collection.query.bm25_calls[0]["query"] == "admissions"
+    assert collection.query.bm25_calls[0]["limit"] == 3
 
 
 def test_model_config_keeps_master_defaults_and_budgets(monkeypatch):

tests/test_programme_positioning_real_agent.py

@@ -19,16 +19,13 @@ def _has_real_agent_prerequisites() -> tuple[bool, str]:
     if not llm_api_key:
         return False, "No LLM API key configured for the real agent positioning test."
 
-    if config.weaviate.LOCAL_DATABASE:
-        return True, ""
-
     missing = []
     if not config.weaviate.CLUSTER_URL:
         missing.append("WEAVIATE_CLUSTER_URL")
     if not config.weaviate.WEAVIATE_API_KEY:
         missing.append("WEAVIATE_API_KEY")
-    if not config.weaviate.HUGGING_FACE_API_KEY:
-        missing.append("HUGGING_FACE_API_KEY")
+    if not config.processing.EMBEDDING_API_KEY:
+        missing.append("OPEN_ROUTER_API_KEY")
 
     if missing:
         return False, f"Missing Weaviate configuration for real agent positioning test: {', '.join(missing)}"

tests/test_reply_speed_real_agent.py

@@ -20,16 +20,13 @@ def _has_real_agent_prerequisites() -> tuple[bool, str]:
     if not llm_api_key:
         return False, "No LLM API key configured for the real agent test."
 
-    if config.weaviate.LOCAL_DATABASE:
-        return True, ""
-
     missing = []
     if not config.weaviate.CLUSTER_URL:
         missing.append("WEAVIATE_CLUSTER_URL")
     if not config.weaviate.WEAVIATE_API_KEY:
         missing.append("WEAVIATE_API_KEY")
-    if not config.weaviate.HUGGING_FACE_API_KEY:
-        missing.append("HUGGING_FACE_API_KEY")
+    if not config.processing.EMBEDDING_API_KEY:
+        missing.append("OPEN_ROUTER_API_KEY")
 
     if missing:
         return False, f"Missing Weaviate configuration for real agent test: {', '.join(missing)}"