feat: improve query transformer and enable multi-language response

data/jrg_data.json

@@ -63,7 +63,7 @@
     }
   },
   {
-    "content": "Anh Đỗ Phi Long (Long Do) là đại gia Long Thành Đồng Nai. Anh ấy hiện đang làm việc với vai trò Data Analyst tại Pizza Hut Vietnam.",
+    "content": "Anh Đỗ Phi Long (Long Do) là đại gia Biên Hòa Đồng Nai. Anh ấy hiện đang làm việc với vai trò Data Analyst tại Pizza Hut Vietnam.",
     "metadata": {
       "source": "Employee Directory",
       "category": "Employee_Directory"

data/save_to_vectordb.py

@@ -1,5 +1,6 @@
 import json
 import os
+import glob
 from dotenv import load_dotenv
 from qdrant_client import models
 from langchain_qdrant import QdrantVectorStore, FastEmbedSparse, RetrievalMode
@@ -7,54 +8,86 @@ from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 
-# Safely resolve absolute paths to avoid directory execution issues
+# 1. Setup paths
 current_dir = os.path.dirname(os.path.abspath(__file__))
-env_path = os.path.join(current_dir, '..', '.env')
+# Assuming .env is located in the parent directory (pizzahut/)
+env_path = os.path.join(current_dir, '..', '.env') 
 load_dotenv(env_path)
 
 QDRANT_COLLECTION_NAME = os.getenv('QDRANT_COLLECTION_NAME', 'jrg_bot_collection')
 QDRANT_URL = os.getenv('QDRANT_URL')
 QDRANT_API_KEY = os.getenv('QDRANT_API_KEY')
 
-def load_and_chunk_data(file_path):
-    """Load JSON data and split into smaller chunks for optimal retrieval."""
-    with open(file_path, 'r', encoding='utf-8') as f:
-        data = json.load(f)
-
+def load_all_json_files(data_dir):
+    """Scan and convert all .json files in the data directory into Langchain Documents"""
     raw_documents = []
-    for item in data:
-        source = item.get('metadata', {}).get('source', 'Unknown')
-        print(f"Loading document from source: {source} ...")
+    
+    # Find all .json files in the current directory
+    json_files = glob.glob(os.path.join(data_dir, "*.json"))
+    
+    print(f"--- STARTING DATA INGESTION: FOUND {len(json_files)} JSON FILES ---")
+    
+    for file_path in json_files:
+        file_name = os.path.basename(file_path)
+        print(f"Reading file: {file_name}...")
         
-        raw_documents.append(Document(
-            page_content=item.get('content', ''),
-            metadata=item.get('metadata', {})
-        ))
-
-    # Text Splitter to prevent context window overflow when data grows
+        with open(file_path, 'r', encoding='utf-8') as f:
+            try:
+                data = json.load(f)
+                for item in data:
+                    content = item.get('content', '').strip()
+                    if not content:
+                        continue
+                        
+                    raw_documents.append(Document(
+                        page_content=content,
+                        metadata=item.get('metadata', {})
+                    ))
+            except json.JSONDecodeError:
+                print(f"ERROR: JSON syntax error in file: {file_name}")
+    
+    print(f"Successfully loaded {len(raw_documents)} raw documents.")
+    
+    # Text Splitter: Crucial for optimizing context window and memory limits
+    print("Initializing Text Splitter for chunking...")
     text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=500,
-        chunk_overlap=50
+        chunk_size=800,      
+        chunk_overlap=100    
     )
     
-    docs = text_splitter.split_documents(raw_documents)
-    print(f"Successfully loaded and split into {len(docs)} chunks.")
-    return docs
+    final_docs = text_splitter.split_documents(raw_documents)
+    print(f"Data has been split into {len(final_docs)} optimal chunks ready for VectorDB.")
+    
+    return final_docs
 
 def main():
-    json_path = os.path.join(current_dir, 'jrg_data.json')
-    documents = load_and_chunk_data(json_path)
+    if not QDRANT_URL or not QDRANT_API_KEY:
+        print("CRITICAL ERROR: QDRANT_URL or QDRANT_API_KEY not found in .env file.")
+        return
+
+    # 2. Process data
+    documents = load_all_json_files(current_dir)
 
-    print("Initializing Embedding Models (Dense & Sparse)...")
-    # Dense Embedding for semantic search
+    if not documents:
+        print("WARNING: No data found to ingest. Please check your JSON files.")
+        return
+
+    # 3. Initialize AI Embedding Models
+    print("Initializing Hybrid Embedding Models (Dense & Sparse)...")
+    
+    # Dense Model: For semantic search (Vietnamese optimized)
     model_name = 'bkai-foundation-models/vietnamese-bi-encoder'
+    print(f"Loading Dense Model: {model_name}...")
     embedding_model = HuggingFaceEmbeddings(model_name=model_name)
     
-    # Sparse Embedding for keyword search (BM25)
+    # Sparse Model: For exact keyword matching (BM25)
+    print("Loading Sparse Model: Qdrant/BM25...")
     sparse_embeddings = FastEmbedSparse(model_name="Qdrant/BM25")
 
-    print("Uploading data to Qdrant Cloud...")
-    # Save to Qdrant using Hybrid Search
+    # 4. Upload to Qdrant Cloud
+    print(f"Uploading vectors to Qdrant Collection: '{QDRANT_COLLECTION_NAME}'...")
+    print("This process may take a few minutes depending on the data size. Please do not close the terminal...")
+    
     vectorstore = QdrantVectorStore.from_documents(
         documents, 
         embedding_model, 
@@ -64,10 +97,10 @@ def main():
         api_key=QDRANT_API_KEY,
         collection_name=QDRANT_COLLECTION_NAME,
         distance=models.Distance.COSINE,
-        force_recreate=True  # Avoid duplicating data on multiple runs
+        force_recreate=True  # WARNING: This will drop the existing collection and recreate it
     )
 
-    print(f"SUCCESS: Data has been saved to Qdrant collection '{QDRANT_COLLECTION_NAME}'.")
+    print("SUCCESS: The entire Knowledge Base has been uploaded to Qdrant Cloud!")
 
 if __name__ == "__main__":
     main()