“…I”

VectorDB/.lock

@@ -1 +0,0 @@
-tmp lock file

VectorDB/collection/chung/storage.sqlite

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c1d8a04276ee97871976635bc7d339d32dfd5d4df556206c6f64f1a8bc4e189c
-size 12288

VectorDB/collection/doanh_nghiep/storage.sqlite

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c1d8a04276ee97871976635bc7d339d32dfd5d4df556206c6f64f1a8bc4e189c
-size 12288

VectorDB/collection/ky_thuat/storage.sqlite

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c1d8a04276ee97871976635bc7d339d32dfd5d4df556206c6f64f1a8bc4e189c
-size 12288

VectorDB/collection/rag_input/storage.sqlite

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ebc272d4c79930cce8098f4a96b3f45ae10c375e0015eeaa08e6973ac7151cfb
-size 212992

VectorDB/meta.json

@@ -1 +0,0 @@
-{"collections": {"ky_thuat": {"vectors": {"dense": {"size": 384, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "quantization_config": null, "sparse_vectors": {"sparse": {"index": null, "modifier": null}}, "strict_mode_config": null, "metadata": null}, "doanh_nghiep": {"vectors": {"dense": {"size": 384, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "quantization_config": null, "sparse_vectors": {"sparse": {"index": null, "modifier": null}}, "strict_mode_config": null, "metadata": null}, "chung": {"vectors": {"dense": {"size": 384, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "quantization_config": null, "sparse_vectors": {"sparse": {"index": null, "modifier": null}}, "strict_mode_config": null, "metadata": null}, "rag_input": {"vectors": {"dense": {"size": 384, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "quantization_config": null, "sparse_vectors": {"sparse": {"index": null, "modifier": null}}, "strict_mode_config": null, "metadata": null}}, "aliases": {}}

app/core/config.py

@@ -12,7 +12,7 @@ QDRANT_PATH = os.path.join(BASE_DIR, "VectorDB")
 COLLECTION_NAME = "rag_input"
 CATEGORIES = ["ky_thuat", "doanh_nghiep", "chung"]
 
-EMBED_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
+EMBED_MODEL_NAME = "bkai-foundation-models/vietnamese-bi-encoder"
 LLM_MODEL_NAME = "gemini-2.5-flash"
 RERANK_MODEL_NAME = "cross-encoder/ms-marco-MiniLM-L-6-v2"
 

app/service/ingest.py

@@ -1,53 +1,77 @@
 import os
-from llama_index.core import SimpleDirectoryReader
-from langchain_core.documents import Document
+from langchain_community.document_loaders import (
+    TextLoader,
+    PyPDFLoader,
+    Docx2txtLoader,
+    CSVLoader,
+    UnstructuredMarkdownLoader
+)
 from langchain_experimental.text_splitter import SemanticChunker
 from langchain_qdrant import QdrantVectorStore
 from qdrant_client import QdrantClient
 from qdrant_client.http import models
 from app.core.config import DATA_PATH, QDRANT_PATH, COLLECTION_NAME, get_embeddings, CATEGORIES
 
-def load_document():
-    # 1. Load documents using LlamaIndex
-    llama_docs = SimpleDirectoryReader(
-        input_dir=DATA_PATH,
-        recursive=True
-    ).load_data()
+LOADER_MAPPING = {
+    ".pdf": PyPDFLoader,
+    ".docx": Docx2txtLoader,
+    ".doc": Docx2txtLoader,
+    ".txt": TextLoader,
+    ".csv": CSVLoader,
+    ".md": UnstructuredMarkdownLoader,
+}
 
-    # 2. Convert to LangChain documents
+def load_document():
     langchain_docs = []
-    for doc in llama_docs:
-        lc_doc = Document(page_content=doc.text, metadata=doc.metadata)
-        langchain_docs.append(lc_doc)
+    
+    if not os.path.exists(DATA_PATH):
+        os.makedirs(DATA_PATH)
+        return None
+
+    for root, dirs, files in os.walk(DATA_PATH):
+        for file in files:
+            ext = os.path.splitext(file)[1].lower()
+            if ext in LOADER_MAPPING:
+                file_path = os.path.join(root, file)
+                try:
+                    loader_cls = LOADER_MAPPING[ext]
+                    loader = loader_cls(file_path)
+                    langchain_docs.extend(loader.load())
+                except Exception as e:
+                    print(f"[Error")
+            else:
+                if not file.startswith('.'):
+                    print(f"Ignore format : {file}")
 
     if not langchain_docs:
         return None
 
-    # 3. Chunking
+    # 2. Chunking
     embeddings = get_embeddings()
-    from langchain_text_splitters import RecursiveCharacterTextSplitter
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-    chunks = text_splitter.split_documents(langchain_docs)
+    semantic_chunker = SemanticChunker(
+        embeddings,
+        breakpoint_threshold_amount=0.8
+    )
+    chunks = semantic_chunker.split_documents(langchain_docs)
 
-    # 4. Vector Store setup
+    # 3. Vector Store setup
     client = QdrantClient(path=QDRANT_PATH)
-    
-    # Initialize collections for all categories + default
     all_collections = CATEGORIES + [COLLECTION_NAME]
+    embed_dim = len(embeddings.embed_query("test"))
     
     for coll in all_collections:
         if not client.collection_exists(coll):
             client.create_collection(
                 collection_name=coll,
                 vectors_config={
-                    "dense": models.VectorParams(size=len(embeddings.embed_query("test")), distance=models.Distance.COSINE)
+                    "dense": models.VectorParams(size=embed_dim, distance=models.Distance.COSINE)
                 },
                 sparse_vectors_config={
                     "sparse": models.SparseVectorParams()
                 }
             )
 
-    # 5. Ingest into default collection
+    # 4. Ingest into default collection
     vector_store = QdrantVectorStore(
         client=client,
         collection_name=COLLECTION_NAME,
@@ -58,5 +82,4 @@ def load_document():
     return vector_store
 
 if __name__ == "__main__":
-    os.makedirs(DATA_PATH, exist_ok=True)
     load_document()

requirements.txt

@@ -9,6 +9,8 @@ llama-index
 qdrant-client
 sentence-transformers
 python-dotenv
+unstructured[all-docs]
+numpy<2.0.0
 pydantic
 fastapi
 uvicorn

run.sh

@@ -17,9 +17,10 @@ fi
 echo -e "Chọn một tùy chọn:"
 echo -e "${GREEN}1)${NC} Ingest Data (Nạp dữ liệu từ thư mục /data vào VectorDB)"
 echo -e "${GREEN}2)${NC} Run Web API (Bắt đầu Web Chatbot trên http://localhost:8000)"
-echo -e "${GREEN}3)${NC} Thoát"
+echo -e "${GREEN}3)${NC} Clear VectorDB (Xóa dữ liệu cũ để đổi mô hình)"
+echo -e "${GREEN}4)${NC} Thoát"
 
-read -p "Nhập lựa chọn của bạn [1-3]: " choice
+read -p "Nhập lựa chọn của bạn [1-4]: " choice
 
 case $choice in
     1)
@@ -39,6 +40,10 @@ case $choice in
         PYTHONPATH=. python3 api.py
         ;;
     3)
+        echo -e "${RED}[!] Cảnh báo: Tất cả dữ liệu vector sẽ bị xóa.${NC}"
+        PYTHONPATH=. python3 scripts/clear_db.py
+        ;;
+    4)
         echo -e "Tạm biệt!"
         exit 0
         ;;

scripts/clear_db.py

@@ -0,0 +1,39 @@
+import shutil
+import os
+from app.core.config import QDRANT_PATH, COLLECTION_NAME, CATEGORIES
+from qdrant_client import QdrantClient
+
+def clear_vector_db():
+    print(f"Checking Qdrant database at: {QDRANT_PATH}")
+    
+    # Method 1: Delete via Client (Cleaner if Qdrant is running or using local path)
+    try:
+        client = QdrantClient(path=QDRANT_PATH)
+        all_collections = CATEGORIES + [COLLECTION_NAME]
+        
+        for coll in all_collections:
+            if client.collection_exists(coll):
+                print(f"Deleting collection: {coll}")
+                client.delete_collection(coll)
+            else:
+                print(f"Collection {coll} does not exist.")
+    except Exception as e:
+        print(f"Error deleting collections via client: {e}")
+        
+    # Method 2: Force delete the storage directory if client method is not enough
+    if os.path.exists(QDRANT_PATH):
+        print(f"Removing storage directory: {QDRANT_PATH}")
+        try:
+            shutil.rmtree(QDRANT_PATH)
+            print("Successfully removed VectorDB directory.")
+        except Exception as e:
+            print(f"Error removing directory: {e}")
+    else:
+        print("VectorDB directory already cleared.")
+
+if __name__ == "__main__":
+    confirm = input("This will delete all existing vector data. Are you sure? (y/n): ")
+    if confirm.lower() == 'y':
+        clear_vector_db()
+    else:
+        print("Operation cancelled.")

static/index.html

@@ -15,7 +15,7 @@
                     <img src="https://api.dicebear.com/7.x/bottts/svg?seed=ragbot&backgroundColor=1e1e2f" alt="Bot Avatar" class="avatar">
                 </div>
                 <div class="header-text">
-                    <h1>Enterprise AI Assistant</h1>
+                    <h1>AI Assistant</h1>
                     <span class="status"><span class="dot"></span> Trực tuyến</span>
                 </div>
             </div>
@@ -30,7 +30,7 @@
             <div class="message bot">
                 <img src="https://api.dicebear.com/7.x/bottts/svg?seed=ragbot&backgroundColor=1e1e2f" alt="Bot" class="msg-avatar">
                 <div class="msg-content">
-                    <p>Xin chào! Tôi là AI Assistant được tích hợp hệ thống RAG (Retrieval-Augmented Generation). Tôi có thể giúp gì cho bạn hôm nay?</p>
+                    <p>Xin chào! Tôi là AI Assistant được tích hợp hệ thống RAG (Retrieval-Augmented Generation). Tôi có thể giúp gì cho bạn?</p>
                 </div>
             </div>
         </main>

static/style.css

@@ -1,13 +1,13 @@
 :root {
-    --bg-color: #0f172a;
-    --container-bg: #1e293b;
-    --text-main: #f8fafc;
+    --bg-color: #abdef2;
+    --container-bg: #ffffff;
+    --text-main: #373e45;
     --text-muted: #94a3b8;
-    --accent: #3b82f6;
+    --accent: #f9f9f9;
     --accent-hover: #60a5fa;
-    --bot-msg-bg: #334155;
-    --user-msg-bg: #2563eb;
-    --border-color: #334155;
+    --bot-msg-bg: #ffffff;
+    --user-msg-bg: #ffffff;
+    --border-color: #010101;
 }
 
 * {
@@ -44,7 +44,7 @@ body {
 
 .chat-header {
     padding: 20px;
-    background: rgba(30, 41, 59, 0.8);
+    background: rgba(255, 255, 255, 0.8);
     backdrop-filter: blur(10px);
     border-bottom: 1px solid var(--border-color);
     display: flex;
@@ -67,7 +67,7 @@ body {
     content: '';
     position: absolute;
     top: -2px; left: -2px; right: -2px; bottom: -2px;
-    background: linear-gradient(45deg, #3b82f6, #8b5cf6);
+    background: linear-gradient(45deg, #3b82f6, #5ce9f6);
     border-radius: 50%;
     z-index: -1;
     animation: pulse 2s infinite;
@@ -185,7 +185,7 @@ body {
 .message.user .msg-content {
     background: var(--user-msg-bg);
     border-top-right-radius: 4px;
-    box-shadow: 0 4px 15px rgba(37, 99, 235, 0.3);
+    box-shadow: 0 4px 15px rgba(0, 0, 0, 0.3);
 }
 
 .typing-indicator {
@@ -216,14 +216,14 @@ body {
 
 .chat-input-area {
     padding: 20px;
-    background: rgba(30, 41, 59, 0.95);
+    background: rgba(255, 255, 255, 0.95);
     border-top: 1px solid var(--border-color);
 }
 
 #chatForm {
     display: flex;
     gap: 15px;
-    background: #0f172a;
+    background: #ffffff;
     padding: 8px 8px 8px 20px;
     border-radius: 30px;
     border: 1px solid var(--border-color);
@@ -232,7 +232,7 @@ body {
 
 #chatForm:focus-within {
     border-color: var(--accent);
-    box-shadow: 0 0 0 2px rgba(37, 99, 235, 0.2);
+    box-shadow: 0 0 0 2px rgba(0, 0, 0, 0.2);
 }
 
 #userInput {
@@ -249,7 +249,7 @@ body {
 }
 
 #sendBtn {
-    background: linear-gradient(45deg, var(--accent), #8b5cf6);
+    background: linear-gradient(45deg, var(--accent), #5baed2);
     border: none;
     width: 44px;
     height: 44px;
@@ -264,7 +264,7 @@ body {
 
 #sendBtn:hover {
     transform: scale(1.05);
-    box-shadow: 0 0 15px rgba(139, 92, 246, 0.5);
+    box-shadow: 0 0 15px rgba(109, 73, 192, 0.5);
 }
 
 #sendBtn:active {