unroll table and project restructuring

core/ai_provider.py

@@ -0,0 +1,38 @@
+import os
+import logging
+import threading
+import groq
+
+logger = logging.getLogger(__name__)
+
+class AIProviderManager:
+    def __init__(self):
+        # Lấy danh sách keys 
+        self.groq_keys = [k.strip() for k in os.getenv("GROQ_API_KEYS", "").split(",") if k.strip()]
+        self.gemini_keys = [k.strip() for k in os.getenv("GEMINI_API_KEYS", "").split(",") if k.strip()]
+        self.groq_idx = 0
+        self.gemini_idx = 0
+        self._lock = threading.Lock() # Đảm bảo Thread-Safe khi có nhiều Request cùng lúc
+
+    def get_groq_client(self):
+        if not self.groq_keys: return None
+        # Chỉ lấy key, không thay đổi state nên không cần lock
+        return groq.Groq(api_key=self.groq_keys[self.groq_idx])
+
+    def rotate_groq(self):
+        with self._lock: # Khóa luồng khi xoay tua để tránh xung đột
+            if len(self.groq_keys) > 1:
+                self.groq_idx = (self.groq_idx + 1) % len(self.groq_keys)
+                logger.info(f"Đã xoay sang Groq Key thứ {self.groq_idx + 1}")
+
+    def get_gemini_key(self):
+        if not self.gemini_keys: return None
+        return self.gemini_keys[self.gemini_idx]
+
+    def rotate_gemini(self):
+        with self._lock:
+            if len(self.gemini_keys) > 1:
+                self.gemini_idx = (self.gemini_idx + 1) % len(self.gemini_keys)
+                logger.info("Đã xoay sang Gemini Key dự phòng")
+
+api_manager = AIProviderManager()

core/chunking.py

@@ -1,63 +0,0 @@
-import re
-from typing import List
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from .config import CHUNK_SIZE, CHUNK_OVERLAP
-
-def extract_and_protect_tables(text: str) -> tuple[str, dict]:
-    """Tìm và bọc các bảng Markdown để bảo vệ chúng khỏi việc bị cắt gãy."""
-    # Pattern tìm bảng Markdown (các dòng bắt đầu và chứa ký tự | liên tiếp)
-    table_pattern = re.compile(r'(?:\|.*\|[\r\n]+)+')
-    tables = {}
-    
-    def replace_table(match):
-        table_id = f"<TABLE_{len(tables)}>"
-        tables[table_id] = match.group(0)
-        return f"\n{table_id}\n"
-
-    protected_text = re.sub(table_pattern, replace_table, text)
-    return protected_text, tables
-
-def smart_chunking(docs: List) -> List:
-    print("Đang áp dụng Smart Chunking (Bảo toàn Bảng & Danh sách)...")
-    legal_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=CHUNK_SIZE,
-        chunk_overlap=CHUNK_OVERLAP,
-        separators=[
-            "\nĐiều ", "\nChương ", "\nMục ", "\nKhoản ",
-            "\n\n", "\n", ". ", " ", ""
-        ],
-        length_function=len,
-        is_separator_regex=False
-    )
-    
-    chunks = []
-    for doc in docs:
-        # 1. Bảo vệ List đang có
-        protected_text = doc.page_content.replace('\na.', '<LIST_a>') \
-                                         .replace('\nb.', '<LIST_b>') \
-                                         .replace('\nc.', '<LIST_c>')
-        
-        # 2. Bảo vệ Table
-        protected_text, tables = extract_and_protect_tables(protected_text)
-        
-        # 3. Tiến hành cắt
-        doc_chunks = legal_splitter.split_text(protected_text)
-        
-        # 4. Phục hồi dữ liệu
-        for chunk_text in doc_chunks:
-            restored = chunk_text.replace('<LIST_a>', '\na.') \
-                                 .replace('<LIST_b>', '\nb.') \
-                                 .replace('<LIST_c>', '\nc.')
-            
-            for table_id, table_content in tables.items():
-                if table_id in restored:
-                    restored = restored.replace(table_id, table_content)
-                    
-            new_doc = type(doc)(
-                page_content=restored,
-                metadata=doc.metadata.copy()
-            )
-            chunks.append(new_doc)
-            
-    print(f" Đã tạo {len(chunks)} chunks thông minh (giữ nguyên cấu trúc bảng)")
-    return chunks

core/llm_utils.py

@@ -1,31 +0,0 @@
-import time
-import logging 
-from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
-
-logger = logging.getLogger(__name__)
-
-def safe_stream(llm, prompt) :
-    try:
-        for chunk in llm.stream(prompt):
-            text = getattr(chunk, "content", str(chunk))
-            if text:
-                yield text
-    except Exception :
-        logger.exception("Lỗi khi stream LLM:")
-        yield "Lỗi khi stream LLM "
-def safe_invoke(llm ,prompt : str, timeout : int =30, retries: int =2):
-    last_error = None
-    for attempt in range(1, retries+1):
-        try:
-            with ThreadPoolExecutor(max_workers=1) as pool:
-                fut = pool.submit(llm.invoke, prompt)
-                return fut.result(timeout=timeout)
-        except FuturesTimeoutError as e:
-            last_error = e
-            logger.warning(f" Lần {attempt}: LLM invoke timeout sau {timeout} giây. Đang thử lại...")
-        except Exception as e:
-            last_error = e
-            logger.error(f"Lần {attempt}: Lỗi khi gọi LLM: {e}. Đang thử lại...")
-        time.sleep(0.6*attempt)
-        
-    raise RuntimeError (f"LLM failed after {retries} attempts: {last_error}")  # Thêm delay nhỏ trước khi thử lại

core/retriever.py

@@ -1,73 +0,0 @@
-from typing import List
-import hashlib
-from rank_bm25 import BM25Okapi
-
-class HybridRetriever:
-    """Kết hợp BM25 và Vector Search."""
-    def __init__(self, vectorstore, documents):
-        self.vectorstore = vectorstore
-        self.documents = documents
-        print(" Đang khởi tạo BM25...")
-        tokenized_docs = [doc.page_content.lower().split() for doc in documents]
-        self.bm25 = BM25Okapi(tokenized_docs, k1=1.5, b=0.5)
-        self.rrf_c = 60
-        print(" BM25 sẵn sàng!")
-
-    @staticmethod
-    def _doc_key(doc) -> str:
-        metadata = doc.metadata if isinstance(doc.metadata, dict) else {}
-        source = str(metadata.get("source_relpath") or metadata.get("source_file") or metadata.get("source") or "")
-        page = str(metadata.get("page_number") or metadata.get("page") or "")
-        content = (doc.page_content or "").strip()
-        digest = hashlib.sha1(content.encode("utf-8")).hexdigest() if content else "empty"
-        return f"{source}|{page}|{digest}"
-
-    def search(self, query: str, k: int = 10, alpha: float = 0.6, year_scope: str | None = None) -> List:
-        del year_scope
-        if not self.documents or k <= 0:
-            return []
-
-        alpha = max(0.0, min(1.0, float(alpha)))
-        bm25_weight = 1.0 - alpha
-        vector_weight = alpha
-
-        # Lấy top k từ BM25
-        tokenized_query = query.lower().split()
-        candidate_k = min(max(k * 4, k), len(self.documents))
-        bm25_top_docs = self.bm25.get_top_n(tokenized_query, self.documents, n=candidate_k)
-
-        bm25_ranked = {}
-        all_retrieved = {}
-        for rank, doc in enumerate(bm25_top_docs, 1):
-            key = self._doc_key(doc)
-            bm25_ranked[key] = rank
-            all_retrieved[key] = doc
-
-        # Lấy top k từ Vector
-        try:
-            vector_results = self.vectorstore.similarity_search(query, k=candidate_k)
-        except Exception as e:
-            print(f"Lỗi Vector Search: {e}")
-            return [doc for doc in bm25_top_docs[:k]]
-
-        vector_ranked = {}
-        for rank, doc in enumerate(vector_results, 1):
-            key = self._doc_key(doc)
-            vector_ranked[key] = rank
-            all_retrieved[key] = doc
-
-        rrf_results = []
-        
-        for content, doc in all_retrieved.items():
-            score = 0.0
-            if content in bm25_ranked:
-                score += bm25_weight / (self.rrf_c + bm25_ranked[content])
-            if content in vector_ranked:
-                score += vector_weight / (self.rrf_c + vector_ranked[content])
-
-            if score > 0:
-                rrf_results.append((score, doc))
-
-        rrf_results.sort(key=lambda x: x[0], reverse=True)
-        return [doc for score, doc in rrf_results[:k]]
-

data/Sổ tay sinh viên 2023-2024/0. Mục lục Sổ tay sinh viên K65.docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:725638f2ebed983fb38354fa9d2073937a4df89418b4c4c6958c68f29c868113
-size 14530

data/Sổ tay sinh viên 2023-2024/1. QĐ-1226-Quy che dao tao dai hoc-DHTL (ban hanh).docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aaf33dda956f69966bbe64864432618068eaded7d3016644c4acfe8889f4dd5a
-size 120225

data/Sổ tay sinh viên 2023-2024/10. QĐ 1089 thi OLP môn học (Final 10-5-2023).docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8616d772baf6ff2225f2163e12a88418f2a6b46c1770fcd7d306da952ec3a94e
-size 52007

data/Sổ tay sinh viên 2023-2024/11. QĐ về Học phí final (25-10-2021).docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:17a91329f9e31308df57a53e36a69665fe37f3aea37fc32bc69a7fe984fbf3c5
-size 56350

data/Sổ tay sinh viên 2023-2024/12. QD ngoại trú.docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:06e7018cfa51e785f90009a66a2dc442a17f55ae52f5f28ddce208e61a6ba7d6
-size 26065

data/Sổ tay sinh viên 2023-2024/2. QĐ về tiếng anh CTTT.300921.QD.1315.docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e7b27a9545255d7d69158cc8e8d42d560f00c6865f665317619822159af7e90c
-size 40894

data/Sổ tay sinh viên 2023-2024/3. QD1767.TA tăng cường ban hanh.docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1dc5e1e6aef63069f673606c5083fb9dab484fa1b9cb10f955121bc4042c7131
-size 37167

data/Sổ tay sinh viên 2023-2024/4. QD411_QD_DHTL-Chuan_Dau_Ra_CNTT.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:544fe7b5b71f4fb407175fe0bcdc8853e0943eb410a4947b2ac8d891f359619b
-size 369669

data/Sổ tay sinh viên 2023-2024/4.1. QuyDinh_Ve_CDR_CNTT_Ban_hanh_theo_QD411-06-4-2022.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:17ebf4761e56b3d3336cfeaeab86bee4de7614fd2c27c20d848a0fed7650abfa
-size 498348

data/Sổ tay sinh viên 2023-2024/5. QD_1038_16.07.2021_GDTC.docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ee1a9d836579a08c6cbde24f63f9c8eb64aa70f52cfeec4c5e1c6ed789993da3
-size 47518

data/Sổ tay sinh viên 2023-2024/6. Quy định về tổ chức thi trực tuyến.docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:80c185d0fdb75571f7f9fafd974e4ac4a4006a890ca3ce975ca7ca70181848a3
-size 65814

data/Sổ tay sinh viên 2023-2024/7. QĐ đánh giá KQRL (Final 18-8-2016).docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4905773937cd8d12a8d24224146dc1442418ed3a57b60e3348a0e1b8eeb5ea2b
-size 74575

data/Sổ tay sinh viên 2023-2024/8. QĐ ve HBKKHT, HBCS (final 12-5-2021).docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:da1cd4f217a697635a5dae85285f46aa9a19f217f3fd03c981a9f47d79e31f2d
-size 30512

data/Sổ tay sinh viên 2023-2024/9. QĐ Khen thưởng - KL (Final 10-8-2016).docx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:46e6bdba5671d6aabfda753efa4010b8fb828d2c9180ab58882d7361c0200c67
-size 76052

{core → database}/document_db.py

@@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional
 from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, create_engine, func, inspect, or_, text
 from sqlalchemy.orm import Session, declarative_base, relationship, sessionmaker
 
-from .config import DOCUMENTS_DATABASE_URL
+from core.config import DOCUMENTS_DATABASE_URL
 
 Base = declarative_base()
 logger = logging.getLogger(__name__)

main.py

@@ -26,12 +26,12 @@ from core.config import (
     SUPABASE_SYNC_SNAPSHOT_FILE,
     SUPABASE_URL,
 )
-from core.document_db import init_document_db
-from core.supabase_sync_service import SupabaseStorageSyncService, SupabaseSyncCoordinator
-from core.collection_router_retriever import CollectionRouterRetriever
-from core.vectorstore import build_vectorstore_improved, load_vectorstore_improved
-from core.models import embeddings
-from core.qa_pipeline import ask_ai_improved, ask_ai_stream_delta
+from database.document_db import init_document_db
+from services.supabase_sync_service import SupabaseStorageSyncService, SupabaseSyncCoordinator
+from rag.collection_router_retriever import CollectionRouterRetriever
+from rag.vectorstore import build_vectorstore_improved, load_vectorstore_improved
+from rag.models import embeddings
+from rag.qa_pipeline import ask_ai_improved, ask_ai_stream_delta
 from api.admin_sync_router import router as admin_sync_router
 
 # Hàm log lỗi an toàn

{core → rag}/analyze_and_expand.py

@@ -21,8 +21,7 @@ def clean_json_string(text: str) -> str:
 def analyze_and_expand_query(question: str) -> Dict[str, Any]:
     print(" Phân tích & Mở rộng câu hỏi...")
     
-    # Import cục bộ để tránh lỗi vòng lặp import (circular import) với qa_pipeline
-    from .qa_pipeline import api_manager
+    from core.ai_provider import api_manager  
     
     # Prompt được tối ưu để ép AI trả về JSON chuẩn
     prompt = f"""

rag/chunking.py

@@ -0,0 +1,38 @@
+from typing import List
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from core.config import CHUNK_SIZE, CHUNK_OVERLAP
+
+def smart_chunking(docs: List) -> List:
+    print("Đang áp dụng Smart Chunking (Regex Lookahead)...")
+    
+    # Cấu hình Regex bắt cấu trúc phân cấp hành chính (Chương -> Điều -> Khoản)
+    legal_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP,
+        separators=[
+            "\nChương ",      
+            "\nĐiều ",        
+            "\nKhoản ",       
+            "\n\n",           
+            r"\n(?=\d+\.)",   
+            r"\n(?=[a-z]\.)", 
+            r"\n(?=-|\+)",    
+            "\n", " ", ""
+        ],
+        length_function=len,
+        is_separator_regex=True  
+    )
+    
+    chunks = []
+    for doc in docs:
+        doc_chunks = legal_splitter.split_text(doc.page_content)
+        
+        for chunk_text in doc_chunks:
+            new_doc = type(doc)(
+                page_content=chunk_text,
+                metadata=doc.metadata.copy()  
+            )
+            chunks.append(new_doc)
+            
+    print(f"Đã tạo {len(chunks)} chunks thông minh.")
+    return chunks

{core → rag}/collection_router_retriever.py

@@ -5,25 +5,39 @@ from typing import List
 from langchain_core.documents import Document as LangChainDocument
 from rank_bm25 import BM25Okapi
 
+
+try:
+    from pyvi import ViTokenizer
+except Exception:
+    ViTokenizer = None
+
 from .collection_utils import collection_matches_cohort
-from .document_db import SessionLocal, list_active_collection_names
+from database.document_db import SessionLocal, list_active_collection_names
 
 logger = logging.getLogger(__name__)
 
 
+def _vi_tokenize(text: str) -> List[str]:
+    normalized = (text or "").lower().strip()
+    if not normalized:
+        return []
+    if ViTokenizer is None:
+        return normalized.split()
+    return ViTokenizer.tokenize(normalized).split()
+
+
 class CollectionRouterRetriever:
     def __init__(
         self,
-        base_retriever,
-        qdrant_client,
+        qdrant_client,          
         embeddings_model,
         top_n_collections: int = 3,
     ) -> None:
-        self.base_retriever = base_retriever
         self.qdrant_client = qdrant_client
         self.embeddings_model = embeddings_model
         self.top_n_collections = max(1, int(top_n_collections or 3))
-        self._bm25_cache = {}  # {collection_name -> BM25Okapi instance}
+        # Cache giờ đây lưu một dict: { 'bm25': obj, 'corpus_docs': list, 'count': int }
+        self._bm25_cache = {}  
 
     @staticmethod
     def _doc_key(doc) -> str:
@@ -63,63 +77,91 @@ class CollectionRouterRetriever:
 
         return active_collections[: self.top_n_collections]
 
-    def _ensure_bm25_loaded(self, collection_name: str) -> BM25Okapi | None:
-        """Lazy load and cache BM25 index for a collection.
-        
-        First time: fetch all docs from Qdrant, build BM25, cache it (~0.3s)
-        Subsequent times: reuse from cache (~0.001s)
-        """
-        # Check if already cached
-        if collection_name in self._bm25_cache:
-            return self._bm25_cache[collection_name]
+    def _ensure_bm25_loaded(self, collection_name: str) -> tuple[BM25Okapi, List[LangChainDocument]] | None:
+        """Lazy load and cache BM25 index and corpus for a collection (với cơ chế tự động làm mới Cache)"""
         
+        # 1. Lấy tổng số chunks hiện tại trong Qdrant (Rất nhanh, tốn < 10ms)
         try:
-            # Fetch ALL documents from collection (no query vector, get full corpus)
-            all_points = self.qdrant_client.scroll(
-                collection_name=collection_name,
-                limit=10000,  # Batch size
-            )
+            collection_info = self.qdrant_client.get_collection(collection_name)
+            current_count = collection_info.points_count
+        except Exception:
+            logger.exception("Failed to get collection info for %s", collection_name)
+            return None
+
+        # 2. Kiểm tra Cache: Nếu chưa có hoặc số lượng thay đổi -> Xóa cache build lại
+        cached_data = self._bm25_cache.get(collection_name)
+        if cached_data and cached_data.get('count') == current_count:
+            # Tái sử dụng (Phải trả về cả bm25 VÀ corpus_docs để map điểm)
+            return cached_data['bm25'], cached_data['corpus_docs'] 
             
-            points_list = all_points[0] if isinstance(all_points, tuple) else all_points
+        logger.info(f"Phát hiện dữ liệu mới hoặc chưa có cache cho {collection_name} (Count: {current_count}). Đang build lại BM25...")
+        
+        try:
+            points_list = []
+            offset = None
+            # Phân trang để lấy TOÀN BỘ documents từ collection
+            while True:
+                response = self.qdrant_client.scroll(
+                    collection_name=collection_name,
+                    limit=10000,
+                    offset=offset,
+                    with_payload=True,     
+                    with_vectors=False     
+                )
+                batch_points, next_offset = response
+                points_list.extend([p for p in batch_points if p is not None])
+                
+                offset = next_offset
+                if offset is None:  
+                    break
             
             if not points_list:
                 logger.warning("No documents found in collection=%s for BM25 indexing", collection_name)
                 return None
             
-            # Filter out None values
-            points_list = [p for p in points_list if p is not None]
-            if not points_list:
-                logger.warning("No valid points found in collection=%s after filtering", collection_name)
-                return None
-            
-            # Extract documents and tokenize for BM25
-            docs_for_bm25 = []
+            # Trích xuất content và build documents
+            corpus_docs = []
             for point in points_list:
                 payload = point.payload if isinstance(point.payload, dict) else {}
                 content = str(payload.get("content") or "").strip()
                 if content:
-                    docs_for_bm25.append(content)
+                    metadata = {
+                        "source": payload.get("path") or payload.get("object_path") or payload.get("stored_name") or "",
+                        "source_file": payload.get("filename") or payload.get("stored_name") or "",
+                        "source_relpath": payload.get("object_path") or payload.get("path") or "",
+                        "object_path": payload.get("object_path") or "",
+                        "folder_key": payload.get("folder_key") or "",
+                        "collection_name": collection_name,
+                        "academic_year": payload.get("academic_year") or "",
+                        "chunk_index": payload.get("chunk_index"),
+                        "page_number": payload.get("page_number"),
+                    }
+                    doc = LangChainDocument(page_content=content, metadata=metadata)
+                    corpus_docs.append(doc)
             
-            if not docs_for_bm25:
+            if not corpus_docs:
                 logger.warning("No valid content found in collection=%s for BM25 indexing", collection_name)
                 return None
             
-            # Build BM25 index
-            tokenized_docs = [doc.lower().split() for doc in docs_for_bm25]
+            tokenized_docs = [_vi_tokenize(doc.page_content) for doc in corpus_docs]
             bm25 = BM25Okapi(tokenized_docs, k1=1.5, b=0.5)
             
-            # Cache it
-            self._bm25_cache[collection_name] = bm25
-            logger.info("BM25 index built and cached for collection=%s (docs=%d)", collection_name, len(docs_for_bm25))
+            # 3. Lưu lại Cache kèm the con số count và corpus_docs để đối chiếu lần sau
+            self._bm25_cache[collection_name] = {
+                'bm25': bm25,
+                'corpus_docs': corpus_docs,
+                'count': current_count
+            }
+            logger.info("BM25 index built and cached for collection=%s (docs=%d)", collection_name, len(corpus_docs))
             
-            return bm25
+            return bm25, corpus_docs
             
         except Exception:
             logger.exception("Failed to build BM25 index for collection=%s", collection_name)
             return None
 
     def _search_target_collections(self, query: str, collections: List[str], limit: int, alpha: float = 0.6) -> List:
-        """Hybrid search: BM25 + Vector + RRF (Option 2 with cached BM25)"""
+        """Hybrid search: BM25 + Vector + RRF"""
         if not collections:
             return []
 
@@ -129,9 +171,9 @@ class CollectionRouterRetriever:
             logger.exception("Failed to embed query for collection routing")
             return []
 
-        # Step 1: Vector search (từ Qdrant)
-        all_docs_dict = {}  # {doc_key -> LangChainDocument}
-        vector_ranked = {}  # {doc_key -> rank}
+        # Step 1: Vector search 
+        all_docs_dict = {}  
+        vector_ranked = {}  
         
         vector_rank = 0
         for collection_name in collections:
@@ -171,60 +213,45 @@ class CollectionRouterRetriever:
                     vector_rank += 1
                     vector_ranked[doc_key] = vector_rank
 
-        # Step 2: BM25 search (lexical) - using CACHED index
-        bm25_ranked = {}  # {doc_key -> rank}
-        if all_docs_dict:
-            try:
-                # Validate query is not empty
-                if not query.strip():
-                    logger.warning("Query is empty, skipping BM25 search")
-                else:
-                    tokenized_query = query.lower().split()
-                
-                # For each collection, use cached BM25 index
+        # Step 2: BM25 search 
+        bm25_ranked = {}  
+        try:
+            tokenized_query = _vi_tokenize(query)
+            
+            if not tokenized_query:
+                logger.warning("Query is empty after tokenization, skipping BM25 search")
+            else:
                 for collection_name in collections:
-                    # Load cached BM25 (or build if first time)
-                    bm25 = self._ensure_bm25_loaded(collection_name)
-                    if bm25 is None:
+                    bm25_data = self._ensure_bm25_loaded(collection_name)
+                    if bm25_data is None:
                         continue
-                    
-                    # Get BM25 scores for vector results
-                    docs_from_collection = [
-                        doc for doc in all_docs_dict.values()
-                        if doc.metadata.get("collection_name") == collection_name
-                    ]
-                    
-                    if not docs_from_collection:
-                        continue
-                    
-                    # Extract content strings for BM25 scoring
-                    content_for_bm25 = [doc.page_content for doc in docs_from_collection]
-                    
-                    # Build BM25 index for this subset and score
-                    if content_for_bm25:
-                        tokenized_subset = [content.lower().split() for content in content_for_bm25]
-                        bm25_subset = BM25Okapi(tokenized_subset, k1=1.5, b=0.5)
-                        bm25_results = bm25_subset.get_top_n(tokenized_query, content_for_bm25, n=len(content_for_bm25))
+
+                    bm25, corpus_docs = bm25_data
+
+                    scores = bm25.get_scores(tokenized_query)
+                    scored_docs = sorted(zip(corpus_docs, scores), key=lambda x: x[1], reverse=True)
+
+                    bm25_rank = 0
+                    for doc, score in scored_docs:
+                        if score <= 0:  
+                            break
+                            
+                        doc_key = self._doc_key(doc)
                         
-                        bm25_rank = 0
-                        for content in bm25_results:  # bm25_results contains strings
-                            # Find matching doc by content (handles duplicates)
-                            matched_doc = None
-                            for doc in docs_from_collection:
-                                if doc.page_content == content:
-                                    matched_doc = doc
-                                    break
+                        if doc_key not in all_docs_dict:
+                            all_docs_dict[doc_key] = doc
                             
-                            if matched_doc:
-                                doc_key = self._doc_key(matched_doc)
-                                if doc_key not in bm25_ranked:
-                                    bm25_rank += 1
-                                    bm25_ranked[doc_key] = bm25_rank
+                        if doc_key not in bm25_ranked:
+                            bm25_rank += 1
+                            bm25_ranked[doc_key] = bm25_rank
                             
-            except Exception:
-                logger.exception("BM25 search failed, falling back to vector-only")
+                        if bm25_rank >= limit:
+                            break
+                            
+        except Exception:
+            logger.exception("BM25 search failed, falling back to vector-only")
 
-        # Step 3: RRF combination (Reciprocal Rank Fusion)
+        # Step 3: RRF combination
         alpha = max(0.0, min(1.0, float(alpha)))
         bm25_weight = 1.0 - alpha
         vector_weight = alpha
@@ -234,18 +261,15 @@ class CollectionRouterRetriever:
         for doc_key, doc in all_docs_dict.items():
             score = 0.0
             
-            # Vector score
             if doc_key in vector_ranked:
                 score += vector_weight / (rrf_c + vector_ranked[doc_key])
             
-            # BM25 score
             if doc_key in bm25_ranked:
                 score += bm25_weight / (rrf_c + bm25_ranked[doc_key])
             
             if score > 0:
                 rrf_scores[doc_key] = score
         
-        # Sort by RRF score
         sorted_results = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True)
         return [all_docs_dict[doc_key] for doc_key, _ in sorted_results[:limit]]
 
@@ -267,54 +291,19 @@ class CollectionRouterRetriever:
             alpha=alpha,
         )
         
-        # Log warning if no documents found
         if not routed_docs:
             logger.warning("No documents found for query=%s, cohort=%s", query[:50], cohort_key)
-
-        if cohort_scoped:
-            deduplicated = []
-            seen = set()
-            for doc in routed_docs:
-                key = self._doc_key(doc)
-                if key in seen:
-                    continue
-                seen.add(key)
-                deduplicated.append(doc)
-                if len(deduplicated) >= candidate_k:
-                    break
-            return deduplicated[:k]
-
-        fallback_docs = []
-        if self.base_retriever is not None:
-            try:
-                fallback_docs = self.base_retriever.search(
-                    query,
-                    k=candidate_k,
-                    alpha=alpha,
-                    cohort_key=cohort_key,
-                )
-            except TypeError:
-                fallback_docs = self.base_retriever.search(
-                    query,
-                    k=candidate_k,
-                    alpha=alpha,
-                )
-            except Exception:
-                logger.exception("Base retriever fallback failed")
+            return []
 
         deduplicated = []
         seen = set()
-        
-        # Safe handling of fallback_docs which might be None
-        fallback_docs_list = list(fallback_docs) if fallback_docs else []
-
-        for doc in routed_docs + fallback_docs_list:
+        for doc in routed_docs:
             key = self._doc_key(doc)
             if key in seen:
                 continue
             seen.add(key)
             deduplicated.append(doc)
-            if len(deduplicated) >= candidate_k:
+            if len(deduplicated) >= k:
                 break
-
-        return deduplicated[:k]
+                
+        return deduplicated

{core → rag}/models.py

@@ -1,6 +1,6 @@
 from langchain_huggingface import HuggingFaceEmbeddings
 from sentence_transformers import CrossEncoder 
-from .config import EMBED_MODEL, CROSS_ENCODER_MODEL
+from core.config import EMBED_MODEL, CROSS_ENCODER_MODEL
 
 # Khởi tạo Embedding model - Chạy trên CPU của Hugging Face
 embeddings = HuggingFaceEmbeddings(

{core → rag}/prompting.py

@@ -1,4 +1,4 @@
-def create_advanced_prompt(question: str, context: str, question_type: str, topic: str = None, year_scope: str = None) -> str:
+def create_advanced_prompt(question: str, context: str, question_type: str, topic: str = None) -> str:
     # Base system - Định nghĩa tư duy cho AI
     base_system = """Bạn là Trợ lý AI chuyên gia về Pháp chế và Quy định Đại học. Nhiệm vụ của bạn là hỗ trợ tra cứu thông tin chính xác từ các văn bản quy phạm nội bộ (Quyết định, Thông tư, Quy định...).
 
@@ -97,25 +97,12 @@ Về vấn đề [Chủ đề], theo **Điều [Số]**, các trường hợp ng
     else:
         topic_instr = ""
 
-    # [YEAR-AWARE CHANGE] Rang buoc cau tra loi theo nam hoc duoc hoi.
-    if year_scope:
-        year_instr = (
-            f"\n\n **RÀNG BUỘC NĂM HỌC (BẮT BUỘC):**\n"
-            f"- Người dùng đang hỏi trong phạm vi năm: **{year_scope}**.\n"
-            f"- Ưu tiên các đoạn có nhãn nguồn cùng năm trong context (ví dụ: [Năm 2022-2023 | ...]).\n"
-            f"- Nếu chưa đủ bằng chứng đúng năm, được phép dùng đoạn có nhãn 'Áp dụng nhiều năm' hoặc quy định gần nhất và phải ghi chú rõ phạm vi áp dụng.\n"
-            f"- Không kết luận 'không có dữ liệu' chỉ vì thiếu đúng nhãn năm nếu vẫn có quy định bao quát liên quan.\n"
-        )
-    else:
-        year_instr = ""
-
     # 4. Gộp Prompt
     full_prompt = f"""{base_system}
 ----------------
 {example}
 ----------------
 {topic_instr}
-{year_instr}
 
 **TÀI LIỆU THAM KHẢO (CONTEXT):**
 {context}

{core → rag}/qa_pipeline.py

@@ -1,128 +1,24 @@
 from typing import List, Generator
 import os, re, hashlib
 import logging 
-import groq
 import google.generativeai as genai
 import json 
 from concurrent.futures import ThreadPoolExecutor
 from threading import Lock
-
-from .models import llm
-from .config import TOP_K_RESULTS, FINAL_TOP_K
+from core.ai_provider import api_manager
+from core.config import TOP_K_RESULTS, FINAL_TOP_K
 from .rerank import advanced_rerank
 from .prompting import create_advanced_prompt
-from .retriever import HybridRetriever
 from .analyze_and_expand import analyze_and_expand_query
-from .llm_utils import safe_invoke, safe_stream
 
 logger = logging.getLogger(__name__)
 
-# Giữ nguyên các hằng số
 MAX_CONTEXT_CHARS = 12000
 MAX_DOC_CHARS = 1800 
 MAX_OUT_CHARS = 3000
-# [YEAR-AWARE CHANGE] Pattern nhan dien nam hoc trong cau hoi.
-ACADEMIC_YEAR_PATTERN = re.compile(r"\b(20\d{2})\s*[-_/]\s*(20\d{2})\b")
-SINGLE_YEAR_PATTERN = re.compile(r"\b(20\d{2})\b")
 
 # Quản lý API Keys cho Groq và Gemini với xoay tua tự động khi gặp lỗi hoặc hết hạn
-class AIProviderManager:
-    def __init__(self):
-        # Lấy danh sách keys 
-        self.groq_keys = [k.strip() for k in os.getenv("GROQ_API_KEYS", "").split(",") if k.strip()]
-        self.gemini_keys = [k.strip() for k in os.getenv("GEMINI_API_KEYS", "").split(",") if k.strip()]
-        self.groq_idx = 0
-        self.gemini_idx = 0
-
-    def get_groq_client(self):
-        if not self.groq_keys: return None
-        return groq.Groq(api_key=self.groq_keys[self.groq_idx])
-
-    def rotate_groq(self):
-        if len(self.groq_keys) > 1:
-            self.groq_idx = (self.groq_idx + 1) % len(self.groq_keys)
-            logger.info(f" Đã xoay sang Groq Key thứ {self.groq_idx + 1}")
-
-    def get_gemini_key(self):
-        if not self.gemini_keys: return None
-        return self.gemini_keys[self.gemini_idx]
-
-    def rotate_gemini(self):
-        if len(self.gemini_keys) > 1:
-            self.gemini_idx = (self.gemini_idx + 1) % len(self.gemini_keys)
-            logger.info(f"Đã xoay sang Gemini Key dự phòng")
-
-api_manager = AIProviderManager()
-
-
-def normalize_academic_year(start_year: str, end_year: str) -> str:
-    return f"{int(start_year):04d}-{int(end_year):04d}"
-
-
-# [YEAR-AWARE CHANGE] Trich xuat nam yeu cau tu cau hoi.
-def detect_requested_year(text: str) -> tuple[str, set]:
-    """Phat hien nam hoc duoc nhac den trong cau hoi."""
-    requested_range = ""
-    mentioned_years = set()
-
-    for start_year, end_year in ACADEMIC_YEAR_PATTERN.findall(text or ""):
-        requested_range = normalize_academic_year(start_year, end_year)
-        mentioned_years.add(start_year)
-        mentioned_years.add(end_year)
-
-    for year in SINGLE_YEAR_PATTERN.findall(text or ""):
-        mentioned_years.add(year)
-
-    return requested_range, mentioned_years
-
-
-def infer_doc_academic_year(doc) -> str:
-    metadata = doc.metadata if isinstance(doc.metadata, dict) else {}
-    existing_year = metadata.get("academic_year")
-    if existing_year:
-        return existing_year
 
-    source_text = " ".join(
-        str(x) for x in [
-            metadata.get("source_relpath"),
-            metadata.get("source"),
-            metadata.get("source_file"),
-        ]
-        if x
-    )
-    match = ACADEMIC_YEAR_PATTERN.search(source_text)
-    if match:
-        year = normalize_academic_year(match.group(1), match.group(2))
-        metadata["academic_year"] = year
-        doc.metadata = metadata
-        return year
-
-    metadata["academic_year"] = "ALL"
-    doc.metadata = metadata
-    return "ALL"
-
-
-# [YEAR-AWARE CHANGE] Loc tai lieu theo metadata nam hoc.
-def filter_docs_by_year(docs: List, requested_range: str, mentioned_years: set) -> List:
-    if not requested_range and not mentioned_years:
-        return docs
-
-    filtered_docs = []
-    for doc in docs:
-        doc_year = infer_doc_academic_year(doc)
-        if doc_year == "ALL":
-            filtered_docs.append(doc)
-            continue
-
-        if requested_range and doc_year == requested_range:
-            filtered_docs.append(doc)
-            continue
-
-        doc_year_tokens = set(SINGLE_YEAR_PATTERN.findall(doc_year))
-        if doc_year_tokens.intersection(mentioned_years):
-            filtered_docs.append(doc)
-
-    return filtered_docs
 
 def sanitize_for_prompt(text: str) -> str:
     """Lọc bỏ prompt injection và PII """
@@ -261,7 +157,6 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever, cohort_ke
     if cohort_key:
         logger.info(f"Sử dụng cohort_key: {cohort_key}")
     
-    # Gửi song song các truy vấn đến Qdrant 
     def search_query(query: str):
         current_alpha = 0.4 if "CNTT" in query.upper() else 0.5
         return hybrid_retriever.search(
@@ -293,18 +188,21 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever, cohort_ke
     final_docs = advanced_rerank(question, all_docs, top_k=FINAL_TOP_K)
 
     context_parts = []
-    context_docs = []  # Lưu metadata để trích dẫn ở cuối
+    context_docs = []  
     total_chars = 0
+    
     for doc in final_docs:
         page = doc.metadata.get('page_number', 'N/A')
         file_name = doc.metadata.get('source_file') or doc.metadata.get('source')
         source = f"[{os.path.basename(file_name)} | Trang {page}]" if file_name else f"[Trang {page}]"
         block = f"{source}\n{doc.page_content}"
+        
+        #  Dùng continue để nhét tối đa các chunk ngắn thay vì break làm đứt gánh
         if total_chars + len(block) > MAX_CONTEXT_CHARS:
-            break
+            continue
+            
         total_chars += len(block)
         context_parts.append(block)
-        # Lưu metadata cho phần tài liệu tham khảo ở cuối
         context_docs.append({
             'source': file_name or "Không rõ",
             'page': page
@@ -318,7 +216,6 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever, cohort_ke
     logger.info("Đang tạo câu trả lời cuối cùng ...")
     
     success = False
-    # Ưu tiên Groq (tiết kiệm token)
     for _ in range(len(api_manager.groq_keys) if api_manager.groq_keys else 1):
         try:
             client = api_manager.get_groq_client()
@@ -336,13 +233,12 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever, cohort_ke
             success = True
             break
         except Exception as e:
-            if "429" in str(e):  # Rate Limit
+            if "429" in str(e): 
                 api_manager.rotate_groq()
                 continue
             logger.error(f"Lỗi Groq: {e}")
             break
     
-    # Fallback sang Gemini nếu Groq lỗi
     if not success:
         logger.warning("Chuyển sang Gemini ...")
         for _ in range(len(api_manager.gemini_keys) if api_manager.gemini_keys else 1):
@@ -363,7 +259,6 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever, cohort_ke
         yield "Đã xảy ra lỗi hệ thống hoặc quá tải. Vui lòng thử lại sau giây lát!"
         return
     
-    # Thêm phần Tài liệu tham khảo ở cuối
     if context_docs:
         yield "\n\n---\n\n"
         yield "## 📚 Tài liệu tham khảo\n\n"

{core → rag}/vectorstore.py

@@ -1,7 +1,6 @@
 import asyncio
 import logging
 import os
-import re
 from typing import Any, Dict, List
 
 import pdfplumber
@@ -13,43 +12,60 @@ from docx.table import Table, _Cell
 from docx.text.paragraph import Paragraph
 from langchain_core.documents import Document as LangChainDocument
 
-from .text_utils import clean_text
+from utils.text_utils import clean_text
 
 logger = logging.getLogger(__name__)
 
-ACADEMIC_YEAR_PATTERN = re.compile(r"(20\d{2})\s*[-_]\s*(20\d{2})")
-
-
-def normalize_academic_year(start_year: str, end_year: str) -> str:
-    return f"{int(start_year):04d}-{int(end_year):04d}"
-
-
-def extract_academic_year(text: str) -> str:
-    if not text:
-        return ""
-    match = ACADEMIC_YEAR_PATTERN.search(text)
-    if not match:
-        return ""
-    return normalize_academic_year(match.group(1), match.group(2))
-
-
-def table_to_markdown(data: List[List[str]]) -> str:
+def table_to_unrolled_text(data: List[List[str]], is_docx: bool = False) -> str:
+    
     if not data or len(data) < 2:
         return ""
 
-    header = [str(cell).replace("\n", " ").strip() if cell else "" for cell in data[0]]
-    separator = ["---"] * len(header)
-    markdown_lines = [
-        "| " + " | ".join(header) + " |",
-        "| " + " | ".join(separator) + " |",
-    ]
-
-    for row in data[1:]:
-        clean_row = [str(cell).replace("\n", "<br>").strip() if cell else "" for cell in row]
-        markdown_lines.append("| " + " | ".join(clean_row) + " |")
-
-    return "\n".join(markdown_lines) + "\n\n"
-
+    # Làm sạch dữ liệu ban đầu chuyển None thành chuỗi rỗng
+    cleaned_data = []
+    for row in data:
+        cleaned_row = [str(cell).strip() if cell else "" for cell in row]
+        cleaned_data.append(cleaned_row)
+
+    num_cols = len(cleaned_data[0])
+    header_row = cleaned_data[0]
+
+    # CHỈ CHẠY FORWARD FILL NẾU KHÔNG PHẢI FILE WORD
+    if not is_docx:
+        # 2. Kỹ thuật Forward-Fill cho khu vực Header (Xử lý gộp cột - Colspan)
+        # Giả định hàng đầu tiên chắc chắn là Header
+        for i in range(1, num_cols):
+            if not header_row[i] and header_row[i-1]:
+                header_row[i] = header_row[i-1] # Kéo giá trị từ trái sang phải
+
+        # 3. Kỹ thuật Forward-Fill cho khu vực Dữ liệu (Xử lý gộp hàng - Rowspan)
+        for r in range(1, len(cleaned_data)):
+            for c in range(num_cols):
+                # Nếu ô hiện tại rỗng, kéo giá trị từ ô ngay bên trên xuống
+                if not cleaned_data[r][c] and cleaned_data[r-1][c]:
+                    cleaned_data[r][c] = cleaned_data[r-1][c]
+
+    # 4. Trải phẳng bảng (Unrolling)
+    headers = cleaned_data[0]
+    unrolled_rows = []
+
+    for r in range(1, len(cleaned_data)):
+        row_values = cleaned_data[r]
+        row_text_parts = []
+        
+        # Chỉ ghép những ô có dữ liệu thực sự (khác Header)
+        for c in range(min(len(headers), len(row_values))):
+            header_val = headers[c]
+            cell_val = row_values[c]
+            
+            # Tránh lặp lại nếu dữ liệu vô tình giống hệt Header
+            if cell_val and cell_val != header_val:
+                row_text_parts.append(f"{header_val}: {cell_val}")
+                
+        if row_text_parts:
+            unrolled_rows.append("- " + " | ".join(row_text_parts))
+
+    return "\n" + "\n".join(unrolled_rows) + "\n\n"
 
 def read_pdf_with_tables(filepath: str) -> List[LangChainDocument]:
     docs: List[LangChainDocument] = []
@@ -61,9 +77,10 @@ def read_pdf_with_tables(filepath: str) -> List[LangChainDocument]:
                 table_texts: List[str] = []
                 if tables:
                     for table in tables:
-                        md_table = table_to_markdown(table)
-                        if md_table:
-                            table_texts.append(md_table)
+                        # Vẫn chạy Forward-Fill bình thường cho PDF
+                        unrolled_table = table_to_unrolled_text(table, is_docx=False)
+                        if unrolled_table:
+                            table_texts.append(unrolled_table)
 
                 full_content = text + "\n\n[BANG DU LIEU TRICH XUAT]:\n" + "\n".join(table_texts)
                 if full_content.strip():
@@ -74,11 +91,10 @@ def read_pdf_with_tables(filepath: str) -> List[LangChainDocument]:
                         )
                     )
     except Exception as error:
-        logger.error("Loi doc PDF (pdfplumber) %s: %s", os.path.basename(filepath), error)
+        logger.error("Lỗi đọc PDF  %s: %s", os.path.basename(filepath), error)
 
     return docs
 
-
 def iter_block_items(parent):
     if isinstance(parent, _Document):
         parent_elm = parent.element.body
@@ -92,7 +108,6 @@ def iter_block_items(parent):
         elif isinstance(child, CT_Tbl):
             yield Table(child, parent)
 
-
 def read_docx_with_tables(filepath: str) -> str:
     doc = Document(filepath)
     full_text: List[str] = []
@@ -108,13 +123,13 @@ def read_docx_with_tables(filepath: str) -> str:
                     row_data.append(clean_text(cell.text))
                 table_data.append(row_data)
 
-            md_table = table_to_markdown(table_data)
-            if md_table:
-                full_text.append(f"\n{md_table}\n")
+            # CẮT FORWARD-FILL TẠI ĐÂY BẰNG is_docx=True
+            unrolled_table = table_to_unrolled_text(table_data, is_docx=True)
+            if unrolled_table:
+                full_text.append(f"\n{unrolled_table}\n")
 
     return "\n".join(full_text)
 
-
 def load_documents_from_file(filepath: str, filename: str) -> List[LangChainDocument]:
     docs: List[LangChainDocument] = []
     lower_name = filename.lower()
@@ -140,12 +155,10 @@ def load_documents_from_file(filepath: str, filename: str) -> List[LangChainDocu
         logger.error("Loi doc %s: %s", filename, str(error)[:120])
         return []
 
-
 async def build_vectorstore_improved(
     sync_coordinator: Any,
     startup_wait_seconds: int = 5,
 ) -> Dict[str, Any]:
-    """Supabase build step: trigger one initial sync and optionally wait for completion."""
     if sync_coordinator is None:
         raise ValueError("sync_coordinator is required")
 
@@ -180,12 +193,9 @@ async def build_vectorstore_improved(
             "timed_out": True,
         }
 
-
 def load_vectorstore_improved(sync_coordinator: Any) -> Dict[str, Any]:
-    """Supabase load step: return current coordinator health snapshot."""
     if sync_coordinator is None:
         return {}
-
     try:
         state = sync_coordinator.get_health_snapshot()
         return state if isinstance(state, dict) else {}

requirements.txt

@@ -30,4 +30,5 @@ langchain-huggingface>=0.0.3,<0.1.0
 #File Loaders
 python-docx
 pdfplumber
-pypdf
+pypdf
+pyvi 

{core → services}/document_ingest_service.py

@@ -3,7 +3,6 @@ import os
 import uuid
 from datetime import datetime, timezone
 from typing import List, Optional
-
 from langchain_core.documents import Document as LangChainDocument
 from qdrant_client import QdrantClient
 from qdrant_client.http.exceptions import UnexpectedResponse
@@ -17,12 +16,12 @@ from qdrant_client.models import (
     VectorParams,
 )
 
-from .chunking import smart_chunking
-from .config import QDRANT_API_KEY, QDRANT_COLLECTION, QDRANT_URL
-from .document_db import Document, DocumentChunk, SessionLocal
-from .models import embeddings
-from .text_utils import clean_text
-from .vectorstore import extract_academic_year, load_documents_from_file
+from rag.chunking import smart_chunking
+from core.config import QDRANT_API_KEY, QDRANT_COLLECTION, QDRANT_URL
+from database.document_db import Document, DocumentChunk, SessionLocal
+from rag.models import embeddings
+from utils.text_utils import clean_text
+from rag.vectorstore import load_documents_from_file
 
 logger = logging.getLogger(__name__)
 
@@ -74,11 +73,9 @@ def chunk_documents_for_ingest(
     if not cleaned_docs:
         return []
 
-    academic_year = extract_academic_year(source_relpath) or "ALL"
     for doc in cleaned_docs:
         metadata = doc.metadata.copy() if isinstance(doc.metadata, dict) else {}
         metadata["source_relpath"] = source_relpath
-        metadata["academic_year"] = academic_year
         doc.metadata = metadata
 
     return [doc for doc in smart_chunking(cleaned_docs) if (doc.page_content or "").strip()]
@@ -252,7 +249,6 @@ def process_document_ingest(
                 "collection_name": target_collection,
                 "source_file": metadata.get("source_file") or source_name,
                 "source_relpath": metadata.get("source_relpath") or source_relpath,
-                "academic_year": metadata.get("academic_year") or "ALL",
                 "page_number": metadata.get("page_number"),
                 "source_updated_at": source_updated_at,
                 "source_etag": source_etag,
@@ -365,4 +361,4 @@ def delete_vectors_for_object_path(collection_name: str, object_path: str) -> bo
             wait=True,
         )
 
-    return True
+    return True

{core → services}/supabase_sync_service.py

@@ -2,15 +2,14 @@ import asyncio
 import json
 import logging
 import os
-import re
 import tempfile
 import time
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional
 from urllib import error, parse, request
 
-from .collection_utils import build_collection_name
-from .document_db import (
+from rag.collection_utils import build_collection_name
+from database.document_db import (
     Document,
     DocumentChunk,
     SessionLocal,
@@ -20,7 +19,7 @@ from .document_db import (
     mark_document_sync_error_resolved,
     utcnow,
 )
-from .document_ingest_service import delete_vectors_for_object_path, process_document_ingest
+from services.document_ingest_service import delete_vectors_for_object_path, process_document_ingest
 
 logger = logging.getLogger(__name__)
 

{core → utils}/text_utils.py

@@ -7,8 +7,10 @@ def clean_text(text: str) -> str:
     # Nối các từ bị gãy ngang do xuống dòng 
     text = re.sub(r'(\w+)-\s*\n\s*(\w+)', r'\1\2', text)
     
-    # \| và < > vào để bảo vệ khung Bảng Markdown và các Placeholder
-    text = re.sub(r'[^\w\s\.,;:!?\-$$\"\'\À-ỹ\n\|<>]', ' ', text)
+    # Loại bỏ các ký tự điều khiển không mong muốn, nhưng giữ lại các dấu câu thông thường
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text)
+    # Xóa các ký tự không nhìn thấy và các ký tự đặc biệt như zero-width space và BOM
+    text = text.replace('\u200b', '').replace('\ufeff', '')
     
     # Chuẩn hóa khoảng trắng 
     text = re.sub(r'[ \t]+', ' ', text)