update data

.gitattributes

@@ -33,21 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-data/1.[[:space:]]QĐ-1226-Quy[[:space:]]che[[:space:]]dao[[:space:]]tao[[:space:]]dai[[:space:]]hoc-DHTL[[:space:]](ban[[:space:]]hanh).docx filter=lfs diff=lfs merge=lfs -text
-data/10.[[:space:]]QĐ[[:space:]]1089[[:space:]]thi[[:space:]]OLP[[:space:]]môn[[:space:]]học[[:space:]](Final[[:space:]]10-5-2023).doc filter=lfs diff=lfs merge=lfs -text
-data/11.[[:space:]]QĐ[[:space:]]về[[:space:]]Học[[:space:]]phí[[:space:]]final[[:space:]](25-10-2021).doc filter=lfs diff=lfs merge=lfs -text
-data/3.[[:space:]]QD1767.TA[[:space:]]tăng[[:space:]]cường[[:space:]]ban[[:space:]]hanh.doc filter=lfs diff=lfs merge=lfs -text
-data/4.[[:space:]]QD411_QD_DHTL-Chuan_Dau_Ra_CNTT.pdf filter=lfs diff=lfs merge=lfs -text
-data/4.1.[[:space:]]QuyDinh_Ve_CDR_CNTT_Ban_hanh_theo_QD411-06-4-2022.pdf filter=lfs diff=lfs merge=lfs -text
-data/7.[[:space:]]QĐ[[:space:]]đánh[[:space:]]giá[[:space:]]KQRL[[:space:]](Final[[:space:]]18-8-2016).doc filter=lfs diff=lfs merge=lfs -text
-data/9.[[:space:]]QĐ[[:space:]]Khen[[:space:]]thưởng[[:space:]]-[[:space:]]KL[[:space:]](Final[[:space:]]10-8-2016).doc filter=lfs diff=lfs merge=lfs -text
-data/10.[[:space:]]QĐ[[:space:]]1089[[:space:]]thi[[:space:]]OLP[[:space:]]môn[[:space:]]học[[:space:]](Final[[:space:]]10-5-2023).pdf filter=lfs diff=lfs merge=lfs -text
-data/11.[[:space:]]QĐ[[:space:]]về[[:space:]]Học[[:space:]]phí[[:space:]]final[[:space:]](25-10-2021).pdf filter=lfs diff=lfs merge=lfs -text
-data/12.[[:space:]]QD[[:space:]]ngoại[[:space:]]trú.pdf filter=lfs diff=lfs merge=lfs -text
-data/2.[[:space:]]QĐ[[:space:]]về[[:space:]]tiếng[[:space:]]anh[[:space:]]CTTT.300921.QD.1315.pdf filter=lfs diff=lfs merge=lfs -text
-data/3.[[:space:]]QD1767.TA[[:space:]]tăng[[:space:]]cường[[:space:]]ban[[:space:]]hanh.pdf filter=lfs diff=lfs merge=lfs -text
-data/7.[[:space:]]QĐ[[:space:]]đánh[[:space:]]giá[[:space:]]KQRL[[:space:]](Final[[:space:]]18-8-2016).pdf filter=lfs diff=lfs merge=lfs -text
-data/9.[[:space:]]QĐ[[:space:]]Khen[[:space:]]thưởng[[:space:]]-[[:space:]]KL[[:space:]](Final[[:space:]]10-8-2016).pdf filter=lfs diff=lfs merge=lfs -text
-data/3784QD-DHTL__NCKH.pdf filter=lfs diff=lfs merge=lfs -text
-data/4079QD-DHTL_dd_Olympia.pdf filter=lfs diff=lfs merge=lfs -text
-data/8.[[:space:]]QĐ[[:space:]]ve[[:space:]]HBKKHT,[[:space:]]HBCS[[:space:]](final[[:space:]]12-5-2021).pdf filter=lfs diff=lfs merge=lfs -text
+
+# Track source regulation files in all year folders under data/
+data/**/*.pdf filter=lfs diff=lfs merge=lfs -text
+data/**/*.doc filter=lfs diff=lfs merge=lfs -text
+data/**/*.docx filter=lfs diff=lfs merge=lfs -text

core/prompting.py

@@ -96,6 +96,7 @@ Về vấn đề [Chủ đề], theo **Điều [Số]**, các trường hợp ng
     else:
         topic_instr = ""
 
+    # [YEAR-AWARE CHANGE] Rang buoc cau tra loi theo nam hoc duoc hoi.
     if year_scope:
         year_instr = (
             f"\n\n **RÀNG BUỘC NĂM HỌC (BẮT BUỘC):**\n"

core/qa_pipeline.py

@@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
 MAX_CONTEXT_CHARS = 12000
 MAX_DOC_CHARS = 1800 
 MAX_OUT_CHARS = 3000
+# [YEAR-AWARE CHANGE] Pattern nhan dien nam hoc trong cau hoi.
 ACADEMIC_YEAR_PATTERN = re.compile(r"\b(20\d{2})\s*[-_/]\s*(20\d{2})\b")
 SINGLE_YEAR_PATTERN = re.compile(r"\b(20\d{2})\b")
 
@@ -56,6 +57,7 @@ def normalize_academic_year(start_year: str, end_year: str) -> str:
     return f"{int(start_year):04d}-{int(end_year):04d}"
 
 
+# [YEAR-AWARE CHANGE] Trich xuat nam yeu cau tu cau hoi.
 def detect_requested_year(text: str) -> tuple[str, set]:
     """Phat hien nam hoc duoc nhac den trong cau hoi."""
     requested_range = ""
@@ -98,6 +100,7 @@ def infer_doc_academic_year(doc) -> str:
     return "ALL"
 
 
+# [YEAR-AWARE CHANGE] Loc tai lieu theo metadata nam hoc.
 def filter_docs_by_year(docs: List, requested_range: str, mentioned_years: set) -> List:
     if not requested_range and not mentioned_years:
         return docs
@@ -238,6 +241,7 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever) -> Genera
 
     logger.info(f" CÂU HỎI GỐC: {message}")
     question = generate_standalone_query(message, history)
+    # [YEAR-AWARE CHANGE] Xac dinh pham vi nam ma nguoi dung yeu cau.
     requested_year_range, mentioned_years = detect_requested_year(f"{message}\n{question}")
     if requested_year_range:
         logger.info(f"Lọc theo năm học yêu cầu: {requested_year_range}")
@@ -272,6 +276,7 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever) -> Genera
         yield "Không tìm thấy thông tin liên quan trong tài liệu."
         return
 
+    # [YEAR-AWARE CHANGE] Loc tap docs theo nam truoc khi rerank.
     year_filtered_docs = filter_docs_by_year(all_docs, requested_year_range, mentioned_years)
     if (requested_year_range or mentioned_years) and not year_filtered_docs:
         if requested_year_range:
@@ -292,6 +297,7 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever) -> Genera
     for doc in final_docs:
         page = doc.metadata.get('page_number', 'N/A')
         file_name = doc.metadata.get('source_file') or doc.metadata.get('source')
+        # [YEAR-AWARE CHANGE] Gan nhan nam trong context de LLM bam dung nguon.
         doc_year = infer_doc_academic_year(doc)
         year_label = f"Năm {doc_year}" if doc_year != "ALL" else "Áp dụng nhiều năm"
         source = f"[{year_label} | {os.path.basename(file_name)} | Trang {page}]" if file_name else f"[{year_label} | Trang {page}]"
@@ -303,6 +309,7 @@ def ask_ai_stream_delta(message: str, history: List, hybrid_retriever) -> Genera
     
     context = "\n\n---\n\n".join(context_parts)
     topic_hint = processed_data.get('topic') or processed_data.get('root_question') or question
+    # [YEAR-AWARE CHANGE] Truyen rang buoc nam vao prompt.
     if requested_year_range:
         year_scope = requested_year_range
     elif mentioned_years:

core/vectorstore.py

@@ -26,6 +26,7 @@ logger = logging.getLogger(__name__)
 
 CHUNKS_PICKLE = os.path.join(VECTOR_DIR, "chunks.pkl")
 COLLECTION_NAME = "quy_che_db"
+# [YEAR-AWARE CHANGE] Ho tro quet de quy va gan metadata nam hoc.
 SUPPORTED_FORMATS = ('.pdf', '.doc', '.docx')
 ACADEMIC_YEAR_PATTERN = re.compile(r"(20\d{2})\s*[-_]\s*(20\d{2})")
 
@@ -112,6 +113,7 @@ def enrich_chunk_metadata(chunks: List) -> bool:
     return changed
 
 
+# [YEAR-AWARE CHANGE] Gom doc tu toan bo thu muc data theo cau truc nam hoc.
 def load_and_clean_all_docs() -> List[LangChainDocument]:
     docs: List[LangChainDocument] = []
     file_entries = discover_data_files()
@@ -285,6 +287,7 @@ def load_documents_from_file(filepath: str, filename: str) -> List:
         logger.error(f" Lỗi đọc {filename}: {str(e)[:60]}")
         return []
 
+# [YEAR-AWARE CHANGE] Cho phep tao lai collection khi phat hien file moi.
 def build_vectorstore_improved(recreate_collection: bool = False) -> Tuple[QdrantVectorStore, List]:
     logger.info(" Đang xây dựng vectorstore...")
     docs = load_and_clean_all_docs()
@@ -362,6 +365,7 @@ def load_vectorstore_improved() -> Tuple[QdrantVectorStore, List]:
                 except Exception as e:
                     logger.error(f" Không thể cập nhật {CHUNKS_PICKLE}: {e}")
 
+            # [YEAR-AWARE CHANGE] Neu co file moi theo nam hoc, rebuild de dong bo Qdrant.
             discovered_relpaths = {os.path.normpath(relpath) for _, _, relpath, _ in discover_data_files()}
             chunk_relpaths = collect_chunk_relpaths(chunks)
             missing_relpaths = sorted(discovered_relpaths - chunk_relpaths)