fix metadata

app/law_document_chunker.py

@@ -133,10 +133,9 @@ class LawDocumentChunker:
 
     def _create_chunk_metadata(self, content: str, level: str, level_value: Optional[str], 
                               parent_id: Optional[str], vanbanid: int, 
-                              document_title: str, chunk_stack: List[Tuple[str, str, Optional[str], str]] = []) -> ChunkMetadata:
+                              document_title: str, chunk_stack: List[Tuple[str, str, Optional[str], str]], chunk_dict: dict) -> 'ChunkMetadata':
         """Tạo metadata cho chunk."""
         chunk_id = str(uuid.uuid4())
-        
         metadata = ChunkMetadata(
             id=chunk_id,
             content=content,
@@ -144,7 +143,6 @@ class LawDocumentChunker:
             cha=parent_id,
             document_title=document_title
         )
-        
         # Điền metadata từ chunk hiện tại
         if level == "DIEU" and level_value:
             metadata.article_number = int(level_value) if level_value.isdigit() else None
@@ -153,85 +151,39 @@ class LawDocumentChunker:
             metadata.clause_number = level_value
         elif level == "DIEM" and level_value:
             metadata.sub_clause_letter = level_value
-        
         # Điền metadata từ parent chunks nếu có
         logger.debug(f"[CHUNKER] Creating chunk with level: {level}, parent_id: {parent_id}, stack_size: {len(chunk_stack)}")
-        if chunk_stack and parent_id:
-            self._fill_metadata_from_parents(metadata, chunk_stack, parent_id)
+        if chunk_dict is not None and parent_id:
+            self._fill_metadata_from_parents(metadata, parent_id, chunk_dict)
         else:
-            logger.debug(f"[CHUNKER] Skipping metadata fill - no parent_id or empty stack")
-        
-        # Debug final metadata
+            logger.debug(f"[CHUNKER] Skipping metadata fill - no parent_id or chunk_dict")
         logger.debug(f"[CHUNKER] Final metadata for chunk {chunk_id[:8]}... - Level: {level}, Article: {metadata.article_number}, Clause: {metadata.clause_number}, Point: {metadata.sub_clause_letter}")
-        
         return metadata
 
-    def _fill_metadata_from_parents(self, metadata: ChunkMetadata, chunk_stack: List[Tuple[str, str, Optional[str], str]], parent_id: str):
+    def _fill_metadata_from_parents(self, metadata: ChunkMetadata, parent_id: str, chunk_dict: Dict[str, ChunkMetadata]):
         """
-        Điền metadata từ parent chunks (Điều, Khoản) nếu chunk hiện tại có cha hoặc ông là Điều/Khoản.
+        Điền metadata từ parent và ancestor (cha, ông, ...), sử dụng dict id->chunk.
         """
-        if not parent_id:
-            return
-            
-        # Debug logging
-        logger.debug(f"[CHUNKER] Filling metadata for chunk with parent_id: {parent_id}")
-        logger.debug(f"[CHUNKER] Chunk stack has {len(chunk_stack)} items")
-        
-        # Tìm parent chunk trong stack
-        parent_chunk = None
-        for chunk_id, level, level_value, content in chunk_stack:
-            if chunk_id == parent_id:
-                parent_chunk = (level, level_value, content)
-                break
-        
-        if not parent_chunk:
-            logger.warning(f"[CHUNKER] Parent chunk {parent_id} not found in stack")
+        parent = chunk_dict.get(parent_id)
+        if not parent:
+            logger.warning(f"[CHUNKER] Parent chunk {parent_id} not found in chunk_dict")
             return
-        
-        parent_level, parent_value, parent_content = parent_chunk
-        
-        # Điền metadata từ parent trực tiếp
-        if parent_level == "DIEU" and parent_value:
-            if not metadata.article_number:
-                metadata.article_number = int(parent_value) if parent_value.isdigit() else None
-                logger.debug(f"[CHUNKER] Set article_number from parent: {metadata.article_number}")
-            if not metadata.article_title:
-                first_line = parent_content.split('\n')[0].strip() if parent_content else ""
-                metadata.article_title = first_line
-                logger.debug(f"[CHUNKER] Set article_title from parent: {metadata.article_title}")
-        
-        elif parent_level == "KHOAN" and parent_value:
-            if not metadata.clause_number:
-                metadata.clause_number = parent_value
-                logger.debug(f"[CHUNKER] Set clause_number from parent: {metadata.clause_number}")
-        
-        # Tìm grandparent (ông) nếu cần
-        # Tìm parent của parent trong stack
-        grandparent_id = None
-        for chunk_id, level, level_value, content in chunk_stack:
-            if chunk_id == parent_id:
-                # Tìm parent của chunk này
-                for cid, lvl, lv, cont in reversed(chunk_stack):
-                    if cid == chunk_id:
-                        break
-                    if lvl == "DIEU" and parent_level == "KHOAN":
-                        grandparent_id = cid
-                        break
-                break
-        
-        if grandparent_id:
-            # Điền metadata từ grandparent
-            for chunk_id, level, level_value, content in chunk_stack:
-                if chunk_id == grandparent_id:
-                    if level == "DIEU" and level_value:
-                        if not metadata.article_number:
-                            metadata.article_number = int(level_value) if level_value.isdigit() else None
-                            logger.debug(f"[CHUNKER] Set article_number from grandparent: {metadata.article_number}")
-                        if not metadata.article_title:
-                            first_line = content.split('\n')[0].strip() if content else ""
-                            metadata.article_title = first_line
-                            logger.debug(f"[CHUNKER] Set article_title from grandparent: {metadata.article_title}")
-                    break
+        # Điền từ cha
+        if parent.article_number and not metadata.article_number:
+            metadata.article_number = parent.article_number
+        if parent.article_title and not metadata.article_title:
+            metadata.article_title = parent.article_title
+        if parent.clause_number and not metadata.clause_number:
+            metadata.clause_number = parent.clause_number
+        if parent.sub_clause_letter and not metadata.sub_clause_letter:
+            metadata.sub_clause_letter = parent.sub_clause_letter
+        # Nếu cha là Khoản, tìm ông là Điều
+        if parent.clause_number and not metadata.article_number:
+            grandparent = chunk_dict.get(parent.cha) if parent.cha else None
+            if grandparent and grandparent.article_number:
+                metadata.article_number = grandparent.article_number
+            if grandparent and grandparent.article_title:
+                metadata.article_title = grandparent.article_title
 
     def _split_into_chunks(self, text: str, chunk_size: int, overlap: int) -> List[str]:
         """Chia text thành các chunk với overlap."""
@@ -267,6 +219,7 @@ class LawDocumentChunker:
         lines = content.split('\n')
         chunks = []
         chunk_stack = []  # (chunk_id, level, level_value, content)
+        chunk_dict = {}   # id -> ChunkMetadata
         current_chunk_content = ""
         current_level = None
         current_level_value = None
@@ -304,10 +257,12 @@ class LawDocumentChunker:
                         None,
                         vanbanid,
                         document_title,
-                        chunk_stack
+                        chunk_stack,
+                        chunk_dict
                     )
                     chunks.append(metadata)
                     chunk_stack.append((metadata.id, "CONTENT", None, current_chunk_content.strip()))
+                    chunk_dict[metadata.id] = metadata
                 preamble_done = True
                 current_chunk_content = ""
                 current_level = level
@@ -328,10 +283,12 @@ class LawDocumentChunker:
                             current_parent,
                             vanbanid,
                             document_title,
-                            chunk_stack
+                            chunk_stack,
+                            chunk_dict
                         )
                         chunks.append(metadata)
                         chunk_stack.append((metadata.id, str(current_level), current_level_value, current_chunk_content.strip()))
+                        chunk_dict[metadata.id] = metadata
                     # Bắt đầu chunk mới
                     current_parent = self._find_parent_for_level(chunk_stack, level, level_priority)
                     current_chunk_content = line + "\n"
@@ -348,10 +305,12 @@ class LawDocumentChunker:
                             current_parent,
                             vanbanid,
                             document_title,
-                            chunk_stack
+                            chunk_stack,
+                            chunk_dict
                         )
                         chunks.append(metadata)
                         chunk_stack.append((metadata.id, str(current_level), current_level_value, current_chunk_content.strip()))
+                        chunk_dict[metadata.id] = metadata
                     current_parent = self._find_parent_for_level(chunk_stack, level, level_priority)
                     current_chunk_content = line + "\n"
                     current_level = level
@@ -371,10 +330,12 @@ class LawDocumentChunker:
                             current_parent,
                             vanbanid,
                             document_title,
-                            chunk_stack
+                            chunk_stack,
+                            chunk_dict
                         )
                         chunks.append(metadata)
                         chunk_stack.append((metadata.id, str(current_level), current_level_value, sub_chunk.strip()))
+                        chunk_dict[metadata.id] = metadata
                     current_chunk_content = ""
         # Lưu chunk cuối cùng
         if current_chunk_content.strip() and current_level is not None:
@@ -385,10 +346,12 @@ class LawDocumentChunker:
                 current_parent,
                 vanbanid,
                 document_title,
-                chunk_stack
+                chunk_stack,
+                chunk_dict
             )
             chunks.append(metadata)
             chunk_stack.append((metadata.id, str(current_level), current_level_value, current_chunk_content.strip()))
+            chunk_dict[metadata.id] = metadata
         root_count = sum(1 for chunk in chunks if chunk.cha is None)
         logger.info(f"[CHUNKER] Created {len(chunks)} chunks, {root_count} root chunks")
         for i, chunk in enumerate(chunks[:10]):
@@ -400,12 +363,22 @@ class LawDocumentChunker:
     def _find_parent_for_level(self, chunk_stack: List[Tuple[str, str, Optional[str], str]], 
                               current_level: str, level_priority: Dict[str, int]) -> Optional[str]:
         """
-        Tìm parent gần nhất có level cao hơn (priority thấp hơn) cho level hiện tại.
+        Tìm parent gần nhất có level cao hơn (priority thấp hơn) cho level hiện tại, kiểm tra hợp lệ cha-con.
         """
         current_priority = level_priority.get(current_level, 999)
+        valid_parents = {
+            "MUC": ["CHUONG", "PHAN"],
+            "DIEU": ["MUC", "CHUONG", "PHAN"],
+            "CHUONG": ["PHAN"],
+            # Các level khác giữ nguyên logic cũ
+        }
         for chunk_id, level, level_value, content in reversed(chunk_stack):
             if level_priority.get(level, 999) < current_priority:
-                return chunk_id
+                if current_level in valid_parents:
+                    if level in valid_parents[current_level]:
+                        return chunk_id
+                else:
+                    return chunk_id
         return None
 
     async def _create_embeddings_for_chunks(self, chunks: List[ChunkMetadata]) -> int: