fix metadata

app/law_document_chunker.py

@@ -263,20 +263,15 @@ class LawDocumentChunker:
 
     def _process_document_recursive(self, content: str, vanbanid: int, 
                                   document_title: str) -> List[ChunkMetadata]:
-        """Xử lý văn bản theo cấu trúc phân cấp."""##
+        """Xử lý văn bản theo cấu trúc phân cấp."""
         lines = content.split('\n')
         chunks = []
-        
-        # Stack để theo dõi các chunks theo thứ tự xuất hiện
-        # Mỗi item là (chunk_id, level, level_value, content)
-        chunk_stack = []
-        
+        chunk_stack = []  # (chunk_id, level, level_value, content)
         current_chunk_content = ""
-        current_level = "CONTENT"
+        current_level = None
         current_level_value = None
         current_parent = None
-        
-        # Định nghĩa thứ tự ưu tiên của các level (số càng nhỏ càng cao)
+        current_level_priority = None
         level_priority = {
             "PHAN": 1,
             "PHU_LUC": 1,
@@ -287,54 +282,91 @@ class LawDocumentChunker:
             "DIEM": 6,
             "CONTENT": 7
         }
-        
+        preamble_done = False
         for line in lines:
-            level, level_value, level_content = self._detect_structure_level(line)
-            
-            # Debug logging
-            if level != "CONTENT" and level_value:
-                logger.debug(f"[CHUNKER] Line: '{line.strip()}' -> Level: {level}, Value: {level_value}")
-            
-            # Nếu phát hiện cấp độ mới
-            if level != "CONTENT" and level_value:
-                # Lưu chunk hiện tại nếu có
+            level, level_value, _ = self._detect_structure_level(line)
+            line_priority = level_priority.get(level, 7)
+            # Nếu là dòng đầu tiên hoặc preamble
+            if not preamble_done and (level == "CONTENT" or not level_value):
+                current_chunk_content += line + "\n"
+                current_level = "CONTENT"
+                current_level_value = None
+                current_parent = None
+                current_level_priority = 7
+                continue
+            if not preamble_done and (level != "CONTENT" and level_value):
+                # Kết thúc preamble
                 if current_chunk_content.strip():
                     metadata = self._create_chunk_metadata(
                         current_chunk_content.strip(),
-                        current_level,
-                        current_level_value,
-                        current_parent,
+                        "CONTENT",
+                        None,
+                        None,
                         vanbanid,
                         document_title,
                         chunk_stack
                     )
                     chunks.append(metadata)
-                    
-                    # Thêm vào stack NGAY LẬP TỨC
-                    chunk_stack.append((metadata.id, current_level, current_level_value, current_chunk_content.strip()))
-                    logger.debug(f"[CHUNKER] Created chunk: {metadata.id[:8]}... Level: {current_level}, Parent: {current_parent}")
-                
-                # Tìm parent cho level mới TRƯỚC KHI tạo chunk mới
-                current_parent = self._find_parent_for_level(chunk_stack, level, level_priority)
-                logger.debug(f"[CHUNKER] Found parent for {level}: {current_parent}")
-                
-                # Bắt đầu chunk mới
-                current_chunk_content = line + "\n"
+                    chunk_stack.append((metadata.id, "CONTENT", None, current_chunk_content.strip()))
+                preamble_done = True
+                current_chunk_content = ""
                 current_level = level
                 current_level_value = level_value
+                current_level_priority = line_priority
+                current_parent = self._find_parent_for_level(chunk_stack, level, level_priority)
+                current_chunk_content += line + "\n"
+                continue
+            # Nếu gặp level mới
+            if level != "CONTENT" and level_value:
+                if current_level is not None and current_level_priority is not None and line_priority <= current_level_priority:
+                    # Kết thúc chunk hiện tại
+                    if current_chunk_content.strip():
+                        metadata = self._create_chunk_metadata(
+                            current_chunk_content.strip(),
+                            str(current_level),
+                            current_level_value,
+                            current_parent,
+                            vanbanid,
+                            document_title,
+                            chunk_stack
+                        )
+                        chunks.append(metadata)
+                        chunk_stack.append((metadata.id, str(current_level), current_level_value, current_chunk_content.strip()))
+                    # Bắt đầu chunk mới
+                    current_parent = self._find_parent_for_level(chunk_stack, level, level_priority)
+                    current_chunk_content = line + "\n"
+                    current_level = level
+                    current_level_value = level_value
+                    current_level_priority = line_priority
+                else:
+                    # Level mới nhưng priority cao hơn (ví dụ: Mục trong Chương)
+                    if current_chunk_content.strip() and current_level is not None:
+                        metadata = self._create_chunk_metadata(
+                            current_chunk_content.strip(),
+                            str(current_level),
+                            current_level_value,
+                            current_parent,
+                            vanbanid,
+                            document_title,
+                            chunk_stack
+                        )
+                        chunks.append(metadata)
+                        chunk_stack.append((metadata.id, str(current_level), current_level_value, current_chunk_content.strip()))
+                    current_parent = self._find_parent_for_level(chunk_stack, level, level_priority)
+                    current_chunk_content = line + "\n"
+                    current_level = level
+                    current_level_value = level_value
+                    current_level_priority = line_priority
             else:
-                # Thêm vào chunk hiện tại
+                # CONTENT nối vào chunk hiện tại
                 current_chunk_content += line + "\n"
-                
-                # Kiểm tra nếu chunk quá lớn
-                if len(current_chunk_content) > self.CHUNK_SIZE:
-                    # Chia chunk hiện tại
+                # Nếu chunk quá lớn thì chia nhỏ
+                if len(current_chunk_content) > self.CHUNK_SIZE and current_level is not None:
                     sub_chunks = self._split_into_chunks(current_chunk_content, self.CHUNK_SIZE, self.CHUNK_OVERLAP)
-                    
-                    for i, sub_chunk in enumerate(sub_chunks):
+                    for sub_chunk in sub_chunks:
                         metadata = self._create_chunk_metadata(
                             sub_chunk.strip(),
-                            current_level,
+                            str(current_level),
                             current_level_value,
                             current_parent,
                             vanbanid,
@@ -342,18 +374,13 @@ class LawDocumentChunker:
                             chunk_stack
                         )
                         chunks.append(metadata)
-                        
-                        # Thêm vào stack NGAY LẬP TỨC
-                        chunk_stack.append((metadata.id, current_level, current_level_value, sub_chunk.strip()))
-                        logger.debug(f"[CHUNKER] Created sub-chunk: {metadata.id[:8]}... Level: {current_level}, Parent: {current_parent}")
-                    
+                        chunk_stack.append((metadata.id, str(current_level), current_level_value, sub_chunk.strip()))
                     current_chunk_content = ""
-        
         # Lưu chunk cuối cùng
-        if current_chunk_content.strip():
+        if current_chunk_content.strip() and current_level is not None:
             metadata = self._create_chunk_metadata(
                 current_chunk_content.strip(),
-                current_level,
+                str(current_level),
                 current_level_value,
                 current_parent,
                 vanbanid,
@@ -361,22 +388,13 @@ class LawDocumentChunker:
                 chunk_stack
             )
             chunks.append(metadata)
-            
-            # Thêm vào stack NGAY LẬP TỨC
-            chunk_stack.append((metadata.id, current_level, current_level_value, current_chunk_content.strip()))
-            logger.debug(f"[CHUNKER] Created final chunk: {metadata.id[:8]}... Level: {current_level}, Parent: {current_parent}")
-        
-        # Debug: Kiểm tra kết quả
+            chunk_stack.append((metadata.id, str(current_level), current_level_value, current_chunk_content.strip()))
         root_count = sum(1 for chunk in chunks if chunk.cha is None)
         logger.info(f"[CHUNKER] Created {len(chunks)} chunks, {root_count} root chunks")
-        
-        # Debug: Log chi tiết từng chunk
-        for i, chunk in enumerate(chunks[:10]):  # Log 10 chunks đầu tiên
+        for i, chunk in enumerate(chunks[:10]):
             logger.debug(f"[CHUNKER] Chunk {i+1}: {chunk.content[:100]}... -> Parent: {chunk.cha}")
-        
         if len(chunks) > 10:
             logger.debug(f"[CHUNKER] ... and {len(chunks) - 10} more chunks")
-        
         return chunks
     
     def _find_parent_for_level(self, chunk_stack: List[Tuple[str, str, Optional[str], str]], 
@@ -385,20 +403,9 @@ class LawDocumentChunker:
         Tìm parent gần nhất có level cao hơn (priority thấp hơn) cho level hiện tại.
         """
         current_priority = level_priority.get(current_level, 999)
-        
-        # Tìm từ cuối stack (gần nhất) đến đầu stack
         for chunk_id, level, level_value, content in reversed(chunk_stack):
-            # Kiểm tra cả priority và quan hệ hợp lệ
             if level_priority.get(level, 999) < current_priority:
-                # Kiểm tra quan hệ hợp lệ: Điều -> Khoản -> Điểm
-                if current_level == "KHOAN" and level == "DIEU":
-                    return chunk_id
-                elif current_level == "DIEM" and level == "KHOAN":
-                    return chunk_id
-                elif current_level == "CONTENT":
-                    # Content có thể có parent là bất kỳ level nào
-                    return chunk_id
-        
+                return chunk_id
         return None
 
     async def _create_embeddings_for_chunks(self, chunks: List[ChunkMetadata]) -> int: