add chunker

app/law_document_chunker.py

@@ -261,12 +261,12 @@ class LawDocumentChunker:
         logger.info(f"[CHUNKER] Created {len(chunks)} chunks from document")
         return chunks
 
-    async def _create_embeddings_for_chunks(self, chunks: List[ChunkMetadata]) -> List[Dict]:
-        """Tạo embeddings cho các chunks."""
-        logger.info(f"[CHUNKER] Creating embeddings for {len(chunks)} chunks")
+    async def _create_embeddings_for_chunks(self, chunks: List[ChunkMetadata]) -> int:
+        """Tạo embeddings cho các chunks và lưu ngay lập tức vào Supabase."""
+        logger.info(f"[CHUNKER] Creating embeddings and storing {len(chunks)} chunks")
         
-        chunk_data = []
-        for chunk in chunks:
+        success_count = 0
+        for i, chunk in enumerate(chunks, 1):
             try:
                 # Tạo embedding
                 embedding = await self.embedding_client.create_embedding(chunk.content)
@@ -285,35 +285,26 @@ class LawDocumentChunker:
                     'sub_clause_letter': chunk.sub_clause_letter,
                     'context_summary': chunk.context_summary
                 }
-                chunk_data.append(chunk_dict)
                 
-                logger.debug(f"[CHUNKER] Created embedding for chunk {chunk.id[:8]}...")
+                # Lưu ngay lập tức vào Supabase
+                success = self.supabase_client.store_document_chunk(chunk_dict)
+                if success:
+                    success_count += 1
+                    logger.info(f"[CHUNKER] Stored chunk {i}/{len(chunks)}: {chunk.id[:8]}...")
+                else:
+                    logger.error(f"[CHUNKER] Failed to store chunk {chunk.id}")
                 
             except Exception as e:
-                logger.error(f"[CHUNKER] Error creating embedding for chunk {chunk.id}: {e}")
+                logger.error(f"[CHUNKER] Error processing chunk {chunk.id}: {e}")
                 continue
         
-        logger.info(f"[CHUNKER] Successfully created embeddings for {len(chunk_data)} chunks")
-        return chunk_data
+        logger.info(f"[CHUNKER] Successfully processed {success_count}/{len(chunks)} chunks")
+        return success_count
 
     async def _store_chunks_to_supabase(self, chunk_data: List[Dict]) -> bool:
-        """Lưu chunks vào Supabase."""
-        try:
-            logger.info(f"[CHUNKER] Storing {len(chunk_data)} chunks to Supabase")
-            
-            # Lưu từng chunk
-            for chunk in chunk_data:
-                success = self.supabase_client.store_document_chunk(chunk)
-                if not success:
-                    logger.error(f"[CHUNKER] Failed to store chunk {chunk['id']}")
-                    return False
-            
-            logger.info(f"[CHUNKER] Successfully stored all chunks to Supabase")
-            return True
-            
-        except Exception as e:
-            logger.error(f"[CHUNKER] Error storing chunks to Supabase: {e}")
-            return False
+        """Legacy method - không còn sử dụng."""
+        logger.warning("[CHUNKER] _store_chunks_to_supabase is deprecated, use _create_embeddings_for_chunks instead")
+        return True
 
     async def process_law_document(self, file_path: str, document_id: int) -> bool:
         """
@@ -351,21 +342,14 @@ class LawDocumentChunker:
                 return False
             
             # 6. Tạo embeddings
-            chunk_data = await self._create_embeddings_for_chunks(chunks)
+            success_count = await self._create_embeddings_for_chunks(chunks)
             
-            if not chunk_data:
+            if success_count == 0:
                 logger.error(f"[CHUNKER] No embeddings created for document {document_id}")
                 return False
             
-            # 7. Lưu vào Supabase
-            success = await self._store_chunks_to_supabase(chunk_data)
-            
-            if success:
-                logger.info(f"[CHUNKER] Successfully processed document {document_id} with {len(chunk_data)} chunks")
-            else:
-                logger.error(f"[CHUNKER] Failed to store chunks for document {document_id}")
-            
-            return success
+            logger.info(f"[CHUNKER] Successfully processed document {document_id} with {success_count} chunks")
+            return True
             
         except Exception as e:
             logger.error(f"[CHUNKER] Error processing document {document_id}: {e}")

app/supabase_db.py

@@ -86,13 +86,45 @@ class SupabaseClient:
         Output: bool (True nếu thành công, False nếu lỗi)
         """
         try:
-            response = self.client.table('document_chunks').insert(chunk_data).execute()
+            # Xử lý các giá trị null/empty cho integer fields
+            processed_data = chunk_data.copy()
+            
+            # Xử lý article_number - chỉ gửi nếu có giá trị hợp lệ
+            if 'article_number' in processed_data:
+                if processed_data['article_number'] is None or processed_data['article_number'] == "":
+                    processed_data['article_number'] = None
+                elif isinstance(processed_data['article_number'], str):
+                    try:
+                        processed_data['article_number'] = int(processed_data['article_number'])
+                    except (ValueError, TypeError):
+                        processed_data['article_number'] = None
+            
+            # Xử lý vanbanid - đảm bảo là integer
+            if 'vanbanid' in processed_data:
+                if isinstance(processed_data['vanbanid'], str):
+                    try:
+                        processed_data['vanbanid'] = int(processed_data['vanbanid'])
+                    except (ValueError, TypeError):
+                        logger.error(f"Invalid vanbanid: {processed_data['vanbanid']}")
+                        return False
+            
+            # Xử lý các trường text - chuyển empty string thành None
+            text_fields = ['document_title', 'article_title', 'clause_number', 'sub_clause_letter', 'context_summary']
+            for field in text_fields:
+                if field in processed_data and processed_data[field] == "":
+                    processed_data[field] = None
+            
+            # Xử lý cha field - chuyển empty string thành None
+            if 'cha' in processed_data and processed_data['cha'] == "":
+                processed_data['cha'] = None
+            
+            response = self.client.table('document_chunks').insert(processed_data).execute()
             
             if response.data:
-                logger.info(f"Successfully stored chunk {chunk_data.get('id', 'unknown')}")
+                logger.info(f"Successfully stored chunk {processed_data.get('id', 'unknown')}")
                 return True
             else:
-                logger.error(f"Failed to store chunk {chunk_data.get('id', 'unknown')}")
+                logger.error(f"Failed to store chunk {processed_data.get('id', 'unknown')}")
                 return False
                 
         except Exception as e:
@@ -106,7 +138,8 @@ class SupabaseClient:
         Output: bool (True nếu thành công, False nếu lỗi)
         """
         try:
-            response = self.client.table('document_chunks').delete().neq('id', '').execute()
+            # Xóa tất cả records trong bảng
+            response = self.client.table('document_chunks').delete().execute()
             logger.info(f"Successfully deleted all document chunks")
             return True
         except Exception as e: