add data viewer

app/supabase_db.py

@@ -197,46 +197,40 @@ class SupabaseClient:
             total_count = count_response.count if hasattr(count_response, 'count') else 'unknown'
             logger.info(f"[SUPABASE] Total records in table: {total_count}")
             
-            # Thử lấy tất cả với limit lớn
-            logger.info("[SUPABASE] Trying to fetch all records with large limit")
-            response = self.client.table('document_chunks').select('*').limit(10000).execute()
-            
-            if response.data:
-                actual_count = len(response.data)
-                logger.info(f"[SUPABASE] Large limit fetched {actual_count} document chunks (expected: {total_count})")
-                
-                if isinstance(total_count, int) and actual_count >= total_count:
-                    logger.info("[SUPABASE] Successfully fetched all records")
-                    return response.data
-                else:
-                    logger.warning(f"[SUPABASE] Only got {actual_count}/{total_count} records, trying pagination")
-            
-            # Fallback to pagination with very small page size
             all_chunks = []
-            page_size = 300  # Rất nhỏ để test
-            start = 0
+            page_size = 1000
+            last_id = 0
             page_count = 0
             
             while True:
                 page_count += 1
-                response = self.client.table('document_chunks').select('*').range(start, start + page_size - 1).execute()
+                
+                # Sử dụng cursor-based pagination với id
+                if last_id == 0:
+                    # Lần đầu: lấy từ đầu
+                    response = self.client.table('document_chunks').select('*').order('id').limit(page_size).execute()
+                else:
+                    # Các lần sau: lấy từ id > last_id
+                    response = self.client.table('document_chunks').select('*').order('id').gt('id', last_id).limit(page_size).execute()
                 
                 actual_count = len(response.data) if response.data else 0
-                logger.info(f"[SUPABASE] Page {page_count}: start={start}, end={start + page_size - 1}, requested={page_size}, actual={actual_count}")
+                logger.info(f"[SUPABASE] Page {page_count}: last_id={last_id}, requested={page_size}, actual={actual_count}")
                 
                 if not response.data:
-                    logger.info(f"[SUPABASE] No more data at start {start}")
+                    logger.info(f"[SUPABASE] No more data after id {last_id}")
                     break
                 
                 all_chunks.extend(response.data)
                 
+                # Cập nhật last_id cho page tiếp theo
+                if response.data:
+                    last_id = max(chunk.get('id', 0) for chunk in response.data)
+                
                 if actual_count < page_size:
                     logger.info(f"[SUPABASE] Last page with {actual_count} records")
                     break
-                
-                start += page_size
             
-            logger.info(f"[SUPABASE] Pagination fetched {len(all_chunks)} document chunks (expected: {total_count})")
+            logger.info(f"[SUPABASE] Cursor-based pagination fetched {len(all_chunks)} document chunks (expected: {total_count})")
             logger.info(f"[SUPABASE] Fetched {page_count} pages with page_size={page_size}")
             return all_chunks