Upload src/chunk.py with huggingface_hub

src/chunk.py

@@ -0,0 +1,140 @@
+"""
+Paragraphization and fixed-size character chunking with overlap.
+"""
+from typing import List, Dict
+
+
+def split_into_paragraphs(cleaned: str) -> list:
+    """
+    Split cleaned text into paragraphs.
+
+    # TODO hints:
+    # - Split on double newlines; strip; drop very short fragments.
+
+    # Acceptance:
+    # - Returns a list of paragraph strings.
+    """
+    paragraphs = cleaned.split('\n\n')
+    return [p.strip() for p in paragraphs if len(p.strip()) > 0]
+
+
+def chunk_paragraphs(paragraphs: list, size: int, overlap: int, book: str) -> List[Dict]:
+    """
+    Make fixed-size chunks with overlap; attach source metadata (book, para_idx, char_span).
+
+    # TODO hints:
+    # - Accumulate paragraph text until >= size; slide by size-overlap.
+    # - Keep 'source_id' for citations; store start/end character indices.
+
+    # Acceptance:
+    # - Returns list of dicts: {id, text, meta:{book, para_idx_start, para_idx_end, span}}
+    """
+    chunks = []
+    chunk_id = 0
+    i = 0  # Start from first paragraph
+    total_paragraphs = len(paragraphs)
+    last_i = -1  # Track last position to detect infinite loops
+    
+    print(f"📚 Chunking '{book}': {total_paragraphs} paragraphs, size={size}, overlap={overlap}")
+    
+    while i < len(paragraphs):
+        # Safety check: prevent infinite loops
+        if i == last_i:
+            print(f"⚠️  Warning: Stuck at paragraph {i}, forcing forward progress")
+            i += 1
+            if i >= len(paragraphs):
+                break
+        last_i = i
+        # Accumulate paragraphs until we reach the target size
+        chunk_paras = []
+        chunk_text = ""
+        para_start_idx = i
+        
+        # Add paragraphs until we reach or exceed the target size
+        while i < len(paragraphs) and len(chunk_text) < size:
+            para = paragraphs[i]
+            # Add paragraph with separator
+            if chunk_text:
+                chunk_text += "\n\n" + para
+            else:
+                chunk_text = para
+            chunk_paras.append(i)
+            i += 1
+        
+        # If we have content, create a chunk
+        if chunk_text:
+            para_end_idx = chunk_paras[-1] if chunk_paras else para_start_idx
+            
+            chunks.append({
+                'id': f'{book}_chunk_{chunk_id}',
+                'text': chunk_text,
+                'meta': {
+                    'book': book,
+                    'para_idx_start': para_start_idx,
+                    'para_idx_end': para_end_idx,
+                    'char_count': len(chunk_text)
+                }
+            })
+            
+            # Print progress
+            progress_pct = (i / total_paragraphs) * 100
+            print(f"  Chunk {chunk_id}: paras {para_start_idx}-{para_end_idx}, {len(chunk_text)} chars ({progress_pct:.1f}% complete)")
+            
+            chunk_id += 1
+            
+            # Slide back by (size - overlap) characters for next chunk
+            # This creates overlapping chunks
+            if i < len(paragraphs) and overlap > 0:
+                slide_back = size - overlap
+                
+                # Find which paragraph to start the next chunk from
+                # We want to keep 'overlap' characters from the end of current chunk
+                if len(chunk_text) > slide_back:
+                    # Work backwards from the end of chunk_text to find where overlap starts
+                    # Count characters from the end backwards
+                    chars_from_end = 0
+                    para_idx_back = len(chunk_paras) - 1
+                    
+                    # Find the paragraph that contains the start of the overlap region
+                    while para_idx_back >= 0:
+                        para_idx = chunk_paras[para_idx_back]
+                        para_len = len(paragraphs[para_idx])
+                        # Add separator length (2 chars for \n\n) if not the last para
+                        separator_len = 2 if para_idx_back < len(chunk_paras) - 1 else 0
+                        chars_from_end += para_len + separator_len
+                        
+                        # If we've covered at least 'overlap' chars, we found our starting point
+                        if chars_from_end >= overlap:
+                            # Start next chunk from this paragraph (included in overlap)
+                            next_start_idx = para_idx
+                            # CRITICAL: Only move backwards if:
+                            # 1. We're going to a different position
+                            # 2. It's before the current 'i' (which is already past this chunk)
+                            # 3. It's different from where we just were (prevents getting stuck)
+                            if next_start_idx < para_end_idx and next_start_idx != last_i:
+                                i = next_start_idx
+                            else:
+                                # Can't safely move backwards, just continue forward from current i
+                                # This prevents infinite loops
+                                pass
+                            break
+                        
+                        para_idx_back -= 1
+                    
+                    # Safety check: if we didn't find a good position, just continue forward
+                    # This prevents infinite loops
+                    if para_idx_back < 0:
+                        # Couldn't find overlap point, just continue from where we are
+                        # Ensure we at least move forward by 1 paragraph to prevent infinite loops
+                        if i <= para_end_idx:
+                            i = para_end_idx + 1
+                else:
+                    # Chunk is smaller than slide_back, can't create meaningful overlap
+                    # Just continue from current position
+                    pass
+        else:
+            # No more content
+            break
+    
+    print(f"✅ Created {len(chunks)} chunks from '{book}'")
+    return chunks