Upload src/labdaps/ingestion/chunker.py with huggingface_hub

src/labdaps/ingestion/chunker.py

@@ -0,0 +1,65 @@
+from dataclasses import dataclass
+from src.labdaps.ingestion.pdf_extractor import RawPage
+from src.labdaps.config import CHUNK_SIZE_CHARS, CHUNK_OVERLAP_CHARS, MIN_CHUNK_CHARS
+
+SEPARATORS = ["\n\n\n", "\n\n", "\n", ". ", "! ", "? ", "; ", ", ", " ", ""]
+
+
+@dataclass
+class Chunk:
+    text: str
+    source_file: str
+    page_number: int
+    chunk_index: int
+
+
+def _split_text(text: str, size: int, overlap: int) -> list[str]:
+    if len(text) <= size:
+        return [text]
+
+    chunks = []
+    for sep in SEPARATORS:
+        if sep == "":
+            pieces = [text[i:i+size] for i in range(0, len(text), size - overlap)]
+            return [p for p in pieces if len(p) >= MIN_CHUNK_CHARS]
+
+        parts = text.split(sep)
+        if len(parts) == 1:
+            continue
+
+        current = ""
+        for part in parts:
+            candidate = current + (sep if current else "") + part
+            if len(candidate) <= size:
+                current = candidate
+            else:
+                if len(current) >= MIN_CHUNK_CHARS:
+                    chunks.append(current)
+                if current and overlap > 0:
+                    overlap_text = current[-overlap:]
+                    current = overlap_text + (sep if overlap_text else "") + part
+                else:
+                    current = part
+
+        if len(current) >= MIN_CHUNK_CHARS:
+            chunks.append(current)
+
+        if chunks:
+            return chunks
+
+    return [text] if len(text) >= MIN_CHUNK_CHARS else []
+
+
+def chunk_pages(pages: list[RawPage]) -> list[Chunk]:
+    chunks = []
+    global_index = 0
+    for page in pages:
+        for text in _split_text(page.text, CHUNK_SIZE_CHARS, CHUNK_OVERLAP_CHARS):
+            chunks.append(Chunk(
+                text=text,
+                source_file=page.source_file,
+                page_number=page.page_number,
+                chunk_index=global_index,
+            ))
+            global_index += 1
+    return chunks