feat: implement recursive RAG text section-aware chunking utility

backend/app/rag/chunker.py

@@ -1 +1,120 @@
-"""Section-aware recursive chunking of PDF pages."""
+from dataclasses import dataclass
+from pathlib import Path
+
+import pymupdf
+
+from app.config import settings
+
+_SEPARATORS = ["\n\n", "\n", ". ", " "]
+
+
+@dataclass
+class Chunk:
+    text: str
+    filename: str
+    page: int
+    section: str
+    chunk_index: int
+
+
+def _is_heading(line: str) -> bool:
+    line = line.strip()
+    if not line or not 3 <= len(line) <= 80:
+        return False
+    if line[-1] in ".,:;!?":
+        return False
+    if not (line[0].isupper() or line[0].isdigit()):
+        return False
+    if len(line.split()) > 8:
+        return False
+    return True
+
+
+def _merge_pieces(pieces: list[str], chunk_size: int, chunk_overlap: int) -> list[str]:
+    """Merge small pieces into chunks with piece-boundary overlap (LangChain-style)."""
+    chunks: list[str] = []
+    window: list[str] = []
+    window_len = 0
+
+    for piece in pieces:
+        piece_len = len(piece)
+        if window and window_len + 1 + piece_len > chunk_size:
+            chunks.append(" ".join(window))
+            # Slide window back: drop pieces from the front until overlap fits
+            while window and (window_len > chunk_overlap or window_len + 1 + piece_len > chunk_size):
+                window_len -= len(window[0]) + (1 if len(window) > 1 else 0)
+                window.pop(0)
+        window.append(piece)
+        window_len += piece_len + (1 if len(window) > 1 else 0)
+
+    if window:
+        chunks.append(" ".join(window))
+    return chunks
+
+
+def _split_recursive(text: str, separators: list[str], chunk_size: int, chunk_overlap: int) -> list[str]:
+    if not separators:
+        step = max(1, chunk_size - chunk_overlap)
+        return [text[i : i + chunk_size] for i in range(0, len(text), step)]
+
+    # Find the first separator that actually appears in the text
+    sep = separators[-1]
+    rest: list[str] = []
+    for i, s in enumerate(separators):
+        if s in text:
+            sep, rest = s, separators[i + 1:]
+            break
+
+    pieces: list[str] = []
+    for part in text.split(sep):
+        part = part.strip()
+        if not part:
+            continue
+        if len(part) <= chunk_size:
+            pieces.append(part)
+        else:
+            pieces.extend(_split_recursive(part, rest, chunk_size, chunk_overlap))
+
+    return _merge_pieces(pieces, chunk_size, chunk_overlap)
+
+
+def chunk_pdf(path: Path) -> list[Chunk]:
+    doc = pymupdf.open(str(path))
+    all_chunks: list[Chunk] = []
+    chunk_index = 0
+    current_section = ""
+
+    for page_num in range(doc.page_count):
+        text = doc[page_num].get_text("text")
+        if not text.strip():
+            continue
+
+        for line in text.splitlines():
+            if _is_heading(line):
+                current_section = line.strip()
+                break
+
+        page_chunks = _split_recursive(
+            text,
+            _SEPARATORS,
+            settings.chunk_size,
+            settings.chunk_overlap,
+        )
+
+        for chunk_text in page_chunks:
+            chunk_text = chunk_text.strip()
+            if not chunk_text:
+                continue
+            all_chunks.append(
+                Chunk(
+                    text=chunk_text,
+                    filename=path.name,
+                    page=page_num + 1,
+                    section=current_section,
+                    chunk_index=chunk_index,
+                )
+            )
+            chunk_index += 1
+
+    doc.close()
+    return all_chunks

backend/app/routes/upload.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-import fitz
+import pymupdf
 from fastapi import APIRouter, File, HTTPException, UploadFile
 from pydantic import BaseModel
 
@@ -36,7 +36,7 @@ async def upload(files: list[UploadFile] = File(...)) -> list[DocumentMeta]:
         dest = UPLOAD_DIR / (file.filename or "upload.pdf")
         dest.write_bytes(contents)
 
-        doc = fitz.open(stream=contents, filetype="pdf")
+        doc = pymupdf.open(stream=contents, filetype="pdf")
         pages = doc.page_count
         doc.close()
 

backend/tests/conftest.py

@@ -1,11 +1,20 @@
 from pathlib import Path
 
-import fitz
+import pymupdf
 import pytest
 from fastapi.testclient import TestClient
 
 from app.main import app
 
+_PARA = (
+    "Retrieval-augmented generation combines a dense retrieval step with a "
+    "generative language model to produce grounded, cited answers. "
+    "The retriever selects the most relevant passages from a large corpus, "
+    "and the generator conditions its output on those passages. "
+    "Hybrid retrieval systems that combine dense vector search with sparse "
+    "keyword methods such as BM25 consistently outperform either approach alone. "
+)
+
 
 @pytest.fixture
 def client() -> TestClient:
@@ -14,11 +23,19 @@ def client() -> TestClient:
 
 @pytest.fixture
 def sample_pdf(tmp_path: Path) -> Path:
-    doc = fitz.open()
-    page = doc.new_page()
-    page.insert_text((72, 72), "Page one of a test document.")
-    doc.new_page()
-    doc[-1].insert_text((72, 72), "Page two of a test document.")
+    doc = pymupdf.open()
+
+    for heading, body in [
+        ("Introduction", _PARA * 5),
+        ("Methods", _PARA * 5),
+    ]:
+        page = doc.new_page()
+        page.insert_textbox(
+            pymupdf.Rect(72, 72, 540, 720),
+            f"{heading}\n\n{body}",
+            fontsize=11,
+        )
+
     path = tmp_path / "sample.pdf"
     doc.save(str(path))
     doc.close()

backend/tests/test_chunker.py

@@ -1 +1,50 @@
-"""Tests for rag/chunker.py — added in Phase 7."""
+from pathlib import Path
+
+from app.config import settings
+from app.rag.chunker import chunk_pdf
+
+
+def test_produces_chunks(sample_pdf: Path) -> None:
+    chunks = chunk_pdf(sample_pdf)
+    assert len(chunks) > 0
+
+
+def test_no_chunk_exceeds_max_size(sample_pdf: Path) -> None:
+    max_allowed = settings.chunk_size + settings.chunk_overlap
+    for chunk in chunk_pdf(sample_pdf):
+        assert len(chunk.text) <= max_allowed, (
+            f"Chunk {chunk.chunk_index} has {len(chunk.text)} chars"
+        )
+
+
+def test_chunk_index_is_sequential(sample_pdf: Path) -> None:
+    chunks = chunk_pdf(sample_pdf)
+    assert [c.chunk_index for c in chunks] == list(range(len(chunks)))
+
+
+def test_page_numbers_are_valid(sample_pdf: Path) -> None:
+    chunks = chunk_pdf(sample_pdf)
+    pages = {c.page for c in chunks}
+    assert pages <= {1, 2}
+    assert all(c.page >= 1 for c in chunks)
+
+
+def test_no_chunk_spans_pages(sample_pdf: Path) -> None:
+    # Chunking is done per page so every chunk carries exactly one page number.
+    chunks = chunk_pdf(sample_pdf)
+    assert all(isinstance(c.page, int) for c in chunks)
+
+
+def test_filename_is_preserved(sample_pdf: Path) -> None:
+    chunks = chunk_pdf(sample_pdf)
+    assert all(c.filename == "sample.pdf" for c in chunks)
+
+
+def test_section_heading_detected(sample_pdf: Path) -> None:
+    chunks = chunk_pdf(sample_pdf)
+    sections = {c.section for c in chunks}
+    assert any(s in ("Introduction", "Methods") for s in sections)
+
+
+def test_no_empty_chunk_text(sample_pdf: Path) -> None:
+    assert all(c.text.strip() for c in chunk_pdf(sample_pdf))