from __future__ import annotations from backend.config import settings from backend.core.ingest import _chunk_reference_text, _chunk_text from backend.core.reference_chunker import build_reference_chunks def test_chunk_text_respects_min_and_max(monkeypatch): monkeypatch.setattr(settings, "paragraph_min_chars", 80) monkeypatch.setattr(settings, "paragraph_max_chars", 120) monkeypatch.setattr(settings, "chunk_overlap", 20) short_a = "a" * 40 short_b = "b" * 40 long_para = "x" * 250 text = f"{short_a}\n\n{short_b}\n\n{long_para}" chunks = _chunk_text(text) assert chunks assert all(len(c) <= 120 for c in chunks) assert any(len(c) >= 80 for c in chunks) def test_reference_chunk_keeps_whole_section_under_max(monkeypatch): monkeypatch.setattr(settings, "reference_paragraph_max_chars", 8000) monkeypatch.setattr(settings, "reference_chunk_overlap", 1500) body = "The roof comprises slate tiles laid to pitched timber rafters. " * 40 text = f"D2 Roof coverings\n\n{body}" chunks = build_reference_chunks(text, source_filename="past.docx") assert len(chunks) == 1 assert len(chunks[0].text) <= 8000 assert "slate tiles" in chunks[0].text def test_reference_chunk_splits_when_exceeding_max(monkeypatch): monkeypatch.setattr(settings, "reference_paragraph_max_chars", 8000) monkeypatch.setattr(settings, "reference_chunk_overlap", 1500) body = "x" * 9000 text = f"D2 Roof coverings\n\n{body}" chunks = build_reference_chunks(text, source_filename="past.docx") assert len(chunks) >= 2 assert all(len(c.text) <= 8000 for c in chunks) def test_chunk_reference_text_uses_long_form_limits(monkeypatch): monkeypatch.setattr(settings, "paragraph_min_chars", 80) monkeypatch.setattr(settings, "reference_paragraph_max_chars", 8000) monkeypatch.setattr(settings, "reference_chunk_overlap", 1500) para = "y" * 500 text = f"{para}\n\n{para}" chunks = _chunk_reference_text(text) assert chunks assert all(len(c) <= 8000 for c in chunks)