Spaces:
Runtime error
Runtime error
| from app.bm25 import BM25Index | |
| from app.chunking import chunk_text | |
| from app.query import extract_pages, match_docs | |
| def test_chunks_respect_size_and_overlap(): | |
| text = " ".join(f"This is sentence number {i} about topic {i % 5}." for i in range(80)) | |
| chunks = chunk_text(text, size=300, overlap=80) | |
| assert len(chunks) > 5 | |
| assert all(len(c) <= 300 + 80 for c in chunks) | |
| # overlap: the end of one chunk reappears at the start of the next | |
| assert chunks[0].split(". ")[-1] in chunks[1] | |
| def test_chunk_handles_giant_unbroken_text(): | |
| chunks = chunk_text("word " * 2000, size=500, overlap=50) | |
| assert chunks and all(len(c) <= 560 for c in chunks) | |
| def test_extract_pages_variants(): | |
| assert extract_pages("what is written on page 20?") == [20] | |
| assert extract_pages("page no. 7 pe kya hai") == [7] | |
| assert extract_pages("20 no ke page pe kya likha hai") == [20] | |
| assert extract_pages("20th page summary") == [20] | |
| assert extract_pages("explain pages 20-23") == [20, 21, 22, 23] | |
| assert extract_pages("page 5 to 7") == [5, 6, 7] | |
| assert extract_pages("20 se 22 pages") == [20, 21, 22] | |
| assert extract_pages("page 3, 9 and 12") == [3, 9, 12] | |
| assert extract_pages("what is the refund policy?") == [] | |
| assert extract_pages("who was born in 1990?") == [] | |
| def test_match_docs_by_filename(): | |
| docs = [ | |
| {"doc_id": "a", "filename": "HR_Policy.pdf"}, | |
| {"doc_id": "b", "filename": "annual-report-2024.pdf"}, | |
| ] | |
| assert match_docs("what does the hr policy say about leave", docs) == ["a"] | |
| assert match_docs("summary of annual report 2024", docs) == ["b"] | |
| assert match_docs("what about leave", docs) == [] | |
| def test_bm25_scopes_by_doc_and_page(): | |
| idx = BM25Index() | |
| idx.build( | |
| [ | |
| {"id": "1", "doc_id": "A", "page": 1, "text": "zebra migration across the plains"}, | |
| {"id": "2", "doc_id": "B", "page": 2, "text": "zebra stripes and lions"}, | |
| {"id": "3", "doc_id": "B", "page": 3, "text": "completely unrelated tax filing"}, | |
| ] | |
| ) | |
| idx._dirty = False | |
| assert {h["id"] for h in idx.search("zebra", limit=5)} == {"1", "2"} | |
| assert [h["id"] for h in idx.search("zebra", doc_ids=["B"])] == ["2"] | |
| assert idx.search("zebra", pages=[1])[0]["id"] == "1" | |