from backend.parser import chunk_text, chunk_pages, page_offsets, char_pos_to_page def test_chunk_text_covers_all_words(): text = " ".join(f"w{i}" for i in range(1200)) chunks = chunk_text(text, size=512, overlap=50) assert len(chunks) == 3 assert chunks[0].split()[0] == "w0" assert chunks[-1].split()[-1] == "w1199" def test_chunk_pages_carry_page_numbers(): pages = ["first page text here", "", "third page text here"] chunks = chunk_pages(pages) assert [c["page"] for c in chunks] == [1, 3] # empty page skipped assert chunks[0]["text"] == "first page text here" def test_page_offsets_and_mapping(): pages = ["aaaa", "bbbbbb", "cc"] offsets = page_offsets(pages) assert offsets == [0, 5, 12] # +1 for each joining newline joined = "\n".join(pages) assert char_pos_to_page(0, offsets) == 1 assert char_pos_to_page(5, offsets) == 2 assert char_pos_to_page(len(joined) - 1, offsets) == 3 def test_char_pos_to_page_handles_none(): assert char_pos_to_page(None, [0, 5]) is None assert char_pos_to_page(3, []) is None