File size: 1,098 Bytes
d4f8959 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | from backend.parser import chunk_text, chunk_pages, page_offsets, char_pos_to_page
def test_chunk_text_covers_all_words():
text = " ".join(f"w{i}" for i in range(1200))
chunks = chunk_text(text, size=512, overlap=50)
assert len(chunks) == 3
assert chunks[0].split()[0] == "w0"
assert chunks[-1].split()[-1] == "w1199"
def test_chunk_pages_carry_page_numbers():
pages = ["first page text here", "", "third page text here"]
chunks = chunk_pages(pages)
assert [c["page"] for c in chunks] == [1, 3] # empty page skipped
assert chunks[0]["text"] == "first page text here"
def test_page_offsets_and_mapping():
pages = ["aaaa", "bbbbbb", "cc"]
offsets = page_offsets(pages)
assert offsets == [0, 5, 12] # +1 for each joining newline
joined = "\n".join(pages)
assert char_pos_to_page(0, offsets) == 1
assert char_pos_to_page(5, offsets) == 2
assert char_pos_to_page(len(joined) - 1, offsets) == 3
def test_char_pos_to_page_handles_none():
assert char_pos_to_page(None, [0, 5]) is None
assert char_pos_to_page(3, []) is None
|