| from app.services.text_chunker import chunk_text, get_chunk_size_limit | |
| class TestGetChunkSizeLimit: | |
| def test_supertonic_limit(self): | |
| assert get_chunk_size_limit("supertonic-2") == 4500 | |
| def test_supertonic_alias(self): | |
| assert get_chunk_size_limit("tts-1") == 4500 | |
| def test_kokoro_limit(self): | |
| assert get_chunk_size_limit("kokoro") == 2000 | |
| def test_kitten_limit(self): | |
| assert get_chunk_size_limit("kitten-tts-micro-0.8") == 3000 | |
| def test_kokoro_onnx(self): | |
| assert get_chunk_size_limit("kokoro-onnx") == 2000 | |
| def test_unknown_model_default(self): | |
| assert get_chunk_size_limit("unknown-model") == 3000 | |
| def test_none_input(self): | |
| assert get_chunk_size_limit(None) == 3000 | |
| class TestChunkText: | |
| def test_short_text_returns_single_chunk(self): | |
| text = "This is a short piece of text." | |
| chunks = chunk_text(text, "supertonic-2") | |
| assert len(chunks) == 1 | |
| assert chunks[0]["text"] == text | |
| assert "index" in chunks[0] | |
| assert "char_count" in chunks[0] | |
| def test_removes_cleaning_markup(self): | |
| text = "# Chapter One\n\nThis is the first chapter. It has some content." | |
| chunks = chunk_text(text, "supertonic-2") | |
| assert len(chunks) == 1 | |
| assert "#" not in chunks[0]["text"] | |
| def test_merges_short_paragraphs(self): | |
| text = "Para one.\n\nPara two.\n\nPara three." | |
| chunks = chunk_text(text, "supertonic-2") | |
| assert len(chunks) == 1 | |
| def test_chunks_long_paragraph(self): | |
| long_text = "Word " * 2000 | |
| chunks = chunk_text(long_text, "supertonic-2") | |
| assert len(chunks) > 1 | |
| for c in chunks: | |
| assert len(c["text"]) <= 4500 | |
| def test_kokoro_uses_lower_limit(self): | |
| long_text = "Word " * 600 | |
| chunks = chunk_text(long_text, "kokoro") | |
| assert len(chunks) > 1 | |
| for c in chunks: | |
| assert len(c["text"]) <= 2000 | |
| def test_chunk_indices_are_sequential(self): | |
| text = "Para one.\n\nPara two.\n\n" + "Word " * 2000 | |
| chunks = chunk_text(text, "supertonic-2") | |
| indices = [c["index"] for c in chunks] | |
| assert indices == list(range(1, len(chunks) + 1)) | |
| def test_empty_text_returns_empty_list(self): | |
| chunks = chunk_text("", "supertonic-2") | |
| assert len(chunks) == 0 | |
| def test_already_cleaned_skips_second_markdown_pass(self): | |
| text = "Keep {literal braces}" | |
| chunks = chunk_text(text, "supertonic-2", already_cleaned=True) | |
| assert chunks[0]["text"] == text | |