from app.services.text_cleaner import clean, clean_markdown, clean_raw_text, detect_type
class TestDetectType:
def test_detects_md_heading(self):
assert detect_type("# Hello World") == "md"
def test_detects_md_code_block(self):
assert detect_type("```python\nprint('hi')\n```") == "md"
def test_detects_md_link(self):
assert detect_type("[click here](https://example.com)") == "md"
def test_detects_md_bullet(self):
assert detect_type("- item one\n- item two") == "md"
def test_detects_txt_plain(self):
assert detect_type("Just some plain text.") == "txt"
def test_detects_txt_empty(self):
assert detect_type("") == "txt"
def test_detects_txt_with_punctuation(self):
assert detect_type("Hello, world! How are you?") == "txt"
class TestCleanRawText:
def test_normalizes_whitespace(self):
result = clean_raw_text(" Hello world ")
assert result == "Hello world"
def test_collapses_blank_lines(self):
result = clean_raw_text("Line one\n\n\n\nLine two")
assert result == "Line one\n\nLine two"
def test_strips_control_chars(self):
result = clean_raw_text("Hello\x00World")
assert "\x00" not in result
def test_normalizes_unicode_punctuation(self):
result = clean_raw_text("Hello\u2014world")
assert "\u2014" not in result
assert "-" in result
def test_handles_empty(self):
assert clean_raw_text("") == ""
assert clean_raw_text(" \n\t ") == ""
class TestCleanMarkdown:
def test_removes_fenced_code(self):
result = clean_markdown("Some text\n```python\ncode\n```\nMore text")
assert "```" not in result
assert "code" not in result
assert "Some text" in result
assert "More text" in result
def test_removes_inline_code_markers(self):
result = clean_markdown("Use `print()` to output")
assert "`" not in result
assert "print()" in result
def test_converts_headings(self):
result = clean_markdown("# Section Title")
assert "Section: Section Title" in result
def test_removes_bullet_markers(self):
result = clean_markdown("- first item\n- second item")
assert "- " not in result
assert "first item" in result
def test_keeps_link_text_removes_url(self):
result = clean_markdown("[Click here](https://example.com)")
assert "Click here" in result
assert "example.com" not in result
def test_removes_images(self):
result = clean_markdown("Text  more text")
assert "![alt]" not in result
assert "image.png" not in result
def test_normalizes_bold_italic(self):
result = clean_markdown("**bold** and *italic* and __bold2__ and _italic2_")
assert "**" not in result
assert "__" not in result
assert "bold" in result
assert "italic" in result
def test_removes_html(self):
result = clean_markdown("Hello world!")
assert "" not in result
def test_handles_empty(self):
assert clean_markdown("") == ""
assert clean_markdown("```\ncode\n```\n") == ""
def test_removes_markdown_attribute_and_footnote_symbols(self):
result = clean_markdown(
"Heading {#intro}\n\nParagraph with ^footnote^ and {custom} markers."
)
assert "^" not in result
assert "{" not in result
assert "}" not in result
assert "Paragraph with" in result
def test_preserves_literal_braces_and_normalizes_entities(self):
result = clean_markdown("Keep {braces} & full-width:text")
assert "{braces}" in result
assert "&" not in result
def test_removes_markdown_footnote_reference_but_keeps_content(self):
result = clean_markdown("Paragraph[^1]\n\n[^1]: Footnote content")
assert "[^1]" not in result
assert "Footnote content" in result
class TestClean:
def test_routes_to_markdown_cleaner(self):
result = clean("# Hello")
assert "Section:" in result
def test_routes_to_txt_cleaner(self):
result = clean("Hello world")
assert result == "Hello world"
def test_honors_explicit_markdown_source_type(self):
result = clean("Paragraph with ^note^ and {#id}", source_type="md")
assert "^" not in result
assert "{" not in result
assert "}" not in result