tests/test_text_cleaner.py

from app.services.text_cleaner import clean, clean_markdown, clean_raw_text, detect_type


class TestDetectType:
    def test_detects_md_heading(self):
        assert detect_type("# Hello World") == "md"

    def test_detects_md_code_block(self):
        assert detect_type("```python\nprint('hi')\n```") == "md"

    def test_detects_md_link(self):
        assert detect_type("[click here](https://example.com)") == "md"

    def test_detects_md_bullet(self):
        assert detect_type("- item one\n- item two") == "md"

    def test_detects_txt_plain(self):
        assert detect_type("Just some plain text.") == "txt"

    def test_detects_txt_empty(self):
        assert detect_type("") == "txt"

    def test_detects_txt_with_punctuation(self):
        assert detect_type("Hello, world! How are you?") == "txt"


class TestCleanRawText:
    def test_normalizes_whitespace(self):
        result = clean_raw_text("  Hello   world  ")
        assert result == "Hello world"

    def test_collapses_blank_lines(self):
        result = clean_raw_text("Line one\n\n\n\nLine two")
        assert result == "Line one\n\nLine two"

    def test_strips_control_chars(self):
        result = clean_raw_text("Hello\x00World")
        assert "\x00" not in result

    def test_normalizes_unicode_punctuation(self):
        result = clean_raw_text("Hello\u2014world")
        assert "\u2014" not in result
        assert "-" in result

    def test_handles_empty(self):
        assert clean_raw_text("") == ""
        assert clean_raw_text("   \n\t  ") == ""


class TestCleanMarkdown:
    def test_removes_fenced_code(self):
        result = clean_markdown("Some text\n```python\ncode\n```\nMore text")
        assert "```" not in result
        assert "code" not in result
        assert "Some text" in result
        assert "More text" in result

    def test_removes_inline_code_markers(self):
        result = clean_markdown("Use `print()` to output")
        assert "`" not in result
        assert "print()" in result

    def test_converts_headings(self):
        result = clean_markdown("# Section Title")
        assert "Section: Section Title" in result

    def test_removes_bullet_markers(self):
        result = clean_markdown("- first item\n- second item")
        assert "- " not in result
        assert "first item" in result

    def test_keeps_link_text_removes_url(self):
        result = clean_markdown("[Click here](https://example.com)")
        assert "Click here" in result
        assert "example.com" not in result

    def test_removes_images(self):
        result = clean_markdown("Text ![alt](image.png) more text")
        assert "![alt]" not in result
        assert "image.png" not in result

    def test_normalizes_bold_italic(self):
        result = clean_markdown("**bold** and *italic* and __bold2__ and _italic2_")
        assert "**" not in result
        assert "__" not in result
        assert "bold" in result
        assert "italic" in result

    def test_removes_html(self):
        result = clean_markdown("Hello <b>world</b>!")
        assert "<b>" not in result

    def test_handles_empty(self):
        assert clean_markdown("") == ""
        assert clean_markdown("```\ncode\n```\n") == ""

    def test_removes_markdown_attribute_and_footnote_symbols(self):
        result = clean_markdown(
            "Heading {#intro}\n\nParagraph with ^footnote^ and {custom} markers."
        )
        assert "^" not in result
        assert "{" not in result
        assert "}" not in result
        assert "Paragraph with" in result

    def test_preserves_literal_braces_and_normalizes_entities(self):
        result = clean_markdown("Keep {braces} &amp; full-width：text")
        assert "{braces}" in result
        assert "&amp;" not in result

    def test_removes_markdown_footnote_reference_but_keeps_content(self):
        result = clean_markdown("Paragraph[^1]\n\n[^1]: Footnote content")
        assert "[^1]" not in result
        assert "Footnote content" in result


class TestClean:
    def test_routes_to_markdown_cleaner(self):
        result = clean("# Hello")
        assert "Section:" in result

    def test_routes_to_txt_cleaner(self):
        result = clean("Hello  world")
        assert result == "Hello world"

    def test_honors_explicit_markdown_source_type(self):
        result = clean("Paragraph with ^note^ and {#id}", source_type="md")
        assert "^" not in result
        assert "{" not in result
        assert "}" not in result