from app.services.text_cleaner import clean, clean_markdown, clean_raw_text, detect_type class TestDetectType: def test_detects_md_heading(self): assert detect_type("# Hello World") == "md" def test_detects_md_code_block(self): assert detect_type("```python\nprint('hi')\n```") == "md" def test_detects_md_link(self): assert detect_type("[click here](https://example.com)") == "md" def test_detects_md_bullet(self): assert detect_type("- item one\n- item two") == "md" def test_detects_txt_plain(self): assert detect_type("Just some plain text.") == "txt" def test_detects_txt_empty(self): assert detect_type("") == "txt" def test_detects_txt_with_punctuation(self): assert detect_type("Hello, world! How are you?") == "txt" class TestCleanRawText: def test_normalizes_whitespace(self): result = clean_raw_text(" Hello world ") assert result == "Hello world" def test_collapses_blank_lines(self): result = clean_raw_text("Line one\n\n\n\nLine two") assert result == "Line one\n\nLine two" def test_strips_control_chars(self): result = clean_raw_text("Hello\x00World") assert "\x00" not in result def test_normalizes_unicode_punctuation(self): result = clean_raw_text("Hello\u2014world") assert "\u2014" not in result assert "-" in result def test_handles_empty(self): assert clean_raw_text("") == "" assert clean_raw_text(" \n\t ") == "" class TestCleanMarkdown: def test_removes_fenced_code(self): result = clean_markdown("Some text\n```python\ncode\n```\nMore text") assert "```" not in result assert "code" not in result assert "Some text" in result assert "More text" in result def test_removes_inline_code_markers(self): result = clean_markdown("Use `print()` to output") assert "`" not in result assert "print()" in result def test_converts_headings(self): result = clean_markdown("# Section Title") assert "Section: Section Title" in result def test_removes_bullet_markers(self): result = clean_markdown("- first item\n- second item") assert "- " not in result assert "first item" in result def test_keeps_link_text_removes_url(self): result = clean_markdown("[Click here](https://example.com)") assert "Click here" in result assert "example.com" not in result def test_removes_images(self): result = clean_markdown("Text ![alt](image.png) more text") assert "![alt]" not in result assert "image.png" not in result def test_normalizes_bold_italic(self): result = clean_markdown("**bold** and *italic* and __bold2__ and _italic2_") assert "**" not in result assert "__" not in result assert "bold" in result assert "italic" in result def test_removes_html(self): result = clean_markdown("Hello world!") assert "" not in result def test_handles_empty(self): assert clean_markdown("") == "" assert clean_markdown("```\ncode\n```\n") == "" def test_removes_markdown_attribute_and_footnote_symbols(self): result = clean_markdown( "Heading {#intro}\n\nParagraph with ^footnote^ and {custom} markers." ) assert "^" not in result assert "{" not in result assert "}" not in result assert "Paragraph with" in result def test_preserves_literal_braces_and_normalizes_entities(self): result = clean_markdown("Keep {braces} & full-width:text") assert "{braces}" in result assert "&" not in result def test_removes_markdown_footnote_reference_but_keeps_content(self): result = clean_markdown("Paragraph[^1]\n\n[^1]: Footnote content") assert "[^1]" not in result assert "Footnote content" in result class TestClean: def test_routes_to_markdown_cleaner(self): result = clean("# Hello") assert "Section:" in result def test_routes_to_txt_cleaner(self): result = clean("Hello world") assert result == "Hello world" def test_honors_explicit_markdown_source_type(self): result = clean("Paragraph with ^note^ and {#id}", source_type="md") assert "^" not in result assert "{" not in result assert "}" not in result