Spaces:
Running
Running
File size: 818 Bytes
e323466 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | """Tests for text preprocessing utilities."""
import pytest
from src.data.preprocessing import clean_text, clean_texts
def test_clean_text_lowercase():
assert clean_text("Hello WORLD") == "hello world"
def test_clean_text_whitespace():
assert clean_text(" too many spaces ") == "too many spaces"
def test_clean_text_non_ascii():
assert clean_text("caf\u00e9 latte") == "caf latte"
def test_clean_text_preserves_punctuation():
result = clean_text("I can't log in!")
assert "'" in result
assert "!" in result
def test_clean_text_non_string():
assert isinstance(clean_text(42), str)
def test_clean_texts_batch():
texts = ["Hello World", " extra spaces "]
results = clean_texts(texts)
assert results[0] == "hello world"
assert results[1] == "extra spaces"
|