Spaces:
Sleeping
Sleeping
File size: 6,355 Bytes
bfcc872 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 | """
Tests for src/analyzer/utils/text.py
Tests cover:
- clean(): whitespace normalization
- to_number(): flexible number parsing
- extract_numbers(): finding all numbers in text
- safe_truncate_chars(): character truncation with ellipsis
- safe_truncate_tokens(): token-aware truncation
"""
import pytest
from src.analyzer.utils.text import (
clean,
to_number,
extract_numbers,
safe_truncate_chars,
safe_truncate_tokens,
)
class TestClean:
"""Test clean() function for text normalization."""
def test_clean_basic(self):
assert clean("hello world") == "hello world"
def test_clean_multiple_spaces(self):
assert clean("hello world") == "hello world"
def test_clean_newlines_and_tabs(self):
assert clean("hello\n\t world") == "hello world"
def test_clean_leading_trailing(self):
assert clean(" hello world ") == "hello world"
def test_clean_none(self):
assert clean(None) == ""
def test_clean_empty_string(self):
assert clean("") == ""
def test_clean_only_whitespace(self):
assert clean(" \n\t ") == ""
def test_clean_numbers(self):
assert clean(123) == "123"
def test_clean_complex_whitespace(self):
result = clean("hello \n\n world\t\tfoo bar")
assert result == "hello world foo bar"
class TestToNumber:
"""Test to_number() function for flexible number parsing."""
def test_to_number_int(self):
assert to_number(42) == 42.0
def test_to_number_float(self):
assert to_number(3.14) == 3.14
def test_to_number_string_int(self):
assert to_number("42") == 42.0
def test_to_number_string_float(self):
assert to_number("3.14") == 3.14
def test_to_number_with_commas(self):
assert to_number("1,234,567") == 1234567.0
def test_to_number_with_currency(self):
assert to_number("£1,234.50") == 1234.50
def test_to_number_negative(self):
assert to_number("-42.5") == -42.5
def test_to_number_none(self):
assert to_number(None) is None
def test_to_number_empty_string(self):
assert to_number("") is None
def test_to_number_no_numbers(self):
assert to_number("hello") is None
def test_to_number_mixed_text(self):
# Should extract first number from text
assert to_number("Project cost: £25,000.00") == 25000.0
def test_to_number_zero(self):
assert to_number(0) == 0.0
assert to_number("0") == 0.0
class TestExtractNumbers:
"""Test extract_numbers() for finding all numbers in text."""
def test_extract_numbers_single(self):
assert extract_numbers("42") == [42.0]
def test_extract_numbers_multiple(self):
assert extract_numbers("10 20 30") == [10.0, 20.0, 30.0]
def test_extract_numbers_mixed_text(self):
result = extract_numbers("Budget: £1,234 Duration: 12 months")
assert result == [1.0, 234.0, 12.0]
def test_extract_numbers_floats(self):
assert extract_numbers("3.14 2.71 1.41") == [3.14, 2.71, 1.41]
def test_extract_numbers_negative(self):
assert extract_numbers("-10 20 -30") == [-10.0, 20.0, -30.0]
def test_extract_numbers_none(self):
assert extract_numbers(None) == []
def test_extract_numbers_empty(self):
assert extract_numbers("") == []
def test_extract_numbers_no_numbers(self):
assert extract_numbers("hello world") == []
class TestSafeTruncateChars:
"""Test safe_truncate_chars() for character-based truncation."""
def test_truncate_chars_short(self):
assert safe_truncate_chars("hello", 10) == "hello"
def test_truncate_chars_exact(self):
assert safe_truncate_chars("hello", 5) == "hello"
def test_truncate_chars_long(self):
result = safe_truncate_chars("hello world", 8)
assert result == "hello w…"
assert len(result) == 8
def test_truncate_chars_very_short(self):
result = safe_truncate_chars("hello", 3)
assert result == "he…"
def test_truncate_chars_zero(self):
result = safe_truncate_chars("hello", 0)
assert result == "…"
def test_truncate_chars_none(self):
assert safe_truncate_chars(None, 10) == ""
def test_truncate_chars_empty(self):
assert safe_truncate_chars("", 10) == ""
class TestSafeTruncateTokens:
"""Test safe_truncate_tokens() for token-aware truncation."""
def test_truncate_tokens_short(self):
result = safe_truncate_tokens("hello world", 100)
assert result == "hello world"
def test_truncate_tokens_fallback(self):
# Test fallback mode (when tiktoken not available or errors)
# Should use char-based approximation (~4 chars per token)
text = "a" * 100
result = safe_truncate_tokens(text, 10)
# Should truncate to ~40 chars (10 tokens * 4 chars)
assert len(result) <= 41 # 40 + ellipsis
def test_truncate_tokens_none(self):
assert safe_truncate_tokens(None, 10) == ""
def test_truncate_tokens_empty(self):
assert safe_truncate_tokens("", 10) == ""
def test_truncate_tokens_exact(self):
# Simple short text should not be truncated
result = safe_truncate_tokens("hi", 10)
assert result == "hi"
class TestEdgeCases:
"""Test edge cases and corner scenarios."""
def test_clean_with_unicode(self):
assert clean("hello 世界") == "hello 世界"
def test_to_number_with_unicode_currency(self):
# Test with various currency symbols
assert to_number("€100") == 100.0
def test_extract_numbers_scientific_notation(self):
# Should handle basic scientific notation
result = extract_numbers("1e5")
# Depends on implementation - might be [1.0, 5.0] or handle it
assert len(result) >= 1
def test_truncate_chars_unicode(self):
result = safe_truncate_chars("hello 世界", 7)
assert len(result) == 7
assert result.endswith("…")
def test_clean_converts_types(self):
# Test that clean handles various types
assert clean([1, 2, 3]) == "[1, 2, 3]"
assert clean({"a": 1}) in ["{'a': 1}", "{'a':1}"] # Dict repr may vary
if __name__ == "__main__":
pytest.main([__file__, "-v"])
|