""" utils/text.py — small text helpers (no hard deps) - clean(s): collapse whitespace - to_number(s): parse first numeric like "£1,234.50" -> 1234.5 - extract_numbers(s): list of floats found in text - safe_truncate_chars(s, n): hard char limit with ellipsis - safe_truncate_tokens(s, max_tokens): uses tiktoken if installed; else char fallback """ from __future__ import annotations import re from typing import Any, List, Optional _NUM_RE = re.compile(r"-?\d+(?:\.\d+)?") def clean(s: Any) -> str: return re.sub(r"\s+", " ", str(s or "")).strip() def to_number(x: Any) -> Optional[float]: if x is None: return None if isinstance(x, (int, float)): return float(x) s = str(x).replace(",", "").replace("£", "").strip() m = _NUM_RE.findall(s) return float(m[0]) if m else None def extract_numbers(s: Any) -> List[float]: return [float(m) for m in _NUM_RE.findall(str(s or ""))] def safe_truncate_chars(s: str, n: int) -> str: s = s or "" if len(s) <= n: return s return s[: max(0, n - 1)] + "…" def safe_truncate_tokens(s: str, max_tokens: int, *, model: str = "gpt-4o-mini") -> str: """ Best effort token truncation. If tiktoken is available, use it; otherwise approximate by ~4 chars/token heuristic. """ s = s or "" try: import tiktoken # type: ignore enc = tiktoken.encoding_for_model(model) toks = enc.encode(s) if len(toks) <= max_tokens: return s toks = toks[:max_tokens] return enc.decode(toks) except Exception: # rough fallback: ~4 chars per token return safe_truncate_chars(s, max_tokens * 4)