src/analyzer/utils/text.py

"""
utils/text.py — small text helpers (no hard deps)

- clean(s): collapse whitespace
- to_number(s): parse first numeric like "£1,234.50" -> 1234.5
- extract_numbers(s): list of floats found in text
- safe_truncate_chars(s, n): hard char limit with ellipsis
- safe_truncate_tokens(s, max_tokens): uses tiktoken if installed; else char fallback
"""
from __future__ import annotations
import re
from typing import Any, List, Optional

_NUM_RE = re.compile(r"-?\d+(?:\.\d+)?")

def clean(s: Any) -> str:
    return re.sub(r"\s+", " ", str(s or "")).strip()

def to_number(x: Any) -> Optional[float]:
    if x is None:
        return None
    if isinstance(x, (int, float)):
        return float(x)
    s = str(x).replace(",", "").replace("£", "").strip()
    m = _NUM_RE.findall(s)
    return float(m[0]) if m else None

def extract_numbers(s: Any) -> List[float]:
    return [float(m) for m in _NUM_RE.findall(str(s or ""))]

def safe_truncate_chars(s: str, n: int) -> str:
    s = s or ""
    if len(s) <= n:
        return s
    return s[: max(0, n - 1)] + "…"

def safe_truncate_tokens(s: str, max_tokens: int, *, model: str = "gpt-4o-mini") -> str:
    """
    Best effort token truncation. If tiktoken is available, use it;
    otherwise approximate by ~4 chars/token heuristic.
    """
    s = s or ""
    try:
        import tiktoken  # type: ignore
        enc = tiktoken.encoding_for_model(model)
        toks = enc.encode(s)
        if len(toks) <= max_tokens:
            return s
        toks = toks[:max_tokens]
        return enc.decode(toks)
    except Exception:
        # rough fallback: ~4 chars per token
        return safe_truncate_chars(s, max_tokens * 4)