utils/string_utils.py

"""String normalization and text processing utilities."""

from __future__ import annotations

import unicodedata


def normalize_text_lower(text: str) -> str:
    """Normalize text for case-insensitive comparisons."""
    return text.strip().lower()


def remove_accents(text: str) -> str:
    """Remove accents/diacritics from text."""
    normalized = unicodedata.normalize("NFD", text)
    return "".join(ch for ch in normalized if not unicodedata.category(ch).startswith("M"))


USDA_QUERY_KEY_PUNCTUATION_TRANSLATION = str.maketrans(
    {
        "\u2018": "'",  # left single quote
        "\u2019": "'",  # right single quote
        "\u02bc": "'",  # modifier letter apostrophe
        "\uff07": "'",  # fullwidth apostrophe
        "\u201c": '"',  # left double quote
        "\u201d": '"',  # right double quote
        "\u2010": "-",  # hyphen
        "\u2011": "-",  # non-breaking hyphen
        "\u2012": "-",  # figure dash
        "\u2013": "-",  # en dash
        "\u2014": "-",  # em dash
        "\u2212": "-",  # minus sign
    },
)


def normalize_usda_query_text(query_text: str) -> str:
    """Canonicalize USDA query text for DB keys.

    Repo policy: no migrations, and these tables are used as caches / learned mappings.
    We enforce a single normalized key form at write time so lookups are deterministic.
    """
    normalized = remove_accents(query_text.strip().lower())
    normalized = normalized.translate(USDA_QUERY_KEY_PUNCTUATION_TRANSLATION)
    return " ".join(normalized.split())