"""Text normalisation, Unicode validation and tokenisation shared by data prep, baselines and eval."""
import re
import unicodedata

ZW = "\u200c\u200d\u200b\ufeff\u00ad"  # ZWNJ, ZWJ, ZWSP, BOM, soft hyphen
_ZW_RE = re.compile(f"[{ZW}]")
_WS_RE = re.compile(r"\s+")

# Devanagari character classes
CONS = set(chr(c) for c in range(0x0915, 0x093A)) | set(chr(c) for c in range(0x0958, 0x0960)) | set("ॹॺॻॼॽॾॿ")
INDEP_V = set(chr(c) for c in range(0x0904, 0x0915)) | set("ॠॡॲॳॴॵॶॷ")
MATRA = set(chr(c) for c in range(0x093E, 0x094D)) | set("ॢॣॏॕॖॗ\u093a\u093b\u094e")
VIRAMA = "\u094d"
NUKTA = "\u093c"
NASAL = set("\u0901\u0902")      # chandrabindu, anusvara
VISARGA = "\u0903"
DEVA_DIGITS = "०१२३४५६७८९"
DEVA_RANGE = re.compile(r"[\u0900-\u097F]")
DANDA = "।॥"


def nfc(s: str) -> str:
    return unicodedata.normalize("NFC", s)


def is_malformed(s: str) -> bool:
    """True if the string contains characters we treat as malformed/garbage:
    replacement char, private use, surrogates, unassigned, or control chars other than tab/newline."""
    for ch in s:
        if ch == "\ufffd":
            return True
        cat = unicodedata.category(ch)
        if cat in ("Co", "Cs", "Cn"):
            return True
        if cat == "Cc" and ch not in "\t\n":
            return True
    return False


def clean(s: str) -> str:
    """NFC + strip zero-width/format chars + collapse whitespace."""
    s = nfc(s)
    s = _ZW_RE.sub("", s)
    s = "".join(ch for ch in s if unicodedata.category(ch) != "Cf")
    return _WS_RE.sub(" ", s).strip()


def valid_devanagari_word(w: str) -> bool:
    """Orthographic sanity check for a single Devanagari word (letters only).
    Rejects: non-Devanagari chars, leading combining marks, matra/virama/nukta after a non-consonant,
    two dependent vowel signs in a row, nukta not on a consonant, dangling virama+matra."""
    if not w:
        return False
    prev = None
    for i, ch in enumerate(w):
        if not ("\u0900" <= ch <= "\u097F"):
            return False
        if ch in DEVA_DIGITS or ch in DANDA:
            return False
        if ch in MATRA or ch == VIRAMA or ch == NUKTA:
            if prev is None:
                return False
            if ch == NUKTA and prev not in CONS:
                return False
            if ch in MATRA and not (prev in CONS or prev == NUKTA):
                return False
            if ch == VIRAMA and not (prev in CONS or prev == NUKTA):
                return False
        if ch in NASAL or ch == VISARGA:
            if prev is None or prev in NASAL or prev == VIRAMA:
                return False
        prev = ch
    return True


def valid_roman_word(w: str) -> bool:
    return bool(re.fullmatch(r"[a-z]+", w))


# --- tokenisation -----------------------------------------------------------
# Words (Latin or Devanagari letters incl. marks), numbers, URLs/emails/handles, punctuation.
TOKEN_RE = re.compile(
    r"(https?://\S+|www\.\S+|[\w.+-]+@[\w-]+\.[\w.]+|[@#][\w_]+"
    r"|[A-Za-z]+(?:'[A-Za-z]+)?"
    r"|[\u0900-\u0963\u0971-\u097F]+"
    r"|[0-9\u0966-\u096F]+(?:[.,:][0-9\u0966-\u096F]+)*"
    r"|[\u0964\u0965]|[^\sA-Za-z0-9\u0900-\u097F])"
)


def tokenize(s: str):
    return TOKEN_RE.findall(s)


PUNCT_RE = re.compile(r"[^\w\u0900-\u097F]+", re.UNICODE)


# --- scoring normalisation --------------------------------------------------
_DIGIT_TR = str.maketrans(DEVA_DIGITS, "0123456789")


def norm_for_score(s: str) -> str:
    """'norm' scoring view: NFC, no zero-width chars, Devanagari digits -> ASCII, punctuation removed
    (incl. danda), whitespace collapsed. Spelling is otherwise untouched (strict on letters)."""
    s = clean(s).translate(_DIGIT_TR)
    s = re.sub(r"[^\w\s\u0900-\u097F]|[\u0964\u0965]", " ", s)
    return _WS_RE.sub(" ", s).strip()


_FOLD_MAP = {
    "\u0901": "\u0902",  # chandrabindu -> anusvara (हूँ ~ हूं)
    NUKTA: "",           # drop nukta (ज़ ~ ज)
    "\u0945": "\u0947",  # candra e (ॅ) -> e
    "\u0949": "\u094b",  # candra o (ॉ) -> o (ऑफिस ~ ओफिस)
    "\u0911": "\u0913",  # independent candra O -> O
    "\u090d": "\u090f",
}


def fold(s: str) -> str:
    """'fold' scoring view (variant-tolerant, reported separately): norm_for_score + nukta dropped,
    chandrabindu->anusvara, candra vowels -> plain. Never used as the primary metric."""
    s = unicodedata.normalize("NFD", norm_for_score(s))  # decompose precomposed nukta letters
    s = "".join(_FOLD_MAP.get(ch, ch) for ch in s)
    return unicodedata.normalize("NFC", s)