ui/parsing/language.py

"""Lightweight, dependency-free language gate.

We only need a binary "is this English?" decision to split scored vs counted-only turns.
Adding langdetect/langid/fasttext would violate the gradio-only dependency constraint, so
this is a deliberate heuristic placeholder (logged in DECISIONS_LOG.md): a turn counts as
English when it is mostly Latin script AND hits a few common English function words. Crude
but stable and zero-dep; the backend can swap in a real detector behind `is_english`.
"""
from __future__ import annotations

import re

_STOPWORDS = {
    "the", "a", "an", "and", "or", "but", "if", "to", "of", "in", "on", "for", "with",
    "is", "are", "was", "were", "be", "this", "that", "it", "you", "i", "we", "they",
    "can", "could", "would", "should", "how", "what", "why", "do", "does", "please",
    "me", "my", "your", "have", "has", "not", "no", "yes", "give", "make", "want",
}
_WORD_RE = re.compile(r"[a-zA-Z']+")


def latin_ratio(text: str) -> float:
    letters = [c for c in text if c.isalpha()]
    if not letters:
        return 0.0
    latin = sum(1 for c in letters if "a" <= c.lower() <= "z")
    return latin / len(letters)


def is_english(text: str) -> bool:
    """Best-effort English check for a single turn."""
    text = (text or "").strip()
    if len(text) < 2:
        return False
    if latin_ratio(text) < 0.6:
        return False  # mostly non-Latin script (CJK, Cyrillic, Arabic, …)
    words = [w.lower() for w in _WORD_RE.findall(text)]
    if not words:
        return False
    if len(words) <= 3:
        return True  # too short to judge by stopwords; Latin script is enough
    hits = sum(1 for w in words if w in _STOPWORDS)
    return hits / len(words) >= 0.08