app.py

# app.py
# Text Normalizer with step-by-step trace (Gradio)
# Designed to run even in sandboxed envs (no ssl, missing NLTK data).

import sys, types, re, unicodedata, string

# --- Safe SSL stub for restricted environments ---
try:
    import ssl  # noqa: F401
except Exception:  # pragma: no cover
    ssl_stub = types.ModuleType("ssl")
    class _SSLContext:  # minimal placeholder
        def __init__(self, *a, **k): pass
    def _noop(*a, **k): return None
    ssl_stub.SSLContext = _SSLContext
    ssl_stub.create_default_context = _noop
    sys.modules["ssl"] = ssl_stub

# --- Try NLTK, but provide graceful fallbacks ---
USE_NLTK = True
try:
    import nltk
    from nltk.tokenize import word_tokenize as _nltk_word_tokenize
    from nltk.corpus import stopwords
    from nltk.stem import WordNetLemmatizer
    # Attempt downloads if missing (ignore network errors)
    try:
        nltk.data.find("tokenizers/punkt")
    except LookupError:
        try: nltk.download("punkt", quiet=True)
        except Exception: pass
    # newer NLTK splits out language tables
    try:
        nltk.data.find("tokenizers/punkt_tab")
    except LookupError:
        try: nltk.download("punkt_tab", quiet=True)
        except Exception: pass
    try:
        nltk.data.find("corpora/stopwords")
    except LookupError:
        try: nltk.download("stopwords", quiet=True)
        except Exception: pass
    try:
        nltk.data.find("corpora/wordnet")
    except LookupError:
        try: nltk.download("wordnet", quiet=True)
        except Exception: pass
except Exception:
    USE_NLTK = False


if USE_NLTK:
    _lemmatizer = WordNetLemmatizer()

    try:
        _stop_set = set(stopwords.words("english"))
    except Exception:
        _stop_set = set("""a an and are as at be but by for if in into is it its no nor not of on or such that the their then there these they this to was will with you your i we he she from""".split())
else:
    # Fallbacks
    def word_tokenize(text):
        return re.findall(r"[A-Za-z']+|[0-9]+", text)
    _stop_set = set("""a an and are as at be but by for if in into is it its no nor not of on or such that the their then there these they this to was will with you your i we he she from""".split())
    class _DummyLem:
        def lemmatize(self, w): return w
    _lemmatizer = _DummyLem()

# --- Normalization steps ---
def remove_non_ascii(words):
    cleaned = []
    for w in words:
        nfkd = unicodedata.normalize("NFKD", w)
        only_ascii = "".join(ch for ch in nfkd if ord(ch) < 128)
        if only_ascii:
            cleaned.append(only_ascii)
    return cleaned

def to_lowercase(words):
    return [w.lower() for w in words]

_punct_tbl = str.maketrans("", "", string.punctuation)

def remove_punctuation(words):
    return [w.translate(_punct_tbl) for w in words if w.translate(_punct_tbl) != ""]

def remove_stopwords(words):
    return [w for w in words if w not in _stop_set]

def lemmatize_list(words):
    return [_lemmatizer.lemmatize(w) for w in words]


# Robust tokenizer wrapper: try NLTK, else regex fallback
def safe_word_tokenize(text: str):
    if not USE_NLTK:
        return re.findall(r"[A-Za-z']+|[0-9]+", text)
    try:
        return _nltk_word_tokenize(text)
    except LookupError:
        # Last-ditch attempt to fetch punkt_tab/punkt; otherwise fallback
        try:
            import nltk
            nltk.download("punkt_tab", quiet=True)
            nltk.download("punkt", quiet=True)
            return _nltk_word_tokenize(text)
        except Exception:
            return re.findall(r"[A-Za-z']+|[0-9]+", text)

def normalize_trace(text: str):

    trace = []

    tokens = safe_word_tokenize(text or "")
    trace.append(("1) Tokenize", " ".join(tokens)))

    step = remove_non_ascii(tokens)
    trace.append(("2) Remove non‑ASCII", " ".join(step)))

    step = to_lowercase(step)
    trace.append(("3) Lowercase", " ".join(step)))

    step = remove_punctuation(step)
    trace.append(("4) Remove punctuation", " ".join(step)))

    step = remove_stopwords(step)
    trace.append(("5) Remove stopwords", " ".join(step)))

    step = lemmatize_list(step)
    trace.append(("6) Lemmatize", " ".join(step)))

    final_text = " ".join(step)
    return final_text, trace

# --- Gradio UI ---
import gradio as gr

EXAMPLES = [
    "NLTK’s Punkt tokenizer handles abbreviations (e.g., Dr., U.S.A.) better than simple splits!",
    "The QUICK brown foxes were jumping over 13 lazy dogs in 2025...",
    "Text normalization cleans, standardizes, and simplifies raw text for NLP tasks.",
    "Hello—world! Café naïve façade coöperate résumé.",
]

with gr.Blocks(title="Gradio Text Normalizer", theme=gr.themes.Soft()) as demo:
    gr.Markdown("# 🧹 Text Normalizer (Step‑by‑Step)\nPaste text or pick an example, then click **Normalize** to see each step of the pipeline.")
    with gr.Row():
        with gr.Column():
            inp = gr.Textbox(label="Input text", lines=6, placeholder="Type or paste text here")
            ex = gr.Dropdown(EXAMPLES, label="Examples (optional)")
            def _use_example(x): return x
            ex.change(_use_example, ex, inp)
            btn = gr.Button("Normalize", variant="primary")
        with gr.Column():
            out = gr.Textbox(label="Final normalized text", lines=3)
            steps = gr.Dataframe(
                headers=["Step", "Output"],
                datatype=["str", "str"],
                row_count=(6, "fixed"),
                col_count=(2, "fixed"),
                label="Step-by-step trace",
            )

    def _run(text):
        final, trace = normalize_trace(text)
        # Convert trace list of tuples to rows for Dataframe
        rows = [[s, o] for (s, o) in trace]
        return final, rows

    btn.click(_run, inputs=inp, outputs=[out, steps])

    gr.Markdown(
        "### Notes\n"
        "- Uses NLTK when available; falls back to light tokenization/lemmatization otherwise.\n"
        "- Safe SSL stub included for environments missing the `ssl` module.\n"
        "- Pipeline: tokenize → remove non‑ASCII → lowercase → remove punctuation → remove stopwords → lemmatize."
    )

# Avoid deprecated/argued signatures; simple launch for Spaces/Local
if __name__ == '__main__':
    demo.launch()