Create app.py

app.py

@@ -0,0 +1,163 @@
+import sys
+from pathlib import Path
+import html
+
+import gradio as gr
+from transformers import PreTrainedTokenizerFast
+
+
+def load_tokenizer():
+    tok_path = Path("kn_bpe/kn_bpe_8000.json")
+    if not tok_path.exists():
+        return None, f"Tokenizer not found at {tok_path}. Run build_kn_bpe.py first."
+    tok = PreTrainedTokenizerFast(
+        tokenizer_file=str(tok_path),
+        unk_token="[UNK]",
+        pad_token="[PAD]",
+        bos_token="[BOS]",
+        eos_token="[EOS]",
+        sep_token="[SEP]",
+        mask_token="[MASK]",
+    )
+    return tok, None
+
+
+tokenizer, load_err = load_tokenizer()
+
+
+def _colored_tokens_html(tokens):
+    if not tokens:
+        return ""
+    palette = [
+        "#e57373", "#64b5f6", "#81c784", "#ffd54f", "#ba68c8",
+        "#4db6ac", "#ff8a65", "#9575cd", "#4fc3f7", "#aed581",
+    ]
+    spans = []
+    for i, t in enumerate(tokens):
+        color = palette[i % len(palette)]
+        safe = html.escape(t)
+        spans.append(
+            f'<span style="background:{color};padding:2px 4px;border-radius:4px;margin-right:2px;display:inline-block">{safe}</span>'
+        )
+    return "".join(spans)
+
+
+def tokenize_single(text: str):
+    if load_err:
+        return "", "", "", "", "", load_err
+    if not text:
+        return [], [], "", {"chars": 0, "tokens": 0, "chars_per_token": 0.0}, "", ""
+    enc = tokenizer(text)
+    tokens = enc.tokens()
+    ids = enc["input_ids"]
+    decoded = tokenizer.decode(ids, skip_special_tokens=True)
+    chars = len(text)
+    tok_count = len(ids)
+    cpt = round(chars / max(tok_count, 1), 3)
+    stats = {"chars": chars, "tokens": tok_count, "chars_per_token": cpt}
+    colored_html = _colored_tokens_html(tokens)
+    return tokens, ids, decoded, stats, colored_html, ""
+
+
+def tokenize_batch(multiline_text: str):
+    if load_err:
+        return "", load_err
+    lines = [ln for ln in multiline_text.splitlines() if ln.strip()]
+    if not lines:
+        return {"input_ids": [], "attention_mask": []}, ""
+    batch = tokenizer(lines, padding=True, truncation=True, max_length=256)
+    # Return compact preview
+    preview = {
+        "num_examples": len(lines),
+        "seq_len": len(batch["input_ids"][0]) if batch["input_ids"] else 0,
+        "sample_input_ids": batch["input_ids"][0][:32] if batch["input_ids"] else [],
+        "sample_attention_mask": batch["attention_mask"][0][:32] if batch["attention_mask"] else [],
+    }
+    return preview, ""
+
+
+def _colored_ids_html(ids):
+    if not ids:
+        return ""
+    palette = [
+        "#e57373", "#64b5f6", "#81c784", "#ffd54f", "#ba68c8",
+        "#4db6ac", "#ff8a65", "#9575cd", "#4fc3f7", "#aed581",
+    ]
+    spans = []
+    for i, idv in enumerate(ids):
+        color = palette[i % len(palette)]
+        safe = html.escape(str(idv))
+        spans.append(
+            f'<span style="background:{color};padding:2px 6px;border-radius:4px;margin-right:2px;display:inline-block">{safe}</span>'
+        )
+    return "".join(spans)
+
+
+def playground_render(text: str, view: str):
+    if load_err:
+        return "0", "0", "0.0", "<em>Tokenizer not loaded</em>"
+    text = text or ""
+    enc = tokenizer(text)
+    tokens = enc.tokens()
+    ids = enc["input_ids"]
+    chars = len(text)
+    toks = len(ids)
+    ratio = round(chars / max(toks, 1), 3)
+    if view == "Text":
+        body = _colored_tokens_html(tokens)
+    elif view == "Token IDs":
+        body = _colored_ids_html(ids)
+    else:
+        body = ""
+    return str(toks), str(chars), str(ratio), body
+
+
+with gr.Blocks(title="The Tokenizer Playground") as app:
+    gr.Markdown("# The Tokenizer Playground")
+    gr.Markdown("Experiment with different tokenizers (running locally in your browser).")
+    if load_err:
+        gr.Markdown(f"**Error:** {load_err}")
+
+    inp = gr.Textbox(lines=10, placeholder="Enter Kannada text here…", show_label=False)
+
+    # Stats row
+    with gr.Row():
+        tokens_count = gr.HTML("<div style='text-align:center'><div>TOKENS</div><div style='font-size:36px;font-weight:700'>0</div></div>")
+        chars_count = gr.HTML("<div style='text-align:center'><div>CHARACTERS</div><div style='font-size:36px;font-weight:700'>0</div></div>")
+        ratio_count = gr.HTML("<div style='text-align:center'><div>COMPRESSION</div><div style='font-size:36px;font-weight:700'>0.0</div></div>")
+
+    view = gr.Radio(["Text", "Token IDs", "Hide"], value="Text", label=None)
+    viz = gr.HTML("")
+
+    def _update(text, mode):
+        tks, chs, ratio, body = playground_render(text, mode)
+        tokens_html = f"<div style='text-align:center'><div>TOKENS</div><div style='font-size:36px;font-weight:700'>{tks}</div></div>"
+        chars_html = f"<div style='text-align:center'><div>CHARACTERS</div><div style='font-size:36px;font-weight:700'>{chs}</div></div>"
+        ratio_html = f"<div style='text-align:center'><div>COMPRESSION</div><div style='font-size:36px;font-weight:700'>{ratio}</div></div>"
+        return tokens_html, chars_html, ratio_html, body
+
+    inp.change(_update, inputs=[inp, view], outputs=[tokens_count, chars_count, ratio_count, viz])
+    view.change(_update, inputs=[inp, view], outputs=[tokens_count, chars_count, ratio_count, viz])
+
+    # Examples
+    examples = [
+        ["೧೯೫೦ರಲ್ಲಿ ಸ್ವಾಮಿ ಭಾರತಕ್ಕೆ ಹಿಂದಿರುಗಿದರು.", "Text"],
+        ["‘ಸ್ವಾಮಿಯಾನ’ ಪುಸ್ತಕದಲ್ಲಿ ಹಲವಾರು ಕ್ಷೇತ್ರದ ಗಣ್ಯರು ಸ್ವಾಮಿಯವರ ಬಗ್ಗೆ ಬರೆದ ಲೇಖನಗಳಿವೆ.", "Text"],
+        ["ವೇದ, ವೇದಾಂತ, ಮೀಮಾಂಸೆ, ಶಾಸ್ತ್ರ, ಆಗಮಶಾಸ್ತ್ರ, ಜ್ಯೋತಿಷ್ಯಶಾಸ್ತ್ರ, ಶಿಲ್ಪಶಾಸ್ತ್ರ, ಸಂಗೀತ ಶಾಸ್ತ್ರ, ಹಾಗೂ ಆಯುರ್ವೇದ ಶಾಸ್ತ್ರ ಮುಂತಾದ ಶಾಸ್ತ್ರಗಳನ್ನು ಆಳವಾಗಿ ಅಭ್ಯಸಿಸಿ ಅವುಗಳಲ್ಲಿ ಮೇರು-ಪಾಂಡಿತ್ಯವನ್ನು ಸಂಪಾದಿಸಿದ್ದರು.", "Text"],
+    ]
+    gr.Examples(
+        examples=examples,
+        inputs=[inp, view],
+        outputs=[tokens_count, chars_count, ratio_count, viz],
+        fn=_update,
+        cache_examples=False,
+    )
+
+if __name__ == "__main__":
+    try:
+        app.launch(server_name="127.0.0.1", server_port=7860)
+    except Exception:
+        # Fallback: default launch
+        app.launch()
+
+