Update app.py

app.py

@@ -1,136 +1,137 @@
-import gradio as gr
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
-from transformers import PreTrainedTokenizerFast
-import os
-
-# --------------------------------------
-# LOAD TOKENIZER
-# --------------------------------------
-
-TOKENIZER_JSON = "tokenizer_hindi_bpe_8k_stream/tokenizer.json"
-HF_DIR = "tokenizer_hindi_bpe_8k_stream/hf"
-
-if os.path.exists(HF_DIR):
-    tokenizer = PreTrainedTokenizerFast.from_pretrained(HF_DIR)
-elif os.path.exists(TOKENIZER_JSON):
-    tokenizer = PreTrainedTokenizerFast(tokenizer_file=TOKENIZER_JSON)
-else:
-    raise ValueError("Tokenizer not found!")
-
-print("Tokenizer loaded: vocab =", tokenizer.vocab_size)
-
-# --------------------------------------
-# ENCODE / DECODE FUNCTIONS
-# --------------------------------------
-
-def encode_text(text: str):
-    """Basic encode: returns tokens + ids."""
-    enc = tokenizer(text, add_special_tokens=False)
-    tokens = tokenizer.convert_ids_to_tokens(enc["input_ids"])
-    return tokens, enc["input_ids"]
-
-def encode_plus(text: str):
-    """HF encode_plus with attention mask etc."""
-    enc = tokenizer(
-        text,
-        truncation=False,
-        return_attention_mask=True,
-        return_offsets_mapping=True,
-        add_special_tokens=True
-    )
-    return enc
-
-def decode_ids(ids: str):
-    """Decode from comma-separated IDs to text."""
-    try:
-        arr = [int(x) for x in ids.split(",") if x.strip()]
-        return tokenizer.decode(arr)
-    except:
-        return "❌ Invalid ID list"
-
-def batch_encode(text_list):
-    """Batch encode multiple lines separated by newline."""
-    lines = [ln.strip() for ln in text_list.split("\n") if ln.strip()]
-    enc = tokenizer(lines, add_special_tokens=False)
-    out = []
-    for i, ids in enumerate(enc["input_ids"]):
-        toks = tokenizer.convert_ids_to_tokens(ids)
-        out.append({
-            "input": lines[i],
-            "tokens": toks,
-            "ids": ids
-        })
-    return out
-
-# --------------------------------------
-# FASTAPI REST BACKEND
-# --------------------------------------
-
-api = FastAPI(title="Hindi Tokenizer API")
-
-api.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_methods=["*"],
-    allow_headers=["*"]
-)
-
-@api.get("/")
-def home():
-    return {
-        "message": "Hindi Tokenizer API",
-        "vocab_size": tokenizer.vocab_size
-    }
-
-@api.get("/tokenize")
-def tokenize_endpoint(text: str):
-    enc = tokenizer(text, add_special_tokens=False)
-    tokens = tokenizer.convert_ids_to_tokens(enc["input_ids"])
-    return {"tokens": tokens, "ids": enc["input_ids"]}
-
-@api.get("/decode")
-def decode_endpoint(ids: str):
-    try:
-        arr = [int(x) for x in ids.split(",") if x.strip()]
-        return {"text": tokenizer.decode(arr)}
-    except:
-        return {"error": "Invalid id list"}
-
-# --------------------------------------
-# GRADIO FRONTEND
-# --------------------------------------
-
-with gr.Blocks(title="Hindi Tokenizer") as demo:
-    gr.Markdown("## 🔡 Hindi BPE Tokenizer — Encode / Decode / Batch")
-
-    with gr.Tab("Encode"):
-        text_in = gr.Textbox(label="Enter text")
-        tokens_out = gr.JSON(label="Tokens")
-        ids_out = gr.JSON(label="Token IDs")
-        btn = gr.Button("Encode")
-        btn.click(encode_text, text_in, [tokens_out, ids_out])
-
-    with gr.Tab("Encode+ (HF full)"):
-        text2_in = gr.Textbox(label="Enter text (HF encode_plus)")
-        enc_plus_out = gr.JSON(label="Output")
-        btn2 = gr.Button("Run encode_plus")
-        btn2.click(encode_plus, text2_in, enc_plus_out)
-
-    with gr.Tab("Decode"):
-        ids_in = gr.Textbox(label="Comma-separated token IDs")
-        text_out = gr.Textbox(label="Decoded text")
-        btn3 = gr.Button("Decode")
-        btn3.click(decode_ids, ids_in, text_out)
-
-    with gr.Tab("Batch Encode"):
-        batch_in = gr.Textbox(
-            label="Enter multiple lines (newline separated)",
-            placeholder="Line 1\nLine 2\nLine 3"
-        )
-        batch_out = gr.JSON(label="Batch output list")
-        btn4 = gr.Button("Batch Encode")
-        btn4.click(batch_encode, batch_in, batch_out)
-
-# Mount FastAPI + Gradio
-app = gr.mount_gradio_app(api, demo, path="/gradio")
+import gradio as gr
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from transformers import PreTrainedTokenizerFast
+import os
+
+# --------------------------------------
+# LOAD TOKENIZER
+# --------------------------------------
+
+TOKENIZER_JSON = "tokenizer_hindi_bpe_8k_stream/tokenizer.json"
+HF_DIR = "tokenizer_hindi_bpe_8k_stream/hf"
+
+if os.path.exists(HF_DIR):
+    tokenizer = PreTrainedTokenizerFast.from_pretrained(HF_DIR)
+elif os.path.exists(TOKENIZER_JSON):
+    tokenizer = PreTrainedTokenizerFast(tokenizer_file=TOKENIZER_JSON)
+else:
+    raise ValueError("Tokenizer not found!")
+
+print("Tokenizer loaded: vocab =", tokenizer.vocab_size)
+
+# --------------------------------------
+# ENCODE / DECODE FUNCTIONS
+# --------------------------------------
+
+def encode_text(text: str):
+    """Basic encode: returns tokens + ids."""
+    enc = tokenizer(text, add_special_tokens=False)
+    tokens = tokenizer.convert_ids_to_tokens(enc["input_ids"])
+    return tokens, enc["input_ids"]
+
+def encode_plus(text: str):
+    """HF encode_plus with attention mask etc."""
+    enc = tokenizer(
+        text,
+        truncation=False,
+        return_attention_mask=True,
+        return_offsets_mapping=True,
+        add_special_tokens=True
+    )
+    return enc
+
+def decode_ids(ids: str):
+    """Decode from comma-separated IDs to text."""
+    try:
+        arr = [int(x) for x in ids.split(",") if x.strip()]
+        return tokenizer.decode(arr)
+    except:
+        return "❌ Invalid ID list"
+
+def batch_encode(text_list):
+    """Batch encode multiple lines separated by newline."""
+    lines = [ln.strip() for ln in text_list.split("\n") if ln.strip()]
+    enc = tokenizer(lines, add_special_tokens=False)
+    out = []
+    for i, ids in enumerate(enc["input_ids"]):
+        toks = tokenizer.convert_ids_to_tokens(ids)
+        out.append({
+            "input": lines[i],
+            "tokens": toks,
+            "ids": ids
+        })
+    return out
+
+# --------------------------------------
+# FASTAPI REST BACKEND
+# --------------------------------------
+
+api = FastAPI(title="Hindi Tokenizer API")
+
+api.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"]
+)
+
+@api.get("/")
+def home():
+    return {
+        "message": "Hindi Tokenizer API",
+        "vocab_size": tokenizer.vocab_size
+    }
+
+@api.get("/tokenize")
+def tokenize_endpoint(text: str):
+    enc = tokenizer(text, add_special_tokens=False)
+    tokens = tokenizer.convert_ids_to_tokens(enc["input_ids"])
+    return {"tokens": tokens, "ids": enc["input_ids"]}
+
+@api.get("/decode")
+def decode_endpoint(ids: str):
+    try:
+        arr = [int(x) for x in ids.split(",") if x.strip()]
+        return {"text": tokenizer.decode(arr)}
+    except:
+        return {"error": "Invalid id list"}
+
+# --------------------------------------
+# GRADIO FRONTEND
+# --------------------------------------
+
+with gr.Blocks(title="Hindi Tokenizer") as demo:
+    gr.Markdown("## 🔡 Hindi BPE Tokenizer — Encode / Decode / Batch")
+
+    with gr.Tab("Encode"):
+        text_in = gr.Textbox(label="Enter text")
+        tokens_out = gr.JSON(label="Tokens")
+        ids_out = gr.JSON(label="Token IDs")
+        btn = gr.Button("Encode")
+        btn.click(encode_text, text_in, [tokens_out, ids_out])
+
+    with gr.Tab("Encode+ (HF full)"):
+        text2_in = gr.Textbox(label="Enter text (HF encode_plus)")
+        enc_plus_out = gr.JSON(label="Output")
+        btn2 = gr.Button("Run encode_plus")
+        btn2.click(encode_plus, text2_in, enc_plus_out)
+
+    with gr.Tab("Decode"):
+        ids_in = gr.Textbox(label="Comma-separated token IDs")
+        text_out = gr.Textbox(label="Decoded text")
+        btn3 = gr.Button("Decode")
+        btn3.click(decode_ids, ids_in, text_out)
+
+    with gr.Tab("Batch Encode"):
+        batch_in = gr.Textbox(
+            label="Enter multiple lines (newline separated)",
+            placeholder="Line 1\nLine 2\nLine 3"
+        )
+        batch_out = gr.JSON(label="Batch output list")
+        btn4 = gr.Button("Batch Encode")
+        btn4.click(batch_encode, batch_in, batch_out)
+
+# Mount FastAPI + Gradio
+if "app" not in globals():
+    app = gr.mount_gradio_app(api, demo, path="/gradio")