Update app.py

app.py

@@ -28,10 +28,11 @@ def encode_text(text: str):
     """Basic encode: returns tokens + ids."""
     enc = tokenizer(text, add_special_tokens=False)
     tokens = tokenizer.convert_ids_to_tokens(enc["input_ids"])
-    return tokens, enc["input_ids"]
+    # return tokens, enc["input_ids"]
+    csv_ids = ",".join(str(x) for x in enc["input_ids"])
+    return tokens, csv_ids
 
 def encode_plus(text: str):
-    """HF encode_plus with attention mask etc."""
     enc = tokenizer(
         text,
         truncation=False,
@@ -39,8 +40,10 @@ def encode_plus(text: str):
         return_offsets_mapping=True,
         add_special_tokens=True
     )
+    enc["input_ids_csv"] = ",".join(str(x) for x in enc["input_ids"])
     return enc
 
+
 def decode_ids(ids: str):
     """Decode from comma-separated IDs to text."""
     try:
@@ -56,11 +59,11 @@ def batch_encode(text_list):
     out = []
     for i, ids in enumerate(enc["input_ids"]):
         toks = tokenizer.convert_ids_to_tokens(ids)
-        out.append({
-            "input": lines[i],
-            "tokens": toks,
-            "ids": ids
-        })
+    out.append({
+        "input": lines[i],
+        "tokens": toks,
+        "ids_csv": ",".join(str(x) for x in ids)
+    })
     return out
 
 # --------------------------------------
@@ -107,7 +110,7 @@ with gr.Blocks(title="Hindi Tokenizer") as demo:
     with gr.Tab("Encode"):
         text_in = gr.Textbox(label="Enter text")
         tokens_out = gr.JSON(label="Tokens")
-        ids_out = gr.JSON(label="Token IDs")
+        ids_out = gr.Textbox(label="Token IDs (CSV)")
         btn = gr.Button("Encode")
         btn.click(encode_text, text_in, [tokens_out, ids_out])
 
@@ -128,10 +131,15 @@ with gr.Blocks(title="Hindi Tokenizer") as demo:
             label="Enter multiple lines (newline separated)",
             placeholder="Line 1\nLine 2\nLine 3"
         )
-        batch_out = gr.JSON(label="Batch output list")
+        batch_out = gr.JSON(label="Batch output (CSV per line)")
+
         btn4 = gr.Button("Batch Encode")
         btn4.click(batch_encode, batch_in, batch_out)
 
 # Mount FastAPI + Gradio
+
 if "app" not in globals():
-    app = gr.mount_gradio_app(api, demo, path="/gradio")
+    app = gr.mount_gradio_app(api, demo, path="/gradio")
+
+if __name__ == "__main__":
+    demo.launch(server_port=7860, share=False)