first commit

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
-import numpy as np
 import pickle
 from typing import List, Dict, Tuple
+import numpy as np
 
 class OptimizedBPETokenizer:
     def __init__(self, merges: Dict[Tuple[int, int], int]):
@@ -71,45 +71,46 @@ class OptimizedBPETokenizer:
             expanded.extend(self._expand_token(t))
         return expanded
 
-# Load the BPE merges
-def load_tokenizer():
-    try:
-        with open("bpe_vocab_350_merges.pkl", "rb") as f:
-            merges = pickle.load(f)
-        return OptimizedBPETokenizer(merges)
-    except FileNotFoundError:
-        raise Exception("Tokenizer merges file not found!")
+# Load the pre-trained merges
+with open("bpe_vocab_350_merges.pkl", "rb") as f:
+    merges = pickle.load(f)
 
-tokenizer = load_tokenizer()
+tokenizer = OptimizedBPETokenizer(merges)
 
-def process_text(text, mode):
-    if mode == "Encode":
-        encoded = tokenizer.encode(text)
-        return f"Encoded tokens: {encoded}\nToken count: {len(encoded)}"
+def process_text(text: str, operation: str) -> str:
+    if operation == "Encode":
+        tokens = tokenizer.encode(text)
+        return f"Encoded tokens: {tokens}\nToken count: {len(tokens)}"
     else:  # Decode
         try:
             # Convert string of numbers to list of integers
-            tokens = [int(t) for t in text.strip('[]').split(',')]
-            decoded = tokenizer.decode(tokens)
-            return decoded
+            tokens = [int(x) for x in text.strip('[]').split(',')]
+            decoded_text = tokenizer.decode(tokens)
+            return f"Decoded text: {decoded_text}"
         except:
-            return "Error: Please provide tokens as comma-separated numbers"
+            return "Error: Please provide a valid list of integers for decoding"
 
-# Create the interface
+# Create the Gradio interface
 iface = gr.Interface(
     fn=process_text,
     inputs=[
-        gr.Textbox(label="Input Text", lines=5),
-        gr.Radio(["Encode", "Decode"], label="Mode", value="Encode")
+        gr.Textbox(label="Input Text", placeholder="Enter text to encode or tokens to decode..."),
+        gr.Radio(["Encode", "Decode"], label="Operation", value="Encode")
     ],
-    outputs=gr.Textbox(label="Output", lines=5),
+    outputs=gr.Textbox(label="Output"),
     title="Telugu BPE Tokenizer",
-    description="Encode Telugu text into BPE tokens or decode tokens back to text.",
-    examples=[
-        ["నమస్కారం", "Encode"],
-        ["[224, 176, 184, 224, 176, 184]", "Decode"]
-    ]
+    description="A byte-pair encoding tokenizer trained on Telugu text. For encoding, enter Telugu text. For decoding, enter a list of integers (e.g., [256, 257, 258])."
 )
 
-if __name__ == "__main__":
-    iface.launch() 
+# if __name__ == "__main__":
+#     # Test encoding
+#     test_text = "నమస్కారం"  # Telugu "Hello"
+#     encoded = tokenizer.encode(test_text)
+#     print(f"Test Encode: '{test_text}' -> {encoded}")
+    
+#     # Test decoding
+#     decoded = tokenizer.decode(encoded)
+#     print(f"Test Decode: {encoded} -> '{decoded}'")
+    
+    # Launch the interface
+iface.launch()