first commit

app.py

@@ -0,0 +1,115 @@
+import gradio as gr
+import numpy as np
+import pickle
+from typing import List, Dict, Tuple
+
+class OptimizedBPETokenizer:
+    def __init__(self, merges: Dict[Tuple[int, int], int]):
+        self.merges = merges
+        self.idx_to_pair = {idx: pair for pair, idx in merges.items()}
+        
+        # Create lookup table for faster encoding
+        self.merge_lookup = {}
+        for (first, second), idx in merges.items():
+            if first not in self.merge_lookup:
+                self.merge_lookup[first] = {}
+            self.merge_lookup[first][second] = idx
+    
+    def encode(self, text: str, chunk_size: int = 1000000) -> List[int]:
+        if not isinstance(text, str):
+            return []
+        
+        ids = np.array(list(text.encode('utf-8')), dtype=np.uint16)
+        
+        result = []
+        for i in range(0, len(ids), chunk_size):
+            chunk = ids[i:i + chunk_size]
+            processed_chunk = self._encode_chunk(chunk)
+            result.extend(processed_chunk)
+            
+        return result
+    
+    def _encode_chunk(self, ids: np.ndarray) -> List[int]:
+        output = []
+        i = 0
+        while i < len(ids):
+            if i < len(ids) - 1:
+                first, second = ids[i], ids[i + 1]
+                if first in self.merge_lookup and second in self.merge_lookup[first]:
+                    output.append(self.merge_lookup[first][second])
+                    i += 2
+                    continue
+            output.append(ids[i])
+            i += 1
+        return output
+    
+    def decode(self, ids: List[int], chunk_size: int = 1000000) -> str:
+        byte_tokens = []
+        for i in range(0, len(ids), chunk_size):
+            chunk = ids[i:i + chunk_size]
+            decoded_chunk = self._decode_chunk(chunk)
+            byte_tokens.extend(decoded_chunk)
+            
+        return bytes(byte_tokens).decode('utf-8')
+    
+    def _decode_chunk(self, ids: List[int]) -> List[int]:
+        result = []
+        for token in ids:
+            if token < 256:
+                result.append(token)
+            else:
+                result.extend(self._expand_token(token))
+        return result
+    
+    def _expand_token(self, token: int) -> List[int]:
+        if token < 256:
+            return [token]
+            
+        pair = self.idx_to_pair[token]
+        expanded = []
+        for t in pair:
+            expanded.extend(self._expand_token(t))
+        return expanded
+
+# Load the BPE merges
+def load_tokenizer():
+    try:
+        with open("bpe_vocab_350_merges.pkl", "rb") as f:
+            merges = pickle.load(f)
+        return OptimizedBPETokenizer(merges)
+    except FileNotFoundError:
+        raise Exception("Tokenizer merges file not found!")
+
+tokenizer = load_tokenizer()
+
+def process_text(text, mode):
+    if mode == "Encode":
+        encoded = tokenizer.encode(text)
+        return f"Encoded tokens: {encoded}\nToken count: {len(encoded)}"
+    else:  # Decode
+        try:
+            # Convert string of numbers to list of integers
+            tokens = [int(t) for t in text.strip('[]').split(',')]
+            decoded = tokenizer.decode(tokens)
+            return decoded
+        except:
+            return "Error: Please provide tokens as comma-separated numbers"
+
+# Create the interface
+iface = gr.Interface(
+    fn=process_text,
+    inputs=[
+        gr.Textbox(label="Input Text", lines=5),
+        gr.Radio(["Encode", "Decode"], label="Mode", value="Encode")
+    ],
+    outputs=gr.Textbox(label="Output", lines=5),
+    title="Telugu BPE Tokenizer",
+    description="Encode Telugu text into BPE tokens or decode tokens back to text.",
+    examples=[
+        ["నమస్కారం", "Encode"],
+        ["[224, 176, 184, 224, 176, 184]", "Decode"]
+    ]
+)
+
+if __name__ == "__main__":
+    iface.launch() 

bpe_vocab_350_merges.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d98eabc47c17d13488f9cd411089290a90f72ea2a3514ac734816fa823faa10
+size 981

requirements.txt

@@ -0,0 +1,2 @@
+gradio
+numpy