Create app.py

app.py

@@ -0,0 +1,138 @@
+import gradio as gr
+import tiktoken
+from transformers import AutoTokenizer
+import os
+
+# Model mappings
+MODEL_MAP = {
+    'llama-2': 'meta-llama/Llama-2-7b-hf',
+    'llama-3': 'meta-llama/Meta-Llama-3-8B', 
+    'gemma-2': 'google/gemma-2-2b',
+    'qwen3': 'Qwen/Qwen2.5-0.5B',
+    'bert': 'bert-base-uncased'
+}
+
+def tokenize_with_tiktoken(text, model):
+    encoding = 'cl100k_base' if model == 'gpt-4' else 'gpt2'
+    enc = tiktoken.get_encoding(encoding)
+    tokens = enc.encode(text)
+    token_texts = [enc.decode([token]) for token in tokens]
+    
+    return {
+        'model': f'GPT-4' if model == 'gpt-4' else 'GPT-2',
+        'token_count': len(tokens),
+        'tokens': token_texts,
+        'token_ids': tokens.tolist()
+    }
+
+def tokenize_with_hf(text, model):
+    try:
+        model_name = MODEL_MAP.get(model, 'gpt2')
+        tokenizer = AutoTokenizer.from_pretrained(model_name, token=os.getenv('HF_TOKEN'))
+        
+        tokens = tokenizer.encode(text)
+        token_texts = [tokenizer.decode([token], skip_special_tokens=False) for token in tokens]
+        
+        return {
+            'model': model.upper(),
+            'token_count': len(tokens),
+            'tokens': token_texts,
+            'token_ids': tokens
+        }
+    except Exception as e:
+        return {
+            'model': model.upper(),
+            'token_count': 0,
+            'tokens': [f"Error: {str(e)}"],
+            'token_ids': []
+        }
+
+def compare_tokenizers(text, selected_models):
+    if not text.strip():
+        return "Please enter some text to tokenize."
+    
+    results = []
+    
+    for model in selected_models:
+        if model in ['gpt-4', 'gpt-2']:
+            result = tokenize_with_tiktoken(text, model)
+        else:
+            result = tokenize_with_hf(text, model)
+        
+        # Format output
+        tokens_display = ' | '.join([f'"{token}"' if token.strip() else '"·"' for token in result['tokens'][:20]])
+        if len(result['tokens']) > 20:
+            tokens_display += f" ... (+{len(result['tokens']) - 20} more)"
+            
+        results.append(f"""
+**{result['model']}**
+- Token Count: **{result['token_count']}**
+- Tokens: {tokens_display}
+- Token IDs: {str(result['token_ids'][:10])}{'...' if len(result['token_ids']) > 10 else ''}
+        """)
+    
+    return "\n\n---\n".join(results)
+
+# Create Gradio interface
+with gr.Blocks(
+    title="🔤 Tokenizer Comparison Tool",
+    theme=gr.themes.Soft()
+) as demo:
+    
+    gr.Markdown("""
+    # 🔤 Tokenizer Comparison Tool
+    
+    Compare how different LLM tokenizers split text into tokens. See the differences between GPT, LLaMA, Gemma, and other models.
+    """)
+    
+    with gr.Row():
+        with gr.Column(scale=2):
+            text_input = gr.Textbox(
+                label="Text to tokenize",
+                placeholder="Hello world! This is a test with some subwords and punctuation.",
+                lines=4,
+                value="Hello world! This is a test with some subwords and punctuation."
+            )
+            
+        with gr.Column(scale=1):
+            model_selector = gr.CheckboxGroup(
+                choices=['gpt-4', 'gpt-2', 'llama-2', 'llama-3', 'gemma-2', 'qwen3', 'bert'],
+                value=['gpt-4', 'llama-3', 'gpt-2'],
+                label="Select tokenizers to compare"
+            )
+    
+    output = gr.Markdown(
+        label="Tokenization Results",
+        value="Enter text above to see tokenization results..."
+    )
+    
+    # Auto-update on text or model change
+    text_input.change(
+        fn=compare_tokenizers,
+        inputs=[text_input, model_selector],
+        outputs=output
+    )
+    
+    model_selector.change(
+        fn=compare_tokenizers,
+        inputs=[text_input, model_selector], 
+        outputs=output
+    )
+    
+    gr.Markdown("""
+    ### Legend:
+    - **Token Count**: Number of tokens the model uses
+    - **Tokens**: The actual text pieces (subwords)
+    - **Token IDs**: Numerical IDs in the vocabulary
+    - **"·"**: Represents spaces/whitespace
+    
+    ### Models:
+    - **GPT-4/GPT-2**: OpenAI tokenizers (tiktoken)
+    - **LLaMA**: Meta's models (SentencePiece)
+    - **Gemma**: Google's models
+    - **Qwen**: Alibaba's models
+    - **BERT**: Google's BERT tokenizer
+    """)
+
+if __name__ == "__main__":
+    demo.launch()