Create app.py

app.py

@@ -0,0 +1,43 @@
+
+import gradio as gr
+from transformers import AutoTokenizer
+
+# Define tokenizers for English and Arabic
+tokenizers = {
+    "English - BERT (bert-base-uncased)": "bert-base-uncased",
+    "Arabic - CAMeL BERT (bert-base-arabic-camelbert-ca)": "CAMeL-Lab/bert-base-arabic-camelbert-ca",
+    "Arabic - AraBERT (asafaya/bert-base-arabic)": "asafaya/bert-base-arabic"
+}
+
+# Tokenization function
+def tokenize_text(text,model_name):
+    # Load the selected tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(tokenizers[model_name])
+    
+    # Tokenize the input text
+    tokens = tokenizer.tokenize(text)
+    
+    # Return tokens and number of tokens
+    return f"Tokens: {tokens}", f"Number of tokens: {len(tokens)}"
+
+# Define Gradio interface components
+model_choice = gr.Dropdown(choices=list(tokenizers.keys()), label="Select Tokenizer", value="English - BERT (bert-base-uncased)")
+text_input = gr.Textbox(label="Enter a sentence to tokenize")
+
+
+# Predefined example sentences
+examples = [
+    ["The quick brown fox jumps over the lazy dog."],  # English sentence
+    ["القمر جميل في السماء."]  # Arabic sentence
+]
+
+# Set up the Gradio interface
+demo = gr.Interface(
+    fn=tokenize_text,
+    inputs=[text_input,model_choice],
+    outputs=[gr.Textbox(label="Tokens"), gr.Textbox(label="Number of Tokens")],  # Properly named outputs
+    title="Hugging Face Tokenizer Explorer",
+    description="Enter a sentence or use one of the example sentences below to see how different tokenizers work.",
+    examples=examples,  # Provide examples that apply to the text input field
+    allow_flagging=False
+)