Update app.py

app.py

@@ -15,9 +15,9 @@ def get_distinct_colors(n):
         colors.append(color)
     return colors
 
-def tokenize_text(hf_model_id, text):
+def tokenize_text(hf_model_id, text, hf_token):
     try:
-        tokenizer = AutoTokenizer.from_pretrained(hf_model_id)
+        tokenizer = AutoTokenizer.from_pretrained(hf_model_id, token=hf_token)
         tokens = tokenizer.tokenize(text)
         token_count = len(tokens)
         colors = get_distinct_colors(token_count)
@@ -35,6 +35,7 @@ demo = gr.Interface(
     fn=tokenize_text,
     inputs=[
         gr.Textbox(label="Hugging Face Model ID", placeholder="unsloth/gemma-3-27b-it", value="unsloth/gemma-3-27b-it"),
+        gr.Textbox(label="Hugging Face Token (optional)", type="password", placeholder="hf_..."),
         gr.Textbox(label="Text to Tokenize", lines=5, placeholder="Enter your text here...")
     ],
     outputs=[
@@ -42,7 +43,7 @@ demo = gr.Interface(
         gr.HTML(label="Tokens", container=True, show_label=True)
     ],
     title="HuggingFace Tokenizer",
-    description="Enter a HuggingFace model ID and text to see how it gets tokenized.",
+    description="Enter a HuggingFace model ID and text to see how it gets tokenized. Provide a HuggingFace token if the model is gated.",
     allow_flagging="never"
 )