Slides 😁

app.py

@@ -4,7 +4,9 @@ import gradio as gr
 from datasets import load_dataset
 from transformers import AutoTokenizer
 
-def ReturnTokens(dataset_name, tokenizer_name="openai-community/gpt2", split="train"):
+def ReturnTokens(dataset_name, tokenizer_name="openai-community/gpt2", split="train", progress=gr.Progress()):
+
+    progress(0, desc="Starting")
 
     # Initialize tokenizer
     tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
@@ -12,19 +14,14 @@ def ReturnTokens(dataset_name, tokenizer_name="openai-community/gpt2", split="tr
     # Load dataset
     dataset = load_dataset(dataset_name, split=split)
     
-    # Function to count tokens in a single example
-    def count_tokens_in_example(example):
-        total_tokens = 0
-        tokenized = tokenizer.batch_encode_plus(example)
-        for i in tokenized:
-            total_tokens+=len(i)
-        return total_tokens
-    
     tokens_=0
 
     for field in dataset[0].keys():
-        tokens_+=count_tokens_in_example(dataset[field])
-    
+        _all=dataset[field]
+
+        for i in progress.tqdm(_all, desc=f"Tokenizing \"{field}\""):
+            tokens_+=len(tokenizer.tokenize(i))
+
     return tokens_
 
 with gr.Blocks(title="Dataset token counter") as app: