Update pages/15_Plus_Detokenizer.py

pages/15_Plus_Detokenizer.py

@@ -122,20 +122,7 @@ components.html(html_content, height=700, scrolling=True)
 # Load the tokenizer
 tokenizer = AutoTokenizer.from_pretrained('gpt2')
 
-# Tokenization section
-st.header("Tokenization")
-sentence = st.text_input("Enter a sentence to tokenize:", "cr8 lg cnvs html js hlds 9 wbs")
 
-def format_token_ids(token_ids):
-    formatted_ids = [str(token_id).zfill(5) for token_id in token_ids]
-    return ''.join(formatted_ids)
-
-if st.button("Tokenize"):
-    input_ids = tokenizer(sentence, return_tensors='pt').input_ids
-    token_ids_list = input_ids[0].tolist()
-    formatted_token_ids = format_token_ids(token_ids_list)
-    st.write("Tokenized input IDs (formatted):")
-    st.write(formatted_token_ids)
 
 # Detokenization section
 st.header("Detokenization")
@@ -158,6 +145,21 @@ if st.button("Detokenize"):
     st.write("Detokenized sentence:")
     st.write(detokenized_sentence)
 
+# Tokenization section
+st.header("Tokenization")
+sentence = st.text_input("Enter a sentence to tokenize:", "cr8 lg")
+
+def format_token_ids(token_ids):
+    formatted_ids = [str(token_id).zfill(5) for token_id in token_ids]
+    return ''.join(formatted_ids)
+
+if st.button("Tokenize"):
+    input_ids = tokenizer(sentence, return_tensors='pt').input_ids
+    token_ids_list = input_ids[0].tolist()
+    formatted_token_ids = format_token_ids(token_ids_list)
+    st.write("Tokenized input IDs (formatted):")
+    st.write(formatted_token_ids)
+
 # Load the model
 gpt2 = AutoModelForCausalLM.from_pretrained('gpt2')