Update pages/15_Plus_Detokenizer.py

pages/15_Plus_Detokenizer.py

@@ -145,20 +145,6 @@ if st.button("Detokenize"):
     st.write("Detokenized sentence:")
     st.write(detokenized_sentence)
 
-# Tokenization section
-st.header("Tokenization")
-sentence = st.text_input("Enter a sentence to tokenize:", "cr8 lg")
-
-def format_token_ids(token_ids):
-    formatted_ids = [str(token_id).zfill(5) for token_id in token_ids]
-    return ''.join(formatted_ids)
-
-if st.button("Tokenize"):
-    input_ids = tokenizer(sentence, return_tensors='pt').input_ids
-    token_ids_list = input_ids[0].tolist()
-    formatted_token_ids = format_token_ids(token_ids_list)
-    st.write("Tokenized input IDs (formatted):")
-    st.write(formatted_token_ids)
 
 # Load the model
 gpt2 = AutoModelForCausalLM.from_pretrained('gpt2')