Update src/streamlit_app.py

src/streamlit_app.py

@@ -26,10 +26,17 @@ sentence = st.text_input("Enter your sentence", "Learning is fun")
 embedding_dim = st.slider("Embedding Dimension (even only)", min_value=4, max_value=64, value=8, step=2)
 
 # --- Load tokenizer ---
+
 # Set custom cache directory within your app's working directory (which is writable on Spaces)
 os.environ['TRANSFORMERS_CACHE'] = './hf_cache'
+
 # Load the tokenizer using the custom cache path
-tokenizer = GPT2TokenizerFast.from_pretrained("gpt2", cache_dir="./hf_cache")
+# tokenizer = GPT2TokenizerFast.from_pretrained("gpt2", cache_dir="./hf_cache")
+from transformers import GPT2TokenizerFast
+# Load tokenizer from bundled local files only
+tokenizer = GPT2TokenizerFast.from_pretrained("./assets/tokenizer", local_files_only=True)
+
+
 # tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
 input_ids = tokenizer.encode(sentence, return_tensors="pt")[0]
 tokens = tokenizer.convert_ids_to_tokens(input_ids)