Update src/streamlit_app.py

src/streamlit_app.py

@@ -1,66 +1,89 @@
 import streamlit as st
 import numpy as np
 import tiktoken
-# import openai
 import os
-
 from openai import OpenAI
+from dotenv import load_dotenv
 
 # Setup
 st.set_page_config(page_title="LLM Token Explorer", layout="centered")
-st.title("LLM Token & Embedding Explorer")
+st.title("🧠 LLM Token & Embedding Explorer")
 
-# OpenAI key from environment
-# openai.api_key = os.getenv("OPENAI_API_KEY")
-
-from dotenv import load_dotenv
 load_dotenv()
+client = OpenAI()  # Automatically uses OPENAI_API_KEY from .env
 
-client = OpenAI()  # Uses OPENAI_API_KEY from env automatically
-
-
-
-# Input text
+# ---------- Input Section ----------
+st.header("✍️ Input Text")
+st.markdown("Enter any short sentence or phrase you'd like to explore. We'll break it down into tokens and explore their structure and meaning.")
 input_text = st.text_area("Enter your text:", height=150)
 
-# Tokenizer selection
+# ---------- Tokenizer Selection ----------
+st.header("🔧 Tokenizer Choice")
+st.markdown("Choose a tokenizer from the available ones in `tiktoken`. Different models use different tokenization strategies.")
 tokenizer_name = st.selectbox("Choose tokenizer:", ["cl100k_base", "p50k_base", "r50k_base", "gpt2"])
 
 if input_text:
-    # Tokenization
-    if st.button("General Token Info"):
+    # ---------- Tokenization Info ----------
+    st.subheader("🔤 Token Information")
+    st.markdown("This shows how your input text is broken down into tokens. Each token is a subword unit that the model processes individually.")
+    
+    if st.button("🔍 Show Token Details"):
         enc = tiktoken.get_encoding(tokenizer_name)
         tokens = enc.encode(input_text)
         token_strings = [enc.decode([t]) for t in tokens]
 
-        with st.expander("Token IDs"):
+        with st.expander("🧾 Token IDs"):
             st.write(tokens)
-        with st.expander("Decoded Tokens"):
+
+        with st.expander("📖 Decoded Tokens"):
             st.write(token_strings)
+
         st.info(f"Token count: {len(tokens)}")
 
-    # OpenAI Embedding
-    if st.button("Generate Embedding using OpenAI"):
-        with st.spinner("Calling OpenAI..."):
+    # ---------- Embedding Section ----------
+    st.subheader("🔗 Token Embeddings (OpenAI)")
+    st.markdown("""
+    Each token is mapped to a high-dimensional vector called an **embedding**. These vectors capture the contextual meaning of words and are the foundation of how language models understand text.
+    
+    We use the `text-embedding-ada-002` model from OpenAI to generate embeddings for each token.
+    """)
+    
+    if st.button("📡 Generate Embeddings"):
+        with st.spinner("Generating embedding for each token..."):
             try:
-                response = client.embeddings.create(
-                    input=[input_text],
-                    model="text-embedding-ada-002"
-                )
-                embedding = response.data[0].embedding
-
-                with st.expander("Embedding Vector"):
-                    st.write(embedding)
-                    st.info(f"Embedding dimension: {len(embedding)}")
+                enc = tiktoken.get_encoding(tokenizer_name)
+                tokens = enc.encode(input_text)
+                token_strings = [enc.decode([t]) for t in tokens]
+
+                for i, token_text in enumerate(token_strings):
+                    response = client.embeddings.create(
+                        input=[token_text],
+                        model="text-embedding-ada-002"
+                    )
+                    embedding = response.data[0].embedding
+
+                    with st.expander(f"🔸 Token {i+1}: '{token_text}'"):
+                        st.write(embedding)
+                        st.caption(f"Embedding dimension: {len(embedding)}")
+
+                st.success(f"Successfully generated embeddings for {len(token_strings)} tokens.")
+
             except Exception as e:
                 st.error(f"OpenAI Error: {str(e)}")
 
-    # Positional Encoding
-    if st.button("Generate Positional Encoding"):
+    # ---------- Positional Encoding Section ----------
+    st.subheader("📍 Positional Encoding")
+    st.markdown("""
+    Transformers have no built-in notion of order, so **positional encoding** adds a signal to each token to tell the model where it occurs in the sequence.
+    
+    We use sinusoidal positional encoding similar to what was introduced in the original Transformer paper.
+    """)
+    
+    if st.button("🌀 Generate Positional Encoding"):
         enc = tiktoken.get_encoding(tokenizer_name)
         tokens = enc.encode(input_text)
         seq_len = len(tokens)
-        dim = st.slider("Encoding dimension:", 16, 512, 64, step=16)
+        dim = st.slider("Select positional encoding dimension:", 16, 512, 64, step=16)
 
         def get_positional_encoding(seq_len, dim):
             PE = np.zeros((seq_len, dim))
@@ -73,6 +96,7 @@ if input_text:
             return PE
 
         PE = get_positional_encoding(seq_len, dim)
-        with st.expander("Positional Encoding Matrix"):
+
+        with st.expander("📐 Positional Encoding Matrix"):
             st.write(PE)
-            st.info(f"Shape: {PE.shape}")
+            st.caption(f"Shape: {PE.shape}")