Update src/streamlit_app.py

src/streamlit_app.py

@@ -1,40 +1,143 @@
-import altair as alt
-import numpy as np
-import pandas as pd
+import os
+# turn off Streamlit’s automatic file-watching
+os.environ["STREAMLIT_SERVER_ENABLE_FILE_WATCHER"] = "false"
+
+import sys
+import types
+import torch               # now safe to import
 import streamlit as st
+import numpy as np
+
+# Prevent Streamlit from trying to walk torch.classes' non-standard __path__
+if isinstance(getattr(sys.modules.get("torch"), "classes", None), types.ModuleType):
+    torch.classes.__path__ = []
+
+# pip install tiktoken transformers
+import tiktoken
+from transformers import GPT2TokenizerFast
+
+st.set_page_config(page_title="Embedding Dimension Visualizer", layout="wide")
+st.title("🔍 Embedding Dimension Visualizer")
+
+# ---- THEORY EXPANDER ----
+with st.expander("📖 Theory: Tokenization, BPE & Positional Encoding"):
+    st.markdown("""
+**1️⃣ Tokenization**  
+Splits raw text into atomic units (“tokens”).  
+
+**2️⃣ Byte-Pair Encoding (BPE)**  
+Iteratively merges the most frequent pair of symbols to build a subword vocabulary.  
+E.g.  "embedding" → ["em", "bed", "ding"]
+
+**3️⃣ Positional Encoding**  
+We add a deterministic sinusoidal vector to each token embedding so the model knows position.
+""")
+    st.markdown("For embedding dimension \(d\), position \(pos\) and channel index \(i\):")
+    st.latex(r"""\mathrm{PE}_{(pos,\,2i)}   = \sin\!\Bigl(\frac{pos}{10000^{2i/d}}\Bigr)""")
+    st.latex(r"""\mathrm{PE}_{(pos,\,2i+1)} = \cos\!\Bigl(\frac{pos}{10000^{2i/d}}\Bigr)""")
+    st.markdown("""
+- \(pos\) starts at 0 for the first token  
+- Even channels use \(\sin\), odd channels use \(\cos\)  
+- This injects unique, smoothly varying positional signals into each embedding  
+""")
+
+
+# ---- Sidebar ----
+with st.sidebar:
+    st.header("Settings")
+    input_text = st.text_input("Enter text to embed", value="Hello world!")
+    dim = st.number_input(
+        "Embedding dimensions",
+        min_value=2,
+        max_value=1536,
+        value=3,
+        step=1,
+        help="Choose 2, 3, 512, 768, 1536, etc."
+    )
+    tokenizer_choice = st.selectbox(
+        "Choose tokenizer",
+        ["tiktoken", "openai", "huggingface"],
+        help="Which tokenization scheme to demo."
+    )
+    generate = st.button("Generate / Reset Embedding")
+
+if not generate:
+    st.info("Adjust the settings in the sidebar and click **Generate / Reset Embedding** to see the tokens and sliders.")
+    st.stop()
+
+# ---- Tokenize ----
+if tokenizer_choice in ("tiktoken", "openai"):
+    model_name = "gpt2" if tokenizer_choice=="tiktoken" else "gpt-3.5-turbo"
+    enc = tiktoken.encoding_for_model(model_name)
+    token_ids = enc.encode(input_text)
+    token_strs = [enc.decode([tid]) for tid in token_ids]
+else:
+    hf_tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
+    token_ids = hf_tokenizer.encode(input_text)
+    token_strs = hf_tokenizer.convert_ids_to_tokens(token_ids)
+
+st.subheader("🪶 Tokens and IDs")
+for i, (tok, tid) in enumerate(zip(token_strs, token_ids), start=1):
+    st.write(f"**{i}.** `{tok}` → ID **{tid}**")
+
+st.write("---")
+st.subheader("📊 Embedding + Positional Encoding per Token")
+st.write(f"Input: `{input_text}` | Tokenizer: **{tokenizer_choice}** | Dims per token: **{dim}**")
+if dim > 20:
+    st.warning("Showing >20 sliders per block may be unwieldy; consider smaller dims for teaching.")
+
+# helper for sinusoidal positional encoding
+def get_positional_encoding(position: int, d_model: int) -> np.ndarray:
+    pe = np.zeros(d_model, dtype=float)
+    for i in range(d_model):
+        angle = position / np.power(10000, (2 * (i // 2)) / d_model)
+        pe[i] = np.sin(angle) if (i % 2 == 0) else np.cos(angle)
+    return pe
+
+# ---- For each token, three slider‐blocks ----
+for t_idx, tok in enumerate(token_strs, start=1):
+    emb = np.random.uniform(-1.0, 1.0, size=dim)
+    pe  = get_positional_encoding(t_idx - 1, dim)
+    combined = emb + pe
+
+    with st.expander(f"Token {t_idx}: `{tok}`"):
+        st.markdown("**1️⃣ Embedding**")
+        for d in range(dim):
+            st.slider(
+                label=f"Emb Dim {d+1}",
+                min_value=-1.0, max_value=1.0,
+                value=float(emb[d]),
+                key=f"t{t_idx}_emb{d+1}",
+                disabled=True
+            )
+
+        st.markdown("**2️⃣ Positional Encoding (sin / cos)**")
+        for d in range(dim):
+            st.slider(
+                label=f"PE Dim {d+1}",
+                min_value=-1.0, max_value=1.0,
+                value=float(pe[d]),
+                key=f"t{t_idx}_pe{d+1}",
+                disabled=True
+            )
+
+        st.markdown("**3️⃣ Embedding + Positional Encoding**")
+        for d in range(dim):
+            st.slider(
+                label=f"Sum Dim {d+1}",
+                min_value=-2.0, max_value=2.0,
+                value=float(combined[d]),
+                key=f"t{t_idx}_sum{d+1}",
+                disabled=True
+            )
+
+# ---- NEW FINAL SECTION ----
+st.write("---")
+st.subheader("Final Input Embedding Plus Positional Encoding Ready to Send to ATtention Heads")
 
-"""
-# Welcome to Streamlit!
-
-Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
-
-In the meantime, below is an example of what you can do with just a few lines of code:
-"""
-
-num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
-num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
-
-indices = np.linspace(0, 1, num_points)
-theta = 2 * np.pi * num_turns * indices
-radius = indices
-
-x = radius * np.cos(theta)
-y = radius * np.sin(theta)
-
-df = pd.DataFrame({
-    "x": x,
-    "y": y,
-    "idx": indices,
-    "rand": np.random.randn(num_points),
-})
-
-st.altair_chart(alt.Chart(df, height=700, width=700)
-    .mark_point(filled=True)
-    .encode(
-        x=alt.X("x", axis=None),
-        y=alt.Y("y", axis=None),
-        color=alt.Color("idx", legend=None, scale=alt.Scale()),
-        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
-    ))
+for t_idx, tid in enumerate(token_ids, start=1):
+    with st.expander(f"Token ID {tid}"):
+        for d in range(1, dim+1):
+            # pull the “sum” value out of session state
+            val = st.session_state.get(f"t{t_idx}_sum{d}", None)
+            st.write(f"Dim {d}: {val:.4f}" if val is not None else f"Dim {d}: N/A")