Create app.py

app.py

@@ -0,0 +1,80 @@
+### app.py
+import streamlit as st
+from transformers import BertTokenizer, BertModel
+import torch
+import pandas as pd
+
+# Load BERT tokenizer and model
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+model = BertModel.from_pretrained('bert-base-uncased')
+
+# Streamlit app setup
+st.title("✨ BERT Token Analyzer 🧠")
+st.write("🔍 This application uses **BERT** to tokenize and encode input text, providing embeddings and token details.")
+st.markdown("---")
+
+# Input field
+user_input = st.text_input("📝 Enter a word or sentence:", "")
+
+if user_input:
+    # Tokenize input
+    st.write("⏳ Tokenizing and encoding input... 🛠️")
+    inputs = tokenizer(user_input, return_tensors="pt", add_special_tokens=True)
+    tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'][0])
+
+    # Get embeddings
+    with torch.no_grad():
+        outputs = model(**inputs)
+    embeddings = outputs.last_hidden_state.squeeze(0)
+
+    # Prepare DataFrame for display
+    token_data = []
+    for i, token in enumerate(tokens):
+        token_data.append({
+            "Token": token,
+            "Token ID": inputs['input_ids'][0][i].item(),
+            "Embedding (first 5 dims)": embeddings[i][:5].tolist()
+        })
+
+    df = pd.DataFrame(token_data)
+
+    # Display token data
+    st.write("### 🧾 Token Details 📜")
+    st.dataframe(df)
+
+    # Option to download the DataFrame as CSV
+    csv = df.to_csv(index=False)
+    st.download_button(
+        label="⬇️ Download Token Data as CSV",
+        data=csv,
+        file_name="token_data.csv",
+        mime="text/csv"
+    )
+
+    # Additional statistics and details
+    st.write("### 📊 Token Statistics 📈")
+    st.markdown(f"- **Number of Tokens:** {len(tokens)}")
+    st.markdown(f"- **Unique Tokens:** {len(set(tokens))}")
+    st.markdown(f"- **Longest Token:** `{max(tokens, key=len)}` ({len(max(tokens, key=len))} characters)")
+    st.markdown(f"- **Shortest Token:** `{min(tokens, key=len)}` ({len(min(tokens, key=len))} characters)")
+
+    st.write("### 🔍 Embedding Analysis 🌌")
+    embedding_magnitudes = embeddings.norm(dim=1).tolist()
+    st.markdown(f"- **Average Embedding Magnitude:** {sum(embedding_magnitudes)/len(embedding_magnitudes):.4f}")
+    st.markdown(f"- **Max Embedding Magnitude:** {max(embedding_magnitudes):.4f}")
+    st.markdown(f"- **Min Embedding Magnitude:** {min(embedding_magnitudes):.4f}")
+
+    st.write("### 🛠 Embedding Tensor Details")
+    st.write("**Shape:**", embeddings.shape)
+    st.write(embeddings)
+
+    # Display tokens and embeddings in Markdown format
+    st.write("### 📝 Token and Embedding Summary")
+    for i, token in enumerate(tokens):
+        st.markdown(f"- **Token {i+1}:** `{token}`")
+        st.markdown(f"  - **Token ID:** {inputs['input_ids'][0][i].item()}")
+        st.markdown(f"  - **Embedding (first 5 dims):** {embeddings[i][:5].tolist()}")
+
+st.markdown("---")
+st.write("👨‍💻 **Replika AI Solutions** - Powered by **Gemini** 🪐")
+st.write("📍 Developed by *Elias Andrade* - Maringá, Paraná 🇧🇷")