a working version using tiktoken

app.py

@@ -1,4 +1,83 @@
 import streamlit as st
+import tiktoken
+from utils.metrics import calculate_compression
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+st.set_page_config(
+    page_title="Malayalam Tokenizer",
+    page_icon="🔤",
+    layout="wide"
+)
+
+st.title("Malayalam Tokenizer 🔤")
+st.markdown("""
+A tool to analyze text tokenization using OpenAI's tiktoken tokenizer. 
+This helps understand how text is processed for language models.
+""")
+
+
+default_text = """ഇതൊരു Malayalam ടോക്കനൈസർ ആണ് 🪧. 
+വാചകത്തെ ടോക്കണുകൾ എന്ന് വിളിക്കുന്ന ചെറിയ കഷണങ്ങളായി വിഭജിക്കുന്ന ഒരു method aanu ടോക്കനൈസർ. 
+ഈ ടോക്കണുകൾ വാക്കുകളോ വാക്കുകളുടെ ഭാഗങ്ങളോ പ്രതീകങ്ങളോ ആകാം. 
+ടെക്‌സ്‌റ്റ് കൂടുതൽ കാര്യക്ഷമമായി മനസ്സിലാക്കാനും പ്രോസസ്സ് ചെയ്യാനും ഇത് language modelukale സഹായിക്കുന്നു.🎂"""
+
+# Text input
+text = st.text_area(
+    "Enter your text:",
+    value=default_text,
+    height=150,
+    help="Enter any text you want to analyze. The text will be tokenized using tiktoken's cl100k_base encoding."
+)
+
+if text:
+    col1, col2, col3 = st.columns(3)
+    
+    unicode_bytes = text.encode("utf-8")
+    enc = tiktoken.get_encoding("cl100k_base")
+    tokens = enc.encode(text)
+    
+    with col1:
+        st.metric("Raw Text Length", f"{len(text)} chars")
+    
+    with col2:
+        st.metric("UTF-8 Encoded Length", f"{len(unicode_bytes)} bytes")
+    
+    with col3:
+        st.metric("Token Count", f"{len(tokens)} tokens")
+    
+    compression = calculate_compression(text, tokens)
+    st.markdown("### Compression Analysis")
+    
+    comp_col1, comp_col2 = st.columns(2)
+    with comp_col1:
+        st.metric(
+            "Compression Ratio",
+            f"{compression['compression_ratio']:.2f}x",
+            help="Higher ratio means better compression. Shows how many characters are represented by each token on average."
+        )
+    
+    with comp_col2:
+        st.metric(
+            "Space Saving",
+            f"{compression['space_saving_percentage']:.1f}%",
+            help="Percentage of space saved by using tokens instead of raw text."
+        )
+    
+    st.markdown("### Token Details")
+    token_col1, token_col2 = st.columns(2)
+    
+    with token_col1:
+        st.markdown("#### First 10 Token IDs")
+        st.code(str(tokens[:10]), language="python")
+    
+    with token_col2:
+        st.markdown("#### First 10 UTF-8 Bytes")
+        st.code(str(list(unicode_bytes[:10])), language="python")
+    
+    with st.expander("ℹ️ How to interpret these results"):
+        st.markdown("""
+        - **Raw Text Length**: Number of characters in your input text
+        - **UTF-8 Encoded Length**: Size of text when encoded in UTF-8 format
+        - **Token Count**: Number of tokens the text is broken into
+        - **Compression Ratio**: How many characters are represented by each token on average
+        - **Space Saving**: Percentage reduction in size when using tokens vs raw text
+        """)

dev_notebooks/01_out_of_box.ipynb

@@ -44,9 +44,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "text = \"\"\"This is a malayalam tokeniser. A tokenizer is a tool that breaks text into smaller pieces called tokens. These tokens can be words, parts of words, or even characters. It helps computers understand and process text more efficiently.\n",
-    "\n",
-    "Think of it like cutting a big cake into slices. so it's easier to eat in mouth 🎂 \"\"\""
+    "text = \"\"\"ഇതൊരു Malayalam ടോക്കനൈസർ ആണ് 🪧. \n",
+    "വാചകത്തെ ടോക്കണുകൾ എന്ന് വിളിക്കുന്ന ചെറിയ കഷണങ്ങളായി വിഭജിക്കുന്ന ഒരു method aanu ടോക്കനൈസർ. \n",
+    "ഈ ടോക്കണുകൾ വാക്കുകളോ വാക്കുകളുടെ ഭാഗങ്ങളോ പ്രതീകങ്ങളോ ആകാം. \n",
+    "ടെക്‌സ്‌റ്റ് കൂടുതൽ കാര്യക്ഷമമായി മനസ്സിലാക്കാനും പ്രോസസ്സ് ചെയ്യാനും ഇത് language modelukale സഹായിക്കുന്നു.🎂\"\"\""
    ]
   },
   {
@@ -58,7 +59,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Length of the input text:  316\n"
+      "Length of the input text:  300\n"
      ]
     }
    ],
@@ -75,8 +76,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Length of the encoded text:  319\n",
-      "First few bytes:  [84, 104, 105, 115, 32, 105, 115, 32, 97, 32]\n"
+      "Length of the encoded text:  750\n",
+      "First few bytes:  [224, 180, 135, 224, 180, 164, 224, 181, 138, 224]\n"
      ]
     }
    ],
@@ -122,8 +123,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Length of the tokenized text:  70\n",
-      "First few token ids:  [2028, 374, 264, 8811, 352, 17243, 4037, 12329, 13, 362]\n"
+      "Length of the tokenized text:  428\n",
+      "First few token ids:  [34839, 229, 34839, 97, 51211, 232, 34839, 108, 51211, 223]\n"
      ]
     }
    ],
@@ -142,8 +143,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Compression ratio:  4.514285714285714\n",
-      "Space saving percentage:  77.84810126582279\n"
+      "Compression ratio:  0.7009345794392523\n",
+      "Space saving percentage:  -42.66666666666667\n"
      ]
     }
    ],

requirements.txt

@@ -0,0 +1,2 @@
+streamlit==1.42.0
+tiktoken==0.6.0