Update app.py

app.py

@@ -1,6 +1,120 @@
 import streamlit as st
 
 
+
+from urllib.request import urlopen
+from bs4 import BeautifulSoup
+
+from urllib.request import urlopen
+from bs4 import BeautifulSoup
+
+url = "https://raw.githubusercontent.com/cltk/hindi_text_ltrc/master/tulasidaas/Raamacharita_maanasa/1/main.txt"
+html = urlopen(url).read()
+soup = BeautifulSoup(html, features="html.parser")
+
+# kill all script and style elements
+for script in soup(["script", "style"]):
+    script.extract()    # rip it out
+
+# get text
+text = soup.get_text()
+ramayana_text = text
+print(type(text))
+
+print(text[:1000])
+
+# here are all the unique characters that occur in this text
+chars = sorted(list(set(text)))
+vocab_size = len(chars)
+print(''.join(chars))
+print(vocab_size)
+
+
+def get_stats(ids):
+    counts = {}
+    for pair in zip(ids, ids[1:]):
+        counts[pair] = counts.get(pair, 0) + 1
+    return counts
+
+def merge(ids, pair, idx):
+  newids = []
+  i = 0
+  while i < len(ids):
+    if i < len(ids) - 1 and ids[i] == pair[0] and ids[i+1] == pair[1]:
+      newids.append(idx)
+      i += 2
+    else:
+      newids.append(ids[i])
+      i += 1
+  return newids
+
+# ---
+#text = "नाम जीहँ जपि जागहिं जोगी। बिरति बिरंचि प्रपंच बियोगी॥"
+tokens = text.encode("utf-8") # raw bytes
+tokens = list(map(int, tokens)) # convert to a list of integers in range 0..255 for convenience
+
+vocab_size = 1000 # the desired final vocabulary size
+num_merges = vocab_size - 256
+ids = list(tokens) # copy so we don't destroy the original list
+
+merges = {} # (int, int) -> int
+for i in range(num_merges):
+  stats = get_stats(ids)
+  pair = max(stats, key=stats.get)
+  idx = 256 + i
+  # print(f"merging {pair} into a new token {idx}")
+  ids = merge(ids, pair, idx)
+  merges[pair] = idx
+
+print("tokens length:", len(tokens))
+print("ids length:", len(ids))
+print(f"compression ratio: {len(tokens) / len(ids):.2f}X")
+
+
+vocab = {idx: bytes([idx]) for idx in range(256)}
+for (p0, p1), idx in merges.items():
+    vocab[idx] = vocab[p0] + vocab[p1]
+
+def decode(ids):
+  # given ids (list of integers), return Python string
+  tokens = b"".join(vocab[idx] for idx in ids)
+  text = tokens.decode("utf-8", errors="replace")
+  return text
+
+print(decode([261]))
+
+
+def encode(text):
+  # given a string, return list of integers (the tokens)
+  tokens = list(text.encode("utf-8"))
+  while len(tokens) >= 2:
+    stats = get_stats(tokens)
+    pair = min(stats, key=lambda p: merges.get(p, float("inf")))
+    if pair not in merges:
+      break # nothing else can be merged
+    idx = merges[pair]
+    tokens = merge(tokens, pair, idx)
+  return tokens
+
+msg = "पुलिस की मानें तो ये वारदात सुलिभंजन इलाके की है"
+tk = list(encode(msg))
+
+print("tokens length:", len(tk))
+print(decode(encode(msg)))
+print(tk)
+#print("Total length:", len(ids))
+#print(f"compression ratio: {len(tokens) / len(ids):.2f}X")
+
+
+
+
+
+
+
+
+
+
+
 # Sidebar contents
 with st.sidebar:
     st.title("The School of AI Tokenization App")
@@ -22,10 +136,21 @@ question = st.text_input("Please enter Hindi text: ")
 
 if question:
     response = "Your text is " + question
-    st.header("Answer")
+    st.header("Tokenization:")
     st.write(response)
 
 
+    msg = "पुलिस की मानें तो ये वारदात सुलिभंजन इलाके की है"
+    tk = list(encode(question))
+    response = "Tokens length:", len(tk), decode(encode(msg))
+    print("tokens length:", len(tk))
+    print(decode(encode(msg)))
+    print(tk)
+    st.write(response)
+
+
+
+