app.py

import streamlit as st



from urllib.request import urlopen
from bs4 import BeautifulSoup

from urllib.request import urlopen
from bs4 import BeautifulSoup

url = "https://raw.githubusercontent.com/cltk/hindi_text_ltrc/master/tulasidaas/Raamacharita_maanasa/1/main.txt"
html = urlopen(url).read()
soup = BeautifulSoup(html, features="html.parser")

# kill all script and style elements
for script in soup(["script", "style"]):
    script.extract()    # rip it out

# get text
text = soup.get_text()
ramayana_text = text
print(type(text))

print(text[:1000])

# here are all the unique characters that occur in this text
chars = sorted(list(set(text)))
vocab_size = len(chars)
print(''.join(chars))
print(vocab_size)


def get_stats(ids):
    counts = {}
    for pair in zip(ids, ids[1:]):
        counts[pair] = counts.get(pair, 0) + 1
    return counts

def merge(ids, pair, idx):
  newids = []
  i = 0
  while i < len(ids):
    if i < len(ids) - 1 and ids[i] == pair[0] and ids[i+1] == pair[1]:
      newids.append(idx)
      i += 2
    else:
      newids.append(ids[i])
      i += 1
  return newids

# ---
#text = "नाम जीहँ जपि जागहिं जोगी। बिरति बिरंचि प्रपंच बियोगी॥"
tokens = text.encode("utf-8") # raw bytes
tokens = list(map(int, tokens)) # convert to a list of integers in range 0..255 for convenience

vocab_size = 1000 # the desired final vocabulary size
num_merges = vocab_size - 256
ids = list(tokens) # copy so we don't destroy the original list

merges = {} # (int, int) -> int
for i in range(num_merges):
  stats = get_stats(ids)
  pair = max(stats, key=stats.get)
  idx = 256 + i
  # print(f"merging {pair} into a new token {idx}")
  ids = merge(ids, pair, idx)
  merges[pair] = idx

print("tokens length:", len(tokens))
print("ids length:", len(ids))
print(f"compression ratio: {len(tokens) / len(ids):.2f}X")


vocab = {idx: bytes([idx]) for idx in range(256)}
for (p0, p1), idx in merges.items():
    vocab[idx] = vocab[p0] + vocab[p1]

def decode(ids):
  # given ids (list of integers), return Python string
  tokens = b"".join(vocab[idx] for idx in ids)
  text = tokens.decode("utf-8", errors="replace")
  return text

print(decode([261]))


def encode(text):
  # given a string, return list of integers (the tokens)
  tokens = list(text.encode("utf-8"))
  while len(tokens) >= 2:
    stats = get_stats(tokens)
    pair = min(stats, key=lambda p: merges.get(p, float("inf")))
    if pair not in merges:
      break # nothing else can be merged
    idx = merges[pair]
    tokens = merge(tokens, pair, idx)
  return tokens

msg = "पुलिस की मानें तो ये वारदात सुलिभंजन इलाके की है"
tk = list(encode(msg))

print("tokens length:", len(tk))
print(decode(encode(msg)))
print(tk)
#print("Total length:", len(ids))
#print(f"compression ratio: {len(tokens) / len(ids):.2f}X")











# Sidebar contents
with st.sidebar:
    st.title("The School of AI Tokenization App")
    st.markdown(
        """
    ## About
    This app is an LLM-powered chatbot built using:
    - [Streamlit](https://streamlit.io/)
    - [LangChain](https://python.langchain.com/)
    - [PaLM](https://makersuite.google.com/app/home) Embeddings & LLM model

    """
    )
    st.write("By Ajit Kumar Singh")

st.title("The School of AI 💬")

question = st.text_input("Please enter text and press Enter key: ")

if question:
    response = "Your text is " + question
    st.header("Tokenization:")
    st.write(response)


    msg = "पुलिस की मानें तो ये वारदात सुलिभंजन इलाके की है"
    tk = list(encode(question))
    response = "Tokens length:", len(tk), decode(encode(question)) , list(encode(question))
    print("tokens length:", len(tk))
    print(decode(encode(msg)))
    print(tk)
    st.write(response)