add all required files

app.py

@@ -0,0 +1,226 @@
+import streamlit as st
+import os
+import regex as re
+import json
+import ast
+import pandas as pd
+
+def clean_text(text, valid_chars_set, replaced_char=None):
+    text = replace_unwanted_chars(text, valid_chars_set, replaced_char)
+    #gpt2pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
+    gpt2pat = re.compile(r"""
+        's|'t|'re|'ve|'m|'ll|'d|                     # English contractions
+        \s*[\u0900-\u097F]+(?:[\u093E-\u094D\u0950-\u0954\u0962-\u0963]+)*|  # Devanagari letters and diacritics with leading spaces
+        \s*\d+|                                      # Digits with leading spaces
+        [^\s\w\u0900-\u097F]+|                       # Punctuation and symbols
+        \s+                                          # Whitespace
+    """, re.VERBOSE)
+    return re.findall(gpt2pat, text)
+
+def replace_unwanted_chars(text, valid_chars_set, replaced_char=None):
+    # Use list comprehension to quickly replace unwanted characters
+    if replaced_char == None:
+        replaced_char = ''
+    result = ''.join([char if char in valid_chars_set else replaced_char for char in text])
+    return result
+
+def get_stats(ids, counts=None):
+    counts = {} if counts is None else counts
+    for pair in zip(ids, ids[1:]):
+        counts[pair] = counts.get(pair, 0) + 1
+    return counts
+
+def merge(ids, pair, idx):
+    newids = []
+    i = 0
+    #print(f"ids {ids} pair {pair} idx {idx}")
+    #print("lllr")
+    while i < len(ids):
+        #print("newids ",newids)
+        #if i < len(id) and id[i] == pair[0] and id[i+1] == pair[1]:
+        if ids[i] == pair[0] and i < len(ids) - 1 and ids[i+1] == pair[1]:
+            newids.append(idx)
+            i += 2
+        else:
+            #print("mer ids-i ",ids[i], i, newids)
+            newids.append(ids[i])
+            i += 1
+    return newids
+
+def get_vocab(merges, univ_vocab):
+    #vocab = {sr: chr(idx) for sr, idx in enumerate (uni_chars)}
+    for (p0, p1), idx in merges.items():
+        univ_vocab[idx] = univ_vocab[p0] + univ_vocab[p1]
+    return univ_vocab
+
+def get_init_vocab(u_ids):
+    vocab = {idx: chr(idx) for idx in u_ids}
+    #print("init vocab", vocab)
+    #print(u_ids)
+    return vocab
+
+def decode(ids, univ_vocab):
+    # given ids , return Python strings
+    text = "".join(univ_vocab[idx] for idx in ids)
+    return text
+
+def encode(text, merges):
+    global  char_to_int
+    # given string, return list of ints
+    #print(text)
+    tokens = [char_to_int[char] for char in text]
+    # list(map(ord, text)) #list(text.encode("utf-8"))
+    #print(tokens)
+    while len(tokens) >= 2:
+        stats = get_stats(tokens)
+        #print("stats ", stats)
+        pair = min(stats, key=lambda p: merges.get(p, float("inf")))
+        if pair not in merges:
+            break # nothing else can be merged
+        #print("merges",merges)
+        idx = merges[pair]
+        #print("idx ",idx)
+        tokens = merge(tokens, pair, idx)
+        #print("encode token ",tokens)
+    #print("done endode tok ", tokens)
+    return tokens
+
+def encode_ordinary(text, merges, valid_char_set):
+    """Encoding that ignores any special tokens."""
+    # split text into chunks of text by categories defined in regex pattern
+    replace_char = chr(191) # inverted char
+    text_chunks = clean_text(text,valid_char_set,replace_char)
+    # all chunks of text are encoded separately, then results are joined
+    ids = []
+    for chunk in text_chunks:
+        #chunk_bytes = chunk.encode("utf-8") # raw bytes
+        chunk_ids = encode(chunk, merges)
+        ids.extend(chunk_ids)
+        #print("encode ord ",ids)
+    return ids
+
+def unicode_chars_range(start, end):
+    return [chr(i) for i in range(start, end+1)]
+
+def list_unicode_chars(sp_list):
+    return [chr(i) for i in sp_list]
+
+def prepare_init_vocab():
+    # functions to list characters in a given Unicode range
+    # Devanagari
+    special_chars = unicode_chars_range(0x0020, 0x0040)
+    punchuation1_chars = unicode_chars_range(0x005B, 0x0060)
+    punchuation2_chars = unicode_chars_range(0x007B, 0x007E)
+
+    # Devanagari
+    devanagari_chars = unicode_chars_range(0x0900, 0x097F)
+    #print(f"devanagari_chars : {devanagari_chars}")
+
+    # Devanagari Extended
+    devanagari_extended_chars = unicode_chars_range(0xA8E0, 0xA8FF)
+    #print(f"devanagari_extended_chars : {devanagari_extended_chars}")
+
+    # General Punctuations list from wiki page
+    # (–,—,―,‗,‛,“,”,„,†,‡,•,…,‰,′,″,‹,›,‼,‾,⁄)
+    pun_list = [0x2013,0x2014,0x2015,0x2017,0x2018,0x2019,0x201A,0x201B,0x201C,0x201D,0x201E,0x2020\
+                ,0x2021,0x2022,0x2026,0x2030,0x2032,0x2033,0x2039,0x203A,0x203C,0x203E,0x2044,0x204A]
+    # append inverted-? and newline
+    pun_list.append(0x00BF)
+    pun_list.append(10)
+    punctuation_chars = list_unicode_chars(pun_list)
+
+    # Superscripts and Subscripts
+    #super_subscript_chars = unicode_chars_range(0x2070, 0x209F)
+
+    # Combine all characters
+    all_chars_list = (devanagari_chars + devanagari_extended_chars + special_chars + punchuation1_chars + \
+                      punchuation2_chars + punctuation_chars)
+
+    # Print all characters with their Unicode code points
+    #for char in all_chars:
+    #    print(f"Character: {char}, Unicode: {ord(char)}")
+    #init_vocab = {ord(ch1): ch1 for ch1 in (all_chars_list)}
+    init_vocab = {ii: ch1 for ii, ch1 in enumerate(all_chars_list)}
+    char_to_int = {ch1: ii for ii, ch1 in enumerate(all_chars_list)}
+    return set(all_chars_list), init_vocab, char_to_int
+
+
+valid_char_set, univ_vocab, char_to_int = prepare_init_vocab()
+n_vocab_init = len(univ_vocab)
+
+# Function to read and print the contents of a JSON file
+def read_json_file(filename):
+    with open(filename, 'r', encoding='utf-8') as file:
+        data = json.load(file)
+    
+    converted_data = {ast.literal_eval(k): v for k, v in data.items()}
+    return converted_data
+
+# File names
+vocab_filename = 'vocab_15000.json'
+merges_filename = 'merges_15000.json'
+
+# Read the vocabulary JSON file
+univ_vocab = read_json_file(vocab_filename)
+#print("Vocabulary Data:")
+#print(vocab_data)
+
+# Read the merges JSON file
+merges = read_json_file(merges_filename)
+#print("\nMerges Data:")
+#print(merges_data)
+
+def tokenize(text):
+    global n_orig_corpus_chars, merges
+    global valid_char_set, univ_vocab
+    #print(" n merges ", num_merges)
+    n_orig_corpus_chars = len(text)
+    
+    tokens_corpus = encode_ordinary(text, merges, valid_char_set)
+    n_tokens_corpus = len(tokens_corpus)
+    #print(" n_tokens_corpus for voc size ", vocab_size, " -- ", n_tokens_corpus)
+
+    ids_tokens = encode_ordinary(text, merges, valid_char_set)
+    #out_text = decode(out_tokens, univ_vocab)
+    txt_tokens = [univ_vocab[tok1] for tok1 in ids_tokens]
+    
+    return ids_tokens, txt_tokens
+    
+#in_text = input("provide some text : ")
+#print(in_text)
+#tokenize(in_text)
+
+
+
+# Streamlit app
+st.title("Marathi Language Tokenizer")
+
+# Input text
+input_text = st.text_area("Enter text to tokenize:")
+
+st.write("""
+This app can tokenize your input Marathi text. It recognizes devnagari and special characters [unrecognizable input characters appear as inverted-?(question mark)] 
+Enter any text in the box below and click "Tokenize" to see the tokens and their corresponding IDs. e.g. \"क्रिकेट हा जगभरातला आणि त्यातही भारतात विशेष लोकप्रिय असलेला खेळ आहे. त्यात यंदा क्रिकेट
+            वर्ल्ड कप भारतात होणार असल्याने क्रिकेटरसिकांच्या उत्साहाला उधाण आलं आहे.\"
+""")
+
+if st.button("Tokenize"):
+    if input_text:
+        # Tokenize the input text
+        tokens, token_ids = tokenize(input_text)
+        
+        st.write(f"Stats | Number of input characters : {len(input_text)} | Number of tokens : {len(tokens)} | Compression : {len(input_text)/len(tokens)} |" )
+        
+        # Display the tokens and their IDs
+        # Create a DataFrame for better readability
+        df = pd.DataFrame(list(zip(tokens, token_ids)), columns=["Token", "Token ID"])
+
+        # Display the tokens and their IDs in a table
+        st.write("Tokens and Token IDs:")
+        st.dataframe(df)
+
+    else:
+        st.write("Please enter some text to tokenize.")
+
+
+

requirements.txt

@@ -0,0 +1,4 @@
+pandas
+json
+regex
+streamlit