updated

app.py

@@ -1,8 +1,26 @@
 import streamlit as st
+import encoder_parallel_telugu as encode_parallel
+from consecutive_tokens import get_consecutive_tokens, search_consecutive_tokens
+import tokenizer
 
 def encode(text):
-    # Placeholder for encoding logic
-    return f"Encoded: {text}"
+    if text == "":
+        return "Enter text to encode..."
+    encoded_tokens = [token.encode('utf-8') for token in text]
+    consective_tokens = get_consecutive_tokens(encoded_tokens,window_size=4)
+    # Reading vocabulary from file
+    formatted_vocab = tokenizer.read_vocab_from_file()
+    # Invert vocabulary
+    inverted_vocab = {v: k for k, v in formatted_vocab.items()}
+    # Expand vocabulary
+    decoder_map = tokenizer.expand_vocab(inverted_vocab)
+    # Invert back again after expansion
+    re_inverted_vocab = {k: v for v, k in decoder_map.items()}
+    
+    # encoded_tokens = [re_inverted_vocab.get(token) for token in consective_tokens]
+    encoded_tokens = search_consecutive_tokens(consective_tokens, re_inverted_vocab)
+    print(encoded_tokens)    
+    return f"Encoded: {encoded_tokens}"
 
 def decode(text):
     # Placeholder for decoding logic

consecutive_tokens.py

@@ -0,0 +1,51 @@
+from collections import OrderedDict
+
+def get_consecutive_tokens(li, window_size=4):
+    if len(li) == 0:
+        return []
+    final_token_dict = OrderedDict((token, []) for token in range(len(li)))
+    i = 0
+    while i <= len(li)-1:
+        j = 1
+        while j <= window_size:
+            final_token_dict[i].append(tuple(li[i:i+j]))
+            j+=1
+        i+=1
+    
+    reversed_token_dict = {key: [tuple(tup) for tup in reversed(value)] for key, value in final_token_dict.items()}
+    return reversed_token_dict
+
+
+
+def search_consecutive_tokens(ordered_dict, encoded_token_dict):
+    final_encoded_tokens = []
+    keys = list(ordered_dict.keys())
+    i = 0
+    while i < len(keys):
+        key = keys[i]
+        j = 0
+        jump = False
+        while j<len(ordered_dict[key]):
+            if ordered_dict[key][j] in encoded_token_dict:
+                final_encoded_tokens.append(encoded_token_dict[ordered_dict[key][j]])
+                i+=len(ordered_dict[key][j])
+                jump = True
+                j = 0
+                break
+            j+=1
+        if not jump:
+            i+=1
+    return final_encoded_tokens
+
+if __name__ == "__main__":
+    text = "తెలుగు భాష ఒక ద్రావిడ భాష."
+    op_li = get_consecutive_tokens([1,2,3,4,5])
+    print(op_li)
+
+    dict = {(1,2):9,(3,):10, (4,5):11}
+    print(search_consecutive_tokens(op_li, dict))
+
+    # encoded_tokens = encode_tokens_parallel(text, chunk_size=1_000_000, max_workers=2)
+    # encoded_tokens = [token.encode('utf-8') for token in text]
+    # decoded_tokens = [i.decode('utf-8') for i in encoded_tokens]
+    # print(get_consecutive_tokens(decoded_tokens))

encoder_parallel_telugu.py

@@ -0,0 +1,69 @@
+import time
+import concurrent.futures
+from tqdm import tqdm
+import pandas as pd
+import re
+
+# Function to encode a chunk of tokens into UTF-8 and return as bytes
+def encode_chunk(chunk):
+    # Encode each token in the chunk to UTF-8
+    return [token.encode('utf-8') for token in chunk]
+
+# Main function to handle parallel encoding and return concatenated results
+def encode_tokens_parallel(tokens, chunk_size=1_000_000, max_workers=10):
+    # Split the tokens into chunks of size chunk_size (1 million tokens per chunk)
+    chunks = [tokens[i:i + chunk_size] for i in range(0, len(tokens), chunk_size)]
+    
+    # Prepare the progress bar
+    total_chunks = len(chunks)
+    
+    # Use ProcessPoolExecutor to process chunks in parallel
+    with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+        # Use tqdm to show a progress bar while processing chunks
+        encoded_chunks = list(tqdm(executor.map(encode_chunk, chunks), total=total_chunks, desc="Processing Chunks"))
+    
+    # Concatenate all encoded chunks into a single list
+    concatenated_encoded = [token for chunk in encoded_chunks for token in chunk]
+    
+    return concatenated_encoded
+
+def load_telugu_texts():
+    file_paths = [
+    '/Users/anvesh/codebase/llm/data/telugu_books/telugu_books.csv',
+    '/Users/anvesh/codebase/llm/data/telugu_news/1_telugu_news.csv',
+    '/Users/anvesh/codebase/llm/data/telugu_news/2_telugu_news.csv'
+    ]
+
+    # Combine data from all files
+    telugu_texts = []
+    for file_path in file_paths:
+        df = pd.read_csv(file_path)
+        if 'text' in df.columns:
+            telugu_texts.append(' '.join(df['text'].astype(str).tolist()))
+        elif 'body' in df.columns:
+            telugu_texts.append(' '.join(df['body'].astype(str).tolist()))
+    # Concatenate all texts and remove all English, numerical values, quotes, and characters outside the UTF-8 range 0x0C00 to 0x0C7F, including special characters like @, #, $, and %.
+    telugu_text = ' '.join(telugu_texts)
+    telugu_text = re.sub(r'[^\u0C00-\u0C7F@#$%]', '', telugu_text)  # Remove characters outside the specified UTF-8 range and special characters
+    telugu_text = re.sub(r'[\r\n\xa0]', '', telugu_text)  # Remove line breaks and non-breaking spaces
+    return telugu_text
+
+# Main script
+if __name__ == '__main__':
+    # Load the Telugu texts
+    tokens = load_telugu_texts()
+    # Start the timer
+    start_time = time.time()
+
+    # Encode the tokens in parallel and get concatenated results
+    encoded_tokens = encode_tokens_parallel(tokens, chunk_size=1_000_000, max_workers=10)
+    print(encoded_tokens[:100])
+    print(len(encoded_tokens))
+    # End the timer
+    end_time = time.time()
+
+    # Calculate the time taken
+    time_taken = end_time - start_time
+
+    print(f"Time taken to encode and process tokens in parallel: {time_taken:.4f} seconds")
+    print("Encoding and processing completed!")

merges_vocab.json

@@ -0,0 +1 @@
+{"merges": {"(b'\\xe0\\xb0\\xa8', b'\\xe0\\xb0\\xbf')": 256}, "vocab": {"(b'\\xe0\\xb0\\xa8', b'\\xe0\\xb0\\xbf')": 256, "b'\\xe0\\xb0\\x80'": 0, "b'\\xe0\\xb0\\x81'": 1, "b'\\xe0\\xb0\\x82'": 2, "b'\\xe0\\xb0\\x83'": 3, "b'\\xe0\\xb0\\x84'": 4, "b'\\xe0\\xb0\\x85'": 5, "b'\\xe0\\xb0\\x86'": 6, "b'\\xe0\\xb0\\x87'": 7, "b'\\xe0\\xb0\\x88'": 8, "b'\\xe0\\xb0\\x89'": 9, "b'\\xe0\\xb0\\x8a'": 10, "b'\\xe0\\xb0\\x8b'": 11, "b'\\xe0\\xb0\\x8c'": 12, "b'\\xe0\\xb0\\x8d'": 13, "b'\\xe0\\xb0\\x8e'": 14, "b'\\xe0\\xb0\\x8f'": 15, "b'\\xe0\\xb0\\x90'": 16, "b'\\xe0\\xb0\\x91'": 17, "b'\\xe0\\xb0\\x92'": 18, "b'\\xe0\\xb0\\x93'": 19, "b'\\xe0\\xb0\\x94'": 20, "b'\\xe0\\xb0\\x95'": 21, "b'\\xe0\\xb0\\x96'": 22, "b'\\xe0\\xb0\\x97'": 23, "b'\\xe0\\xb0\\x98'": 24, "b'\\xe0\\xb0\\x99'": 25, "b'\\xe0\\xb0\\x9a'": 26, "b'\\xe0\\xb0\\x9b'": 27, "b'\\xe0\\xb0\\x9c'": 28, "b'\\xe0\\xb0\\x9d'": 29, "b'\\xe0\\xb0\\x9e'": 30, "b'\\xe0\\xb0\\x9f'": 31, "b'\\xe0\\xb0\\xa0'": 32, "b'\\xe0\\xb0\\xa1'": 33, "b'\\xe0\\xb0\\xa2'": 34, "b'\\xe0\\xb0\\xa3'": 35, "b'\\xe0\\xb0\\xa4'": 36, "b'\\xe0\\xb0\\xa5'": 37, "b'\\xe0\\xb0\\xa6'": 38, "b'\\xe0\\xb0\\xa7'": 39, "b'\\xe0\\xb0\\xa8'": 40, "b'\\xe0\\xb0\\xa9'": 41, "b'\\xe0\\xb0\\xaa'": 42, "b'\\xe0\\xb0\\xab'": 43, "b'\\xe0\\xb0\\xac'": 44, "b'\\xe0\\xb0\\xad'": 45, "b'\\xe0\\xb0\\xae'": 46, "b'\\xe0\\xb0\\xaf'": 47, "b'\\xe0\\xb0\\xb0'": 48, "b'\\xe0\\xb0\\xb1'": 49, "b'\\xe0\\xb0\\xb2'": 50, "b'\\xe0\\xb0\\xb3'": 51, "b'\\xe0\\xb0\\xb4'": 52, "b'\\xe0\\xb0\\xb5'": 53, "b'\\xe0\\xb0\\xb6'": 54, "b'\\xe0\\xb0\\xb7'": 55, "b'\\xe0\\xb0\\xb8'": 56, "b'\\xe0\\xb0\\xb9'": 57, "b'\\xe0\\xb0\\xba'": 58, "b'\\xe0\\xb0\\xbb'": 59, "b'\\xe0\\xb0\\xbc'": 60, "b'\\xe0\\xb0\\xbd'": 61, "b'\\xe0\\xb0\\xbe'": 62, "b'\\xe0\\xb0\\xbf'": 63, "b'\\xe0\\xb1\\x80'": 64, "b'\\xe0\\xb1\\x81'": 65, "b'\\xe0\\xb1\\x82'": 66, "b'\\xe0\\xb1\\x83'": 67, "b'\\xe0\\xb1\\x84'": 68, "b'\\xe0\\xb1\\x85'": 69, "b'\\xe0\\xb1\\x86'": 70, "b'\\xe0\\xb1\\x87'": 71, "b'\\xe0\\xb1\\x88'": 72, "b'\\xe0\\xb1\\x89'": 73, "b'\\xe0\\xb1\\x8a'": 74, "b'\\xe0\\xb1\\x8b'": 75, "b'\\xe0\\xb1\\x8c'": 76, "b'\\xe0\\xb1\\x8d'": 77, "b'\\xe0\\xb1\\x8e'": 78, "b'\\xe0\\xb1\\x8f'": 79, "b'\\xe0\\xb1\\x90'": 80, "b'\\xe0\\xb1\\x91'": 81, "b'\\xe0\\xb1\\x92'": 82, "b'\\xe0\\xb1\\x93'": 83, "b'\\xe0\\xb1\\x94'": 84, "b'\\xe0\\xb1\\x95'": 85, "b'\\xe0\\xb1\\x96'": 86, "b'\\xe0\\xb1\\x97'": 87, "b'\\xe0\\xb1\\x98'": 88, "b'\\xe0\\xb1\\x99'": 89, "b'\\xe0\\xb1\\x9a'": 90, "b'\\xe0\\xb1\\x9b'": 91, "b'\\xe0\\xb1\\x9c'": 92, "b'\\xe0\\xb1\\x9d'": 93, "b'\\xe0\\xb1\\x9e'": 94, "b'\\xe0\\xb1\\x9f'": 95, "b'\\xe0\\xb1\\xa0'": 96, "b'\\xe0\\xb1\\xa1'": 97, "b'\\xe0\\xb1\\xa2'": 98, "b'\\xe0\\xb1\\xa3'": 99, "b'\\xe0\\xb1\\xa4'": 100, "b'\\xe0\\xb1\\xa5'": 101, "b'\\xe0\\xb1\\xa6'": 102, "b'\\xe0\\xb1\\xa7'": 103, "b'\\xe0\\xb1\\xa8'": 104, "b'\\xe0\\xb1\\xa9'": 105, "b'\\xe0\\xb1\\xaa'": 106, "b'\\xe0\\xb1\\xab'": 107, "b'\\xe0\\xb1\\xac'": 108, "b'\\xe0\\xb1\\xad'": 109, "b'\\xe0\\xb1\\xae'": 110, "b'\\xe0\\xb1\\xaf'": 111, "b'\\xe0\\xb1\\xb0'": 112, "b'\\xe0\\xb1\\xb1'": 113, "b'\\xe0\\xb1\\xb2'": 114, "b'\\xe0\\xb1\\xb3'": 115, "b'\\xe0\\xb1\\xb4'": 116, "b'\\xe0\\xb1\\xb5'": 117, "b'\\xe0\\xb1\\xb6'": 118, "b'\\xe0\\xb1\\xb7'": 119, "b'\\xe0\\xb1\\xb8'": 120, "b'\\xe0\\xb1\\xb9'": 121, "b'\\xe0\\xb1\\xba'": 122, "b'\\xe0\\xb1\\xbb'": 123, "b'\\xe0\\xb1\\xbc'": 124, "b'\\xe0\\xb1\\xbd'": 125, "b'\\xe0\\xb1\\xbe'": 126, "b' '": 255, "b'.'": 254}}

tokenizer.py

@@ -0,0 +1,135 @@
+import pandas as pd
+import re
+import encoder_parallel_telugu as encode_parallel
+import time
+import json
+from collections import defaultdict
+from tqdm import tqdm
+
+def load_and_encode_tokens():
+    tokens = encode_parallel.load_telugu_texts()
+    start_time = time.time()
+    encoded_tokens = encode_parallel.encode_tokens_parallel(tokens, chunk_size=1_000_000, max_workers=10)
+    print('encoded_tokens:', encoded_tokens[:100])
+    print(len(encoded_tokens))
+    end_time = time.time()
+    print(f"Time taken to encode and process tokens in parallel: {end_time - start_time:.4f} seconds")
+    print('length of encoded_text:', len(encoded_tokens))
+    print('unique characters in decoded_text:', {token.decode('utf-8') for token in set(encoded_tokens)})
+    # print('unique characters in encoded_text:', set(encoded_tokens))
+    print('unique characters in encoded_text:', len(set(encoded_tokens)))
+    return encoded_tokens
+
+def get_stats(ids):
+    counts = {}
+    for pair in zip(ids, ids[1:]):
+        counts[pair] = counts.get(pair, 0) + 1
+    return counts
+
+def merge(ids, pair, idx):
+    new_ids = []
+    i = 0
+    while i < len(ids):
+        if i < len(ids) - 1 and ids[i] == pair[0] and ids[i + 1] == pair[1]:
+            new_ids.append(idx)
+            i += 2
+        else:
+            new_ids.append(ids[i])
+            i += 1
+    return new_ids
+
+def bpe_process(encoded_tokens,vocab_size=500, encoded_tokens_length=10_00_000):
+    num_merges = vocab_size - 256  # our unique tokens are 194, for our sample text.
+    encoded_tokens = encoded_tokens[:encoded_tokens_length]
+    ids = list(encoded_tokens)  # copy so we don't destroy the original list
+    merges = {}  # (int, int) -> int
+
+    for i in tqdm(range(num_merges), desc="Merging tokens"):
+        stats = get_stats(ids)
+        pair = max(stats, key=stats.get)
+        idx = 256 + i
+        ids = merge(ids, pair, idx)
+        merges[pair] = idx  # merge has a pair of tokens and the new token index
+
+    print("tokens length:", len(encoded_tokens))
+    print("ids length:", len(ids))
+    print("by paired tokens length:", len(set(ids)))
+    print(f"compression ratio: {len(encoded_tokens) / len(ids):.2f}X")
+    # print(f"token size: {len(set(encoded_tokens))}")
+
+    return merges
+
+def build_vocabulary(merges):
+    telugu_unicode_chars = [chr(i) for i in range(0x0C00, 0x0C7F)]  # Telugu Unicode range
+    vocab = {token: idx for token, idx in merges.items()}
+
+    for idx, char in enumerate([chr(i).encode('utf-8') for i in range(0x0C00, 0x0C7F)]):
+        if idx < 256:  # Ensure we only add up to 256 characters
+            vocab[char] = idx  # Map the character to its index
+
+    vocab[b' '] = 255
+    vocab[b'.'] = 254
+
+    with open('merges_vocab.json', 'w') as f:
+        json.dump({'merges': {str(k): v for k, v in merges.items()}, 'vocab': {str(k): v for k, v in vocab.items()}}, f)
+
+def read_vocab_from_file():
+    with open('merges_vocab.json', 'r') as f:
+        data = json.load(f)
+
+    distributed_data = defaultdict(list)
+
+    for key, value in data['vocab'].items():
+        distributed_data['vocab'].append({key: value})
+
+    formatted_vocab = {}
+    for item in distributed_data['vocab']:
+        for k, v in item.items():
+            if ',' not in k:
+                formatted_vocab[(eval(k),)] = v
+            else:
+                formatted_vocab[eval(k)] = v
+
+    return formatted_vocab
+
+def expand_vocab(inverted_vocab):
+    def convert_to_bytes(value):
+        if isinstance(value, bytes):
+            return value
+        elif value in inverted_vocab:
+            return process_tuple(inverted_vocab[value])
+        else:
+            print(f'value not found in inverted_vocab: {value}')
+            return None
+
+    def process_tuple(value_tuple):
+        converted_values = []
+        for v in value_tuple:
+            result = convert_to_bytes(v)
+            if isinstance(result, tuple):
+                converted_values.extend(result)
+            else:
+                converted_values.append(result)
+        return tuple(converted_values)
+
+    decoder_map = {k: process_tuple(v) for k, v in inverted_vocab.items()}
+    print("sample decoder map:", {k: decoder_map[k] for k in list(decoder_map)[:5]})
+    return decoder_map
+
+# # Main execution
+# if __name__ == "__main__":
+#     # 1. Load and encode tokens
+#     encoded_tokens = load_and_encode_tokens()
+#     # 2. Process BPE
+#     merges = bpe_process(encoded_tokens,vocab_size=257, encoded_tokens_length=20_00_000)
+#     # 3. Build vocabulary
+#     build_vocabulary(merges)
+#     # 4. Read vocabulary from file
+#     formatted_vocab = read_vocab_from_file()
+#     # 5. Invert vocabulary
+#     inverted_vocab = {v: k for k, v in formatted_vocab.items()}
+#     # 6. Expand vocabulary
+#     decoder_map = expand_vocab(inverted_vocab)
+#     # 7. Invert back again
+#     re_inverted_vocab = {k: v for v, k in decoder_map.items()}
+#     print(re_inverted_vocab)

tokenizer_backup.py

@@ -0,0 +1,194 @@
+import pandas as pd
+import re
+
+# Data section  start--> 
+# Load the CSV files
+file_paths = [
+    '/Users/anvesh/codebase/llm/data/telugu_books/telugu_books.csv',
+    '/Users/anvesh/codebase/llm/data/telugu_news/1_telugu_news.csv',
+    '/Users/anvesh/codebase/llm/data/telugu_news/2_telugu_news.csv'
+]
+
+# Combine data from all files
+telugu_texts = []
+for file_path in file_paths:
+    df = pd.read_csv(file_path)
+    if 'text' in df.columns:
+        telugu_texts.append(' '.join(df['text'].astype(str).tolist()))
+    elif 'body' in df.columns:
+        telugu_texts.append(' '.join(df['body'].astype(str).tolist()))
+
+# Concatenate all texts and remove all English, numerical values, and quotes
+telugu_text = ' '.join(telugu_texts)
+telugu_text = re.sub(r'[A-Za-z0-9\'"]', '', telugu_text)  # Remove English letters, numbers, and quotes
+telugu_text = re.sub(r'[\r\n\xa0]', '', telugu_text)  # Remove line breaks and non-breaking spaces
+
+print('telugu_text befores utf-8 encoding:', telugu_text[:100])
+
+vocabulary_size = len(set(telugu_text.split()))
+print('Original text size:', len(telugu_text))
+print('Vocabulary size of telugu_text:', vocabulary_size)
+
+unique_characters = set(telugu_text)
+unique_count = len(unique_characters)
+print('Original text size:', len(telugu_text))
+print('Unique character count in telugu_text:', unique_count)
+
+# Data section  end--> 
+
+# utf-8 encoding section start -->
+import encode_parallel_telugu as encode_parallel
+import time
+
+tokens = encode_parallel.load_telugu_texts()
+# Start the timer
+start_time = time.time()
+# Encode the tokens in parallel and get concatenated results
+encoded_tokens = encode_parallel.encode_tokens_parallel(tokens, chunk_size=1_000_000, max_workers=10)
+print('encoded_tokens:', encoded_tokens[:100])
+print(len(encoded_tokens))
+# End the timer
+end_time = time.time()
+print(f"Time taken to encode and process tokens in parallel: {end_time - start_time:.4f} seconds")
+
+print('length of encoded_text:', len(encoded_tokens))
+print('unique characters in encoded_text:', set(encoded_tokens))
+print('unique characters in encoded_text:', len(set(encoded_tokens)))
+# utf-8 encoding section end -->
+
+# BPE section start -->
+#### **BPE implementation**
+
+tokens = encoded_tokens
+
+def get_stats(ids):
+    counts = {}
+    for pair in zip(ids, ids[1:]):
+        counts[pair] = counts.get(pair, 0) + 1
+    return counts
+
+def merge(ids, pair, idx):
+    new_ids = []
+    i = 0
+    while i < len(ids):
+        if i < len(ids) - 1 and ids[i] == pair[0] and ids[i+1] == pair[1]:
+            new_ids.append(idx)
+            i += 2
+        else:
+            new_ids.append(ids[i])
+            i += 1
+    return new_ids
+
+# ---
+vocab_size = 500 # the desired final vocabulary size
+num_merges = vocab_size - 256 ## our unique tokens are 194, for our sample text.
+ids = list(tokens) # copy so we don't destroy the original list
+
+merges = {} # (int, int) -> int
+from tqdm import tqdm  # Import tqdm for progress bar
+
+for i in tqdm(range(num_merges), desc="Merging tokens"):
+    stats = get_stats(ids)
+    pair = max(stats, key=stats.get)
+    idx = 256 + i
+    # print(f"merging {pair} into a new token {idx}")
+    ids = merge(ids, pair, idx)
+    merges[pair] = idx # merge has a pair of tokens and the new token index
+    
+print("tokens length:", len(tokens))
+print("ids length:", len(ids))
+print(f"compression ratio: {len(tokens) / len(ids):.2f}X")
+print(f"token size: {len(set(tokens))}")
+    
+# print(ids)
+# BPE section end -->
+
+# Building the vocabulary section start -->
+telugu_unicode_chars = [chr(i) for i in range(0x0C00, 0x0C7F)]  # Telugu Unicode range
+
+# Add these characters to the vocabulary
+import json
+vocab = {token: idx for token, idx in merges.items()}
+# Add unique Telugu characters to the vocabulary
+for idx, char in enumerate([chr(i).encode('utf-8') for i in range(0x0C00, 0x0C7F)]):
+    if idx < 256:  # Ensure we only add up to 256 characters
+        vocab[char] = idx  # Map the character to its index
+
+vocab[b' '] = 255
+vocab[b'.'] = 254
+# Save merges and vocab to a file
+# with open('merges_vocab.json', 'w') as f:
+#     json.dump({'merges': merges, 'vocab': vocab}, f)
+
+# saving the merges and vocab to a file
+with open('merges_vocab.json', 'w') as f:
+    json.dump({'merges': {str(k): v for k, v in merges.items()}, 'vocab': {str(k): v for k, v in vocab.items()}}, f)
+    
+# Building the vocabulary section end -->
+
+
+# Reading the merges and vocab from a file section start -->
+import json
+from collections import defaultdict
+
+# Read the merges and vocab data from the JSON file
+with open('merges_vocab.json', 'r') as f:
+    data = json.load(f)
+
+# Create a defaultdict to store the data in a distributed manner
+distributed_data = defaultdict(list)
+
+# Distribute the merges and vocab data
+# for key, value in data['merges'].items():
+#     distributed_data['merges'].append({key: value})
+
+for key, value in data['vocab'].items():
+    distributed_data['vocab'].append({key: value})
+
+# Optionally, print the distributed data for verification
+print(distributed_data)
+distributed_data['vocab']
+# Convert the list of dictionaries to a single dictionary
+formatted_vocab = {}
+for item in distributed_data['vocab']:
+    for k, v in item.items():
+        if ',' not in k:
+            formatted_vocab[(eval(k),)] = v
+        else:
+            formatted_vocab[eval(k)] = v
+print(formatted_vocab[:50])
+# inverting the vocab
+inverted_vocab = {v: k for k, v in formatted_vocab.items()}
+inverted_vocab
+
+# Reading the merges and vocab from a file section end -->
+
+# Expanding the vocab section start -->
+def convert_to_bytes(value):
+    if isinstance(value, bytes):
+        return value
+    elif value in inverted_vocab:
+        return process_tuple(inverted_vocab[value])
+    else:
+        print(f'value not found in inverted_vocab: {value}')
+        return None
+
+def process_tuple(value_tuple):
+    # print(f'value_tuple: {value_tuple}')
+    # for vi in value_tuple:
+    #     print(f'v: {vi}')
+    converted_values = []
+    for v in value_tuple:
+        result = convert_to_bytes(v)
+        if isinstance(result, tuple):
+            converted_values.extend(result)
+        else:
+            converted_values.append(result)
+    return tuple(converted_values)
+
+decoder_map = {k: process_tuple(v) for k, v in inverted_vocab.items()}
+
+
+
+
+