updated

app.py

@@ -32,7 +32,7 @@ def decode(text):
     # Expand vocabulary
     decoder_map = tokenizer.expand_vocab(inverted_vocab)
     decoded_tokens = [decoder_map.get(int(token)) for token in toks_li]
-    decoded_tokens = [token[0] for token in decoded_tokens]
+    decoded_tokens = [item for token in decoded_tokens for item in token]
     tokens = [token.decode('utf-8') for token in decoded_tokens]
     decoded_tokens = b''.join(decoded_tokens)
     decoded_tokens = decoded_tokens.decode('utf-8')

consecutive_tokens.py

@@ -38,24 +38,61 @@ def search_consecutive_tokens(ordered_dict, encoded_token_dict):
     return final_encoded_tokens
 
 if __name__ == "__main__":
-    text = "తెలుగు భాష ఒక ద్రావిడ భాష."
-    op_li = get_consecutive_tokens([1,2,3,4,5])
-    print(op_li)
-
-    dict = {(1,2):9,(3,):10, (4,5):11}
-    opp = search_consecutive_tokens(op_li, dict)
-    print(opp)
-    text = "9,10,11"
+    ## Encoder
+    # import tokenizer
+    # text = "తెలుగు భాష ఒక ద్రావిడ భాష."
+    # encoded_tokens = [token.encode('utf-8') for token in text]
+    # consective_tokens = get_consecutive_tokens(encoded_tokens,window_size=4)
+    # # Reading vocabulary from file
+    # formatted_vocab = tokenizer.read_vocab_from_file()
+    # # Invert vocabulary
+    # inverted_vocab = {v: k for k, v in formatted_vocab.items()}
+    # # Expand vocabulary
+    # decoder_map = tokenizer.expand_vocab(inverted_vocab)
+    # # Invert back again after expansion
+    # re_inverted_vocab = {k: v for v, k in decoder_map.items()}
+    
+    # # encoded_tokens = [re_inverted_vocab.get(token) for token in consective_tokens]
+    # encoded_tokens = search_consecutive_tokens(consective_tokens, re_inverted_vocab)
+    # print(encoded_tokens) 
+    
+    ## decoder:
+    import tokenizer
+    text = "573, 312, 255, 255, 419, 55, 255, 255, 394, 255, 255, 624, 62, 291, 33, 255, 255, 419, 55, 254"
     toks_li = [token for token in text.split(',')]
     # Reading vocabulary from file
-    import tokenizer
     formatted_vocab = tokenizer.read_vocab_from_file()
     # Invert vocabulary
     inverted_vocab = {v: k for k, v in formatted_vocab.items()}
     # Expand vocabulary
     decoder_map = tokenizer.expand_vocab(inverted_vocab)
     decoded_tokens = [decoder_map.get(int(token)) for token in toks_li]
+    decoded_tokens = [item for token in decoded_tokens for item in token]
+    tokens = [token.decode('utf-8') for token in decoded_tokens]
+    decoded_tokens = b''.join(decoded_tokens)
+    decoded_tokens = decoded_tokens.decode('utf-8')
     print(decoded_tokens)
+    
+    
+    #op_li = get_consecutive_tokens([1,2,3,4,5])
+    #print(op_li)
+
+    # dict = {(1,2):9,(3,):10, (4,5):11}
+    # opp = search_consecutive_tokens(op_li, dict)
+    # print(opp)
+    
+    
+    # text = "9,10,11"
+    # toks_li = [token for token in text.split(',')]
+    # # Reading vocabulary from file
+    # import tokenizer
+    # formatted_vocab = tokenizer.read_vocab_from_file()
+    # # Invert vocabulary
+    # inverted_vocab = {v: k for k, v in formatted_vocab.items()}
+    # # Expand vocabulary
+    # decoder_map = tokenizer.expand_vocab(inverted_vocab)
+    # decoded_tokens = [decoder_map.get(int(token)) for token in toks_li]
+    # print(decoded_tokens)
     # encoded_tokens = encode_tokens_parallel(text, chunk_size=1_000_000, max_workers=2)
     # encoded_tokens = [token.encode('utf-8') for token in text]
     # decoded_tokens = [i.decode('utf-8') for i in encoded_tokens]

tokenizer.py

@@ -121,7 +121,7 @@ if __name__ == "__main__":
     # 1. Load and encode tokens
     encoded_tokens = load_and_encode_tokens()
     # 2. Process BPE
-    merges = bpe_process(encoded_tokens,vocab_size=1000, encoded_tokens_length=20_00_000)
+    merges = bpe_process(encoded_tokens,vocab_size=1000, encoded_tokens_length=10_00_000)
     # 3. Build vocabulary
     build_vocabulary(merges)
     # 4. Read vocabulary from file