added assamese

app.py

@@ -15,7 +15,7 @@ def load_test_phrases(filename):
 
 models = ["Xenova/claude-tokenizer",                 # Anthropic
           "meta-llama/Llama-2-7b-chat-hf",           # LLAMA-2
-          "beomi/llama-2-ko-7b",                     # LLAMA-2-ko
+#          "beomi/llama-2-ko-7b",                     # LLAMA-2-ko
           "ai4bharat/Airavata",                      # ARIVATA
           "openaccess-ai-collective/tiny-mistral",   # Mistral
           "gpt-3.5-turbo",                           # GPT3.5
@@ -23,7 +23,9 @@ models = ["Xenova/claude-tokenizer",                 # Anthropic
           "CohereForAI/aya-23-8B",                   # AYA
           "google/gemma-1.1-2b-it",                  # GEMMA
           "gpt-4o",                                  # GPT4o
-          "TWO/sutra-mlt256-v2"]                     # SUTRA
+          "TWO/sutra-mlt256-v2",                     # SUTRA
+          "tamang0000/assamese-tokenizer-50k"        # Assamese
+]
 
 test_phrase_set = [
     "I am going for a walk later today",
@@ -111,8 +113,9 @@ def generate_split_token_table(text):
 with gr.Blocks() as sutra_token_count:
     gr.Markdown(
         """
-        # SUTRA Multilingual Tokenizer Specs & Stats.
+        # Multilingual Tokenizer Specs & Stats.
         ## Tokenize paragraphs in multiple languages and compare token counts.
+        Space inspired from [SUTRA](https://huggingface.co/spaces/TWO/sutra-tokenizer-comparison)
         """)
     textbox = gr.Textbox(label="Input Text")
     submit_button = gr.Button("Submit")
@@ -140,9 +143,10 @@ def generate_tokens_table(text):
 with gr.Blocks() as sutra_tokenize:
     gr.Markdown(
         """
-        # SUTRA Multilingual Tokenizer Sentence Inspector.
+        # Multilingual Tokenizer Sentence Inspector.
         ## Tokenize a sentence with various tokenizers and inspect how it's broken down.
-        """)
+        Space inspired from [SUTRA](https://huggingface.co/spaces/TWO/sutra-tokenizer-comparison)
+""")
     textbox = gr.Textbox(label="Input Text")
     submit_button = gr.Button("Submit")
     output = gr.Dataframe()
@@ -156,7 +160,7 @@ if __name__ == '__main__':
         with gr.Row():
             gr.Markdown(
                 """
-                ## <img src="https://playground.two.ai/sutra.svg" height="20"/>
+                ## <img src="https://raw.githubusercontent.com/SAGAR-TAMANG/sagar-tamang-official-website-new/master/img/pi.jpg" height="30"/>
                 """
             )
         with gr.Row():