Add dolma index

app.py

@@ -151,9 +151,10 @@ with gr.Blocks() as demo:
         gr.HTML(
             '''<h1 text-align="center">Infini-gram: An Engine for n-gram / ∞-gram Language Modeling with Trillion-Token Corpora</h1>
 
-            <p style='font-size: 16px;'>This is an engine that processes n-gram / ∞-gram queries on massive text corpora. Please first select the corpus and the type of query, then enter your query and submit. The query is <b>case-sensitive</b>.</p>
+            <p style='font-size: 16px;'>This is an engine that processes n-gram / ∞-gram queries on massive text corpora. Please first select the corpus and the type of query, then enter your query and submit.</p>
             <p style='font-size: 16px;'>The engine is developed by <a href="https://liujch1998.github.io">Jiacheng (Gary) Liu</a> and documented in our paper: <a href="https://huggingface.co/papers/2401.17377">Infini-gram: Scaling Unbounded n-gram Language Models to a Trillion Tokens</a>.</p>
             <p style='font-size: 16px;'><b>API Endpoint:</b> If you'd like to issue batch queries to infini-gram, you may invoke our API endpoint. Please refer to the <a href="https://infini-gram.io/api_doc">API documentation</a>.</p>
+            <p style='font-size: 16px;'><b>Note:</b> The query is <b>case-sensitive</b>. Your query will be tokenized with the Llama-2 tokenizer (unless otherwise specified).</p>
             '''
         )
         with gr.Row():

constants.py

@@ -2,12 +2,23 @@ import os
 
 # options
 CORPUS_BY_DESC = {
-    'RedPajama (LLaMA tokenizer), 1.4T tokens': 'v4_rpj_llama_s4',
-    'Pile-train (LLaMA tokenizer), 380B tokens': 'v4_piletrain_llama',
-    'C4-train (LLaMA tokenizer), 200B tokens': 'v4_c4train_llama',
-    'Pile-val (LLaMA tokenizer), 390M tokens': 'v4_pileval_llama',
-    'Pile-val (GPT-2 tokenizer), 380M tokens': 'v4_pileval_gpt2',
-    'Dolma-sample (OLMo tokenizer), 8.0B tokens': 'v4_dolmasample_olmo',
+    'Dolma (3.1T tokens)': 'v4_dolma-v1_6_llama',
+    'RedPajama (1.4T tokens)': 'v4_rpj_llama_s4',
+    'Pile-train (380B tokens)': 'v4_piletrain_llama',
+    'C4-train (200B tokens)': 'v4_c4train_llama',
+    'Pile-val (390M tokens)': 'v4_pileval_llama',
+    # 'Pile-val (GPT-2 tokenizer), 380M tokens': 'v4_pileval_gpt2',
+    # 'Dolma-v1.6-sample (OLMo tokenizer), 8.0B tokens': 'v4_dolmasample_olmo',
+    # 'Dolma-v1.6-sample (9.2B tokens)': 'v4_dolma-v1_6-sample_llama',
+    # 'Dolma-v1.6-wiki (4.3B tokens)': 'v4_dolma-v1_6-wiki_llama',
+    # 'Dolma-v1.6-books (5.8B tokens)': 'v4_dolma-v1_6-books_llama',
+    # 'Dolma-v1.6-pes2o (69B tokens)': 'v4_dolma-v1_6-pes2o_llama',
+    # 'Dolma-v1.6-reddit (89B tokens)': 'v4_dolma-v1_6-reddit_llama',
+    # 'Dolma-v1.6-c4 (200B tokens)': 'v4_dolma-v1_6-c4_llama',
+    # 'Dolma-v1.6-stack (420B tokens)': 'v4_dolma-v1_6-stack_llama',
+    # 'Dolma-v1.6-cc_en_head (660B tokens): 'v4_dolma-v1_6-cc_en_head_llama',
+    # 'Dolma-v1.6-cc_en_middle (650B tokens): 'v4_dolma-v1_6-cc_en_middle_llama',
+    # 'Dolma-v1.6-cc_en_tail (970B tokens): 'v4_dolma-v1_6-cc_en_tail_llama',
 }
 CORPUS_DESCS = list(CORPUS_BY_DESC.keys())
 ENGINE_BY_DESC = {