Upload 7 files

added_tokens.json

@@ -1,23 +1,13 @@
 {
-  "</cell>": 128264,
   "</citation>": 128013,
   "</code>": 128011,
-  "</header>": 128260,
   "</judgement>": 128020,
-  "</row>": 128262,
-  "</table>": 128258,
   "</think>": 128009,
   "</tool_call>": 128005,
   "</tool_response>": 128007,
-  "<cell>": 128263,
   "<citation>": 128012,
   "<code>": 128010,
-  "<empty>": 128265,
-  "<header>": 128259,
   "<judgement>": 128019,
-  "<mask>": 128256,
-  "<row>": 128261,
-  "<table>": 128257,
   "<think>": 128008,
   "<tool_call>": 128004,
   "<tool_response>": 128006,
@@ -37,7 +27,7 @@
   "<|object_ref_end|>": 128016,
   "<|object_ref_start|>": 128015,
   "<|repo_name|>": 128030,
-  "<|reserved_special_token_0|>": 128032,
+  "<|table_start|>": 128032,
   "<|reserved_special_token_100|>": 128132,
   "<|reserved_special_token_101|>": 128133,
   "<|reserved_special_token_102|>": 128134,
@@ -148,7 +138,7 @@
   "<|reserved_special_token_198|>": 128230,
   "<|reserved_special_token_199|>": 128231,
   "<|reserved_special_token_19|>": 128051,
-  "<|reserved_special_token_1|>": 128033,
+  "<|table_end|>": 128033,
   "<|reserved_special_token_200|>": 128232,
   "<|reserved_special_token_201|>": 128233,
   "<|reserved_special_token_202|>": 128234,
@@ -183,7 +173,7 @@
   "<|reserved_special_token_27|>": 128059,
   "<|reserved_special_token_28|>": 128060,
   "<|reserved_special_token_29|>": 128061,
-  "<|reserved_special_token_2|>": 128034,
+  "<|table_header_start|>": 128034,
   "<|reserved_special_token_30|>": 128062,
   "<|reserved_special_token_31|>": 128063,
   "<|reserved_special_token_32|>": 128064,
@@ -194,7 +184,7 @@
   "<|reserved_special_token_37|>": 128069,
   "<|reserved_special_token_38|>": 128070,
   "<|reserved_special_token_39|>": 128071,
-  "<|reserved_special_token_3|>": 128035,
+  "<|table_header_end|>": 128035,
   "<|reserved_special_token_40|>": 128072,
   "<|reserved_special_token_41|>": 128073,
   "<|reserved_special_token_42|>": 128074,
@@ -205,7 +195,7 @@
   "<|reserved_special_token_47|>": 128079,
   "<|reserved_special_token_48|>": 128080,
   "<|reserved_special_token_49|>": 128081,
-  "<|reserved_special_token_4|>": 128036,
+  "<|table_row_start|>": 128036,
   "<|reserved_special_token_50|>": 128082,
   "<|reserved_special_token_51|>": 128083,
   "<|reserved_special_token_52|>": 128084,
@@ -216,7 +206,7 @@
   "<|reserved_special_token_57|>": 128089,
   "<|reserved_special_token_58|>": 128090,
   "<|reserved_special_token_59|>": 128091,
-  "<|reserved_special_token_5|>": 128037,
+  "<|table_row_end|>": 128037,
   "<|reserved_special_token_60|>": 128092,
   "<|reserved_special_token_61|>": 128093,
   "<|reserved_special_token_62|>": 128094,
@@ -227,7 +217,7 @@
   "<|reserved_special_token_67|>": 128099,
   "<|reserved_special_token_68|>": 128100,
   "<|reserved_special_token_69|>": 128101,
-  "<|reserved_special_token_6|>": 128038,
+  "<|table_cell_start|>": 128038,
   "<|reserved_special_token_70|>": 128102,
   "<|reserved_special_token_71|>": 128103,
   "<|reserved_special_token_72|>": 128104,
@@ -238,7 +228,7 @@
   "<|reserved_special_token_77|>": 128109,
   "<|reserved_special_token_78|>": 128110,
   "<|reserved_special_token_79|>": 128111,
-  "<|reserved_special_token_7|>": 128039,
+  "<|table_cell_end|>": 128039,
   "<|reserved_special_token_80|>": 128112,
   "<|reserved_special_token_81|>": 128113,
   "<|reserved_special_token_82|>": 128114,
@@ -249,7 +239,7 @@
   "<|reserved_special_token_87|>": 128119,
   "<|reserved_special_token_88|>": 128120,
   "<|reserved_special_token_89|>": 128121,
-  "<|reserved_special_token_8|>": 128040,
+  "<|table_pad|>": 128040,
   "<|reserved_special_token_90|>": 128122,
   "<|reserved_special_token_91|>": 128123,
   "<|reserved_special_token_92|>": 128124,

special_tokens_map.json

@@ -1,69 +1,4 @@
 {
-  "additional_special_tokens": [
-    {
-      "content": "<table>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</table>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<header>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</header>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<row>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</row>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<cell>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</cell>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<empty>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
-  ],
   "bos_token": {
     "content": "<|begin_of_text|>",
     "lstrip": false,
@@ -78,13 +13,6 @@
     "rstrip": false,
     "single_word": false
   },
-  "mask_token": {
-    "content": "<mask>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "pad_token": {
     "content": "<|end_of_text|>",
     "lstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:849280aad4f0bc56ac303b5415adb979de8ea15fd32c4dfd379c633ef0310f23
-size 17211324
+oid sha256:bf32d7719f1732eef0b7d9752ec708a65efe14c1d8901b9243fa0dc8af325ece
+size 17209390

tokenizer_config.json

@@ -259,7 +259,7 @@
       "special": true
     },
     "128032": {
-      "content": "<|reserved_special_token_0|>",
+      "content": "<|table_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -267,7 +267,7 @@
       "special": true
     },
     "128033": {
-      "content": "<|reserved_special_token_1|>",
+      "content": "<|table_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -275,7 +275,7 @@
       "special": true
     },
     "128034": {
-      "content": "<|reserved_special_token_2|>",
+      "content": "<|table_header_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -283,7 +283,7 @@
       "special": true
     },
     "128035": {
-      "content": "<|reserved_special_token_3|>",
+      "content": "<|table_header_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -291,7 +291,7 @@
       "special": true
     },
     "128036": {
-      "content": "<|reserved_special_token_4|>",
+      "content": "<|table_row_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -299,7 +299,7 @@
       "special": true
     },
     "128037": {
-      "content": "<|reserved_special_token_5|>",
+      "content": "<|table_row_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -307,7 +307,7 @@
       "special": true
     },
     "128038": {
-      "content": "<|reserved_special_token_6|>",
+      "content": "<|table_cell_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -315,7 +315,7 @@
       "special": true
     },
     "128039": {
-      "content": "<|reserved_special_token_7|>",
+      "content": "<|table_cell_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -323,7 +323,7 @@
       "special": true
     },
     "128040": {
-      "content": "<|reserved_special_token_8|>",
+      "content": "<|table_pad|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -2049,110 +2049,19 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "128256": {
-      "content": "<mask>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128257": {
-      "content": "<table>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128258": {
-      "content": "</table>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128259": {
-      "content": "<header>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128260": {
-      "content": "</header>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128261": {
-      "content": "<row>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128262": {
-      "content": "</row>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128263": {
-      "content": "<cell>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128264": {
-      "content": "</cell>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128265": {
-      "content": "<empty>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "<table>",
-    "</table>",
-    "<header>",
-    "</header>",
-    "<row>",
-    "</row>",
-    "<cell>",
-    "</cell>",
-    "<empty>"
-  ],
   "bos_token": "<|begin_of_text|>",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|im_end|>",
   "errors": "replace",
   "extra_special_tokens": {},
-  "mask_token": "<mask>",
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 131072,
+  "mask_token": "<|end_of_text|>",
   "pad_token": "<|end_of_text|>",
   "padding_side": "left",
   "split_special_tokens": false,