Hugging Face's logo

ocisd4
/
mistral_tokenizer_ext_dataprep

Model card Files
xet
Community
AlexHT_Hung commited on
Commit
5db4fb4
·
1 Parent(s): 1202640

remove dummies

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -2
  2. tokenizer.json +3 -29
  3. tokenizer_config.json +2 -2
added_tokens.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "<|func_end|>": 35712,
3
- "<|func_start|>": 35713
4
  }
 
1
  {
2
+ "<|func_end|>": 35686,
3
+ "<|func_start|>": 35687
4
  }
tokenizer.json CHANGED
@@ -31,7 +31,7 @@
31
  "special": true
32
  },
33
  {
34
- "id": 35712,
35
  "content": "<|func_end|>",
36
  "single_word": true,
37
  "lstrip": true,
@@ -40,7 +40,7 @@
40
  "special": false
41
  },
42
  {
43
- "id": 35713,
44
  "content": "<|func_start|>",
45
  "single_word": true,
46
  "lstrip": true,
@@ -35810,33 +35810,7 @@
35810
  "賅": 35682,
35811
  "簞": 35683,
35812
  "鼴": 35684,
35813
- "躂": 35685,
35814
- "<DUMMY_0>": 35686,
35815
- "<DUMMY_1>": 35687,
35816
- "<DUMMY_2>": 35688,
35817
- "<DUMMY_3>": 35689,
35818
- "<DUMMY_4>": 35690,
35819
- "<DUMMY_5>": 35691,
35820
- "<DUMMY_6>": 35692,
35821
- "<DUMMY_7>": 35693,
35822
- "<DUMMY_8>": 35694,
35823
- "<DUMMY_9>": 35695,
35824
- "<DUMMY_10>": 35696,
35825
- "<DUMMY_11>": 35697,
35826
- "<DUMMY_12>": 35698,
35827
- "<DUMMY_13>": 35699,
35828
- "<DUMMY_14>": 35700,
35829
- "<DUMMY_15>": 35701,
35830
- "<DUMMY_16>": 35702,
35831
- "<DUMMY_17>": 35703,
35832
- "<DUMMY_18>": 35704,
35833
- "<DUMMY_19>": 35705,
35834
- "<DUMMY_20>": 35706,
35835
- "<DUMMY_21>": 35707,
35836
- "<DUMMY_22>": 35708,
35837
- "<DUMMY_23>": 35709,
35838
- "<DUMMY_24>": 35710,
35839
- "<DUMMY_25>": 35711
35840
  },
35841
  "merges": [
35842
  "▁ t",
 
31
  "special": true
32
  },
33
  {
34
+ "id": 35686,
35
  "content": "<|func_end|>",
36
  "single_word": true,
37
  "lstrip": true,
 
40
  "special": false
41
  },
42
  {
43
+ "id": 35687,
44
  "content": "<|func_start|>",
45
  "single_word": true,
46
  "lstrip": true,
 
35810
  "賅": 35682,
35811
  "簞": 35683,
35812
  "鼴": 35684,
35813
+ "躂": 35685
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35814
  },
35815
  "merges": [
35816
  "▁ t",
tokenizer_config.json CHANGED
@@ -26,7 +26,7 @@
26
  "single_word": false,
27
  "special": true
28
  },
29
- "35712": {
30
  "content": "<|func_end|>",
31
  "lstrip": true,
32
  "normalized": false,
@@ -34,7 +34,7 @@
34
  "single_word": true,
35
  "special": false
36
  },
37
- "35713": {
38
  "content": "<|func_start|>",
39
  "lstrip": true,
40
  "normalized": false,
 
26
  "single_word": false,
27
  "special": true
28
  },
29
+ "35686": {
30
  "content": "<|func_end|>",
31
  "lstrip": true,
32
  "normalized": false,
 
34
  "single_word": true,
35
  "special": false
36
  },
37
+ "35687": {
38
  "content": "<|func_start|>",
39
  "lstrip": true,
40
  "normalized": false,