Hugging Face's logo

rainerberger
/
planetn2

Text Generation
Transformers
Safetensors
mistral
trl
sft
text-generation-inference
Model card Files
xet
 Community
rainerberger commited on
Commit
9099915
·
verified ·
1 Parent(s): 8098913

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +10 -6
  2. tokenizer_config.json +8 -0
tokenizer.json CHANGED
@@ -1,11 +1,6 @@
1
  {
2
  "version": "1.0",
3
- "truncation": {
4
- "direction": "Right",
5
- "max_length": 1024,
6
- "strategy": "LongestFirst",
7
- "stride": 0
8
- },
9
  "padding": null,
10
  "added_tokens": [
11
  {
@@ -34,6 +29,15 @@
34
  "rstrip": false,
35
  "normalized": false,
36
  "special": true
 
 
 
 
 
 
 
 
 
37
  }
38
  ],
39
  "normalizer": {
 
1
  {
2
  "version": "1.0",
3
+ "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
 
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
+ },
33
+ {
34
+ "id": 32000,
35
+ "content": "[PAD]",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
  }
42
  ],
43
  "normalizer": {
tokenizer_config.json CHANGED
@@ -25,6 +25,14 @@
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
 
 
 
 
 
 
 
 
28
  }
29
  },
30
  "additional_special_tokens": [],
 
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
+ },
29
+ "32000": {
30
+ "content": "[PAD]",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
  }
37
  },
38
  "additional_special_tokens": [],