Hugging Face's logo

minpeter
/
tiny-ko-random

Text Generation
Transformers
Safetensors
llama
text-generation-inference
Model card Files
xet
 Community
minpeter commited on
Commit
02b0555
·
verified ·
1 Parent(s): 873ecce

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +12 -12
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -26,39 +26,39 @@
26
  "single_word": false,
27
  "special": true
28
  },
29
- "31992": {
30
  "content": "<tool_call>",
31
  "lstrip": false,
32
  "normalized": false,
33
  "rstrip": false,
34
  "single_word": false,
35
- "special": false
36
  },
37
- "31993": {
38
  "content": "</tool_call>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
42
  "single_word": false,
43
- "special": false
44
  },
45
- "31994": {
46
  "content": "<think>",
47
  "lstrip": false,
48
  "normalized": false,
49
  "rstrip": false,
50
  "single_word": false,
51
- "special": false
52
  },
53
- "31995": {
54
  "content": "</think>",
55
  "lstrip": false,
56
  "normalized": false,
57
  "rstrip": false,
58
  "single_word": false,
59
- "special": false
60
  },
61
- "31996": {
62
  "content": "<|unused_special_token_0|>",
63
  "lstrip": false,
64
  "normalized": false,
@@ -66,7 +66,7 @@
66
  "single_word": false,
67
  "special": true
68
  },
69
- "31997": {
70
  "content": "<|unused_special_token_1|>",
71
  "lstrip": false,
72
  "normalized": false,
@@ -74,7 +74,7 @@
74
  "single_word": false,
75
  "special": true
76
  },
77
- "31998": {
78
  "content": "<|unused_special_token_2|>",
79
  "lstrip": false,
80
  "normalized": false,
@@ -82,7 +82,7 @@
82
  "single_word": false,
83
  "special": true
84
  },
85
- "31999": {
86
  "content": "<|unused_special_token_3|>",
87
  "lstrip": false,
88
  "normalized": false,
 
26
  "single_word": false,
27
  "special": true
28
  },
29
+ "3": {
30
  "content": "<tool_call>",
31
  "lstrip": false,
32
  "normalized": false,
33
  "rstrip": false,
34
  "single_word": false,
35
+ "special": true
36
  },
37
+ "4": {
38
  "content": "</tool_call>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
42
  "single_word": false,
43
+ "special": true
44
  },
45
+ "5": {
46
  "content": "<think>",
47
  "lstrip": false,
48
  "normalized": false,
49
  "rstrip": false,
50
  "single_word": false,
51
+ "special": true
52
  },
53
+ "6": {
54
  "content": "</think>",
55
  "lstrip": false,
56
  "normalized": false,
57
  "rstrip": false,
58
  "single_word": false,
59
+ "special": true
60
  },
61
+ "7": {
62
  "content": "<|unused_special_token_0|>",
63
  "lstrip": false,
64
  "normalized": false,
 
66
  "single_word": false,
67
  "special": true
68
  },
69
+ "8": {
70
  "content": "<|unused_special_token_1|>",
71
  "lstrip": false,
72
  "normalized": false,
 
74
  "single_word": false,
75
  "special": true
76
  },
77
+ "9": {
78
  "content": "<|unused_special_token_2|>",
79
  "lstrip": false,
80
  "normalized": false,
 
82
  "single_word": false,
83
  "special": true
84
  },
85
+ "10": {
86
  "content": "<|unused_special_token_3|>",
87
  "lstrip": false,
88
  "normalized": false,