Upload model trained with Unsloth
Browse filesUpload model trained with Unsloth 2x faster
- added_tokens.json +3 -0
- tokenizer.json +2 -2
- tokenizer_config.json +8 -0
added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<BOS_KSL>": 256000
|
| 3 |
+
}
|
tokenizer.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0b3ebfda89adafaed140b2a809d7fcf5b9809ff3a7abc32fa98a19acbcfc666a
|
| 3 |
+
size 34363059
|
tokenizer_config.json
CHANGED
|
@@ -1993,6 +1993,14 @@
|
|
| 1993 |
"rstrip": false,
|
| 1994 |
"single_word": false,
|
| 1995 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1996 |
}
|
| 1997 |
},
|
| 1998 |
"additional_special_tokens": [
|
|
|
|
| 1993 |
"rstrip": false,
|
| 1994 |
"single_word": false,
|
| 1995 |
"special": false
|
| 1996 |
+
},
|
| 1997 |
+
"256000": {
|
| 1998 |
+
"content": "<BOS_KSL>",
|
| 1999 |
+
"lstrip": false,
|
| 2000 |
+
"normalized": true,
|
| 2001 |
+
"rstrip": false,
|
| 2002 |
+
"single_word": false,
|
| 2003 |
+
"special": false
|
| 2004 |
}
|
| 2005 |
},
|
| 2006 |
"additional_special_tokens": [
|