Upload trained BPE tokenizer (vocab 32768)
Browse files- tokenizer/token_bytes.pt +3 -0
- tokenizer/tokenizer.pkl +3 -0
tokenizer/token_bytes.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6b293b04ec1bda26b2f745e5dbb2df2ad4149eb56977a1afbb69f6712c58d24c
|
| 3 |
+
size 132677
|
tokenizer/tokenizer.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:36ca79841b4203beb9d504c18e11ac7d301973982302ff4d233302bb1464c863
|
| 3 |
+
size 412762
|