Add missing tokenizer.json (fast tokenizer for TEI compatibility)
Browse files- .gitattributes +1 -0
- tokenizer.json +3 -0
.gitattributes
CHANGED
|
@@ -42,3 +42,4 @@ onnx/model.onnx_data_4 filter=lfs diff=lfs merge=lfs -text
|
|
| 42 |
onnx/model.onnx_data_5 filter=lfs diff=lfs merge=lfs -text
|
| 43 |
onnx/model.onnx_data_6 filter=lfs diff=lfs merge=lfs -text
|
| 44 |
onnx/model.onnx_data_7 filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 42 |
onnx/model.onnx_data_5 filter=lfs diff=lfs merge=lfs -text
|
| 43 |
onnx/model.onnx_data_6 filter=lfs diff=lfs merge=lfs -text
|
| 44 |
onnx/model.onnx_data_7 filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c6fb5c5bbba5fa5f8332edfb6d8aa67bd7fb3d75365b1765f108201698eaebf5
|
| 3 |
+
size 11422837
|