Upload tokenizer
Browse files- README.md +1 -1
- tokenizer_config.json +1 -1
README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
---
|
| 2 |
license: other
|
| 3 |
-
base_model: facebook/opt-125m
|
| 4 |
tags:
|
| 5 |
- generated_from_trainer
|
|
|
|
| 6 |
model-index:
|
| 7 |
- name: opt_125_wiki_k10
|
| 8 |
results: []
|
|
|
|
| 1 |
---
|
| 2 |
license: other
|
|
|
|
| 3 |
tags:
|
| 4 |
- generated_from_trainer
|
| 5 |
+
base_model: facebook/opt-125m
|
| 6 |
model-index:
|
| 7 |
- name: opt_125_wiki_k10
|
| 8 |
results: []
|
tokenizer_config.json
CHANGED
|
@@ -23,7 +23,7 @@
|
|
| 23 |
"clean_up_tokenization_spaces": true,
|
| 24 |
"eos_token": "</s>",
|
| 25 |
"errors": "replace",
|
| 26 |
-
"model_max_length":
|
| 27 |
"pad_token": "</s>",
|
| 28 |
"tokenizer_class": "GPT2Tokenizer",
|
| 29 |
"unk_token": "</s>"
|
|
|
|
| 23 |
"clean_up_tokenization_spaces": true,
|
| 24 |
"eos_token": "</s>",
|
| 25 |
"errors": "replace",
|
| 26 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 27 |
"pad_token": "</s>",
|
| 28 |
"tokenizer_class": "GPT2Tokenizer",
|
| 29 |
"unk_token": "</s>"
|