George Treacy commited on
Commit ·
9fc7c74
1
Parent(s): 55f9c2f
fix AutoTokenizer auto_map to string not array
Browse files- config.json +1 -1
- tokenization_makemore.py +1 -1
- tokenizer_config.json +1 -1
config.json
CHANGED
|
@@ -8,6 +8,6 @@
|
|
| 8 |
"auto_map": {
|
| 9 |
"AutoConfig": "modeling_makemore.MakemoreConfig",
|
| 10 |
"AutoModel": "modeling_makemore.MakemoreMLP",
|
| 11 |
-
"AutoTokenizer":
|
| 12 |
}
|
| 13 |
}
|
|
|
|
| 8 |
"auto_map": {
|
| 9 |
"AutoConfig": "modeling_makemore.MakemoreConfig",
|
| 10 |
"AutoModel": "modeling_makemore.MakemoreMLP",
|
| 11 |
+
"AutoTokenizer": "tokenization_makemore.MakemoreTokenizer"
|
| 12 |
}
|
| 13 |
}
|
tokenization_makemore.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
from transformers import PreTrainedTokenizer
|
| 2 |
|
| 3 |
AUTO_MAP = {
|
| 4 |
-
"AutoTokenizer":
|
| 5 |
}
|
| 6 |
|
| 7 |
|
|
|
|
| 1 |
from transformers import PreTrainedTokenizer
|
| 2 |
|
| 3 |
AUTO_MAP = {
|
| 4 |
+
"AutoTokenizer": "tokenization_makemore.MakemoreTokenizer"
|
| 5 |
}
|
| 6 |
|
| 7 |
|
tokenizer_config.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"tokenizer_class": "MakemoreTokenizer",
|
| 3 |
"auto_map": {
|
| 4 |
-
"AutoTokenizer":
|
| 5 |
}
|
| 6 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"tokenizer_class": "MakemoreTokenizer",
|
| 3 |
"auto_map": {
|
| 4 |
+
"AutoTokenizer": "tokenization_makemore.MakemoreTokenizer"
|
| 5 |
}
|
| 6 |
}
|