George Treacy commited on
Commit
9fc7c74
·
1 Parent(s): 55f9c2f

fix AutoTokenizer auto_map to string not array

Browse files
config.json CHANGED
@@ -8,6 +8,6 @@
8
  "auto_map": {
9
  "AutoConfig": "modeling_makemore.MakemoreConfig",
10
  "AutoModel": "modeling_makemore.MakemoreMLP",
11
- "AutoTokenizer": ["tokenization_makemore.MakemoreTokenizer", null]
12
  }
13
  }
 
8
  "auto_map": {
9
  "AutoConfig": "modeling_makemore.MakemoreConfig",
10
  "AutoModel": "modeling_makemore.MakemoreMLP",
11
+ "AutoTokenizer": "tokenization_makemore.MakemoreTokenizer"
12
  }
13
  }
tokenization_makemore.py CHANGED
@@ -1,7 +1,7 @@
1
  from transformers import PreTrainedTokenizer
2
 
3
  AUTO_MAP = {
4
- "AutoTokenizer": ["tokenization_makemore.MakemoreTokenizer", None]
5
  }
6
 
7
 
 
1
  from transformers import PreTrainedTokenizer
2
 
3
  AUTO_MAP = {
4
+ "AutoTokenizer": "tokenization_makemore.MakemoreTokenizer"
5
  }
6
 
7
 
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "tokenizer_class": "MakemoreTokenizer",
3
  "auto_map": {
4
- "AutoTokenizer": ["tokenization_makemore.MakemoreTokenizer", null]
5
  }
6
  }
 
1
  {
2
  "tokenizer_class": "MakemoreTokenizer",
3
  "auto_map": {
4
+ "AutoTokenizer": "tokenization_makemore.MakemoreTokenizer"
5
  }
6
  }