nmstech commited on
Commit
be2f46e
·
verified ·
1 Parent(s): fffa764

Add AutoTokenizer support (trust_remote_code)

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +4 -2
tokenizer_config.json CHANGED
@@ -1,10 +1,12 @@
1
  {
2
  "tokenizer_class": "TurkTokenizer",
3
  "model_type": "turk-tokenizer",
 
 
 
4
  "version": "1.0.0",
5
- "description": "Turkish morphological tokenizer — TR-MMLU world record 92%",
6
  "language": "tr",
7
- "authors": "Ethosoft",
8
  "requires_java": true,
9
  "dependencies": ["turkish-tokenizer", "jpype1"]
10
  }
 
1
  {
2
  "tokenizer_class": "TurkTokenizer",
3
  "model_type": "turk-tokenizer",
4
+ "auto_map": {
5
+ "AutoTokenizer": ["tokenization_turk.TurkTokenizer", null]
6
+ },
7
  "version": "1.0.0",
 
8
  "language": "tr",
9
+ "description": "Turkish morphological tokenizer — TR-MMLU world record 92%",
10
  "requires_java": true,
11
  "dependencies": ["turkish-tokenizer", "jpype1"]
12
  }