Spaces:
Sleeping
Sleeping
Qwen3 vocab: add OpenSeek 151851 and SciCore 151670 variants
Browse files
scan.py
CHANGED
|
@@ -72,8 +72,10 @@ KNOWN_BASES = {
|
|
| 72 |
},
|
| 73 |
"qwen3": {
|
| 74 |
"name": "Qwen3",
|
| 75 |
-
"vocab_size": [151936, 152064],
|
| 76 |
"model_type_patterns": ["qwen3"],
|
|
|
|
|
|
|
| 77 |
},
|
| 78 |
"qwen2": {
|
| 79 |
"name": "Qwen2.5 (incl. VL)",
|
|
|
|
| 72 |
},
|
| 73 |
"qwen3": {
|
| 74 |
"name": "Qwen3",
|
| 75 |
+
"vocab_size": [151936, 152064, 151851, 151670],
|
| 76 |
"model_type_patterns": ["qwen3"],
|
| 77 |
+
# 151936/152064 = standard Qwen3; 151851 = BAAI OpenSeek (domain token swap);
|
| 78 |
+
# 151670 = OpenBMB SciCore-Mol (chemistry tokenizer variant)
|
| 79 |
},
|
| 80 |
"qwen2": {
|
| 81 |
"name": "Qwen2.5 (incl. VL)",
|