trohrbaugh commited on
Commit
a1a2d2b
·
verified ·
1 Parent(s): 6bb466a

Qwen3 vocab: add OpenSeek 151851 and SciCore 151670 variants

Browse files
Files changed (1) hide show
  1. scan.py +3 -1
scan.py CHANGED
@@ -72,8 +72,10 @@ KNOWN_BASES = {
72
  },
73
  "qwen3": {
74
  "name": "Qwen3",
75
- "vocab_size": [151936, 152064],
76
  "model_type_patterns": ["qwen3"],
 
 
77
  },
78
  "qwen2": {
79
  "name": "Qwen2.5 (incl. VL)",
 
72
  },
73
  "qwen3": {
74
  "name": "Qwen3",
75
+ "vocab_size": [151936, 152064, 151851, 151670],
76
  "model_type_patterns": ["qwen3"],
77
+ # 151936/152064 = standard Qwen3; 151851 = BAAI OpenSeek (domain token swap);
78
+ # 151670 = OpenBMB SciCore-Mol (chemistry tokenizer variant)
79
  },
80
  "qwen2": {
81
  "name": "Qwen2.5 (incl. VL)",