| // Static lm_head shortlist — which ids can the decoder ever EMIT? | |
| // | |
| // The vocab is shared zh+vi (tied embeddings): 18,105 of MoxhiMT-30's 24,000 | |
| // pieces contain CJK characters and exist for the SOURCE side only; the | |
| // decoder's Vietnamese output draws from the 5,895 non-CJK pieces (syllables, | |
| // latin, digits, punctuation, byte fallbacks, specials). Measured basis | |
| // (2026-07-09, notes-m4-tuning.md "Lexical-shortlist pre-gate"): 2.078M | |
| // tokens emitted over qingshan used 5,232 distinct ids, every one non-CJK; | |
| // cross-corpus probe misses (golden + a second novel vs a frequency list) | |
| // were ALL non-CJK Vietnamese pieces too — frequency lists leak on proper | |
| // names, the script split does not. Classifying by script therefore gives a | |
| // closed emittable set with no offline artifact: computable from | |
| // tokenizer.json at load, per model, in milliseconds. | |
| // | |
| // Selection is by EXCLUSION of ideographs only — NOT of the CJK punctuation | |
| // and fullwidth blocks. Measured the hard way (shortlist_ab, first run): the | |
| // model legitimately emits 《》-style book-title marks inside Vietnamese | |
| // output (71/60,353 rows drifted when the whole U+3000 block was excluded), | |
| // so punctuation, enclosed symbols, and fullwidth forms stay emittable | |
| // (+56 pieces on MT-30 — 5,951 vs 5,895; the win is unchanged). What stays | |
| // out: Han ideographs (URO + ext A), radicals, strokes, bopomofo, | |
| // compatibility ideographs, and the supplementary planes. ASCII-looking | |
| // specials (</s>, <pad>, <unk>, <0xNN> byte pieces) pass by construction. | |
| // The list feeds loadWeights({ lmHeadIds }) which repacks the q8 lm_head to | |
| // just these rows — display/embedding paths never see it. | |
| export const CJK_RE = new RegExp( | |
| '[\\u2E80-\\u2EFF\\u3100-\\u312F\\u31C0-\\u31EF' | |
| + '\\u3400-\\u9FFF\\uF900-\\uFAFF]' | |
| + '|[\\u{20000}-\\u{3FFFF}]', 'u'); | |
| // idToToken: id-indexed array of piece strings (SpmTokenizer.idToToken). | |
| // Returns ascending Uint32Array of emittable ids. | |
| export function emittableIds(idToToken) { | |
| const ids = []; | |
| for (let id = 0; id < idToToken.length; id++) { | |
| const piece = idToToken[id]; | |
| if (typeof piece === 'string' && !CJK_RE.test(piece)) ids.push(id); | |
| } | |
| return Uint32Array.from(ids); | |
| } | |