// SPDX-License-Identifier: Apache-2.0 // Copyright 2026 Kale // make_ids.mjs: Kokoro-table token ids + a word index per token for the text-side sentences (step 1b of textside/), through the RELEASE // front end (../../frontend/words: lexicon + public overrides + g2p model; en-GB: en-gb/ files + enGb.js). d03 / d13 read en-GB ids. // Word index: each word through the same g2p; if the words' ids joined by the space id (16) equal the sentence's ids, token k belongs to // its word (the space after a word takes that word); otherwise proportional over the IPA words. Ids with no 0 pads; sentences with no ids // or more than 500 ids are dropped. // node make_ids.mjs en-us work/sents.jsonl work/sents_us.jsonl (lines {text, prev} -> {text, prev, ids, widx}) // node make_ids.mjs en-gb work/sents.jsonl work/sents_gb.jsonl // needs node >= 18, no npm packages. import { readFileSync, writeFileSync } from "node:fs"; const FE = new URL("../../frontend/words/", import.meta.url).pathname; const { G2p, G2pModel, readTable } = await import(FE + "kaleTts.js"); const { BritishG2p } = await import(FE + "enGb.js"); const [, , accent, inp, out, maxIds = "500"] = process.argv; if (!["en-us", "en-gb"].includes(accent)) throw new Error("usage: node make_ids.mjs en-us|en-gb in.jsonl out.jsonl"); const bin = (p) => { const b = readFileSync(p); return new G2pModel(b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength)); }; const dir = accent === "en-gb" ? FE + "en-gb/" : FE; const lex = readTable(readFileSync(dir + "lexicon.tsv", "utf8")); readTable(readFileSync(dir + "override.public.tsv", "utf8"), lex); const g2p = accent === "en-gb" ? new BritishG2p(lex, bin(dir + "g2p.bin")) : new G2p(lex, bin(dir + "g2p.bin")); const map = JSON.parse(readFileSync(FE + "ipa2kokoro.json", "utf8")); map["ɒ"] ??= 71; const ids = (ipa) => [...ipa].map((c) => map[c]).filter((x) => x !== undefined); let exact = 0; function one(text) { const ipa = g2p.phonemes(text), a = ids(ipa), words = text.split(/\s+/).filter(Boolean); const per = words.map((w) => ids(g2p.phonemes(w))); const joined = []; per.forEach((p, i) => { joined.push(...p); if (i < per.length - 1) joined.push(16); }); const widx = []; if (JSON.stringify(joined) === JSON.stringify(a)) { exact++; per.forEach((p, i) => { widx.push(...p.map(() => i)); if (i < per.length - 1) widx.push(i); }); } else { const nw = Math.max(1, (ipa.match(/ /g) || []).length + 1); let w = 0; for (const c of ipa) { if (map[c] === undefined) continue; widx.push(Math.min(words.length - 1, Math.floor(w * words.length / nw))); if (c === " ") w++; } } return { ids: a, widx }; } const rows = []; for (const line of readFileSync(inp, "utf8").split("\n")) { if (!line.trim()) continue; const r = JSON.parse(line); const g = one(r.text); if (g.ids.length && g.ids.length <= +maxIds) rows.push(JSON.stringify({ text: r.text, prev: r.prev, ...g })); } writeFileSync(out, rows.join("\n") + "\n"); console.log(accent, "sentences", rows.length, "exact word map", exact);