kale-tts-1-tiny / recipe /textside /make_ids.mjs
NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
3.02 kB
// SPDX-License-Identifier: Apache-2.0
// Copyright 2026 Kale
// make_ids.mjs: Kokoro-table token ids + a word index per token for the text-side sentences (step 1b of textside/), through the RELEASE
// front end (../../frontend/words: lexicon + public overrides + g2p model; en-GB: en-gb/ files + enGb.js). d03 / d13 read en-GB ids.
// Word index: each word through the same g2p; if the words' ids joined by the space id (16) equal the sentence's ids, token k belongs to
// its word (the space after a word takes that word); otherwise proportional over the IPA words. Ids with no 0 pads; sentences with no ids
// or more than 500 ids are dropped.
// node make_ids.mjs en-us work/sents.jsonl work/sents_us.jsonl (lines {text, prev} -> {text, prev, ids, widx})
// node make_ids.mjs en-gb work/sents.jsonl work/sents_gb.jsonl
// needs node >= 18, no npm packages.
import { readFileSync, writeFileSync } from "node:fs";
const FE = new URL("../../frontend/words/", import.meta.url).pathname;
const { G2p, G2pModel, readTable } = await import(FE + "kaleTts.js");
const { BritishG2p } = await import(FE + "enGb.js");
const [, , accent, inp, out, maxIds = "500"] = process.argv;
if (!["en-us", "en-gb"].includes(accent)) throw new Error("usage: node make_ids.mjs en-us|en-gb in.jsonl out.jsonl");
const bin = (p) => { const b = readFileSync(p); return new G2pModel(b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength)); };
const dir = accent === "en-gb" ? FE + "en-gb/" : FE;
const lex = readTable(readFileSync(dir + "lexicon.tsv", "utf8")); readTable(readFileSync(dir + "override.public.tsv", "utf8"), lex);
const g2p = accent === "en-gb" ? new BritishG2p(lex, bin(dir + "g2p.bin")) : new G2p(lex, bin(dir + "g2p.bin"));
const map = JSON.parse(readFileSync(FE + "ipa2kokoro.json", "utf8")); map["ɒ"] ??= 71;
const ids = (ipa) => [...ipa].map((c) => map[c]).filter((x) => x !== undefined);
let exact = 0;
function one(text) {
const ipa = g2p.phonemes(text), a = ids(ipa), words = text.split(/\s+/).filter(Boolean);
const per = words.map((w) => ids(g2p.phonemes(w)));
const joined = []; per.forEach((p, i) => { joined.push(...p); if (i < per.length - 1) joined.push(16); });
const widx = [];
if (JSON.stringify(joined) === JSON.stringify(a)) { exact++; per.forEach((p, i) => { widx.push(...p.map(() => i)); if (i < per.length - 1) widx.push(i); }); }
else { const nw = Math.max(1, (ipa.match(/ /g) || []).length + 1); let w = 0;
for (const c of ipa) { if (map[c] === undefined) continue; widx.push(Math.min(words.length - 1, Math.floor(w * words.length / nw))); if (c === " ") w++; } }
return { ids: a, widx };
}
const rows = [];
for (const line of readFileSync(inp, "utf8").split("\n")) {
if (!line.trim()) continue; const r = JSON.parse(line); const g = one(r.text);
if (g.ids.length && g.ids.length <= +maxIds) rows.push(JSON.stringify({ text: r.text, prev: r.prev, ...g }));
}
writeFileSync(out, rows.join("\n") + "\n"); console.log(accent, "sentences", rows.length, "exact word map", exact);