Download recipe/textside/make_ids.mjs from KaleComputer/kale-tts-1-tiny: direct link, hf CLI and curl.
- Browser
- Download file 3.02 kB
-
https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/textside/make_ids.mjs
- Command line
-
hf download hf://KaleComputer/kale-tts-1-tiny/recipe/textside/make_ids.mjs
-
curl -L -o make_ids.mjs https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/textside/make_ids.mjs
3.02 kB
| // SPDX-License-Identifier: Apache-2.0 | |
| // Copyright 2026 Kale | |
| // make_ids.mjs: Kokoro-table token ids + a word index per token for the text-side sentences (step 1b of textside/), through the RELEASE | |
| // front end (../../frontend/words: lexicon + public overrides + g2p model; en-GB: en-gb/ files + enGb.js). d03 / d13 read en-GB ids. | |
| // Word index: each word through the same g2p; if the words' ids joined by the space id (16) equal the sentence's ids, token k belongs to | |
| // its word (the space after a word takes that word); otherwise proportional over the IPA words. Ids with no 0 pads; sentences with no ids | |
| // or more than 500 ids are dropped. | |
| // node make_ids.mjs en-us work/sents.jsonl work/sents_us.jsonl (lines {text, prev} -> {text, prev, ids, widx}) | |
| // node make_ids.mjs en-gb work/sents.jsonl work/sents_gb.jsonl | |
| // needs node >= 18, no npm packages. | |
| import { readFileSync, writeFileSync } from "node:fs"; | |
| const FE = new URL("../../frontend/words/", import.meta.url).pathname; | |
| const { G2p, G2pModel, readTable } = await import(FE + "kaleTts.js"); | |
| const { BritishG2p } = await import(FE + "enGb.js"); | |
| const [, , accent, inp, out, maxIds = "500"] = process.argv; | |
| if (!["en-us", "en-gb"].includes(accent)) throw new Error("usage: node make_ids.mjs en-us|en-gb in.jsonl out.jsonl"); | |
| const bin = (p) => { const b = readFileSync(p); return new G2pModel(b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength)); }; | |
| const dir = accent === "en-gb" ? FE + "en-gb/" : FE; | |
| const lex = readTable(readFileSync(dir + "lexicon.tsv", "utf8")); readTable(readFileSync(dir + "override.public.tsv", "utf8"), lex); | |
| const g2p = accent === "en-gb" ? new BritishG2p(lex, bin(dir + "g2p.bin")) : new G2p(lex, bin(dir + "g2p.bin")); | |
| const map = JSON.parse(readFileSync(FE + "ipa2kokoro.json", "utf8")); map["ɒ"] ??= 71; | |
| const ids = (ipa) => [...ipa].map((c) => map[c]).filter((x) => x !== undefined); | |
| let exact = 0; | |
| function one(text) { | |
| const ipa = g2p.phonemes(text), a = ids(ipa), words = text.split(/\s+/).filter(Boolean); | |
| const per = words.map((w) => ids(g2p.phonemes(w))); | |
| const joined = []; per.forEach((p, i) => { joined.push(...p); if (i < per.length - 1) joined.push(16); }); | |
| const widx = []; | |
| if (JSON.stringify(joined) === JSON.stringify(a)) { exact++; per.forEach((p, i) => { widx.push(...p.map(() => i)); if (i < per.length - 1) widx.push(i); }); } | |
| else { const nw = Math.max(1, (ipa.match(/ /g) || []).length + 1); let w = 0; | |
| for (const c of ipa) { if (map[c] === undefined) continue; widx.push(Math.min(words.length - 1, Math.floor(w * words.length / nw))); if (c === " ") w++; } } | |
| return { ids: a, widx }; | |
| } | |
| const rows = []; | |
| for (const line of readFileSync(inp, "utf8").split("\n")) { | |
| if (!line.trim()) continue; const r = JSON.parse(line); const g = one(r.text); | |
| if (g.ids.length && g.ids.length <= +maxIds) rows.push(JSON.stringify({ text: r.text, prev: r.prev, ...g })); | |
| } | |
| writeFileSync(out, rows.join("\n") + "\n"); console.log(accent, "sentences", rows.length, "exact word map", exact); | |