Clear file names: kale-tts-1-tiny.onnx at the root, PyTorch weights in pytorch/
Browse files- README.md +4 -4
- SHA256SUMS +6 -6
- js/README.md +2 -2
- js/kale-tts.mjs +5 -4
- model.fp32.onnx β kale-tts-1-tiny.fp32.onnx +0 -0
- model.onnx β kale-tts-1-tiny.onnx +0 -0
- python/kale_tts.py +4 -3
- {checkpoints β pytorch}/README.md +5 -3
- checkpoints/teacher/kokoro_teacher.pth β pytorch/teacher/kale-tts-1-teacher.pth +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/d03.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/d13.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/f06.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/h013f.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/m3.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/s057b.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/t071.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/w165.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/w602.pt +0 -0
- {checkpoints/packs β pytorch/teacher/voices}/x328.pt +0 -0
- {checkpoints β pytorch}/tiny/decoder_flkd.pt +0 -0
- {checkpoints β pytorch}/tiny/duration_head_fld.pt +0 -0
- {checkpoints/tiny β pytorch/tiny/earlier}/decoder_t10k.pt +0 -0
- {checkpoints/tiny β pytorch/tiny/earlier}/duration_head_t10d2.pt +0 -0
- {checkpoints/tiny β pytorch/tiny/earlier}/ettin_text_side_t10e.pt +0 -0
- {checkpoints/tiny β pytorch/tiny/earlier}/pitch_heads_t10p.pt +0 -0
- {checkpoints β pytorch}/tiny/ettin_text_side_fle.pt +0 -0
- {checkpoints β pytorch}/tiny/pitch_heads_flp.pt +0 -0
- recipe/distil/README.md +1 -1
- recipe/export/README.md +1 -1
- export-check-model.fp32.json β results/export-check/export-check-model.fp32.json +0 -0
- export-check-model.json β results/export-check/export-check-model.json +0 -0
- export-check-whisper.json β results/export-check/export-check-whisper.json +0 -0
- runtimes/RUNTIMES.md +2 -0
- runtimes/python/kale_tts.py +4 -3
- server/server.mjs +2 -2
- voices.json +1 -0
README.md
CHANGED
|
@@ -18,7 +18,7 @@ kale-tts-1-tiny is a small English text-to-speech model with 10 voices designed
|
|
| 18 |
|
| 19 |
| measure | value |
|
| 20 |
|---|---|
|
| 21 |
-
| size | 79.9 MB on the CPU (int8, `
|
| 22 |
| parameters | 45.7M (16.8M of them the frozen Ettin word encoder) |
|
| 23 |
| unclear words | +1.4 points vs Maya-1's own recordings (Maya-1: 5.82 %) |
|
| 24 |
| melody gap | +7.48 points vs Maya-1's own recordings (Maya-1: 0) |
|
|
@@ -240,7 +240,7 @@ pip install -U huggingface_hub
|
|
| 240 |
|
| 241 |
# only the files the runtime needs
|
| 242 |
hf download KaleComputer/kale-tts-1-tiny --local-dir kale-tts-1-tiny \
|
| 243 |
-
--include "
|
| 244 |
--include "voices/*" --include "js/*" --include "frontend/*" --include "python/*" --include "server/*"
|
| 245 |
|
| 246 |
cd kale-tts-1-tiny
|
|
@@ -381,7 +381,7 @@ What we tried, one line each:
|
|
| 381 |
|
| 382 |
## Make it faster or better
|
| 383 |
- **WebGPU:** keep the six small LSTM layers on the CPU (`forceCpuNodeNames`; our JS API does it by default): about 2.2x faster on onnxruntime-web 1.27 and newer (29x -> 63x through transformers.js, 18 Nov).
|
| 384 |
-
- **Pick the file per backend:** the int8 `
|
| 385 |
- **Threads:** 4 threads on a phone or tablet CPU; on a server 1 thread per process and one process per core.
|
| 386 |
- **Stream sentence by sentence:** the first sentence plays while the rest renders.
|
| 387 |
- **A smaller text reader:** most of tiny's extra time is the Ettin word reader; nano's own small reader is about 2x faster, with a melody further from Maya-1's.
|
|
@@ -395,7 +395,7 @@ What we tried, one line each:
|
|
| 395 |
- Not for impersonating real people or passing the output off as human speech.
|
| 396 |
|
| 397 |
## Files
|
| 398 |
-
`
|
| 399 |
|
| 400 |
## Licence and credits
|
| 401 |
Weights and code Apache-2.0; dataset CC BY 4.0.
|
|
|
|
| 18 |
|
| 19 |
| measure | value |
|
| 20 |
|---|---|
|
| 21 |
+
| size | 79.9 MB on the CPU (int8, `kale-tts-1-tiny.onnx`); 183.2 MB for WebGPU in the browser (float32, `kale-tts-1-tiny.fp32.onnx`) |
|
| 22 |
| parameters | 45.7M (16.8M of them the frozen Ettin word encoder) |
|
| 23 |
| unclear words | +1.4 points vs Maya-1's own recordings (Maya-1: 5.82 %) |
|
| 24 |
| melody gap | +7.48 points vs Maya-1's own recordings (Maya-1: 0) |
|
|
|
|
| 240 |
|
| 241 |
# only the files the runtime needs
|
| 242 |
hf download KaleComputer/kale-tts-1-tiny --local-dir kale-tts-1-tiny \
|
| 243 |
+
--include "kale-tts-1-tiny.onnx" --include "*tokenizer.json" --include "voices.json" --include "config.json" \
|
| 244 |
--include "voices/*" --include "js/*" --include "frontend/*" --include "python/*" --include "server/*"
|
| 245 |
|
| 246 |
cd kale-tts-1-tiny
|
|
|
|
| 381 |
|
| 382 |
## Make it faster or better
|
| 383 |
- **WebGPU:** keep the six small LSTM layers on the CPU (`forceCpuNodeNames`; our JS API does it by default): about 2.2x faster on onnxruntime-web 1.27 and newer (29x -> 63x through transformers.js, 18 Nov).
|
| 384 |
+
- **Pick the file per backend:** the int8 `kale-tts-1-tiny.onnx` on CPU and WASM, `kale-tts-1-tiny.fp32.onnx` on WebGPU.
|
| 385 |
- **Threads:** 4 threads on a phone or tablet CPU; on a server 1 thread per process and one process per core.
|
| 386 |
- **Stream sentence by sentence:** the first sentence plays while the rest renders.
|
| 387 |
- **A smaller text reader:** most of tiny's extra time is the Ettin word reader; nano's own small reader is about 2x faster, with a melody further from Maya-1's.
|
|
|
|
| 395 |
- Not for impersonating real people or passing the output off as human speech.
|
| 396 |
|
| 397 |
## Files
|
| 398 |
+
`kale-tts-1-tiny.onnx` (79.90 MB, int8 / fp16 weights where safe, phoneme encoder and durations float32), `kale-tts-1-tiny.fp32.onnx` (183.25 MB, float32, for WebGPU), `voices/` + `voices.json`, `tokenizer.json`, `ettin-tokenizer.json` (MIT). Inputs: `input_ids`, `style`, `speed`, `ettin_ids`, `ettin_mask`, `wavg`, `widx`, then the voice inputs listed in `voices.json`. PyTorch weights in `pytorch/` (see `pytorch/README.md`). The training recipe is in `recipe/`.
|
| 399 |
|
| 400 |
## Licence and credits
|
| 401 |
Weights and code Apache-2.0; dataset CC BY 4.0.
|
SHA256SUMS
CHANGED
|
@@ -1,13 +1,13 @@
|
|
| 1 |
cf786abdfdf8e89962d1fb888d22decfde1efc6cf43564199ba9468788763c7b config.json
|
| 2 |
9fd55248d51d33976b324fc11592e28071da7d41e0e9401dfb7082e30574b7b1 ettin-tokenizer.json
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
61188314434d9972e42677fb71dba9e19aa8698e980541b1c06a1f9f16086e60 tokenizer.json
|
| 9 |
be1cb066d6ef6b074b3f15e6a6dd21ac88ff3cdaedf325f0aaed686c70f75d20 tokenizer_config.json
|
| 10 |
-
|
| 11 |
179741da5b5fb1f7d49399043f4788f78c28eaf02b00115aa8f3ffee3ad4cb6b voices/d03.bin
|
| 12 |
c026f7cef743b802088f49c6d0790978234336c4b13ca0344b15692dee4838f8 voices/d03.extra.bin
|
| 13 |
add53ad105ba63b83e39bd85f3bfa96b2375bf550caa65d6c99feb7e66e7cee3 voices/d13.bin
|
|
|
|
| 1 |
cf786abdfdf8e89962d1fb888d22decfde1efc6cf43564199ba9468788763c7b config.json
|
| 2 |
9fd55248d51d33976b324fc11592e28071da7d41e0e9401dfb7082e30574b7b1 ettin-tokenizer.json
|
| 3 |
+
885675190d58a5f28f096f4698526fa2a2f2378d9d1897b6b0721596f585f64f kale-tts-1-tiny.fp32.onnx
|
| 4 |
+
8f255ef6f2b22c130cce15bdf92c5d420a821b2e5e39dbdef34064d52d6dc495 kale-tts-1-tiny.onnx
|
| 5 |
+
0ad61cb8bc4899e610a317d1ddbaa4c7e5bf55c2abda41f0c746d41479800deb results/export-check/export-check-model.fp32.json
|
| 6 |
+
27f07f3c212bf5bcbf9a26b6fc1415284245527fd58ccfdce46c456648cfdf3f results/export-check/export-check-model.json
|
| 7 |
+
c535680c3645897b5b1d5e1aa12b839976ef423c05719dc823e2ece90f52f183 results/export-check/export-check-whisper.json
|
| 8 |
61188314434d9972e42677fb71dba9e19aa8698e980541b1c06a1f9f16086e60 tokenizer.json
|
| 9 |
be1cb066d6ef6b074b3f15e6a6dd21ac88ff3cdaedf325f0aaed686c70f75d20 tokenizer_config.json
|
| 10 |
+
4875793c25eb5378bdf74c0fd49f3e1dc643ede4b9bb2db58721e39b0db689cd voices.json
|
| 11 |
179741da5b5fb1f7d49399043f4788f78c28eaf02b00115aa8f3ffee3ad4cb6b voices/d03.bin
|
| 12 |
c026f7cef743b802088f49c6d0790978234336c4b13ca0344b15692dee4838f8 voices/d03.extra.bin
|
| 13 |
add53ad105ba63b83e39bd85f3bfa96b2375bf550caa65d6c99feb7e66e7cee3 voices/d13.bin
|
js/README.md
CHANGED
|
@@ -20,14 +20,14 @@ set offline after preload, all 8 voices speak, 0 console errors).
|
|
| 20 |
|
| 21 |
## Graph and backend
|
| 22 |
`KaleTTS.load(name, { graph, executionProviders, sessionOptions })`:
|
| 23 |
-
- `graph`: `"auto"` (default), `"int8"` (`
|
| 24 |
- `executionProviders`: onnxruntime EPs, e.g. `["webgpu"]`, `["wasm"]`, `["cpu"]`. `sessionOptions` go to `InferenceSession.create`.
|
| 25 |
- Default with nothing set: in a browser with a WebGPU adapter, fp32 on webgpu (checked with one tiny run); if there's no adapter, no fp32
|
| 26 |
file, or the run fails, int8 on wasm. In Node: int8 on cpu. Options you set are used as given, with no fallback.
|
| 27 |
- WebGPU sessions keep the six LSTM layers on the CPU (`forceCpuNodeNames`): since onnxruntime-web 1.27 their WebGPU kernel is slower
|
| 28 |
for this model (nano fp32 85x -> 163-175x, tiny int8 29x -> 58-62x, Mac Chrome). Pass your own `{ name: "webgpu", ... }` object to
|
| 29 |
override. With no fp32 file (tiny), the default tries int8 on webgpu before wasm.
|
| 30 |
-
- `tts.backend` tells what was picked: `{ graph: "
|
| 31 |
- Browser: WebGPU needs onnxruntime-web 1.31 or newer, loaded from its WebGPU build, e.g. an import map entry
|
| 32 |
`"onnxruntime-web": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.31.1/dist/ort.webgpu.bundle.min.mjs"` (that build runs wasm too).
|
| 33 |
With 1.22.0, WebGPU fails at run time and the default falls back to wasm.
|
|
|
|
| 20 |
|
| 21 |
## Graph and backend
|
| 22 |
`KaleTTS.load(name, { graph, executionProviders, sessionOptions })`:
|
| 23 |
+
- `graph`: `"auto"` (default), `"int8"` (`kale-tts-1-tiny.onnx`), `"fp32"` (`kale-tts-1-tiny.fp32.onnx`, nano only for now) or a file name in the model folder.
|
| 24 |
- `executionProviders`: onnxruntime EPs, e.g. `["webgpu"]`, `["wasm"]`, `["cpu"]`. `sessionOptions` go to `InferenceSession.create`.
|
| 25 |
- Default with nothing set: in a browser with a WebGPU adapter, fp32 on webgpu (checked with one tiny run); if there's no adapter, no fp32
|
| 26 |
file, or the run fails, int8 on wasm. In Node: int8 on cpu. Options you set are used as given, with no fallback.
|
| 27 |
- WebGPU sessions keep the six LSTM layers on the CPU (`forceCpuNodeNames`): since onnxruntime-web 1.27 their WebGPU kernel is slower
|
| 28 |
for this model (nano fp32 85x -> 163-175x, tiny int8 29x -> 58-62x, Mac Chrome). Pass your own `{ name: "webgpu", ... }` object to
|
| 29 |
override. With no fp32 file (tiny), the default tries int8 on webgpu before wasm.
|
| 30 |
+
- `tts.backend` tells what was picked: `{ graph: "kale-tts-1-tiny.fp32.onnx", executionProviders: ["webgpu"] }`.
|
| 31 |
- Browser: WebGPU needs onnxruntime-web 1.31 or newer, loaded from its WebGPU build, e.g. an import map entry
|
| 32 |
`"onnxruntime-web": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.31.1/dist/ort.webgpu.bundle.min.mjs"` (that build runs wasm too).
|
| 33 |
With 1.22.0, WebGPU fails at run time and the default falls back to wasm.
|
js/kale-tts.mjs
CHANGED
|
@@ -104,7 +104,7 @@ export class KaleTTS {
|
|
| 104 |
static registerFrontEnd(accent, filesOrLoader) { FRONT_ENDS[accent] = filesOrLoader; }
|
| 105 |
|
| 106 |
/** model: a model folder (path or URL) or a release name looked up next to this file (../models/<name>/).
|
| 107 |
-
* graph: "auto" (default) | "int8"
|
| 108 |
* executionProviders: onnxruntime EPs, e.g. ["webgpu"], ["wasm"], ["cpu"]. sessionOptions: passed to InferenceSession.create.
|
| 109 |
* Default (nothing set): browser with a WebGPU adapter -> fp32 graph on webgpu (int8 on webgpu if there is no fp32 file), otherwise (or if that fails) int8 on wasm; Node -> int8
|
| 110 |
* on cpu. Anything set explicitly is used as given, with no fallback. tts.backend tells what was picked. */
|
|
@@ -112,17 +112,18 @@ export class KaleTTS {
|
|
| 112 |
const base = /[/\\]/.test(model) ? model : (isNode ? new URL(`../models/${model}/`, import.meta.url).pathname : new URL(`../models/${model}/`, import.meta.url).href);
|
| 113 |
const feBase = frontend ?? (isNode ? new URL("../frontend/", import.meta.url).pathname : new URL("../frontend/", import.meta.url).href);
|
| 114 |
ort ??= isNode ? await import("onnxruntime-node") : await import("onnxruntime-web"); // browser: map it to ort.webgpu.bundle for WebGPU
|
| 115 |
-
const file = (g) => ({ int8: "model.onnx", fp32: "model.fp32.onnx" })[g] ?? g;
|
| 116 |
const [voicesJson, tok] = await Promise.all([readText(join(base, "voices.json")), readText(join(base, "tokenizer.json"))]);
|
|
|
|
|
|
|
| 117 |
const webgpuOk = JSON.parse(voicesJson).webgpu !== false; // voices.json "webgpu": false = this graph has no WebGPU path (auto skips it)
|
| 118 |
let plans;
|
| 119 |
if (graph !== "auto" || executionProviders) {
|
| 120 |
const eps = executionProviders ?? [isNode ? "cpu" : "wasm"];
|
| 121 |
plans = [{ graph: file(graph === "auto" ? (eps.includes("webgpu") ? "fp32" : "int8") : graph), eps }];
|
| 122 |
-
} else if (isNode) plans = [{ graph:
|
| 123 |
else {
|
| 124 |
const gpu = webgpuOk && typeof navigator !== "undefined" && navigator.gpu ? await navigator.gpu.requestAdapter().catch(() => null) : null;
|
| 125 |
-
plans = [...(gpu ? [{ graph:
|
| 126 |
}
|
| 127 |
const opts = { logSeverityLevel: 3, ...(threads ? { intraOpNumThreads: threads } : {}), ...sessionOptions }; // quiet: errors only
|
| 128 |
for (const [i, p] of plans.entries()) {
|
|
|
|
| 104 |
static registerFrontEnd(accent, filesOrLoader) { FRONT_ENDS[accent] = filesOrLoader; }
|
| 105 |
|
| 106 |
/** model: a model folder (path or URL) or a release name looked up next to this file (../models/<name>/).
|
| 107 |
+
* graph: "auto" (default) | "int8" | "fp32" (file names from voices.json "graphs", else model.onnx / model.fp32.onnx) | a file name in the model folder.
|
| 108 |
* executionProviders: onnxruntime EPs, e.g. ["webgpu"], ["wasm"], ["cpu"]. sessionOptions: passed to InferenceSession.create.
|
| 109 |
* Default (nothing set): browser with a WebGPU adapter -> fp32 graph on webgpu (int8 on webgpu if there is no fp32 file), otherwise (or if that fails) int8 on wasm; Node -> int8
|
| 110 |
* on cpu. Anything set explicitly is used as given, with no fallback. tts.backend tells what was picked. */
|
|
|
|
| 112 |
const base = /[/\\]/.test(model) ? model : (isNode ? new URL(`../models/${model}/`, import.meta.url).pathname : new URL(`../models/${model}/`, import.meta.url).href);
|
| 113 |
const feBase = frontend ?? (isNode ? new URL("../frontend/", import.meta.url).pathname : new URL("../frontend/", import.meta.url).href);
|
| 114 |
ort ??= isNode ? await import("onnxruntime-node") : await import("onnxruntime-web"); // browser: map it to ort.webgpu.bundle for WebGPU
|
|
|
|
| 115 |
const [voicesJson, tok] = await Promise.all([readText(join(base, "voices.json")), readText(join(base, "tokenizer.json"))]);
|
| 116 |
+
const graphs = { int8: "model.onnx", fp32: "model.fp32.onnx", ...JSON.parse(voicesJson).graphs }; // a hub repo names its graphs in voices.json
|
| 117 |
+
const file = (g) => graphs[g] ?? g;
|
| 118 |
const webgpuOk = JSON.parse(voicesJson).webgpu !== false; // voices.json "webgpu": false = this graph has no WebGPU path (auto skips it)
|
| 119 |
let plans;
|
| 120 |
if (graph !== "auto" || executionProviders) {
|
| 121 |
const eps = executionProviders ?? [isNode ? "cpu" : "wasm"];
|
| 122 |
plans = [{ graph: file(graph === "auto" ? (eps.includes("webgpu") ? "fp32" : "int8") : graph), eps }];
|
| 123 |
+
} else if (isNode) plans = [{ graph: graphs.int8, eps: ["cpu"] }];
|
| 124 |
else {
|
| 125 |
const gpu = webgpuOk && typeof navigator !== "undefined" && navigator.gpu ? await navigator.gpu.requestAdapter().catch(() => null) : null;
|
| 126 |
+
plans = [...(gpu ? [{ graph: graphs.fp32, eps: ["webgpu"] }, { graph: graphs.int8, eps: ["webgpu"] }] : []), { graph: graphs.int8, eps: ["wasm"] }];
|
| 127 |
}
|
| 128 |
const opts = { logSeverityLevel: 3, ...(threads ? { intraOpNumThreads: threads } : {}), ...sessionOptions }; // quiet: errors only
|
| 129 |
for (const [i, p] of plans.entries()) {
|
model.fp32.onnx β kale-tts-1-tiny.fp32.onnx
RENAMED
|
File without changes
|
model.onnx β kale-tts-1-tiny.onnx
RENAMED
|
File without changes
|
python/kale_tts.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs,
|
| 2 |
kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself.
|
| 3 |
-
pip install onnxruntime numpy quickjs (graph: "int8"
|
| 4 |
|
| 5 |
from kale_tts import KaleTTS
|
| 6 |
tts = KaleTTS("models/kale-tts-1-nano")
|
|
@@ -104,12 +104,13 @@ def default_frontend():
|
|
| 104 |
|
| 105 |
class KaleTTS:
|
| 106 |
def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None):
|
| 107 |
-
model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx"}.get(graph, graph)
|
| 108 |
d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir)
|
|
|
|
|
|
|
| 109 |
so = ort.SessionOptions()
|
| 110 |
if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1
|
| 111 |
self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers))
|
| 112 |
-
|
| 113 |
self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {}
|
| 114 |
self.kn = vj.get("phonemes") == "kokoro-native" # Kokoro-native ids (see kn_ipa)
|
| 115 |
self.frontend = default_frontend() if frontend is True else (frontend or None)
|
|
|
|
| 1 |
"""kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs,
|
| 2 |
kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself.
|
| 3 |
+
pip install onnxruntime numpy quickjs (graph: "int8" (default) or "fp32", named in voices.json "graphs", or a file name)
|
| 4 |
|
| 5 |
from kale_tts import KaleTTS
|
| 6 |
tts = KaleTTS("models/kale-tts-1-nano")
|
|
|
|
| 104 |
|
| 105 |
class KaleTTS:
|
| 106 |
def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None):
|
|
|
|
| 107 |
d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir)
|
| 108 |
+
vj = json.loads((d / "voices.json").read_text())
|
| 109 |
+
model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx", **vj.get("graphs", {})}.get(graph, graph)
|
| 110 |
so = ort.SessionOptions()
|
| 111 |
if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1
|
| 112 |
self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers))
|
| 113 |
+
self.voice_inputs, self.voices = vj["voice_inputs"], vj["voices"]
|
| 114 |
self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {}
|
| 115 |
self.kn = vj.get("phonemes") == "kokoro-native" # Kokoro-native ids (see kn_ipa)
|
| 116 |
self.frontend = default_frontend() if frontend is True else (frontend or None)
|
{checkpoints β pytorch}/README.md
RENAMED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
<!-- SPDX-License-Identifier: Apache-2.0 | Copyright 2026 Kale -->
|
| 2 |
# kale-tts-1-tiny training checkpoints (PyTorch)
|
| 3 |
|
| 4 |
-
The trained parts `
|
| 5 |
-
The teacher
|
| 6 |
-
|
| 7 |
|
| 8 |
| file | what |
|
| 9 |
|---|---|
|
|
@@ -13,3 +13,5 @@ the 10 voices); tiny keeps its timing-and-melody planner, frozen.
|
|
| 13 |
| `tiny/decoder_flkd.pt` | tiny's 3.65M sound generator, 10 style rows, with its two critics (`model`, `D`, `J`) |
|
| 14 |
|
| 15 |
Voice order in every per-voice table: s057b, f06, d03, w165, w602, m3, t071, d13, x328, h013f.
|
|
|
|
|
|
|
|
|
| 1 |
<!-- SPDX-License-Identifier: Apache-2.0 | Copyright 2026 Kale -->
|
| 2 |
# kale-tts-1-tiny training checkpoints (PyTorch)
|
| 3 |
|
| 4 |
+
The trained parts `kale-tts-1-tiny.onnx` was exported from, for fine-tuning with `recipe/`. Weights only (no optimiser states), Apache-2.0.
|
| 5 |
+
The teacher both models were distilled from is `teacher/kale-tts-1-teacher.pth` (Kokoro-82M fine-tuned on the 10 voices), with its
|
| 6 |
+
per-voice style packs in `teacher/voices/<id>.pt`; tiny keeps its timing-and-melody planner, frozen.
|
| 7 |
|
| 8 |
| file | what |
|
| 9 |
|---|---|
|
|
|
|
| 13 |
| `tiny/decoder_flkd.pt` | tiny's 3.65M sound generator, 10 style rows, with its two critics (`model`, `D`, `J`) |
|
| 14 |
|
| 15 |
Voice order in every per-voice table: s057b, f06, d03, w165, w602, m3, t071, d13, x328, h013f.
|
| 16 |
+
|
| 17 |
+
`tiny/earlier/` holds the earlier training run's copies of the same parts (the `t10*` files), kept for reference.
|
checkpoints/teacher/kokoro_teacher.pth β pytorch/teacher/kale-tts-1-teacher.pth
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/d03.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/d13.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/f06.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/h013f.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/m3.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/s057b.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/t071.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/w165.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/w602.pt
RENAMED
|
File without changes
|
{checkpoints/packs β pytorch/teacher/voices}/x328.pt
RENAMED
|
File without changes
|
{checkpoints β pytorch}/tiny/decoder_flkd.pt
RENAMED
|
File without changes
|
{checkpoints β pytorch}/tiny/duration_head_fld.pt
RENAMED
|
File without changes
|
{checkpoints/tiny β pytorch/tiny/earlier}/decoder_t10k.pt
RENAMED
|
File without changes
|
{checkpoints/tiny β pytorch/tiny/earlier}/duration_head_t10d2.pt
RENAMED
|
File without changes
|
{checkpoints/tiny β pytorch/tiny/earlier}/ettin_text_side_t10e.pt
RENAMED
|
File without changes
|
{checkpoints/tiny β pytorch/tiny/earlier}/pitch_heads_t10p.pt
RENAMED
|
File without changes
|
{checkpoints β pytorch}/tiny/ettin_text_side_fle.pt
RENAMED
|
File without changes
|
{checkpoints β pytorch}/tiny/pitch_heads_flp.pt
RENAMED
|
File without changes
|
recipe/distil/README.md
CHANGED
|
@@ -14,7 +14,7 @@ The released kale-tts-1-nano (7.5 M parameters, 14.8 MB on the CPU, 30.6 MB for
|
|
| 14 |
earlier nano's `run/t10r/kdec/last.pt`;
|
| 15 |
- student text side `run/flt/r2text/last.pt`: r2distill, 16k steps, lr 3e-4, bs 64, length-bucketed batches with 50 % short lines,
|
| 16 |
started from the earlier nano's `run/t10t2/r2text/last.pt`.
|
| 17 |
-
Its trained parts are in the kale-tts-1-nano repo's `
|
| 18 |
Configs: `config/flrd.json`, `config/flt-r2distill.json` (`"bucketed": true`: `../train/train.py` `bsample`, the batch sampler
|
| 19 |
the run used); `python distil.py --phase nano-b` runs both after phase B.
|
| 20 |
|
|
|
|
| 14 |
earlier nano's `run/t10r/kdec/last.pt`;
|
| 15 |
- student text side `run/flt/r2text/last.pt`: r2distill, 16k steps, lr 3e-4, bs 64, length-bucketed batches with 50 % short lines,
|
| 16 |
started from the earlier nano's `run/t10t2/r2text/last.pt`.
|
| 17 |
+
Its trained parts are in the kale-tts-1-nano repo's `pytorch/nano/` (`student_text_side_flt.pt`, `sound_side_flrd.pt`).
|
| 18 |
Configs: `config/flrd.json`, `config/flt-r2distill.json` (`"bucketed": true`: `../train/train.py` `bsample`, the batch sampler
|
| 19 |
the run used); `python distil.py --phase nano-b` runs both after phase B.
|
| 20 |
|
recipe/export/README.md
CHANGED
|
@@ -5,7 +5,7 @@ Licence: Apache-2.0. Needs `pip install onnx onnxruntime numpy` (and `node` + `m
|
|
| 5 |
`build_release.sh <bundle dir> <out dir>`
|
| 6 |
- in: the training exports in `<bundle dir>` (nano: one graph with a voice-id input; tiny: one graph per voice; voicepacks
|
| 7 |
`vp-<voice>.f32`), plus tokenizer / config from `../../models` (or `RELEASE_MODELS`).
|
| 8 |
-
- out: `<out dir>/kale-tts-1-{nano,tiny}/` with `model.onnx`, `model.fp32.onnx`, `voices/`, `voices.json`, `export-check-*.json`.
|
| 9 |
|
| 10 |
Steps, in order (each keeps the graph's inputs, outputs and node names):
|
| 11 |
1. `make_onefile.py`: one graph per model, voice as inputs (`voice_prosody`, `voice_prosody_col`, `voice_sound`) instead of a voice id;
|
|
|
|
| 5 |
`build_release.sh <bundle dir> <out dir>`
|
| 6 |
- in: the training exports in `<bundle dir>` (nano: one graph with a voice-id input; tiny: one graph per voice; voicepacks
|
| 7 |
`vp-<voice>.f32`), plus tokenizer / config from `../../models` (or `RELEASE_MODELS`).
|
| 8 |
+
- out: `<out dir>/kale-tts-1-{nano,tiny}/` with `model.onnx`, `model.fp32.onnx`, `voices/`, `voices.json`, `export-check-*.json` (on the hub these are `kale-tts-1-<size>.onnx`, `kale-tts-1-<size>.fp32.onnx` and `results/export-check/`).
|
| 9 |
|
| 10 |
Steps, in order (each keeps the graph's inputs, outputs and node names):
|
| 11 |
1. `make_onefile.py`: one graph per model, voice as inputs (`voice_prosody`, `voice_prosody_col`, `voice_sound`) instead of a voice id;
|
export-check-model.fp32.json β results/export-check/export-check-model.fp32.json
RENAMED
|
File without changes
|
export-check-model.json β results/export-check/export-check-model.json
RENAMED
|
File without changes
|
export-check-whisper.json β results/export-check/export-check-whisper.json
RENAMED
|
File without changes
|
runtimes/RUNTIMES.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
| 1 |
# kale-tts-1 runs everywhere: tested runtimes (16 Nov 2026, 09:45β11:15 CET, draft, nothing published)
|
| 2 |
|
|
|
|
|
|
|
| 3 |
Every row below was run today on the models in `../models/` (nano 10 MB, tiny 61 MB, release graphs unchanged). The machine is this Mac
|
| 4 |
(Apple M5 Max, CPU only, 4 threads unless noted) or, where Linux/x86 or Docker was needed, a Modal CPU container (app `kale-tts-runtimes`,
|
| 5 |
ephemeral `modal run`, stopped after, no GPU, no web endpoint). Docker wasn't running on the Mac, so it wasn't used. Nothing was deployed
|
|
|
|
| 1 |
# kale-tts-1 runs everywhere: tested runtimes (16 Nov 2026, 09:45β11:15 CET, draft, nothing published)
|
| 2 |
|
| 3 |
+
The examples here read a release folder's `model.onnx`. In a downloaded model repo that file is `kale-tts-1-<size>.onnx` (`.fp32.onnx` for WebGPU): pass that path or copy it to `model.onnx`.
|
| 4 |
+
|
| 5 |
Every row below was run today on the models in `../models/` (nano 10 MB, tiny 61 MB, release graphs unchanged). The machine is this Mac
|
| 6 |
(Apple M5 Max, CPU only, 4 threads unless noted) or, where Linux/x86 or Docker was needed, a Modal CPU container (app `kale-tts-runtimes`,
|
| 7 |
ephemeral `modal run`, stopped after, no GPU, no web endpoint). Docker wasn't running on the Mac, so it wasn't used. Nothing was deployed
|
runtimes/python/kale_tts.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs,
|
| 2 |
kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself.
|
| 3 |
-
pip install onnxruntime numpy quickjs (graph: "int8"
|
| 4 |
|
| 5 |
from kale_tts import KaleTTS
|
| 6 |
tts = KaleTTS("models/kale-tts-1-nano")
|
|
@@ -104,12 +104,13 @@ def default_frontend():
|
|
| 104 |
|
| 105 |
class KaleTTS:
|
| 106 |
def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None):
|
| 107 |
-
model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx"}.get(graph, graph)
|
| 108 |
d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir)
|
|
|
|
|
|
|
| 109 |
so = ort.SessionOptions()
|
| 110 |
if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1
|
| 111 |
self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers))
|
| 112 |
-
|
| 113 |
self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {}
|
| 114 |
self.kn = vj.get("phonemes") == "kokoro-native" # Kokoro-native ids (see kn_ipa)
|
| 115 |
self.frontend = default_frontend() if frontend is True else (frontend or None)
|
|
|
|
| 1 |
"""kale-tts-1 for Python: onnxruntime + numpy. Text goes through the release front end: in-process via QuickJS (pip install quickjs,
|
| 2 |
kale_frontend.js) or, without quickjs, a node helper (frontend_cli.mjs). Or pass IPA / ids yourself.
|
| 3 |
+
pip install onnxruntime numpy quickjs (graph: "int8" (default) or "fp32", named in voices.json "graphs", or a file name)
|
| 4 |
|
| 5 |
from kale_tts import KaleTTS
|
| 6 |
tts = KaleTTS("models/kale-tts-1-nano")
|
|
|
|
| 104 |
|
| 105 |
class KaleTTS:
|
| 106 |
def __init__(self, model_dir, threads=None, providers=("CPUExecutionProvider",), frontend=True, graph="int8", model_file=None):
|
|
|
|
| 107 |
d = self.dir = Path(model_dir) if Path(model_dir).exists() or "/" in str(model_dir) else _near("models") / str(model_dir)
|
| 108 |
+
vj = json.loads((d / "voices.json").read_text())
|
| 109 |
+
model_file = model_file or {"int8": "model.onnx", "fp32": "model.fp32.onnx", **vj.get("graphs", {})}.get(graph, graph)
|
| 110 |
so = ort.SessionOptions()
|
| 111 |
if threads: so.intra_op_num_threads = threads; so.inter_op_num_threads = 1
|
| 112 |
self.session = ort.InferenceSession(str(d / model_file), so, providers=list(providers))
|
| 113 |
+
self.voice_inputs, self.voices = vj["voice_inputs"], vj["voices"]
|
| 114 |
self.vocab = json.loads((d / "tokenizer.json").read_text())["model"]["vocab"]; self._v = {}
|
| 115 |
self.kn = vj.get("phonemes") == "kokoro-native" # Kokoro-native ids (see kn_ipa)
|
| 116 |
self.frontend = default_frontend() if frontend is True else (frontend or None)
|
server/server.mjs
CHANGED
|
@@ -24,9 +24,9 @@ const ROOT = process.env.KALE_TTS_ROOT ?? new URL("../", import.meta.url).pathna
|
|
| 24 |
const { KaleTTS, toWav } = await import(ROOT + "js/kale-tts.mjs");
|
| 25 |
const ort = await import("onnxruntime-node");
|
| 26 |
const threads = Number(process.env.KALE_TTS_THREADS ?? 1);
|
| 27 |
-
// a downloaded model repo (
|
| 28 |
const { existsSync } = await import("node:fs");
|
| 29 |
-
const ONE = existsSync(ROOT + "
|
| 30 |
const MODELS = ONE ? [ONE] : ["kale-tts-1-nano", "kale-tts-1-tiny"];
|
| 31 |
const tts = {}; for (const m of MODELS) tts[m] = await KaleTTS.load(ONE ? ROOT : m, { ort, threads });
|
| 32 |
for (const m of MODELS) { await tts[m].preload(); for (const v of tts[m].listVoices()) await tts[m].speak("Hi.", { voice: v.id }); } // load voices + warm
|
|
|
|
| 24 |
const { KaleTTS, toWav } = await import(ROOT + "js/kale-tts.mjs");
|
| 25 |
const ort = await import("onnxruntime-node");
|
| 26 |
const threads = Number(process.env.KALE_TTS_THREADS ?? 1);
|
| 27 |
+
// a downloaded model repo (voices.json at its root) serves that one model; the release folder serves both
|
| 28 |
const { existsSync } = await import("node:fs");
|
| 29 |
+
const ONE = existsSync(ROOT + "voices.json") ? (process.env.KALE_TTS_MODEL ?? (existsSync(ROOT + "ettin-tokenizer.json") ? "kale-tts-1-tiny" : "kale-tts-1-nano")) : null;
|
| 30 |
const MODELS = ONE ? [ONE] : ["kale-tts-1-nano", "kale-tts-1-tiny"];
|
| 31 |
const tts = {}; for (const m of MODELS) tts[m] = await KaleTTS.load(ONE ? ROOT : m, { ort, threads });
|
| 32 |
for (const m of MODELS) { await tts[m].preload(); for (const v of tts[m].listVoices()) await tts[m].speak("Hi.", { voice: v.id }); } // load voices + warm
|
voices.json
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
{
|
|
|
|
| 2 |
"layout": "voices/<id>.bin = style table float32 [510, 256], row = number of tokens - 1 -> input 'style' [1,256]; voices/<id>.extra.bin = float32, the inputs below concatenated in this order",
|
| 3 |
"phonemes": "kokoro-native",
|
| 4 |
"ettin_tokenizer": "ettin-tokenizer.json",
|
|
|
|
| 1 |
{
|
| 2 |
+
"graphs": {"int8": "kale-tts-1-tiny.onnx", "fp32": "kale-tts-1-tiny.fp32.onnx"},
|
| 3 |
"layout": "voices/<id>.bin = style table float32 [510, 256], row = number of tokens - 1 -> input 'style' [1,256]; voices/<id>.extra.bin = float32, the inputs below concatenated in this order",
|
| 4 |
"phonemes": "kokoro-native",
|
| 5 |
"ettin_tokenizer": "ettin-tokenizer.json",
|