kale-tts-1-tiny / recipe /render /make_text.py
NERDDISCO's picture
kale-tts-1
f21d6eb
Raw History Blame Contribute Delete
10.2 kB
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Training text pool (15 Nov 2026): public-domain Project Gutenberg books -> single sentences and passages of 2-5 consecutive sentences
of one paragraph, mix by item count 1/2/3/4/5 sentences = 40/20/20/12/8 % (config/render.json "text_mix"), max ~20 s at ~195 wpm
(<= 62 words; 66 for 4-5 sentences). Long groups are scarce in narration-only book text, so every 2-5-sentence group is used and
singles are 40 % of all items; the 4- and 5-sentence shares stay below the target mix.
Every release voice's texts are sampled from this pool (make_singles.py, make_jobs.py). The pool was first rendered as the
storyteller voice v7, so its rows are written as v7 jobs (clip v7_<n>).
Inputs: a folder of plain-text Gutenberg books named pg<id>.txt (ids in BOOKS below; download them yourself), and ban files with
sentences that must never be trained on (your bench lines, test sets, fixed test lines such as the volcano line): .jsonl (field
"text"), .json (list of strings or of {"text"}), or .txt (one line per sentence).
python3 -I make_text.py <gutenberg dir> "<Maya-1 description>" --ban ../bench/script.jsonl --ban my_test_lines.txt
Outputs: work/jobs-v7.jsonl (clip, voice, text, description, kind single|passage, n_sent, source pg<id>), results/text-info.json."""
import json
import random
import re
import sys
from pathlib import Path
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("gutenberg_dir"); ap.add_argument("description")
ap.add_argument("--ban", action="append", default=[], help="file of banned sentences (.jsonl text field, .json list, .txt lines)")
ap.add_argument("--voice", default="v7"); ap.add_argument("--out", default="work/jobs-v7.jsonl")
A = ap.parse_args()
GUT = Path(A.gutenberg_dir)
rnd = random.Random(1411)
norm = lambda t: re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", t.lower())).strip()
split = lambda t: [s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s]
# ---------------------------------------------------------------- banned sentences (held out / bench / test lines / guard lines)
ban_src = []
for f in map(Path, A.ban):
if f.suffix == ".jsonl":
ban_src += [json.loads(l)["text"] for l in f.read_text().splitlines() if l.strip() and "text" in json.loads(l)]
elif f.suffix == ".json":
ban_src += [x["text"] if isinstance(x, dict) else x for x in json.loads(f.read_text())]
else:
ban_src += [l for l in f.read_text().splitlines() if l.strip()]
ban = {norm(s) for t in ban_src for s in split(t) if len(s.split()) >= 3}
banned = lambda t: any(b in norm(t) for b in ban)
OK = re.compile(r"^[A-Za-z ,.!?;:'\-]+$")
ARCHAIC = re.compile(r"\b(thee|thou|thy|thine|hath|doth|dost|shalt|wilt|ye|tis|twas|o'er|ere|nay|whither|hither|thither|wherefore|art thou|aught|naught)\b", re.I)
def good_sentence(s, lo=3, hi=28):
w = s.split()
if not (lo <= len(w) <= hi) or not OK.match(s) or s[-1] not in ".!?" or not s[0].isupper():
return False
if ARCHAIC.search(s) or re.search(r"(^|\s)'|'(\s|$)(?<!s'\s)", s) or re.search(r"\b[A-Z]{2,}\b", s):
return False
if re.search(r"[;:]\s*$|\s-|-\s|--", s) or banned(s):
return False
return True
# ---------------------------------------------------------------- Gutenberg books (id -> title, author, death year, translator)
BOOKS = {
11: ("Alice's Adventures in Wonderland", "Lewis Carroll", 1898), 12: ("Through the Looking-Glass", "Lewis Carroll", 1898),
55: ("The Wonderful Wizard of Oz", "L. Frank Baum", 1919), 54: ("The Marvelous Land of Oz", "L. Frank Baum", 1919),
486: ("Ozma of Oz", "L. Frank Baum", 1919), 289: ("The Wind in the Willows", "Kenneth Grahame", 1932),
2781: ("Just So Stories", "Rudyard Kipling", 1936), 236: ("The Jungle Book", "Rudyard Kipling", 1936),
113: ("The Secret Garden", "Frances Hodgson Burnett", 1924), 146: ("A Little Princess", "Frances Hodgson Burnett", 1924),
479: ("Little Lord Fauntleroy", "Frances Hodgson Burnett", 1924), 1874: ("The Railway Children", "E. Nesbit", 1924),
778: ("Five Children and It", "E. Nesbit", 1924), 501: ("The Story of Doctor Dolittle", "Hugh Lofting", 1947),
21: ("Three hundred Aesop's fables", "Aesop, tr. George Fyler Townsend", 1900), 2591: ("Grimms' Fairy Tales", "J. & W. Grimm, tr. Edgar Taylor & Marian Edwardes", 1932),
1597: ("Andersen's Fairy Tales", "H. C. Andersen (PG translation, 19th c.)", 1875), 271: ("Black Beauty", "Anna Sewell", 1878),
45: ("Anne of Green Gables", "L. M. Montgomery", 1942), 11757: ("The Velveteen Rabbit", "Margery Williams Bianco", 1944),
503: ("The Blue Fairy Book", "Andrew Lang (ed.)", 1912), 14838: ("The Tale of Peter Rabbit", "Beatrix Potter", 1943),
120: ("Treasure Island", "Robert Louis Stevenson", 1894), 514: ("Little Women", "Louisa May Alcott", 1888),
1450: ("Pollyanna", "Eleanor H. Porter", 1920), 498: ("Rebecca of Sunnybrook Farm", "Kate Douglas Wiggin", 1923),
708: ("The Princess and the Goblin", "George MacDonald", 1905), 225: ("At the Back of the North Wind", "George MacDonald", 1905),
1018: ("The Water-Babies", "Charles Kingsley", 1875), 754: ("The Story of Mankind", "Hendrik Willem van Loon", 1944),
22693: ("A Book of Myths", "Jean Lang", 1933),
420: ("Dorothy and the Wizard in Oz", "L. Frank Baum", 1919), 485: ("The Road to Oz", "L. Frank Baum", 1919),
517: ("The Emerald City of Oz", "L. Frank Baum", 1919), 955: ("The Patchwork Girl of Oz", "L. Frank Baum", 1919),
956: ("Tik-Tok of Oz", "L. Frank Baum", 1919), 957: ("The Scarecrow of Oz", "L. Frank Baum", 1919),
958: ("Rinkitink in Oz", "L. Frank Baum", 1919), 959: ("The Lost Princess of Oz", "L. Frank Baum", 1919),
960: ("The Tin Woodman of Oz", "L. Frank Baum", 1919), 1937: ("The Second Jungle Book", "Rudyard Kipling", 1936),
557: ("Puck of Pook's Hill", "Rudyard Kipling", 1936), 836: ("The Phoenix and the Carpet", "E. Nesbit", 1924),
837: ("The Story of the Amulet", "E. Nesbit", 1924), 3536: ("The Enchanted Castle", "E. Nesbit", 1924),
770: ("The Story of the Treasure Seekers", "E. Nesbit", 1924), 23661: ("The Book of Dragons", "E. Nesbit", 1924),
1154: ("The Voyages of Doctor Dolittle", "Hugh Lofting", 1947), 540: ("The Red Fairy Book", "Andrew Lang (ed.)", 1912),
641: ("The Violet Fairy Book", "Andrew Lang (ed.)", 1912), 3027: ("The Orange Fairy Book", "Andrew Lang (ed.)", 1912),
47: ("Anne of Avonlea", "L. M. Montgomery", 1942), 51: ("Anne of the Island", "L. M. Montgomery", 1942),
2788: ("Little Men", "Louisa May Alcott", 1888), 18442: ("Fifty Famous Stories Retold", "James Baldwin", 1925),
4018: ("Japanese Fairy Tales", "Yei Theodora Ozaki", 1932), 7439: ("English Fairy Tales", "Joseph Jacobs (ed.)", 1916),
976: ("Tanglewood Tales", "Nathaniel Hawthorne", 1864), 677: ("The Heroes, or Greek Fairy Tales", "Charles Kingsley", 1875),
902: ("The Happy Prince, and Other Tales", "Oscar Wilde", 1900), 32575: ("The Head Girl at the Gables", "Angela Brazil", 1947),
}
def book_paragraphs(i):
t = (GUT / f"pg{i}.txt").read_text(encoding="utf-8", errors="replace").replace("\r", "")
a, b = t.find("*** START OF"), t.find("*** END OF")
t = t[t.find("\n", a) + 1: b if b > 0 else None]
out = []
for p in re.split(r"\n\s*\n", t):
p = " ".join(p.split())
if not p or '"' in p or "“" in p or "”" in p or re.search(r"\d", p) or "[" in p:
continue
p = p.replace("’", "'").replace("‘", "'").replace("_", "").replace("—", ", ").replace("--", ", ")
p = re.sub(r"\bMr\.", "Mister", p); p = re.sub(r"\bMrs\.", "Missus", p); p = re.sub(r"\bDr\.", "Doctor", p)
p = re.sub(r"\s+,", ",", re.sub(r",\s*,", ",", p))
out.append(p)
return out
MIX = {1: 0.40, 2: 0.20, 3: 0.20, 4: 0.12, 5: 0.08}
N = 28000
MAXW = 62
desc = A.description
cands = {n: [] for n in MIX} # (text, book)
for i in BOOKS:
for para in book_paragraphs(i):
ss = split(para)
ok = [good_sentence(x, 3, 28) for x in ss]
k = 0
while k < len(ss): # each run of consecutive good sentences, cut into the longest groups that fit (5 first)
if not ok[k]:
k += 1; continue
hit = 0
for n in (5, 4, 3, 2):
grp = ss[k:k + n]
w = sum(len(x.split()) for x in grp)
if len(grp) == n and all(ok[k:k + n]) and 10 <= w <= (MAXW + 4 if n >= 4 else MAXW):
hit = n; break
if not hit and good_sentence(ss[k], 5, 28):
hit = 1
if hit:
cands[hit].append((" ".join(ss[k:k + hit]), i)); k += hit
else:
k += 1
avail = {n: len(v) for n, v in cands.items()}
# long groups are scarce in narration-only book text: take EVERY 2-5-sentence group, singles = 40 % of all items
rows = []
for n in (2, 3, 4, 5):
rows += [dict(n=n, text=t, source=f"pg{i}") for t, i in dict.fromkeys(cands[n])]
n1 = int(round(len(rows) * 0.40 / 0.60))
pool = list(dict.fromkeys(cands[1])); rnd.shuffle(pool)
rows += [dict(n=1, text=t, source=f"pg{i}") for t, i in pool[:n1]]
rnd.shuffle(rows)
jobs = [dict(clip=f"{A.voice}_{k:05d}", voice=A.voice, text=r["text"], description=desc, mood="default", tagname="", kind="single" if r["n"] == 1 else "passage",
n_sent=r["n"], source=r["source"]) for k, r in enumerate(rows)]
out = Path(A.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("".join(json.dumps(j) + "\n" for j in jobs))
from collections import Counter # noqa: E402
c = Counter(j["n_sent"] for j in jobs)
words = sum(len(j["text"].split()) for j in jobs)
info = dict(jobs=len(jobs), by_sentences=dict(sorted(c.items())), share={k: round(v / len(jobs), 3) for k, v in sorted(c.items())}, available=avail,
words=words, est_hours_at_195wpm=round(words / 195 / 60, 1), max_words=max(len(j["text"].split()) for j in jobs), banned_sentences=len(ban),
books=len(BOOKS), description=desc)
Path("results").mkdir(exist_ok=True)
Path("results/text-info.json").write_text(json.dumps(info, indent=1))
print(json.dumps(info, indent=1))