Download recipe/render/make_text.py from KaleComputer/kale-tts-1-tiny: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/render/make_text.py
- Command line
-
hf download hf://KaleComputer/kale-tts-1-tiny/recipe/render/make_text.py
-
curl -L -o make_text.py https://huggingface.co/KaleComputer/kale-tts-1-tiny/resolve/main/recipe/render/make_text.py
10.2 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| # Copyright 2026 Kale | |
| """Training text pool (15 Nov 2026): public-domain Project Gutenberg books -> single sentences and passages of 2-5 consecutive sentences | |
| of one paragraph, mix by item count 1/2/3/4/5 sentences = 40/20/20/12/8 % (config/render.json "text_mix"), max ~20 s at ~195 wpm | |
| (<= 62 words; 66 for 4-5 sentences). Long groups are scarce in narration-only book text, so every 2-5-sentence group is used and | |
| singles are 40 % of all items; the 4- and 5-sentence shares stay below the target mix. | |
| Every release voice's texts are sampled from this pool (make_singles.py, make_jobs.py). The pool was first rendered as the | |
| storyteller voice v7, so its rows are written as v7 jobs (clip v7_<n>). | |
| Inputs: a folder of plain-text Gutenberg books named pg<id>.txt (ids in BOOKS below; download them yourself), and ban files with | |
| sentences that must never be trained on (your bench lines, test sets, fixed test lines such as the volcano line): .jsonl (field | |
| "text"), .json (list of strings or of {"text"}), or .txt (one line per sentence). | |
| python3 -I make_text.py <gutenberg dir> "<Maya-1 description>" --ban ../bench/script.jsonl --ban my_test_lines.txt | |
| Outputs: work/jobs-v7.jsonl (clip, voice, text, description, kind single|passage, n_sent, source pg<id>), results/text-info.json.""" | |
| import json | |
| import random | |
| import re | |
| import sys | |
| from pathlib import Path | |
| import argparse | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("gutenberg_dir"); ap.add_argument("description") | |
| ap.add_argument("--ban", action="append", default=[], help="file of banned sentences (.jsonl text field, .json list, .txt lines)") | |
| ap.add_argument("--voice", default="v7"); ap.add_argument("--out", default="work/jobs-v7.jsonl") | |
| A = ap.parse_args() | |
| GUT = Path(A.gutenberg_dir) | |
| rnd = random.Random(1411) | |
| norm = lambda t: re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", t.lower())).strip() | |
| split = lambda t: [s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s] | |
| # ---------------------------------------------------------------- banned sentences (held out / bench / test lines / guard lines) | |
| ban_src = [] | |
| for f in map(Path, A.ban): | |
| if f.suffix == ".jsonl": | |
| ban_src += [json.loads(l)["text"] for l in f.read_text().splitlines() if l.strip() and "text" in json.loads(l)] | |
| elif f.suffix == ".json": | |
| ban_src += [x["text"] if isinstance(x, dict) else x for x in json.loads(f.read_text())] | |
| else: | |
| ban_src += [l for l in f.read_text().splitlines() if l.strip()] | |
| ban = {norm(s) for t in ban_src for s in split(t) if len(s.split()) >= 3} | |
| banned = lambda t: any(b in norm(t) for b in ban) | |
| OK = re.compile(r"^[A-Za-z ,.!?;:'\-]+$") | |
| ARCHAIC = re.compile(r"\b(thee|thou|thy|thine|hath|doth|dost|shalt|wilt|ye|tis|twas|o'er|ere|nay|whither|hither|thither|wherefore|art thou|aught|naught)\b", re.I) | |
| def good_sentence(s, lo=3, hi=28): | |
| w = s.split() | |
| if not (lo <= len(w) <= hi) or not OK.match(s) or s[-1] not in ".!?" or not s[0].isupper(): | |
| return False | |
| if ARCHAIC.search(s) or re.search(r"(^|\s)'|'(\s|$)(?<!s'\s)", s) or re.search(r"\b[A-Z]{2,}\b", s): | |
| return False | |
| if re.search(r"[;:]\s*$|\s-|-\s|--", s) or banned(s): | |
| return False | |
| return True | |
| # ---------------------------------------------------------------- Gutenberg books (id -> title, author, death year, translator) | |
| BOOKS = { | |
| 11: ("Alice's Adventures in Wonderland", "Lewis Carroll", 1898), 12: ("Through the Looking-Glass", "Lewis Carroll", 1898), | |
| 55: ("The Wonderful Wizard of Oz", "L. Frank Baum", 1919), 54: ("The Marvelous Land of Oz", "L. Frank Baum", 1919), | |
| 486: ("Ozma of Oz", "L. Frank Baum", 1919), 289: ("The Wind in the Willows", "Kenneth Grahame", 1932), | |
| 2781: ("Just So Stories", "Rudyard Kipling", 1936), 236: ("The Jungle Book", "Rudyard Kipling", 1936), | |
| 113: ("The Secret Garden", "Frances Hodgson Burnett", 1924), 146: ("A Little Princess", "Frances Hodgson Burnett", 1924), | |
| 479: ("Little Lord Fauntleroy", "Frances Hodgson Burnett", 1924), 1874: ("The Railway Children", "E. Nesbit", 1924), | |
| 778: ("Five Children and It", "E. Nesbit", 1924), 501: ("The Story of Doctor Dolittle", "Hugh Lofting", 1947), | |
| 21: ("Three hundred Aesop's fables", "Aesop, tr. George Fyler Townsend", 1900), 2591: ("Grimms' Fairy Tales", "J. & W. Grimm, tr. Edgar Taylor & Marian Edwardes", 1932), | |
| 1597: ("Andersen's Fairy Tales", "H. C. Andersen (PG translation, 19th c.)", 1875), 271: ("Black Beauty", "Anna Sewell", 1878), | |
| 45: ("Anne of Green Gables", "L. M. Montgomery", 1942), 11757: ("The Velveteen Rabbit", "Margery Williams Bianco", 1944), | |
| 503: ("The Blue Fairy Book", "Andrew Lang (ed.)", 1912), 14838: ("The Tale of Peter Rabbit", "Beatrix Potter", 1943), | |
| 120: ("Treasure Island", "Robert Louis Stevenson", 1894), 514: ("Little Women", "Louisa May Alcott", 1888), | |
| 1450: ("Pollyanna", "Eleanor H. Porter", 1920), 498: ("Rebecca of Sunnybrook Farm", "Kate Douglas Wiggin", 1923), | |
| 708: ("The Princess and the Goblin", "George MacDonald", 1905), 225: ("At the Back of the North Wind", "George MacDonald", 1905), | |
| 1018: ("The Water-Babies", "Charles Kingsley", 1875), 754: ("The Story of Mankind", "Hendrik Willem van Loon", 1944), | |
| 22693: ("A Book of Myths", "Jean Lang", 1933), | |
| 420: ("Dorothy and the Wizard in Oz", "L. Frank Baum", 1919), 485: ("The Road to Oz", "L. Frank Baum", 1919), | |
| 517: ("The Emerald City of Oz", "L. Frank Baum", 1919), 955: ("The Patchwork Girl of Oz", "L. Frank Baum", 1919), | |
| 956: ("Tik-Tok of Oz", "L. Frank Baum", 1919), 957: ("The Scarecrow of Oz", "L. Frank Baum", 1919), | |
| 958: ("Rinkitink in Oz", "L. Frank Baum", 1919), 959: ("The Lost Princess of Oz", "L. Frank Baum", 1919), | |
| 960: ("The Tin Woodman of Oz", "L. Frank Baum", 1919), 1937: ("The Second Jungle Book", "Rudyard Kipling", 1936), | |
| 557: ("Puck of Pook's Hill", "Rudyard Kipling", 1936), 836: ("The Phoenix and the Carpet", "E. Nesbit", 1924), | |
| 837: ("The Story of the Amulet", "E. Nesbit", 1924), 3536: ("The Enchanted Castle", "E. Nesbit", 1924), | |
| 770: ("The Story of the Treasure Seekers", "E. Nesbit", 1924), 23661: ("The Book of Dragons", "E. Nesbit", 1924), | |
| 1154: ("The Voyages of Doctor Dolittle", "Hugh Lofting", 1947), 540: ("The Red Fairy Book", "Andrew Lang (ed.)", 1912), | |
| 641: ("The Violet Fairy Book", "Andrew Lang (ed.)", 1912), 3027: ("The Orange Fairy Book", "Andrew Lang (ed.)", 1912), | |
| 47: ("Anne of Avonlea", "L. M. Montgomery", 1942), 51: ("Anne of the Island", "L. M. Montgomery", 1942), | |
| 2788: ("Little Men", "Louisa May Alcott", 1888), 18442: ("Fifty Famous Stories Retold", "James Baldwin", 1925), | |
| 4018: ("Japanese Fairy Tales", "Yei Theodora Ozaki", 1932), 7439: ("English Fairy Tales", "Joseph Jacobs (ed.)", 1916), | |
| 976: ("Tanglewood Tales", "Nathaniel Hawthorne", 1864), 677: ("The Heroes, or Greek Fairy Tales", "Charles Kingsley", 1875), | |
| 902: ("The Happy Prince, and Other Tales", "Oscar Wilde", 1900), 32575: ("The Head Girl at the Gables", "Angela Brazil", 1947), | |
| } | |
| def book_paragraphs(i): | |
| t = (GUT / f"pg{i}.txt").read_text(encoding="utf-8", errors="replace").replace("\r", "") | |
| a, b = t.find("*** START OF"), t.find("*** END OF") | |
| t = t[t.find("\n", a) + 1: b if b > 0 else None] | |
| out = [] | |
| for p in re.split(r"\n\s*\n", t): | |
| p = " ".join(p.split()) | |
| if not p or '"' in p or "“" in p or "”" in p or re.search(r"\d", p) or "[" in p: | |
| continue | |
| p = p.replace("’", "'").replace("‘", "'").replace("_", "").replace("—", ", ").replace("--", ", ") | |
| p = re.sub(r"\bMr\.", "Mister", p); p = re.sub(r"\bMrs\.", "Missus", p); p = re.sub(r"\bDr\.", "Doctor", p) | |
| p = re.sub(r"\s+,", ",", re.sub(r",\s*,", ",", p)) | |
| out.append(p) | |
| return out | |
| MIX = {1: 0.40, 2: 0.20, 3: 0.20, 4: 0.12, 5: 0.08} | |
| N = 28000 | |
| MAXW = 62 | |
| desc = A.description | |
| cands = {n: [] for n in MIX} # (text, book) | |
| for i in BOOKS: | |
| for para in book_paragraphs(i): | |
| ss = split(para) | |
| ok = [good_sentence(x, 3, 28) for x in ss] | |
| k = 0 | |
| while k < len(ss): # each run of consecutive good sentences, cut into the longest groups that fit (5 first) | |
| if not ok[k]: | |
| k += 1; continue | |
| hit = 0 | |
| for n in (5, 4, 3, 2): | |
| grp = ss[k:k + n] | |
| w = sum(len(x.split()) for x in grp) | |
| if len(grp) == n and all(ok[k:k + n]) and 10 <= w <= (MAXW + 4 if n >= 4 else MAXW): | |
| hit = n; break | |
| if not hit and good_sentence(ss[k], 5, 28): | |
| hit = 1 | |
| if hit: | |
| cands[hit].append((" ".join(ss[k:k + hit]), i)); k += hit | |
| else: | |
| k += 1 | |
| avail = {n: len(v) for n, v in cands.items()} | |
| # long groups are scarce in narration-only book text: take EVERY 2-5-sentence group, singles = 40 % of all items | |
| rows = [] | |
| for n in (2, 3, 4, 5): | |
| rows += [dict(n=n, text=t, source=f"pg{i}") for t, i in dict.fromkeys(cands[n])] | |
| n1 = int(round(len(rows) * 0.40 / 0.60)) | |
| pool = list(dict.fromkeys(cands[1])); rnd.shuffle(pool) | |
| rows += [dict(n=1, text=t, source=f"pg{i}") for t, i in pool[:n1]] | |
| rnd.shuffle(rows) | |
| jobs = [dict(clip=f"{A.voice}_{k:05d}", voice=A.voice, text=r["text"], description=desc, mood="default", tagname="", kind="single" if r["n"] == 1 else "passage", | |
| n_sent=r["n"], source=r["source"]) for k, r in enumerate(rows)] | |
| out = Path(A.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text("".join(json.dumps(j) + "\n" for j in jobs)) | |
| from collections import Counter # noqa: E402 | |
| c = Counter(j["n_sent"] for j in jobs) | |
| words = sum(len(j["text"].split()) for j in jobs) | |
| info = dict(jobs=len(jobs), by_sentences=dict(sorted(c.items())), share={k: round(v / len(jobs), 3) for k, v in sorted(c.items())}, available=avail, | |
| words=words, est_hours_at_195wpm=round(words / 195 / 60, 1), max_words=max(len(j["text"].split()) for j in jobs), banned_sentences=len(ban), | |
| books=len(BOOKS), description=desc) | |
| Path("results").mkdir(exist_ok=True) | |
| Path("results/text-info.json").write_text(json.dumps(info, indent=1)) | |
| print(json.dumps(info, indent=1)) | |