# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Kale """Training text pool (15 Nov 2026): public-domain Project Gutenberg books -> single sentences and passages of 2-5 consecutive sentences of one paragraph, mix by item count 1/2/3/4/5 sentences = 40/20/20/12/8 % (config/render.json "text_mix"), max ~20 s at ~195 wpm (<= 62 words; 66 for 4-5 sentences). Long groups are scarce in narration-only book text, so every 2-5-sentence group is used and singles are 40 % of all items; the 4- and 5-sentence shares stay below the target mix. Every release voice's texts are sampled from this pool (make_singles.py, make_jobs.py). The pool was first rendered as the storyteller voice v7, so its rows are written as v7 jobs (clip v7_). Inputs: a folder of plain-text Gutenberg books named pg.txt (ids in BOOKS below; download them yourself), and ban files with sentences that must never be trained on (your bench lines, test sets, fixed test lines such as the volcano line): .jsonl (field "text"), .json (list of strings or of {"text"}), or .txt (one line per sentence). python3 -I make_text.py "" --ban ../bench/script.jsonl --ban my_test_lines.txt Outputs: work/jobs-v7.jsonl (clip, voice, text, description, kind single|passage, n_sent, source pg), results/text-info.json.""" import json import random import re import sys from pathlib import Path import argparse ap = argparse.ArgumentParser() ap.add_argument("gutenberg_dir"); ap.add_argument("description") ap.add_argument("--ban", action="append", default=[], help="file of banned sentences (.jsonl text field, .json list, .txt lines)") ap.add_argument("--voice", default="v7"); ap.add_argument("--out", default="work/jobs-v7.jsonl") A = ap.parse_args() GUT = Path(A.gutenberg_dir) rnd = random.Random(1411) norm = lambda t: re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", t.lower())).strip() split = lambda t: [s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s] # ---------------------------------------------------------------- banned sentences (held out / bench / test lines / guard lines) ban_src = [] for f in map(Path, A.ban): if f.suffix == ".jsonl": ban_src += [json.loads(l)["text"] for l in f.read_text().splitlines() if l.strip() and "text" in json.loads(l)] elif f.suffix == ".json": ban_src += [x["text"] if isinstance(x, dict) else x for x in json.loads(f.read_text())] else: ban_src += [l for l in f.read_text().splitlines() if l.strip()] ban = {norm(s) for t in ban_src for s in split(t) if len(s.split()) >= 3} banned = lambda t: any(b in norm(t) for b in ban) OK = re.compile(r"^[A-Za-z ,.!?;:'\-]+$") ARCHAIC = re.compile(r"\b(thee|thou|thy|thine|hath|doth|dost|shalt|wilt|ye|tis|twas|o'er|ere|nay|whither|hither|thither|wherefore|art thou|aught|naught)\b", re.I) def good_sentence(s, lo=3, hi=28): w = s.split() if not (lo <= len(w) <= hi) or not OK.match(s) or s[-1] not in ".!?" or not s[0].isupper(): return False if ARCHAIC.search(s) or re.search(r"(^|\s)'|'(\s|$)(? title, author, death year, translator) BOOKS = { 11: ("Alice's Adventures in Wonderland", "Lewis Carroll", 1898), 12: ("Through the Looking-Glass", "Lewis Carroll", 1898), 55: ("The Wonderful Wizard of Oz", "L. Frank Baum", 1919), 54: ("The Marvelous Land of Oz", "L. Frank Baum", 1919), 486: ("Ozma of Oz", "L. Frank Baum", 1919), 289: ("The Wind in the Willows", "Kenneth Grahame", 1932), 2781: ("Just So Stories", "Rudyard Kipling", 1936), 236: ("The Jungle Book", "Rudyard Kipling", 1936), 113: ("The Secret Garden", "Frances Hodgson Burnett", 1924), 146: ("A Little Princess", "Frances Hodgson Burnett", 1924), 479: ("Little Lord Fauntleroy", "Frances Hodgson Burnett", 1924), 1874: ("The Railway Children", "E. Nesbit", 1924), 778: ("Five Children and It", "E. Nesbit", 1924), 501: ("The Story of Doctor Dolittle", "Hugh Lofting", 1947), 21: ("Three hundred Aesop's fables", "Aesop, tr. George Fyler Townsend", 1900), 2591: ("Grimms' Fairy Tales", "J. & W. Grimm, tr. Edgar Taylor & Marian Edwardes", 1932), 1597: ("Andersen's Fairy Tales", "H. C. Andersen (PG translation, 19th c.)", 1875), 271: ("Black Beauty", "Anna Sewell", 1878), 45: ("Anne of Green Gables", "L. M. Montgomery", 1942), 11757: ("The Velveteen Rabbit", "Margery Williams Bianco", 1944), 503: ("The Blue Fairy Book", "Andrew Lang (ed.)", 1912), 14838: ("The Tale of Peter Rabbit", "Beatrix Potter", 1943), 120: ("Treasure Island", "Robert Louis Stevenson", 1894), 514: ("Little Women", "Louisa May Alcott", 1888), 1450: ("Pollyanna", "Eleanor H. Porter", 1920), 498: ("Rebecca of Sunnybrook Farm", "Kate Douglas Wiggin", 1923), 708: ("The Princess and the Goblin", "George MacDonald", 1905), 225: ("At the Back of the North Wind", "George MacDonald", 1905), 1018: ("The Water-Babies", "Charles Kingsley", 1875), 754: ("The Story of Mankind", "Hendrik Willem van Loon", 1944), 22693: ("A Book of Myths", "Jean Lang", 1933), 420: ("Dorothy and the Wizard in Oz", "L. Frank Baum", 1919), 485: ("The Road to Oz", "L. Frank Baum", 1919), 517: ("The Emerald City of Oz", "L. Frank Baum", 1919), 955: ("The Patchwork Girl of Oz", "L. Frank Baum", 1919), 956: ("Tik-Tok of Oz", "L. Frank Baum", 1919), 957: ("The Scarecrow of Oz", "L. Frank Baum", 1919), 958: ("Rinkitink in Oz", "L. Frank Baum", 1919), 959: ("The Lost Princess of Oz", "L. Frank Baum", 1919), 960: ("The Tin Woodman of Oz", "L. Frank Baum", 1919), 1937: ("The Second Jungle Book", "Rudyard Kipling", 1936), 557: ("Puck of Pook's Hill", "Rudyard Kipling", 1936), 836: ("The Phoenix and the Carpet", "E. Nesbit", 1924), 837: ("The Story of the Amulet", "E. Nesbit", 1924), 3536: ("The Enchanted Castle", "E. Nesbit", 1924), 770: ("The Story of the Treasure Seekers", "E. Nesbit", 1924), 23661: ("The Book of Dragons", "E. Nesbit", 1924), 1154: ("The Voyages of Doctor Dolittle", "Hugh Lofting", 1947), 540: ("The Red Fairy Book", "Andrew Lang (ed.)", 1912), 641: ("The Violet Fairy Book", "Andrew Lang (ed.)", 1912), 3027: ("The Orange Fairy Book", "Andrew Lang (ed.)", 1912), 47: ("Anne of Avonlea", "L. M. Montgomery", 1942), 51: ("Anne of the Island", "L. M. Montgomery", 1942), 2788: ("Little Men", "Louisa May Alcott", 1888), 18442: ("Fifty Famous Stories Retold", "James Baldwin", 1925), 4018: ("Japanese Fairy Tales", "Yei Theodora Ozaki", 1932), 7439: ("English Fairy Tales", "Joseph Jacobs (ed.)", 1916), 976: ("Tanglewood Tales", "Nathaniel Hawthorne", 1864), 677: ("The Heroes, or Greek Fairy Tales", "Charles Kingsley", 1875), 902: ("The Happy Prince, and Other Tales", "Oscar Wilde", 1900), 32575: ("The Head Girl at the Gables", "Angela Brazil", 1947), } def book_paragraphs(i): t = (GUT / f"pg{i}.txt").read_text(encoding="utf-8", errors="replace").replace("\r", "") a, b = t.find("*** START OF"), t.find("*** END OF") t = t[t.find("\n", a) + 1: b if b > 0 else None] out = [] for p in re.split(r"\n\s*\n", t): p = " ".join(p.split()) if not p or '"' in p or "“" in p or "”" in p or re.search(r"\d", p) or "[" in p: continue p = p.replace("’", "'").replace("‘", "'").replace("_", "").replace("—", ", ").replace("--", ", ") p = re.sub(r"\bMr\.", "Mister", p); p = re.sub(r"\bMrs\.", "Missus", p); p = re.sub(r"\bDr\.", "Doctor", p) p = re.sub(r"\s+,", ",", re.sub(r",\s*,", ",", p)) out.append(p) return out MIX = {1: 0.40, 2: 0.20, 3: 0.20, 4: 0.12, 5: 0.08} N = 28000 MAXW = 62 desc = A.description cands = {n: [] for n in MIX} # (text, book) for i in BOOKS: for para in book_paragraphs(i): ss = split(para) ok = [good_sentence(x, 3, 28) for x in ss] k = 0 while k < len(ss): # each run of consecutive good sentences, cut into the longest groups that fit (5 first) if not ok[k]: k += 1; continue hit = 0 for n in (5, 4, 3, 2): grp = ss[k:k + n] w = sum(len(x.split()) for x in grp) if len(grp) == n and all(ok[k:k + n]) and 10 <= w <= (MAXW + 4 if n >= 4 else MAXW): hit = n; break if not hit and good_sentence(ss[k], 5, 28): hit = 1 if hit: cands[hit].append((" ".join(ss[k:k + hit]), i)); k += hit else: k += 1 avail = {n: len(v) for n, v in cands.items()} # long groups are scarce in narration-only book text: take EVERY 2-5-sentence group, singles = 40 % of all items rows = [] for n in (2, 3, 4, 5): rows += [dict(n=n, text=t, source=f"pg{i}") for t, i in dict.fromkeys(cands[n])] n1 = int(round(len(rows) * 0.40 / 0.60)) pool = list(dict.fromkeys(cands[1])); rnd.shuffle(pool) rows += [dict(n=1, text=t, source=f"pg{i}") for t, i in pool[:n1]] rnd.shuffle(rows) jobs = [dict(clip=f"{A.voice}_{k:05d}", voice=A.voice, text=r["text"], description=desc, mood="default", tagname="", kind="single" if r["n"] == 1 else "passage", n_sent=r["n"], source=r["source"]) for k, r in enumerate(rows)] out = Path(A.out) out.parent.mkdir(parents=True, exist_ok=True) out.write_text("".join(json.dumps(j) + "\n" for j in jobs)) from collections import Counter # noqa: E402 c = Counter(j["n_sent"] for j in jobs) words = sum(len(j["text"].split()) for j in jobs) info = dict(jobs=len(jobs), by_sentences=dict(sorted(c.items())), share={k: round(v / len(jobs), 3) for k, v in sorted(c.items())}, available=avail, words=words, est_hours_at_195wpm=round(words / 195 / 60, 1), max_words=max(len(j["text"].split()) for j in jobs), banned_sentences=len(ban), books=len(BOOKS), description=desc) Path("results").mkdir(exist_ok=True) Path("results/text-info.json").write_text(json.dumps(info, indent=1)) print(json.dumps(info, indent=1))