File size: 10,218 Bytes
f21d6eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | # SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Training text pool (15 Nov 2026): public-domain Project Gutenberg books -> single sentences and passages of 2-5 consecutive sentences
of one paragraph, mix by item count 1/2/3/4/5 sentences = 40/20/20/12/8 % (config/render.json "text_mix"), max ~20 s at ~195 wpm
(<= 62 words; 66 for 4-5 sentences). Long groups are scarce in narration-only book text, so every 2-5-sentence group is used and
singles are 40 % of all items; the 4- and 5-sentence shares stay below the target mix.
Every release voice's texts are sampled from this pool (make_singles.py, make_jobs.py). The pool was first rendered as the
storyteller voice v7, so its rows are written as v7 jobs (clip v7_<n>).
Inputs: a folder of plain-text Gutenberg books named pg<id>.txt (ids in BOOKS below; download them yourself), and ban files with
sentences that must never be trained on (your bench lines, test sets, fixed test lines such as the volcano line): .jsonl (field
"text"), .json (list of strings or of {"text"}), or .txt (one line per sentence).
python3 -I make_text.py <gutenberg dir> "<Maya-1 description>" --ban ../bench/script.jsonl --ban my_test_lines.txt
Outputs: work/jobs-v7.jsonl (clip, voice, text, description, kind single|passage, n_sent, source pg<id>), results/text-info.json."""
import json
import random
import re
import sys
from pathlib import Path
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("gutenberg_dir"); ap.add_argument("description")
ap.add_argument("--ban", action="append", default=[], help="file of banned sentences (.jsonl text field, .json list, .txt lines)")
ap.add_argument("--voice", default="v7"); ap.add_argument("--out", default="work/jobs-v7.jsonl")
A = ap.parse_args()
GUT = Path(A.gutenberg_dir)
rnd = random.Random(1411)
norm = lambda t: re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", t.lower())).strip()
split = lambda t: [s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s]
# ---------------------------------------------------------------- banned sentences (held out / bench / test lines / guard lines)
ban_src = []
for f in map(Path, A.ban):
if f.suffix == ".jsonl":
ban_src += [json.loads(l)["text"] for l in f.read_text().splitlines() if l.strip() and "text" in json.loads(l)]
elif f.suffix == ".json":
ban_src += [x["text"] if isinstance(x, dict) else x for x in json.loads(f.read_text())]
else:
ban_src += [l for l in f.read_text().splitlines() if l.strip()]
ban = {norm(s) for t in ban_src for s in split(t) if len(s.split()) >= 3}
banned = lambda t: any(b in norm(t) for b in ban)
OK = re.compile(r"^[A-Za-z ,.!?;:'\-]+$")
ARCHAIC = re.compile(r"\b(thee|thou|thy|thine|hath|doth|dost|shalt|wilt|ye|tis|twas|o'er|ere|nay|whither|hither|thither|wherefore|art thou|aught|naught)\b", re.I)
def good_sentence(s, lo=3, hi=28):
w = s.split()
if not (lo <= len(w) <= hi) or not OK.match(s) or s[-1] not in ".!?" or not s[0].isupper():
return False
if ARCHAIC.search(s) or re.search(r"(^|\s)'|'(\s|$)(?<!s'\s)", s) or re.search(r"\b[A-Z]{2,}\b", s):
return False
if re.search(r"[;:]\s*$|\s-|-\s|--", s) or banned(s):
return False
return True
# ---------------------------------------------------------------- Gutenberg books (id -> title, author, death year, translator)
BOOKS = {
11: ("Alice's Adventures in Wonderland", "Lewis Carroll", 1898), 12: ("Through the Looking-Glass", "Lewis Carroll", 1898),
55: ("The Wonderful Wizard of Oz", "L. Frank Baum", 1919), 54: ("The Marvelous Land of Oz", "L. Frank Baum", 1919),
486: ("Ozma of Oz", "L. Frank Baum", 1919), 289: ("The Wind in the Willows", "Kenneth Grahame", 1932),
2781: ("Just So Stories", "Rudyard Kipling", 1936), 236: ("The Jungle Book", "Rudyard Kipling", 1936),
113: ("The Secret Garden", "Frances Hodgson Burnett", 1924), 146: ("A Little Princess", "Frances Hodgson Burnett", 1924),
479: ("Little Lord Fauntleroy", "Frances Hodgson Burnett", 1924), 1874: ("The Railway Children", "E. Nesbit", 1924),
778: ("Five Children and It", "E. Nesbit", 1924), 501: ("The Story of Doctor Dolittle", "Hugh Lofting", 1947),
21: ("Three hundred Aesop's fables", "Aesop, tr. George Fyler Townsend", 1900), 2591: ("Grimms' Fairy Tales", "J. & W. Grimm, tr. Edgar Taylor & Marian Edwardes", 1932),
1597: ("Andersen's Fairy Tales", "H. C. Andersen (PG translation, 19th c.)", 1875), 271: ("Black Beauty", "Anna Sewell", 1878),
45: ("Anne of Green Gables", "L. M. Montgomery", 1942), 11757: ("The Velveteen Rabbit", "Margery Williams Bianco", 1944),
503: ("The Blue Fairy Book", "Andrew Lang (ed.)", 1912), 14838: ("The Tale of Peter Rabbit", "Beatrix Potter", 1943),
120: ("Treasure Island", "Robert Louis Stevenson", 1894), 514: ("Little Women", "Louisa May Alcott", 1888),
1450: ("Pollyanna", "Eleanor H. Porter", 1920), 498: ("Rebecca of Sunnybrook Farm", "Kate Douglas Wiggin", 1923),
708: ("The Princess and the Goblin", "George MacDonald", 1905), 225: ("At the Back of the North Wind", "George MacDonald", 1905),
1018: ("The Water-Babies", "Charles Kingsley", 1875), 754: ("The Story of Mankind", "Hendrik Willem van Loon", 1944),
22693: ("A Book of Myths", "Jean Lang", 1933),
420: ("Dorothy and the Wizard in Oz", "L. Frank Baum", 1919), 485: ("The Road to Oz", "L. Frank Baum", 1919),
517: ("The Emerald City of Oz", "L. Frank Baum", 1919), 955: ("The Patchwork Girl of Oz", "L. Frank Baum", 1919),
956: ("Tik-Tok of Oz", "L. Frank Baum", 1919), 957: ("The Scarecrow of Oz", "L. Frank Baum", 1919),
958: ("Rinkitink in Oz", "L. Frank Baum", 1919), 959: ("The Lost Princess of Oz", "L. Frank Baum", 1919),
960: ("The Tin Woodman of Oz", "L. Frank Baum", 1919), 1937: ("The Second Jungle Book", "Rudyard Kipling", 1936),
557: ("Puck of Pook's Hill", "Rudyard Kipling", 1936), 836: ("The Phoenix and the Carpet", "E. Nesbit", 1924),
837: ("The Story of the Amulet", "E. Nesbit", 1924), 3536: ("The Enchanted Castle", "E. Nesbit", 1924),
770: ("The Story of the Treasure Seekers", "E. Nesbit", 1924), 23661: ("The Book of Dragons", "E. Nesbit", 1924),
1154: ("The Voyages of Doctor Dolittle", "Hugh Lofting", 1947), 540: ("The Red Fairy Book", "Andrew Lang (ed.)", 1912),
641: ("The Violet Fairy Book", "Andrew Lang (ed.)", 1912), 3027: ("The Orange Fairy Book", "Andrew Lang (ed.)", 1912),
47: ("Anne of Avonlea", "L. M. Montgomery", 1942), 51: ("Anne of the Island", "L. M. Montgomery", 1942),
2788: ("Little Men", "Louisa May Alcott", 1888), 18442: ("Fifty Famous Stories Retold", "James Baldwin", 1925),
4018: ("Japanese Fairy Tales", "Yei Theodora Ozaki", 1932), 7439: ("English Fairy Tales", "Joseph Jacobs (ed.)", 1916),
976: ("Tanglewood Tales", "Nathaniel Hawthorne", 1864), 677: ("The Heroes, or Greek Fairy Tales", "Charles Kingsley", 1875),
902: ("The Happy Prince, and Other Tales", "Oscar Wilde", 1900), 32575: ("The Head Girl at the Gables", "Angela Brazil", 1947),
}
def book_paragraphs(i):
t = (GUT / f"pg{i}.txt").read_text(encoding="utf-8", errors="replace").replace("\r", "")
a, b = t.find("*** START OF"), t.find("*** END OF")
t = t[t.find("\n", a) + 1: b if b > 0 else None]
out = []
for p in re.split(r"\n\s*\n", t):
p = " ".join(p.split())
if not p or '"' in p or "“" in p or "”" in p or re.search(r"\d", p) or "[" in p:
continue
p = p.replace("’", "'").replace("‘", "'").replace("_", "").replace("—", ", ").replace("--", ", ")
p = re.sub(r"\bMr\.", "Mister", p); p = re.sub(r"\bMrs\.", "Missus", p); p = re.sub(r"\bDr\.", "Doctor", p)
p = re.sub(r"\s+,", ",", re.sub(r",\s*,", ",", p))
out.append(p)
return out
MIX = {1: 0.40, 2: 0.20, 3: 0.20, 4: 0.12, 5: 0.08}
N = 28000
MAXW = 62
desc = A.description
cands = {n: [] for n in MIX} # (text, book)
for i in BOOKS:
for para in book_paragraphs(i):
ss = split(para)
ok = [good_sentence(x, 3, 28) for x in ss]
k = 0
while k < len(ss): # each run of consecutive good sentences, cut into the longest groups that fit (5 first)
if not ok[k]:
k += 1; continue
hit = 0
for n in (5, 4, 3, 2):
grp = ss[k:k + n]
w = sum(len(x.split()) for x in grp)
if len(grp) == n and all(ok[k:k + n]) and 10 <= w <= (MAXW + 4 if n >= 4 else MAXW):
hit = n; break
if not hit and good_sentence(ss[k], 5, 28):
hit = 1
if hit:
cands[hit].append((" ".join(ss[k:k + hit]), i)); k += hit
else:
k += 1
avail = {n: len(v) for n, v in cands.items()}
# long groups are scarce in narration-only book text: take EVERY 2-5-sentence group, singles = 40 % of all items
rows = []
for n in (2, 3, 4, 5):
rows += [dict(n=n, text=t, source=f"pg{i}") for t, i in dict.fromkeys(cands[n])]
n1 = int(round(len(rows) * 0.40 / 0.60))
pool = list(dict.fromkeys(cands[1])); rnd.shuffle(pool)
rows += [dict(n=1, text=t, source=f"pg{i}") for t, i in pool[:n1]]
rnd.shuffle(rows)
jobs = [dict(clip=f"{A.voice}_{k:05d}", voice=A.voice, text=r["text"], description=desc, mood="default", tagname="", kind="single" if r["n"] == 1 else "passage",
n_sent=r["n"], source=r["source"]) for k, r in enumerate(rows)]
out = Path(A.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("".join(json.dumps(j) + "\n" for j in jobs))
from collections import Counter # noqa: E402
c = Counter(j["n_sent"] for j in jobs)
words = sum(len(j["text"].split()) for j in jobs)
info = dict(jobs=len(jobs), by_sentences=dict(sorted(c.items())), share={k: round(v / len(jobs), 3) for k, v in sorted(c.items())}, available=avail,
words=words, est_hours_at_195wpm=round(words / 195 / 60, 1), max_words=max(len(j["text"].split()) for j in jobs), banned_sentences=len(ban),
books=len(BOOKS), description=desc)
Path("results").mkdir(exist_ok=True)
Path("results/text-info.json").write_text(json.dumps(info, indent=1))
print(json.dumps(info, indent=1))
|