File size: 10,218 Bytes
f21d6eb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Kale
"""Training text pool (15 Nov 2026): public-domain Project Gutenberg books -> single sentences and passages of 2-5 consecutive sentences
of one paragraph, mix by item count 1/2/3/4/5 sentences = 40/20/20/12/8 % (config/render.json "text_mix"), max ~20 s at ~195 wpm
(<= 62 words; 66 for 4-5 sentences). Long groups are scarce in narration-only book text, so every 2-5-sentence group is used and
singles are 40 % of all items; the 4- and 5-sentence shares stay below the target mix.
Every release voice's texts are sampled from this pool (make_singles.py, make_jobs.py). The pool was first rendered as the
storyteller voice v7, so its rows are written as v7 jobs (clip v7_<n>).
Inputs: a folder of plain-text Gutenberg books named pg<id>.txt (ids in BOOKS below; download them yourself), and ban files with
sentences that must never be trained on (your bench lines, test sets, fixed test lines such as the volcano line): .jsonl (field
"text"), .json (list of strings or of {"text"}), or .txt (one line per sentence).
  python3 -I make_text.py <gutenberg dir> "<Maya-1 description>" --ban ../bench/script.jsonl --ban my_test_lines.txt
Outputs: work/jobs-v7.jsonl (clip, voice, text, description, kind single|passage, n_sent, source pg<id>), results/text-info.json."""
import json
import random
import re
import sys
from pathlib import Path

import argparse
ap = argparse.ArgumentParser()
ap.add_argument("gutenberg_dir"); ap.add_argument("description")
ap.add_argument("--ban", action="append", default=[], help="file of banned sentences (.jsonl text field, .json list, .txt lines)")
ap.add_argument("--voice", default="v7"); ap.add_argument("--out", default="work/jobs-v7.jsonl")
A = ap.parse_args()
GUT = Path(A.gutenberg_dir)
rnd = random.Random(1411)
norm = lambda t: re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", t.lower())).strip()
split = lambda t: [s for s in re.split(r"(?<=[.!?])\s+", t.strip()) if s]

# ---------------------------------------------------------------- banned sentences (held out / bench / test lines / guard lines)
ban_src = []
for f in map(Path, A.ban):
    if f.suffix == ".jsonl":
        ban_src += [json.loads(l)["text"] for l in f.read_text().splitlines() if l.strip() and "text" in json.loads(l)]
    elif f.suffix == ".json":
        ban_src += [x["text"] if isinstance(x, dict) else x for x in json.loads(f.read_text())]
    else:
        ban_src += [l for l in f.read_text().splitlines() if l.strip()]
ban = {norm(s) for t in ban_src for s in split(t) if len(s.split()) >= 3}
banned = lambda t: any(b in norm(t) for b in ban)


OK = re.compile(r"^[A-Za-z ,.!?;:'\-]+$")
ARCHAIC = re.compile(r"\b(thee|thou|thy|thine|hath|doth|dost|shalt|wilt|ye|tis|twas|o'er|ere|nay|whither|hither|thither|wherefore|art thou|aught|naught)\b", re.I)


def good_sentence(s, lo=3, hi=28):
    w = s.split()
    if not (lo <= len(w) <= hi) or not OK.match(s) or s[-1] not in ".!?" or not s[0].isupper():
        return False
    if ARCHAIC.search(s) or re.search(r"(^|\s)'|'(\s|$)(?<!s'\s)", s) or re.search(r"\b[A-Z]{2,}\b", s):
        return False
    if re.search(r"[;:]\s*$|\s-|-\s|--", s) or banned(s):
        return False
    return True


# ---------------------------------------------------------------- Gutenberg books (id -> title, author, death year, translator)
BOOKS = {
    11: ("Alice's Adventures in Wonderland", "Lewis Carroll", 1898), 12: ("Through the Looking-Glass", "Lewis Carroll", 1898),
    55: ("The Wonderful Wizard of Oz", "L. Frank Baum", 1919), 54: ("The Marvelous Land of Oz", "L. Frank Baum", 1919),
    486: ("Ozma of Oz", "L. Frank Baum", 1919), 289: ("The Wind in the Willows", "Kenneth Grahame", 1932),
    2781: ("Just So Stories", "Rudyard Kipling", 1936), 236: ("The Jungle Book", "Rudyard Kipling", 1936),
    113: ("The Secret Garden", "Frances Hodgson Burnett", 1924), 146: ("A Little Princess", "Frances Hodgson Burnett", 1924),
    479: ("Little Lord Fauntleroy", "Frances Hodgson Burnett", 1924), 1874: ("The Railway Children", "E. Nesbit", 1924),
    778: ("Five Children and It", "E. Nesbit", 1924), 501: ("The Story of Doctor Dolittle", "Hugh Lofting", 1947),
    21: ("Three hundred Aesop's fables", "Aesop, tr. George Fyler Townsend", 1900), 2591: ("Grimms' Fairy Tales", "J. & W. Grimm, tr. Edgar Taylor & Marian Edwardes", 1932),
    1597: ("Andersen's Fairy Tales", "H. C. Andersen (PG translation, 19th c.)", 1875), 271: ("Black Beauty", "Anna Sewell", 1878),
    45: ("Anne of Green Gables", "L. M. Montgomery", 1942), 11757: ("The Velveteen Rabbit", "Margery Williams Bianco", 1944),
    503: ("The Blue Fairy Book", "Andrew Lang (ed.)", 1912), 14838: ("The Tale of Peter Rabbit", "Beatrix Potter", 1943),
    120: ("Treasure Island", "Robert Louis Stevenson", 1894), 514: ("Little Women", "Louisa May Alcott", 1888),
    1450: ("Pollyanna", "Eleanor H. Porter", 1920), 498: ("Rebecca of Sunnybrook Farm", "Kate Douglas Wiggin", 1923),
    708: ("The Princess and the Goblin", "George MacDonald", 1905), 225: ("At the Back of the North Wind", "George MacDonald", 1905),
    1018: ("The Water-Babies", "Charles Kingsley", 1875), 754: ("The Story of Mankind", "Hendrik Willem van Loon", 1944),
    22693: ("A Book of Myths", "Jean Lang", 1933),
    420: ("Dorothy and the Wizard in Oz", "L. Frank Baum", 1919), 485: ("The Road to Oz", "L. Frank Baum", 1919),
    517: ("The Emerald City of Oz", "L. Frank Baum", 1919), 955: ("The Patchwork Girl of Oz", "L. Frank Baum", 1919),
    956: ("Tik-Tok of Oz", "L. Frank Baum", 1919), 957: ("The Scarecrow of Oz", "L. Frank Baum", 1919),
    958: ("Rinkitink in Oz", "L. Frank Baum", 1919), 959: ("The Lost Princess of Oz", "L. Frank Baum", 1919),
    960: ("The Tin Woodman of Oz", "L. Frank Baum", 1919), 1937: ("The Second Jungle Book", "Rudyard Kipling", 1936),
    557: ("Puck of Pook's Hill", "Rudyard Kipling", 1936), 836: ("The Phoenix and the Carpet", "E. Nesbit", 1924),
    837: ("The Story of the Amulet", "E. Nesbit", 1924), 3536: ("The Enchanted Castle", "E. Nesbit", 1924),
    770: ("The Story of the Treasure Seekers", "E. Nesbit", 1924), 23661: ("The Book of Dragons", "E. Nesbit", 1924),
    1154: ("The Voyages of Doctor Dolittle", "Hugh Lofting", 1947), 540: ("The Red Fairy Book", "Andrew Lang (ed.)", 1912),
    641: ("The Violet Fairy Book", "Andrew Lang (ed.)", 1912), 3027: ("The Orange Fairy Book", "Andrew Lang (ed.)", 1912),
    47: ("Anne of Avonlea", "L. M. Montgomery", 1942), 51: ("Anne of the Island", "L. M. Montgomery", 1942),
    2788: ("Little Men", "Louisa May Alcott", 1888), 18442: ("Fifty Famous Stories Retold", "James Baldwin", 1925),
    4018: ("Japanese Fairy Tales", "Yei Theodora Ozaki", 1932), 7439: ("English Fairy Tales", "Joseph Jacobs (ed.)", 1916),
    976: ("Tanglewood Tales", "Nathaniel Hawthorne", 1864), 677: ("The Heroes, or Greek Fairy Tales", "Charles Kingsley", 1875),
    902: ("The Happy Prince, and Other Tales", "Oscar Wilde", 1900), 32575: ("The Head Girl at the Gables", "Angela Brazil", 1947),
}


def book_paragraphs(i):
    t = (GUT / f"pg{i}.txt").read_text(encoding="utf-8", errors="replace").replace("\r", "")
    a, b = t.find("*** START OF"), t.find("*** END OF")
    t = t[t.find("\n", a) + 1: b if b > 0 else None]
    out = []
    for p in re.split(r"\n\s*\n", t):
        p = " ".join(p.split())
        if not p or '"' in p or "“" in p or "”" in p or re.search(r"\d", p) or "[" in p:
            continue
        p = p.replace("’", "'").replace("‘", "'").replace("_", "").replace("—", ", ").replace("--", ", ")
        p = re.sub(r"\bMr\.", "Mister", p); p = re.sub(r"\bMrs\.", "Missus", p); p = re.sub(r"\bDr\.", "Doctor", p)
        p = re.sub(r"\s+,", ",", re.sub(r",\s*,", ",", p))
        out.append(p)
    return out



MIX = {1: 0.40, 2: 0.20, 3: 0.20, 4: 0.12, 5: 0.08}
N = 28000
MAXW = 62
desc = A.description
cands = {n: [] for n in MIX}            # (text, book)
for i in BOOKS:
    for para in book_paragraphs(i):
        ss = split(para)
        ok = [good_sentence(x, 3, 28) for x in ss]
        k = 0
        while k < len(ss):                   # each run of consecutive good sentences, cut into the longest groups that fit (5 first)
            if not ok[k]:
                k += 1; continue
            hit = 0
            for n in (5, 4, 3, 2):
                grp = ss[k:k + n]
                w = sum(len(x.split()) for x in grp)
                if len(grp) == n and all(ok[k:k + n]) and 10 <= w <= (MAXW + 4 if n >= 4 else MAXW):
                    hit = n; break
            if not hit and good_sentence(ss[k], 5, 28):
                hit = 1
            if hit:
                cands[hit].append((" ".join(ss[k:k + hit]), i)); k += hit
            else:
                k += 1
avail = {n: len(v) for n, v in cands.items()}
# long groups are scarce in narration-only book text: take EVERY 2-5-sentence group, singles = 40 % of all items
rows = []
for n in (2, 3, 4, 5):
    rows += [dict(n=n, text=t, source=f"pg{i}") for t, i in dict.fromkeys(cands[n])]
n1 = int(round(len(rows) * 0.40 / 0.60))
pool = list(dict.fromkeys(cands[1])); rnd.shuffle(pool)
rows += [dict(n=1, text=t, source=f"pg{i}") for t, i in pool[:n1]]
rnd.shuffle(rows)
jobs = [dict(clip=f"{A.voice}_{k:05d}", voice=A.voice, text=r["text"], description=desc, mood="default", tagname="", kind="single" if r["n"] == 1 else "passage",
             n_sent=r["n"], source=r["source"]) for k, r in enumerate(rows)]
out = Path(A.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("".join(json.dumps(j) + "\n" for j in jobs))
from collections import Counter  # noqa: E402
c = Counter(j["n_sent"] for j in jobs)
words = sum(len(j["text"].split()) for j in jobs)
info = dict(jobs=len(jobs), by_sentences=dict(sorted(c.items())), share={k: round(v / len(jobs), 3) for k, v in sorted(c.items())}, available=avail,
            words=words, est_hours_at_195wpm=round(words / 195 / 60, 1), max_words=max(len(j["text"].split()) for j in jobs), banned_sentences=len(ban),
            books=len(BOOKS), description=desc)
Path("results").mkdir(exist_ok=True)
Path("results/text-info.json").write_text(json.dumps(info, indent=1))
print(json.dumps(info, indent=1))