# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Kale """prep_text.py: the text-only distillation set for the Ettin text side (step 1a of textside/). Sentences from the text pools (config/text.json `pools`), split at sentence ends, deduplicated, 3-60 words, bench lines and every guard-line sentence held out, shuffled (seed 0); each sentence keeps the previous sentence of its passage (Ettin's left context). Token ids + word index per token come next, from make_ids.mjs (release front end, en-US and en-GB). Writes (under work/): sents.jsonl {text, prev} training sentences trace.jsonl {text, prev} the trace line's sentences (export trace + test inputs; config `trace_line`) check_texts.json [{id, text}] the first `check_lines` bench lines (export check inputs, check_items.mjs) usage: python3 -I prep_text.py [limit] [--config config/text.json] [--out work]""" import argparse import json import random import re from pathlib import Path HERE = Path(__file__).resolve().parent ap = argparse.ArgumentParser() ap.add_argument("limit", nargs="?", type=int, default=0) ap.add_argument("--config", default=str(HERE / "config" / "text.json")) ap.add_argument("--out", default=str(HERE / "work")) a = ap.parse_args() C = json.loads(Path(a.config).read_text()) rel = lambda p: (HERE / p).resolve() # noqa: E731 SENT = re.compile(C["sentence_split"]) def jsonl(p): return [json.loads(l) for l in open(p) if l.strip()] bench = [x for p in C["held_out_jsonl"] for x in jsonl(rel(p)) if "text" in x] guard = json.loads(rel(C["held_out_guard"]).read_text())["guard_lines"] # [[name, text], ...] held = {x["text"].strip() for x in bench} held |= {s.strip() for _, t in guard for s in SENT.split(t)} texts = [] for p in C["pools"]: texts += [x["text"] for x in jsonl(rel(p))] seen, items = set(), [] for t in texts: sents = [s.strip() for s in SENT.split(t.strip()) if s.strip()] prev = "" for s in sents: if s not in seen and s not in held and C["min_words"] <= len(s.split()) <= C["max_words"]: seen.add(s); items.append((s, prev)) prev = s random.Random(C["shuffle_seed"]).shuffle(items) lim = a.limit or len(items) od = Path(a.out); od.mkdir(parents=True, exist_ok=True) with open(od / "sents.jsonl", "w") as f: for s, prev in items[:lim]: f.write(json.dumps(dict(text=s, prev=prev)) + "\n") tl = dict(guard)[C["trace_line"]] with open(od / "trace.jsonl", "w") as f: prev = "" for s in [s.strip() for s in SENT.split(tl) if s.strip()]: f.write(json.dumps(dict(text=s, prev=prev)) + "\n"); prev = s (od / "check_texts.json").write_text(json.dumps([dict(id=x["id"], text=x["text"]) for x in bench[:C["check_lines"]]])) print("sentences", min(lim, len(items)), "of", len(items), "candidates; held out", len(held))