| """Fresh held-out titles for the final confirmation (runs/queue/PLAN.md step 4; runs/CSEL/CONFIRM_PREREG.md). |
| |
| The reserved set, as recorded when the dev pilot was drawn (runs/CSEL/WORDRHYME_PREREG.md): the hygiene-passing |
| TEST-split titles outside data/prompts/test_topics.jsonl, never used in development. Each needs a human control |
| built exactly as for test150 (runs/SPLIT/judge/p*_htest.jsonl): the opening post's first stanza under that title |
| (data/modern_bot_test_prov.jsonl: post == 0, stanza == 0), spaces stripped; with several such stanzas the one |
| from the lowest thread id. |
| |
| Exclusions, all by the decoration-free title key of scripts/split_titles.py: every text in data/prompts/*.jsonl |
| (test_topics, test150, pilots, seen_topics, benches, train-title lists ...), every "topic:" line in any judge |
| batch under runs/ and any runs/*.jsonl prompt field already written (so a title that was ever decoded, judged or |
| used to tune is out). Selection: seeded shuffle, first N. |
| |
| Usage: fresh_titles.py [--n 120] [--seed 7] (takes up to n; asserts >= 100) -> data/prompts/fresh120.jsonl, data/pools/fresh120_human.jsonl |
| """ |
| import sys, os, re, json, glob, random, argparse, collections, unicodedata |
| sys.path.insert(0, "."); sys.path.insert(0, os.path.dirname(__file__)) |
| from klon8 import prosody as P |
|
|
| THAI_ONLY = re.compile(r"[ก-๛]+") |
| KW = ("สุขสันต์", "วันเกิด", "กลบท", "บอร์ด", "สมาชิก", "ขอบคุณ", "ประกาศ", "ชมรม", "ยินดีต้อนรับ", |
| "อวยพร", "แนะนำตัว", "ทดสอบ", "บทกวี", "เรื่องสั้น", "ตอนที่", "ภาค", "ฝากตัว", "คุยกัน", |
| "ห้อง", "กระทู้", "รับสมัคร", "เชิญ", |
| |
| "Happy", "Birthday", "HBD", "Re:", "RE:", "re:", "หัวข้อนี้") |
| LATIN_OR_DIGIT = re.compile(r"[A-Za-z0-9\u0e50-\u0e59]") |
|
|
|
|
| def key(t): |
| return re.sub(r"[^-]", "", unicodedata.normalize("NFC", t or "")) |
|
|
|
|
| def usable(t): |
| if not THAI_ONLY.fullmatch(unicodedata.normalize("NFC", t)): return False |
| if any(k in t for k in KW): return False |
| return 6 <= len(t) <= 22 and len(P.g2p_line(t)) >= 2 |
|
|
|
|
| def used_keys(): |
| used = set() |
| for f in glob.glob("data/prompts/*.jsonl"): |
| if os.path.basename(f).startswith("fresh"): continue |
| for l in open(f, encoding="utf-8"): |
| if l.strip(): |
| r = json.loads(l) |
| for k in ("text", "prompt", "topic", "title"): |
| if isinstance(r.get(k), str): used.add(key(r[k])) |
| for f in glob.glob("runs/**/batch_*.txt", recursive=True): |
| for l in open(f, encoding="utf-8", errors="ignore"): |
| m = re.match(r"item \S+\s+topic:\s*(.*)", l) |
| if m: used.add(key(m.group(1))) |
| for f in glob.glob("runs/**/*.jsonl", recursive=True): |
| if "/CONFIRM/" in f or os.path.getsize(f) > 200_000_000: continue |
| try: |
| for l in open(f, encoding="utf-8"): |
| if '"prompt"' not in l: continue |
| p = json.loads(l).get("prompt") |
| if isinstance(p, str): used.add(key(p)) |
| except (UnicodeDecodeError, json.JSONDecodeError): |
| continue |
| used.discard("") |
| return used |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser(); ap.add_argument("--n", type=int, default=120); ap.add_argument("--seed", type=int, default=7) |
| ap.add_argument("--out", default="data/prompts/fresh120.jsonl"); ap.add_argument("--human", default="data/pools/fresh120_human.jsonl") |
| a = ap.parse_args() |
| train_keys = {key(json.loads(l)["title"]) for l in open("data/modern_bot_train.jsonl", encoding="utf-8")} |
| prov = [json.loads(l) for l in open("data/modern_bot_test_prov.jsonl", encoding="utf-8")] |
| opening = collections.defaultdict(list) |
| for r in prov: |
| if r.get("post") == 0 and r.get("stanza") == 0: opening[r["title"]].append(r) |
| EX = "data/prompts/fresh_exclusion_keys.json" |
| if os.path.exists(EX): |
| used = set(json.load(open(EX, encoding="utf-8"))) |
| else: |
| used = used_keys(); json.dump(sorted(used), open(EX, "w", encoding="utf-8"), ensure_ascii=False, indent=0) |
| |
| |
| cand = sorted(t for t in opening if not any(k in t for k in KW) and not LATIN_OR_DIGIT.search(t) |
| and usable(key(t)) and key(t) not in used and key(t) not in train_keys) |
| random.Random(a.seed).shuffle(cand) |
| pick, seen = [], set() |
| for t in cand: |
| if key(t) in seen: continue |
| seen.add(key(t)); pick.append(t) |
| if len(pick) == a.n: break |
| assert len(pick) >= 50, f"only {len(pick)} fresh titles available" |
| |
| assert not ({key(t) for t in pick} & used), "a fresh title was used before" |
| assert not ({key(t) for t in pick} & train_keys), "a fresh title is a train title" |
| by_title = collections.defaultdict(list) |
| for r in prov: by_title[r["title"]].append(r) |
| os.makedirs(os.path.dirname(a.human), exist_ok=True) |
| with open(a.out, "w", encoding="utf-8") as f, open(a.human, "w", encoding="utf-8") as h: |
| for i, t in enumerate(pick, 1): |
| op = sorted(opening[t], key=lambda r: r["id"])[0] |
| fid = f"F{i:03d}" |
| f.write(json.dumps({"id": fid, "form": "topic", "text": key(t), "raw_title": t, "set": "FRESH", |
| "raw_clean": bool(usable(t)), |
| "n_human_bot": len(by_title[t]), "syl": len(P.g2p_line(key(t)))}, ensure_ascii=False) + "\n") |
| h.write(json.dumps({"id": fid, "prompt": key(t), "poem": op["poem"].replace(" ", ""), "src_id": op["id"], "board": op.get("board"), |
| "thread": op["thread"]}, ensure_ascii=False) + "\n") |
| P.save_cache() |
| print(f"{len(cand)} unused hygiene-passing held-out titles with an opening-post first stanza; kept {len(pick)} " |
| f"(seed {a.seed}); {len(used)} used title keys excluded -> {a.out}, {a.human}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
