src/scripts/title_clean.py

"""One shared title cleaner for training titles and every prompt set (user 2026-09-29: "remove the decoration from
all topic titles").

Forum titles carry decoration that is not part of the topic: symbol runs (❦ • ♥ ✿ ◆ ~ * ...), repeated punctuation
("หญิงน่ะหรือ.....?"), reply prefixes ("Re:"), bracketed notes, isolated decorative Latin letters (the "O ... O"
frames), and "+ member" signatures. Left in place, the syllable segmenter glues punctuation into syllables
('เหมือน. ..' -> 'เห', 'มือ', 'น.'), which puts junk units into the title vocabulary and the conditioning signal.

clean_title(t) keeps Thai text, Thai digits and meaningful Latin words/digits (VS, VITAS, 2020), replaces every
other character by a space, drops the decorations above, and collapses whitespace. It never changes Thai letters and never returns an empty
title (a title of symbols only is kept as written, stripped). Only a bracket at the start or end is a note;
a bracket inside the title keeps its text (ขอเป็นทาส(หัวใจ)เธอ).

Usage: title_clean.py report OUT.json     # before/after counts for the training titles and every prompt set
"""
import re, sys, json, unicodedata

REPLY = re.compile(r"^\s*(?:re|fw|fwd)\s*(?:[:：]\s*|(?=[“”\"'‘’«]))", re.I)   # "Re: ...", "re“...”"
MEMBER = re.compile(r"\s*\+\s*[A-Za-z0-9_.]+\s*$")          # "... + benuser" signature at the end
BRACKET_LEAD = re.compile(r"^\s*[\(\[\{（][^\)\]\}）]*[\)\]\}）]")   # "[กลอนสด] ..." - a note, not topic
BRACKET_TAIL = re.compile(r"[\(\[\{（][^\)\]\}）]*[\)\]\}）][^ก-๙A-Za-z0-9]*$")   # "... (แฮะๆ)"; mid-title brackets keep their text
EMOTICON = re.compile(r"<3+|\^\^+|[:;]-?[\)\(DP]")                 # "<3", "^^", ":)" - not "3"
KEEP = re.compile(r"[ก-ฺเ-๎๐-๙A-Za-z0-9ๆ ]")
DECOR_LATIN = {"o", "x", "oo", "ooo", "xo", "ox", "xx", "pz", "t"}   # frame letters: "O ขอใจ O", "XX ... XX", lone T


def clean_title(t):
    t = orig = unicodedata.normalize("NFC", t or "")
    t = REPLY.sub("", t)
    t = MEMBER.sub("", t)
    t = EMOTICON.sub(" ", t)
    inner = BRACKET_TAIL.sub(" ", BRACKET_LEAD.sub(" ", t))
    if re.search(r"[ก-๎]", inner): t = inner         # drop a leading/trailing bracketed note unless it IS the title
    t = "".join(ch if KEEP.match(ch) else " " for ch in t)
    words = [w for w in t.split() if w.lower() not in DECOR_LATIN]
    # a Latin word that sits between Thai words without spaces in the raw title (เป็ดVSไก่) was split by the
    # symbol pass only if it was surrounded by symbols; Thai runs and Latin runs are otherwise kept as written
    out = re.sub(r"\s+", " ", " ".join(words)).strip()
    return out or re.sub(r"\s+", " ", orig).strip()    # never empty: a title of symbols only stays as written


def _files():
    import glob
    return ["data/modern_bot_train.jsonl"] + sorted(glob.glob("data/prompts/*.jsonl"))


def report(out):
    import random
    rec, ex = {}, []
    for f in _files():
        n = ch = emp = 0
        for l in open(f, encoding="utf-8"):
            if not l.strip(): continue
            r = json.loads(l)
            for k in ("title", "text", "prompt"):
                if k in r and isinstance(r[k], str):
                    n += 1; c = clean_title(r[k])
                    if c != r[k]:
                        ch += 1
                        if len(ex) < 400: ex.append((f, r[k], c))
                    if not c: emp += 1
                    break
        rec[f] = {"titles": n, "changed": ch, "empty_after": emp}
    random.Random(0).shuffle(ex)
    json.dump({"files": rec, "examples": ex[:30]}, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
    for f, v in rec.items(): print(f"  {f}: {v['titles']} titles, {v['changed']} changed, {v['empty_after']} empty after cleaning")


def _test():
    cases = {
        "• • ♥ ขอใจ..ได้ไหม ♥ • •": "ขอใจ ได้ไหม",
        "Re: อยากอยู่เป็นคู่ทีไรเจ็บทุกที ชิ": "อยากอยู่เป็นคู่ทีไรเจ็บทุกที ชิ",
        "อยากเป็นเด็ก~~~~%?": "อยากเป็นเด็ก",
        "รู้สึกเหมือน. ..ไร้พระจันทร์ ((": "รู้สึกเหมือน ไร้พระจันทร์",
        "เป็ดVSไก่": "เป็ดVSไก่",
        "O ฝากเพลง O": "ฝากเพลง",
        "❦• แอบรักคนเขียนกลอนค๊า •❦": "แอบรักคนเขียนกลอนค๊า",
        "หญิงน่ะหรือ.....?": "หญิงน่ะหรือ",
        "ใครสักคน + benuser": "ใครสักคน",
        "คิดถึง (แฮะๆ)": "คิดถึง",
        "(กลอนสด)": "กลอนสด",
        "ปี 2020 ของฉัน": "ปี 2020 ของฉัน",
        "ทุกๆวัน": "ทุกๆวัน",
        "รักร้าวเพราะโควิด": "รักร้าวเพราะโควิด",
        "": "",
        "ขอเป็นทาส(หัวใจ)เธอ": "ขอเป็นทาส หัวใจ เธอ",
        "[กลอนสด] ฝนตก": "ฝนตก",
        "re“ฝากใจ”": "ฝากใจ",
        "remember ฉัน": "remember ฉัน",
        "XX คิดถึง XX": "คิดถึง",
        "Pz ลาก่อน": "ลาก่อน",
        "T รักเธอ T": "รักเธอ",
        "?": "?",
        "♥♥": "♥♥",
        "รับสมัครคนแก้เหงา (ตามสเปคนะเออ) <3": "รับสมัครคนแก้เหงา",
        "คนนี้แหละ...ใช่เลย ^^": "คนนี้แหละ ใช่เลย",
    }
    bad = [(k, clean_title(k), v) for k, v in cases.items() if clean_title(k) != v]
    assert not bad, bad
    # never changes a Thai letter: the Thai characters of the output are a subsequence of the input's
    for k in cases:
        o = clean_title(k); it = iter(k)
        assert all(ch in it for ch in o if "ก" <= ch <= "๎"), k
        assert o or not k.strip(), k
    print(f"title_clean: {len(cases)} cases pass")


if __name__ == "__main__":
    if sys.argv[1:2] == ["report"]: report(sys.argv[2])
    elif sys.argv[1:2] == ["test"]: _test()
    else: print(__doc__)