File size: 6,696 Bytes
7d3eb8f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | """One shared title cleaner for training titles and every prompt set (user 2026-09-29: "remove the decoration from
all topic titles").
Forum titles carry decoration that is not part of the topic: symbol runs (❦ • ♥ ✿ ◆ ~ * ...), repeated punctuation
("หญิงน่ะหรือ.....?"), reply prefixes ("Re:"), bracketed notes, isolated decorative Latin letters (the "O ... O"
frames), and "+ member" signatures. Left in place, the syllable segmenter glues punctuation into syllables
('เหมือน. ..' -> 'เห', 'มือ', 'น.'), which puts junk units into the title vocabulary and the conditioning signal.
clean_title(t) keeps Thai text, Thai digits and meaningful Latin words/digits (VS, VITAS, 2020), replaces every
other character by a space, drops the decorations above, and collapses whitespace. It never changes Thai letters and never returns an empty
title (a title of symbols only is kept as written, stripped). Only a bracket at the start or end is a note;
a bracket inside the title keeps its text (ขอเป็นทาส(หัวใจ)เธอ).
Usage: title_clean.py report OUT.json # before/after counts for the training titles and every prompt set
"""
import re, sys, json, unicodedata
REPLY = re.compile(r"^\s*(?:re|fw|fwd)\s*(?:[:：]\s*|(?=[“”\"'‘’«]))", re.I) # "Re: ...", "re“...”"
MEMBER = re.compile(r"\s*\+\s*[A-Za-z0-9_.]+\s*$") # "... + benuser" signature at the end
BRACKET_LEAD = re.compile(r"^\s*[\(\[\{（][^\)\]\}）]*[\)\]\}）]") # "[กลอนสด] ..." - a note, not topic
BRACKET_TAIL = re.compile(r"[\(\[\{（][^\)\]\}）]*[\)\]\}）][^ก-๙A-Za-z0-9]*$") # "... (แฮะๆ)"; mid-title brackets keep their text
EMOTICON = re.compile(r"<3+|\^\^+|[:;]-?[\)\(DP]") # "<3", "^^", ":)" - not "3"
KEEP = re.compile(r"[ก-ฺเ-๎๐-๙A-Za-z0-9ๆ ]")
DECOR_LATIN = {"o", "x", "oo", "ooo", "xo", "ox", "xx", "pz", "t"} # frame letters: "O ขอใจ O", "XX ... XX", lone T
def clean_title(t):
t = orig = unicodedata.normalize("NFC", t or "")
t = REPLY.sub("", t)
t = MEMBER.sub("", t)
t = EMOTICON.sub(" ", t)
inner = BRACKET_TAIL.sub(" ", BRACKET_LEAD.sub(" ", t))
if re.search(r"[ก-๎]", inner): t = inner # drop a leading/trailing bracketed note unless it IS the title
t = "".join(ch if KEEP.match(ch) else " " for ch in t)
words = [w for w in t.split() if w.lower() not in DECOR_LATIN]
# a Latin word that sits between Thai words without spaces in the raw title (เป็ดVSไก่) was split by the
# symbol pass only if it was surrounded by symbols; Thai runs and Latin runs are otherwise kept as written
out = re.sub(r"\s+", " ", " ".join(words)).strip()
return out or re.sub(r"\s+", " ", orig).strip() # never empty: a title of symbols only stays as written
def _files():
import glob
return ["data/modern_bot_train.jsonl"] + sorted(glob.glob("data/prompts/*.jsonl"))
def report(out):
import random
rec, ex = {}, []
for f in _files():
n = ch = emp = 0
for l in open(f, encoding="utf-8"):
if not l.strip(): continue
r = json.loads(l)
for k in ("title", "text", "prompt"):
if k in r and isinstance(r[k], str):
n += 1; c = clean_title(r[k])
if c != r[k]:
ch += 1
if len(ex) < 400: ex.append((f, r[k], c))
if not c: emp += 1
break
rec[f] = {"titles": n, "changed": ch, "empty_after": emp}
random.Random(0).shuffle(ex)
json.dump({"files": rec, "examples": ex[:30]}, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
for f, v in rec.items(): print(f" {f}: {v['titles']} titles, {v['changed']} changed, {v['empty_after']} empty after cleaning")
def _test():
cases = {
"• • ♥ ขอใจ..ได้ไหม ♥ • •": "ขอใจ ได้ไหม",
"Re: อยากอยู่เป็นคู่ทีไรเจ็บทุกที ชิ": "อยากอยู่เป็นคู่ทีไรเจ็บทุกที ชิ",
"อยากเป็นเด็ก~~~~%?": "อยากเป็นเด็ก",
"รู้สึกเหมือน. ..ไร้พระจันทร์ ((": "รู้สึกเหมือน ไร้พระจันทร์",
"เป็ดVSไก่": "เป็ดVSไก่",
"O ฝากเพลง O": "ฝากเพลง",
"❦• แอบรักคนเขียนกลอนค๊า •❦": "แอบรักคนเขียนกลอนค๊า",
"หญิงน่ะหรือ.....?": "หญิงน่ะหรือ",
"ใครสักคน + benuser": "ใครสักคน",
"คิดถึง (แฮะๆ)": "คิดถึง",
"(กลอนสด)": "กลอนสด",
"ปี 2020 ของฉัน": "ปี 2020 ของฉัน",
"ทุกๆวัน": "ทุกๆวัน",
"รักร้าวเพราะโควิด": "รักร้าวเพราะโควิด",
"": "",
"ขอเป็นทาส(หัวใจ)เธอ": "ขอเป็นทาส หัวใจ เธอ",
"[กลอนสด] ฝนตก": "ฝนตก",
"re“ฝากใจ”": "ฝากใจ",
"remember ฉัน": "remember ฉัน",
"XX คิดถึง XX": "คิดถึง",
"Pz ลาก่อน": "ลาก่อน",
"T รักเธอ T": "รักเธอ",
"?": "?",
"♥♥": "♥♥",
"รับสมัครคนแก้เหงา (ตามสเปคนะเออ) <3": "รับสมัครคนแก้เหงา",
"คนนี้แหละ...ใช่เลย ^^": "คนนี้แหละ ใช่เลย",
}
bad = [(k, clean_title(k), v) for k, v in cases.items() if clean_title(k) != v]
assert not bad, bad
# never changes a Thai letter: the Thai characters of the output are a subsequence of the input's
for k in cases:
o = clean_title(k); it = iter(k)
assert all(ch in it for ch in o if "ก" <= ch <= "๎"), k
assert o or not k.strip(), k
print(f"title_clean: {len(cases)} cases pass")
if __name__ == "__main__":
if sys.argv[1:2] == ["report"]: report(sys.argv[2])
elif sys.argv[1:2] == ["test"]: _test()
else: print(__doc__)
|