| """Shared bits for thrasher corpus ingest. |
| |
| Row schema (all lanes): |
| {"id", "lane", "family", "source", "messages": [{"role", "content"}]} |
| |
| messages roles are system/user/assistant. The system turn, when present, is the |
| character/task card itself — mockingbird has no persona core to prepend. |
| """ |
|
|
| import re |
|
|
| THINK = re.compile(r"<think>.*?</think>\s*", re.DOTALL) |
| THINK_UNCLOSED = re.compile(r"^\s*<think>.*", re.DOTALL) |
|
|
| ROLE = {"system": "system", "human": "user", "user": "user", |
| "gpt": "assistant", "assistant": "assistant"} |
|
|
| NAME_PREFIX = re.compile(r"^[A-Z][A-Za-z .'\-]{1,30}:\s") |
|
|
| META_BREAK = re.compile( |
| r"no ethical (restrictions|guidelines|constraints)" |
| r"|i understand (all )?the guidelines" |
| r"|i am playing as .{0,40}(npcs|narrator)" |
| r"|i will avoid writing (any )?dialogue" |
| r"|here is (the|my) (next |first )?(rp |roleplay )(response|reply)" |
| r"|('|\")?thinking('|\")? block|guidelines('|\")? xml tag" |
| r"|content policy|jailbreak" |
| r"|i('m| am) sorry,? (but )?i (cannot|can't) (continue|create|generate|write)" |
| r"|violates .{0,30}(policy|guidelines)" |
| r"|\bas an ai\b|\blanguage model\b|\bopenai\b|\banthropic\b", re.I) |
|
|
| |
| _ENG = set("the and to of a in is it you that he she was for on are with as her " |
| "his they at be this have from or had by not but what all were when " |
| "we there can an your which their".split()) |
|
|
|
|
| def is_english(text: str) -> bool: |
| letters = [c for c in text if c.isalpha()] |
| if len(letters) >= 40: |
| if sum(1 for c in letters if ord(c) > 0x24F) / len(letters) > 0.3: |
| return False |
| words = re.findall(r"[a-zA-Z']+", text.lower()) |
| if len(words) < 20: |
| return True |
| return sum(w in _ENG for w in words) / len(words) >= 0.12 |
|
|
|
|
| def strip_think(text: str) -> str: |
| out = THINK.sub("", text) |
| out = THINK_UNCLOSED.sub("", out) |
| return out.strip() |
|
|
|
|
| def sharegpt_to_messages(conversations: list[dict]) -> list[dict] | None: |
| """ShareGPT -> messages, thinking stripped from assistant turns. |
| Returns None if any assistant turn becomes empty after stripping.""" |
| msgs = [] |
| for m in conversations: |
| role = ROLE.get(m["from"]) |
| if role is None: |
| return None |
| content = m["value"] |
| if role == "assistant": |
| content = strip_think(content) |
| if not content: |
| return None |
| msgs.append({"role": role, "content": content}) |
| return msgs |
|
|
