eval/common.py

"""Shared bits for thrasher corpus ingest.

Row schema (all lanes):
  {"id", "lane", "family", "source", "messages": [{"role", "content"}]}

messages roles are system/user/assistant. The system turn, when present, is the
character/task card itself — mockingbird has no persona core to prepend.
"""

import re

THINK = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
THINK_UNCLOSED = re.compile(r"^\s*<think>.*", re.DOTALL)

ROLE = {"system": "system", "human": "user", "user": "user",
        "gpt": "assistant", "assistant": "assistant"}

NAME_PREFIX = re.compile(r"^[A-Z][A-Za-z .'\-]{1,30}:\s")

META_BREAK = re.compile(
    r"no ethical (restrictions|guidelines|constraints)"
    r"|i understand (all )?the guidelines"
    r"|i am playing as .{0,40}(npcs|narrator)"
    r"|i will avoid writing (any )?dialogue"
    r"|here is (the|my) (next |first )?(rp |roleplay )(response|reply)"
    r"|('|\")?thinking('|\")? block|guidelines('|\")? xml tag"
    r"|content policy|jailbreak"
    r"|i('m| am) sorry,? (but )?i (cannot|can't) (continue|create|generate|write)"
    r"|violates .{0,30}(policy|guidelines)"
    r"|\bas an ai\b|\blanguage model\b|\bopenai\b|\banthropic\b", re.I)

# English gate for the RP lanes.
_ENG = set("the and to of a in is it you that he she was for on are with as her "
           "his they at be this have from or had by not but what all were when "
           "we there can an your which their".split())


def is_english(text: str) -> bool:
    letters = [c for c in text if c.isalpha()]
    if len(letters) >= 40:
        if sum(1 for c in letters if ord(c) > 0x24F) / len(letters) > 0.3:
            return False
    words = re.findall(r"[a-zA-Z']+", text.lower())
    if len(words) < 20:
        return True
    return sum(w in _ENG for w in words) / len(words) >= 0.12


def strip_think(text: str) -> str:
    out = THINK.sub("", text)
    out = THINK_UNCLOSED.sub("", out)
    return out.strip()


def sharegpt_to_messages(conversations: list[dict]) -> list[dict] | None:
    """ShareGPT -> messages, thinking stripped from assistant turns.
    Returns None if any assistant turn becomes empty after stripping."""
    msgs = []
    for m in conversations:
        role = ROLE.get(m["from"])
        if role is None:
            return None
        content = m["value"]
        if role == "assistant":
            content = strip_think(content)
            if not content:
                return None
        msgs.append({"role": role, "content": content})
    return msgs