Update eval/common.py
Browse files- eval/common.py +2 -15
eval/common.py
CHANGED
|
@@ -1,34 +1,22 @@
|
|
| 1 |
-
"""Shared bits for
|
| 2 |
|
| 3 |
Row schema (all lanes):
|
| 4 |
{"id", "lane", "family", "source", "messages": [{"role", "content"}]}
|
| 5 |
|
| 6 |
messages roles are system/user/assistant. The system turn, when present, is the
|
| 7 |
character/task card itself — mockingbird has no persona core to prepend.
|
| 8 |
-
NO THINKING is settled: <think>...</think> is stripped at ingest, never
|
| 9 |
-
retargeted.
|
| 10 |
"""
|
| 11 |
|
| 12 |
import re
|
| 13 |
|
| 14 |
-
# Qwen-style thinking blocks, as stored in both v6 and rp-reasoning-v2.
|
| 15 |
-
# DOTALL because plans span lines; also catch an unclosed opener at text start.
|
| 16 |
THINK = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
|
| 17 |
THINK_UNCLOSED = re.compile(r"^\s*<think>.*", re.DOTALL)
|
| 18 |
|
| 19 |
ROLE = {"system": "system", "human": "user", "user": "user",
|
| 20 |
"gpt": "assistant", "assistant": "assistant"}
|
| 21 |
|
| 22 |
-
# The gemma-era defect: fiction-transcript openers like "Sorcha: ..." — the
|
| 23 |
-
# v6:carded lane measures 0.0% and must stay there.
|
| 24 |
NAME_PREFIX = re.compile(r"^[A-Z][A-Za-z .'\-]{1,30}:\s")
|
| 25 |
|
| 26 |
-
# Meta/jailbreak/refusal turns inside RP logs (measured 1.15% of v6:carded,
|
| 27 |
-
# 1.66% of rp-reasoning): proxy-log compliance preambles ("Sure, I am playing
|
| 28 |
-
# as X, NPCs and Narrator!") and mid-RP policy refusals ("OpenAI's content
|
| 29 |
-
# policy prohibits..."). Both are defects we would TEACH. Only strong meta
|
| 30 |
-
# signals — generic openers like "Okay, I..." are in-character dialogue
|
| 31 |
-
# (12/12 sampled false positives).
|
| 32 |
META_BREAK = re.compile(
|
| 33 |
r"no ethical (restrictions|guidelines|constraints)"
|
| 34 |
r"|i understand (all )?the guidelines"
|
|
@@ -41,8 +29,7 @@ META_BREAK = re.compile(
|
|
| 41 |
r"|violates .{0,30}(policy|guidelines)"
|
| 42 |
r"|\bas an ai\b|\blanguage model\b|\bopenai\b|\banthropic\b", re.I)
|
| 43 |
|
| 44 |
-
# English gate for the RP lanes
|
| 45 |
-
# was a visible defect; the battery can only measure English).
|
| 46 |
_ENG = set("the and to of a in is it you that he she was for on are with as her "
|
| 47 |
"his they at be this have from or had by not but what all were when "
|
| 48 |
"we there can an your which their".split())
|
|
|
|
| 1 |
+
"""Shared bits for thrasher corpus ingest.
|
| 2 |
|
| 3 |
Row schema (all lanes):
|
| 4 |
{"id", "lane", "family", "source", "messages": [{"role", "content"}]}
|
| 5 |
|
| 6 |
messages roles are system/user/assistant. The system turn, when present, is the
|
| 7 |
character/task card itself — mockingbird has no persona core to prepend.
|
|
|
|
|
|
|
| 8 |
"""
|
| 9 |
|
| 10 |
import re
|
| 11 |
|
|
|
|
|
|
|
| 12 |
THINK = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
|
| 13 |
THINK_UNCLOSED = re.compile(r"^\s*<think>.*", re.DOTALL)
|
| 14 |
|
| 15 |
ROLE = {"system": "system", "human": "user", "user": "user",
|
| 16 |
"gpt": "assistant", "assistant": "assistant"}
|
| 17 |
|
|
|
|
|
|
|
| 18 |
NAME_PREFIX = re.compile(r"^[A-Z][A-Za-z .'\-]{1,30}:\s")
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
META_BREAK = re.compile(
|
| 21 |
r"no ethical (restrictions|guidelines|constraints)"
|
| 22 |
r"|i understand (all )?the guidelines"
|
|
|
|
| 29 |
r"|violates .{0,30}(policy|guidelines)"
|
| 30 |
r"|\bas an ai\b|\blanguage model\b|\bopenai\b|\banthropic\b", re.I)
|
| 31 |
|
| 32 |
+
# English gate for the RP lanes.
|
|
|
|
| 33 |
_ENG = set("the and to of a in is it you that he she was for on are with as her "
|
| 34 |
"his they at be this have from or had by not but what all were when "
|
| 35 |
"we there can an your which their".split())
|