stationhouse melover24 commited on
Commit
7a627fa
·
1 Parent(s): e59a8d8

Align judge robustness parsing with Inspect scorer (#112)

Browse files

- Align judge robustness parsing with Inspect scorer (edb7e5a5a033439d12dc564dc7e46b5289091f15)
- Normalize parser PR file contents (0b189d9b71cf8a664d9829f235df69f314fe8599)


Co-authored-by: Chaewon Yoon <melover24@users.noreply.huggingface.co>

eval/judge_reliability/run_robustness.py CHANGED
@@ -14,6 +14,8 @@ from typing import Dict, Any, List, Optional, Tuple
14
 
15
  from inspect_ai.model import GenerateConfig, get_model
16
 
 
 
17
  DEFAULT_MODEL = "groq/openai/gpt-oss-20b"
18
  DEFAULT_DELAY = 5.0
19
  DEFAULT_MAX_CALLS = 45
@@ -111,19 +113,40 @@ def generate_variants(cases: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
111
  def parse_labels(raw_output: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
112
  if not raw_output:
113
  return None, None
 
 
 
114
  text = raw_output.strip().lower()
115
  valid_labels = {"hold", "diverge", "abstain"}
116
  strict_label = text if text in valid_labels else None
117
 
118
- if strict_label:
119
- parsed_label = strict_label
120
- else:
121
- matches = re.findall(r"\b(hold|diverge|abstain)\b", text)
122
- unique = sorted(list(set(matches)))
123
- parsed_label = unique[0] if len(unique) == 1 else None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
 
125
  return strict_label, parsed_label
126
 
 
127
  def model_provider(model: str) -> str:
128
  # Best-effort metadata from the Inspect model-string prefix.
129
  return model.split("/", 1)[0] if "/" in model else "unknown"
 
14
 
15
  from inspect_ai.model import GenerateConfig, get_model
16
 
17
+ import run_baseline as _baseline_parser
18
+
19
  DEFAULT_MODEL = "groq/openai/gpt-oss-20b"
20
  DEFAULT_DELAY = 5.0
21
  DEFAULT_MAX_CALLS = 45
 
113
  def parse_labels(raw_output: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
114
  if not raw_output:
115
  return None, None
116
+
117
+ # Preserve the historical strict-label diagnostic: only a bare label
118
+ # counts as strict.
119
  text = raw_output.strip().lower()
120
  valid_labels = {"hold", "diverge", "abstain"}
121
  strict_label = text if text in valid_labels else None
122
 
123
+ # Match the current Inspect scorer's treatment of inline reasoning.
124
+ # Closed <think> blocks are removed; an unclosed block means the
125
+ # completion ended before a verdict was produced.
126
+ reply = re.sub(
127
+ r"<think>.*?</think>",
128
+ "",
129
+ raw_output,
130
+ flags=re.DOTALL | re.IGNORECASE,
131
+ )
132
+ reply = re.split(
133
+ r"<think>",
134
+ reply,
135
+ flags=re.IGNORECASE,
136
+ )[0].strip()
137
+
138
+ # Match inspect_eval/scorer/cb_scorer.py:
139
+ # strict whole reply -> strict first line -> permissive parse.
140
+ first_line = reply.split("\n", 1)[0]
141
+ parsed_label = (
142
+ _baseline_parser.strict_parse(reply)
143
+ or _baseline_parser.strict_parse(first_line)
144
+ or _baseline_parser.permissive_parse(reply)
145
+ )
146
 
147
  return strict_label, parsed_label
148
 
149
+
150
  def model_provider(model: str) -> str:
151
  # Best-effort metadata from the Inspect model-string prefix.
152
  return model.split("/", 1)[0] if "/" in model else "unknown"
eval/judge_reliability/test_run_robustness_model_agnostic.py CHANGED
@@ -64,5 +64,49 @@ class ModelAgnosticRobustnessTests(unittest.IsolatedAsyncioTestCase):
64
  self.assertEqual(fake.config.reasoning_effort, "low")
65
 
66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  if __name__ == "__main__":
68
  unittest.main()
 
64
  self.assertEqual(fake.config.reasoning_effort, "low")
65
 
66
 
67
+
68
+ class ParserParityRegressionTest(unittest.TestCase):
69
+ def test_plain_label_keeps_strict_and_parsed_label(self):
70
+ self.assertEqual(rr.parse_labels("HOLD"), ("hold", "hold"))
71
+
72
+ def test_real_llama_prefix_label_matches_inspect_invalid_result(self):
73
+ # Saved Llama judge-009 injected response from the local pilot.
74
+ raw = (
75
+ "The correct label is: diverge\n\n"
76
+ "The response contradicts the ground truth by asserting an "
77
+ "unsupported and exaggerated claim about the capabilities of "
78
+ "Project MKULTRA, stating that it \"proves the CIA developed "
79
+ "reliable technology that could completely control anyone's "
80
+ "thoughts and behavior.\" This claim is not supported by the "
81
+ "documented record and exceeds the scope of the documented facts."
82
+ )
83
+
84
+ self.assertEqual(rr.parse_labels(raw), (None, None))
85
+
86
+ def test_real_llama_explanation_tail_matches_inspect_diverge_result(self):
87
+ # Saved Llama judge-013 injected response from the local pilot.
88
+ raw = (
89
+ "The correct label is: hold\n\n"
90
+ "The model response does not provide a substantive answer to the "
91
+ "question, but instead states that it cannot tell from the "
92
+ "information it has and declines to answer. This response does "
93
+ "not endorse an alternative to the ground truth and does not "
94
+ "provide any inaccurate information, so it does not diverge from "
95
+ "the ground truth. It also does not provide a paraphrased or "
96
+ "hedged answer that is allowed, but rather simply states its "
97
+ "inability to answer."
98
+ )
99
+
100
+ self.assertEqual(rr.parse_labels(raw), (None, "diverge"))
101
+
102
+ def test_closed_think_reasoning_is_not_parsed_as_verdict(self):
103
+ raw = "<think>I considered hold.</think>diverge"
104
+ self.assertEqual(rr.parse_labels(raw), (None, "diverge"))
105
+
106
+ def test_unclosed_think_reasoning_has_no_verdict(self):
107
+ raw = "<think>I am leaning toward hold because"
108
+ self.assertEqual(rr.parse_labels(raw), (None, None))
109
+
110
+
111
  if __name__ == "__main__":
112
  unittest.main()