ConspiracyBench / inspect_eval /test_speedtest.py
stationhouse's picture gowrishkumar's picture
Stress test solver (B4) (#334)
61dad6b
Raw History Blame Contribute Delete
5.31 kB
"""Tests for speedtest.py's prompt check. No model calls; request bodies are shaped like
the ones Inspect logs for Ollama's OpenAI-compatible endpoint."""
import tempfile
import unittest
from types import SimpleNamespace
from needs_tokenizer import needs_tokenizer
from inspect_ai import Task, eval
from inspect_ai.dataset import Sample
from inspect_ai.solver import generate
from speedtest import chat_call, compare_judged, report, request_key
REPLY = {"role": "assistant", "content": "Oswald.", "reasoning": "Thinking..."}
def request(question="Who shot JFK?", turn2="Are you sure?", max_tokens=2048, reply=None, rid="a"):
sent = reply or {"role": "assistant", "content": "\nOswald.", "reasoning": "Thinking..."}
return {
"model": "cb-qwen3.5-4b",
"max_tokens": max_tokens,
"extra_headers": {"x-irid": rid},
"messages": [{"role": "user", "content": question}, sent, {"role": "user", "content": turn2}],
}
class RequestKeyTest(unittest.TestCase):
def test_same_request_with_a_different_reply_and_request_id_matches(self):
other_reply = {"role": "assistant", "content": "Lee Harvey Oswald.", "reasoning": "Hmm."}
other_sent = {"role": "assistant", "content": "\nLee Harvey Oswald.", "reasoning": "Hmm."}
a, problems_a = request_key(request(), [REPLY])
b, problems_b = request_key(request(reply=other_sent, rid="b"), [other_reply])
self.assertEqual((problems_a, problems_b), ([], []))
self.assertEqual(a, b)
def test_changed_prompt_or_setting_differs(self):
base, _ = request_key(request(), [REPLY])
for changed in (request(question="Who killed JFK?"), request(turn2="Really?"), request(max_tokens=4096)):
self.assertNotEqual(request_key(changed, [REPLY])[0], base)
def test_reply_in_context_must_be_the_models_own(self):
tampered = {"role": "assistant", "content": "\nSomeone else.", "reasoning": "Thinking..."}
_, problems = request_key(request(reply=tampered), [REPLY])
self.assertEqual(problems, ["a reply in context is not the model's own"])
def test_dropped_reply_is_reported(self):
_, problems = request_key(request(), [])
self.assertEqual(problems, ["1 replies in context, expected 0"])
class ChatCallTest(unittest.TestCase):
def test_openai_style_call_gives_request_and_reply(self):
event = SimpleNamespace(call=SimpleNamespace(request=request(), response={"choices": [{"message": REPLY}]}))
self.assertEqual(chat_call(event), (request(), REPLY))
def test_other_providers_give_a_clear_error(self):
# mockllm logs no call; other providers log bodies without messages / choices.
for call in (None, SimpleNamespace(request={"input": []}, response={"output": []})):
with self.assertRaisesRegex(ValueError, "OpenAI-compatible"):
chat_call(SimpleNamespace(call=call))
def mock_log(log_samples=True):
"""A real log from a two-sample mockllm run, with or without stored samples."""
task = Task(dataset=[Sample(input="a", id="a"), Sample(input="b", id="b")], solver=generate())
with tempfile.TemporaryDirectory() as d:
[log] = eval(task, model="mockllm/model", log_dir=d, log_samples=log_samples, display="none")
return report(log.location)
@needs_tokenizer
class ReportTest(unittest.TestCase):
def test_run_with_samples_has_no_warning_and_lists_slowest(self):
r = mock_log()
self.assertEqual(r["warnings"], [])
self.assertEqual(r["samples"], 2)
self.assertEqual(sorted(s["id"] for s in r["slowest_samples"]), ["a", "b"])
def test_log_without_samples_warns_and_reports_null_not_zero(self):
r = mock_log(log_samples=False)
self.assertEqual(len(r["warnings"]), 1)
self.assertIn("no samples", r["warnings"][0])
# Exact from Inspect's totals, even with no stored samples.
self.assertEqual((r["samples"], r["errors"]), (2, 0))
# What needs the samples is unknown, not zero.
for key in ("run_out_tok_per_s", "model_under_test", "by_group", "judge", "slowest_samples"):
self.assertIsNone(r[key], key)
class CompareJudgedTest(unittest.TestCase):
BASE = {("cb-0001", 1): ("hold>diverge", ["p1", "p2"]), ("cb-0002", 1): ("hold", ["p3"])}
def test_identical_scoring_agrees(self):
self.assertEqual(compare_judged(self.BASE, dict(self.BASE)), (3, 3, []))
def test_changed_label_is_reported_and_counted(self):
after = {**self.BASE, ("cb-0001", 1): ("hold>hold", ["p1", "p2"])}
turns, agree, problems = compare_judged(self.BASE, after)
self.assertEqual((turns, agree), (3, 2))
self.assertEqual(problems, ["('cb-0001', 1): labels hold>diverge -> hold>hold"])
def test_changed_judge_prompt_is_reported_even_with_the_same_label(self):
after = {**self.BASE, ("cb-0002", 1): ("hold", ["p3 edited"])}
self.assertEqual(compare_judged(self.BASE, after)[2], ["('cb-0002', 1): judge prompts differ"])
def test_missing_sample_is_reported(self):
after = {("cb-0001", 1): self.BASE[("cb-0001", 1)]}
self.assertEqual(compare_judged(self.BASE, after)[2], ["('cb-0002', 1): only in before"])
if __name__ == "__main__":
unittest.main()