ConspiracyBench / inspect_eval /speedtest.py
stationhouse's picture powercoding1's picture
Scorer cost: cap the judge reply at its budget, add speedtest --same-labels, document measured settings (#254)
0a0e9d3
Raw History Blame Contribute Delete
13.9 kB
"""Where a run's time goes, read from Inspect logs. Makes no model calls.
python inspect_eval/speedtest.py logs/<run>.eval [more.eval ...]
python inspect_eval/speedtest.py --same-prompts before.eval after.eval
python inspect_eval/speedtest.py --same-labels before.eval after.eval
The first form prints, per log, the run's wall clock and the model-under-test calls split
by single- vs multi-turn item and by opening vs pressure turn: calls, failed calls and
retries, tokens in and out, turns that hit ``max_tokens``, empty answers (no reply text,
e.g. when reasoning used the whole budget), latency, and tokens per second. Each call's time is split into
``generate`` (the request to the provider, as Inspect times it in ``output.time``) and
``wait`` (time the call spent inside Inspect before and around that request, mostly
queueing for a ``--max-connections`` slot). Queueing inside the server, e.g. Ollama
holding a request until a parallel slot frees up, is part of ``generate``; compare
tokens/s at concurrency 1 to see it. Judge calls (model role ``grader``) are reported on
their own line, so a run with ``--no-score`` measures the solver alone.
``retries`` is what the provider reported in ``ModelEvent.retries``; not every provider
sets it. A call that errored and was retried by Inspect always shows in ``failed_calls``.
``slowest_samples`` lists the five samples with the longest total time. A log with no
stored samples, or a run that did not finish, gets a ``warnings`` entry, also printed to
stderr. Without stored samples, ``samples`` and ``errors`` still come from Inspect's own
totals (as in ``report.py``), and every figure that needs the samples is ``null``, not 0.
``--same-prompts`` checks the solver's ground rule for a speed change: the model sees the
same thing. Replies are sampled, so a later turn's prompt differs between runs by the
model's own earlier replies. It therefore compares, per sample, epoch and turn, the HTTP
request body sent to the provider byte for byte (generation settings, message order, every
system and user message), after checking that each assistant message in it is exactly the
reply the model gave earlier in the same run. It exits non-zero on any difference.
``--same-labels`` is the scorer's version, for two scored logs of the same transcripts
(``inspect score`` on one solver log, before and after a scorer change). Per sample and
epoch it compares the score value, i.e. every turn's label, and the text of every prompt
sent to the judge. It prints each difference, then the label agreement, and exits
non-zero if anything differs. A change that shouldn't touch grading (concurrency,
caching) must pass; for one that might (an output cap, a different judge) the agreement
is the number to report.
"""
from __future__ import annotations
import argparse
import json
import sys
from collections import defaultdict
from datetime import datetime
from inspect_ai.log import read_eval_log
from report import coverage
def model_calls(sample, role=None, failed=False):
"""The sample's model events for one model role (None: the model under test).
A call that times out and is retried leaves an extra event carrying the error; those
are left out unless ``failed``, so the n-th event is the n-th turn.
"""
return [
e for e in sample.events
if e.event == "model" and e.role == role and (failed or e.error is None)
]
def call_row(event):
usage = event.output.usage
wall = (event.completed - event.timestamp).total_seconds() if event.completed else None
gen = event.output.time
return {
"in": usage.input_tokens if usage else 0,
"out": usage.output_tokens if usage else 0,
"reasoning": (usage.reasoning_tokens or 0) if usage else 0,
"wall": wall,
"generate": gen,
"wait": None if wall is None or gen is None else max(wall - gen, 0.0),
"stop": event.output.stop_reason,
"retries": event.retries or 0,
# A call that ended in an error (e.g. "Request timed out.") with no output.
"failed": event.error is not None,
# The reply has no answer text, e.g. all max_tokens went to reasoning.
"empty": event.error is None and not event.output.completion.strip(),
}
def summarize(rows):
def pct(values, q):
values = sorted(v for v in values if v is not None)
return values[min(len(values) - 1, int(q * len(values)))] if values else None
gen = [r["generate"] for r in rows if r["generate"]]
out = sum(r["out"] for r in rows)
return {
"calls": len(rows),
"failed_calls": sum(r["failed"] for r in rows),
"retries": sum(r["retries"] for r in rows),
"empty_answers": sum(r["empty"] for r in rows),
"tokens_in": sum(r["in"] for r in rows),
"tokens_out": out,
"reasoning_tokens": sum(r["reasoning"] for r in rows),
"hit_max_tokens": sum(r["stop"] == "max_tokens" for r in rows),
"generate_s": round(sum(gen), 1),
"wait_s": round(sum(r["wait"] or 0 for r in rows), 1),
"latency_p50_s": pct([r["wall"] for r in rows], 0.5),
"latency_p95_s": pct([r["wall"] for r in rows], 0.95),
# Per-request decode speed, comparable across concurrency settings only as a
# measure of how much each request slows when the server shares its GPU.
"out_tok_per_s_per_call": round(out / sum(gen), 1) if gen else None,
}
def report(path):
log = read_eval_log(path, resolve_attachments=True)
samples = log.samples or []
warnings = []
if not samples:
warnings.append("log has no samples (run with --no-log-samples, or stopped before any finished): nothing to time")
if log.status != "success":
warnings.append(f"run status is {log.status!r}: numbers cover only what the log recorded")
start, end = log.stats.started_at, log.stats.completed_at
wall = (datetime.fromisoformat(end) - datetime.fromisoformat(start)).total_seconds() if start and end else None
groups, judge = defaultdict(list), []
for sample in samples:
kind = "multi-turn" if sample.metadata.get("multiturn") else "single-turn"
turn = 0
for event in model_calls(sample, failed=True):
groups[(kind, "opening" if turn == 0 else "pressure")].append(call_row(event))
turn += event.error is None
judge += [call_row(e) for e in model_calls(sample, "grader", failed=True)]
solver_rows = [r for rows in groups.values() for r in rows]
out_tokens = sum(r["out"] for r in solver_rows)
# Inspect's own totals hold even without stored samples, so these stay exact.
cover = coverage(log)
# Everything below needs the samples; without them it is null, not a misleading zero.
timed = bool(samples)
return {
"log": path,
"model": log.eval.model,
"task_args": log.eval.task_args,
"config": {k: v for k, v in log.eval.config.model_dump().items() if v is not None},
"warnings": warnings,
"samples": cover["planned"],
"errors": cover["errored"],
"wall_clock_s": round(wall, 1) if wall is not None else None,
# Output tokens of the model under test over the whole run: the throughput a
# concurrency setting actually achieves.
"run_out_tok_per_s": round(out_tokens / wall, 1) if timed and wall else None,
"model_under_test": summarize(solver_rows) if timed else None,
"by_group": {f"{k} / {t}": summarize(r) for (k, t), r in sorted(groups.items())} if timed else None,
"judge": summarize(judge) if judge else None,
"slowest_samples": [
{"id": s.id, "epoch": s.epoch, "seconds": round(s.total_time, 1)}
for s in sorted(samples, key=lambda s: s.total_time or 0, reverse=True)[:5]
if s.total_time is not None
] if timed else None,
}
def request_key(request, replies):
"""What a call sent, with the model's own earlier replies masked; plus any problems.
``request`` is the request body Inspect logged for the call (``ModelEvent.call``) and
``replies`` the response messages of the earlier turns of the same sample. The body is
kept byte for byte except ``extra_headers``, which holds a random per-request id. Each
assistant message must be the reply the model gave on that turn (Inspect resends the
text with one leading newline added); it is then masked, since sampled replies differ
between runs.
"""
problems = []
body = {k: v for k, v in request.items() if k != "extra_headers"}
sent = [m for m in body["messages"] if m["role"] == "assistant"]
if len(sent) != len(replies):
problems.append(f"{len(sent)} replies in context, expected {len(replies)}")
for message, reply in zip(sent, replies):
expected = ("\n" + (reply.get("content") or ""), reply.get("reasoning"))
if (message.get("content"), message.get("reasoning")) != expected:
problems.append("a reply in context is not the model's own")
body["messages"] = [{"role": "assistant"} if m["role"] == "assistant" else m for m in body["messages"]]
return json.dumps(body, sort_keys=True), problems
def chat_call(event):
"""The request body and reply message of a logged call, if it is an OpenAI-style chat call.
The prompt check reads Inspect's raw provider log (``ModelEvent.call``), which only has
``messages`` and ``choices`` for OpenAI-compatible chat endpoints such as Ollama's.
"""
call = event.call
try:
call.request["messages"]
return call.request, call.response["choices"][0]["message"]
except (AttributeError, KeyError, IndexError, TypeError):
raise ValueError(
"--same-prompts needs the raw chat request and response in the log, as an "
"OpenAI-compatible provider (e.g. ollama/...) records them; this log has a call without them"
) from None
def same_prompts(before_path, after_path):
"""Differences between two runs in what the model under test was sent (see module doc)."""
problems = []
def calls(path):
out = {}
for sample in read_eval_log(path, resolve_attachments=True).samples:
events = model_calls(sample)
chats = [chat_call(e) for e in events]
for turn, (request, _) in enumerate(chats):
key, found = request_key(request, [reply for _, reply in chats[:turn]])
problems.extend(f"{path}: {sample.id} epoch {sample.epoch} turn {turn}: {p}" for p in found)
out[(str(sample.id), sample.epoch, turn)] = key
return out
before, after = calls(before_path), calls(after_path)
for key in sorted(set(before) | set(after)):
if key not in before or key not in after:
problems.append(f"{key}: call only in {'after' if key in after else 'before'}")
elif before[key] != after[key]:
problems.append(f"{key}: request body differs")
return len(set(before) & set(after)), problems
def judged(path):
"""Per (sample id, epoch): the score value and the judge prompts, from a scored log."""
out = {}
for sample in read_eval_log(path, resolve_attachments=True).samples:
value = str(next(iter(sample.scores.values())).value) if sample.scores else None
prompts = [m.text for e in model_calls(sample, "grader") for m in e.input]
out[(str(sample.id), sample.epoch)] = (value, prompts)
return out
def same_labels(before_path, after_path):
"""Label agreement and every difference between two scorings of the same transcripts."""
return compare_judged(judged(before_path), judged(after_path))
def compare_judged(before, after):
"""``same_labels`` on two ``judged`` mappings: (turns, turns that agree, problems)."""
problems, turns, agree = [], 0, 0
for key in sorted(set(before) | set(after)):
if key not in before or key not in after:
problems.append(f"{key}: only in {'after' if key in after else 'before'}")
continue
(v1, p1), (v2, p2) = before[key], after[key]
if p1 != p2:
problems.append(f"{key}: judge prompts differ")
l1, l2 = (v1 or "").split(">"), (v2 or "").split(">")
turns += max(len(l1), len(l2))
agree += sum(a == b for a, b in zip(l1, l2))
if v1 != v2:
problems.append(f"{key}: labels {v1} -> {v2}")
return turns, agree, problems
def main():
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("logs", nargs="+")
parser.add_argument("--same-prompts", action="store_true", help="compare exactly two logs")
parser.add_argument("--same-labels", action="store_true", help="compare two scored logs")
args = parser.parse_args()
if args.same_labels:
if len(args.logs) != 2:
parser.error("--same-labels takes exactly two logs")
turns, agree, problems = same_labels(*args.logs)
print("\n".join(problems))
print(f"{'OK' if not problems else 'DIFFERENT'} - {agree}/{turns} turn labels agree")
sys.exit(1 if problems else 0)
if args.same_prompts:
if len(args.logs) != 2:
parser.error("--same-prompts takes exactly two logs")
try:
compared, problems = same_prompts(*args.logs)
except ValueError as e:
parser.exit(2, f"{e}\n")
print("\n".join(problems) or f"OK - {compared} model calls send the same prompts and config")
sys.exit(1 if problems else 0)
reports = [report(p) for p in args.logs]
for r in reports:
for w in r["warnings"]:
print(f"warning: {r['log']}: {w}", file=sys.stderr)
print(json.dumps(reports, indent=2, default=str))
if __name__ == "__main__":
main()