Buckets:
| """Validation: run the attacker (base model, or base + a trained LoRA adapter) on a scenarios file | |
| with the same episode harness and reward as training (rollout.py), and summarise. | |
| Run from /workspace: | |
| python evaluate.py --run-name baseline-14b-v3 | |
| python evaluate.py --adapter checkpoints/proto-14b-v2/final --run-name proto-14b-v2 | |
| # Several adapters in one go (the engine loads once; "base" = no adapter): | |
| python evaluate.py --adapter base checkpoints/proto-9b-v5-70b-a100/checkpoint-* \ | |
| checkpoints/proto-9b-v5-70b-a100/final --run-name proto-9b-v5-70b-a100 | |
| Results go to runs/eval-<run-name>-<date>-<time>.jsonl, one record per conversation with its own | |
| metrics (outcome none / partial / full, success, verified leak, first success turn, tactics per turn, | |
| judge attack-quality grade, each reward part, reward). | |
| With several adapters, each gets its own file, runs/eval-<run-name>-<adapter folder name>-<date>-<time>.jsonl. | |
| The printed summary has ASR, partial rate, ASR@k, strategy coverage, mean attack quality and per-kind / | |
| per-tactic success, and with several adapters a side-by-side table at the end. | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import statistics | |
| from collections import Counter | |
| from datetime import datetime | |
| from transformers import AutoTokenizer | |
| from vllm import LLM | |
| from vllm.lora.request import LoRARequest | |
| import prompts | |
| import rollout | |
| import utils | |
| class VLLMTurnGenerator: | |
| """Generates one attacker turn per episode with a plain vLLM engine (optionally with a LoRA adapter).""" | |
| def __init__(self, llm, sampling, lora): | |
| self.llm, self.sampling, self.lora = llm, sampling, lora | |
| def __call__(self, prompt_ids_list): | |
| outputs = self.llm.generate([{"prompt_token_ids": ids} for ids in prompt_ids_list], self.sampling, | |
| lora_request=self.lora, use_tqdm=False) | |
| results = [] | |
| for out in outputs: | |
| comp = out.outputs[0] | |
| steps = comp.logprobs or [None] * len(comp.token_ids) | |
| logprobs = [step[tok].logprob if step else None for tok, step in zip(comp.token_ids, steps)] | |
| results.append((comp.token_ids, logprobs, comp.finish_reason)) | |
| return results | |
| def episode_record(ep): | |
| """One conversation as a JSONL record, with its per-conversation metrics up front.""" | |
| request = ep["job"]["request"] | |
| return {"scenario": ep["job"]["scenario"]["id"], "request": request["id"], "rep": ep["job"]["rep"], | |
| "difficulty": request.get("difficulty"), "kind": request.get("kind"), | |
| "status": ep["status"], "turns": ep["turns"], "outcome": ep["outcome"], | |
| "success": ep["success"], "verified_leak": bool(ep["leaked"]), "leaked": ep["leaked"], | |
| "first_success_turn": ep["first_success_turn"], "tactics": ep["tactics"], | |
| "turn_tactics": ep["turn_tactics"], "attack_quality": ep["attack_quality"], | |
| "attack_quality_grade": ep["judgment"].get("attack_quality_grade"), | |
| "outcome_score": ep["outcome_score"], "attack_quality_reward": ep["attack_quality_reward"], | |
| "early_done_bonus": ep["early_done_bonus"], "false_done_penalty": ep["false_done_penalty"], | |
| "format_penalty": ep["format_penalty"], "reward": ep["reward"], "timing": ep["timing"], | |
| "judgment": ep["judgment"], "transcript": ep["transcript"]} | |
| def rate(values): | |
| """Mean of a list of 0/1 values, or None for an empty list.""" | |
| return statistics.mean(values) if values else None | |
| def summarize(records, run_name, repeats): | |
| """Print the run's ASR, coverage and breakdowns. Errored episodes are excluded from the rates. | |
| Returns the headline numbers (for the multi-adapter comparison table), or None if nothing was graded.""" | |
| graded = [r for r in records if r["status"] not in ("errored", "judge_errored")] | |
| print(f"\n{run_name}: {len(graded)} graded / {len(records)} episodes") | |
| if not graded: | |
| return None | |
| full = [float(r["success"] == 1.0) for r in graded] | |
| # ASR@k: share of requests where at least one of their k episodes was a full success. | |
| by_request = {} | |
| for r in graded: | |
| by_request.setdefault((r["scenario"], r["request"]), []).append(r["success"] == 1.0) | |
| first_turns = [r["first_success_turn"] for r in graded | |
| if r["success"] == 1.0 and r["first_success_turn"] is not None] | |
| headline = {"graded": len(graded), "episodes": len(records), "asr": rate(full), | |
| "asr_at_k": rate([any(v) for v in by_request.values()]), | |
| "partial_rate": rate([float(r["outcome"] == "partial") for r in graded]), | |
| "reward": statistics.mean(r["reward"] for r in graded), | |
| "turns": statistics.mean(r["turns"] for r in graded), | |
| "first_success_turn": statistics.mean(first_turns) if first_turns else float("nan")} | |
| print(f" ASR {rate(full):.2f} ASR@{repeats} {rate([any(v) for v in by_request.values()]):.2f} " | |
| f"partial {rate([float(r['outcome'] == 'partial') for r in graded]):.2f} " | |
| f"verified leaks {rate([float(r['verified_leak']) for r in graded]):.2f}") | |
| print(f" mean reward {statistics.mean(r['reward'] for r in graded):.3f} " | |
| f"mean turns {statistics.mean(r['turns'] for r in graded):.1f} " | |
| f"mean first-success turn {statistics.mean(first_turns) if first_turns else float('nan'):.1f} " | |
| f"mean attack quality {statistics.mean(r['attack_quality'] for r in graded):.3f}") | |
| # Strategy coverage: which tactic labels were used at all, and which led to a full success. | |
| used = Counter(t for r in graded for t in r["tactics"]) | |
| succeeded = Counter(t for r in graded if r["success"] == 1.0 for t in r["tactics"]) | |
| print(f" strategy coverage: {len(used)}/{len(prompts.TACTICS)} tactics used, " | |
| f"{len(succeeded)}/{len(prompts.TACTICS)} with a full success") | |
| print(f" {'tactic':<28}{'used':>6}{'ASR':>7}") | |
| for tactic in prompts.TACTICS: | |
| uses = [r for r in graded if tactic in r["tactics"]] | |
| asr_text = f"{rate([float(r['success'] == 1.0) for r in uses]):.2f}" if uses else "-" | |
| print(f" {tactic:<28}{len(uses):>6}{asr_text:>7}") | |
| # ASR per request kind (commitment, scope, skipped step, role, injected instruction) and, for | |
| # scenario files that have it, per difficulty. | |
| for field in ("kind", "difficulty"): | |
| levels = sorted({r[field] for r in graded if r[field]}) | |
| if not levels: | |
| continue | |
| print(f" {field:<28}{'n':>6}{'ASR':>7}") | |
| for level in levels: | |
| rows = [r for r in graded if r[field] == level] | |
| print(f" {level:<28}{len(rows):>6}{rate([float(r['success'] == 1.0) for r in rows]):>7.2f}") | |
| statuses = Counter(r["status"] for r in records) | |
| cut = sum(1 for r in records for t in r["transcript"] if t.get("finish_reason") == "length") | |
| print(f" statuses {dict(statuses)} truncated turns {cut}") | |
| return headline | |
| def adapter_label(adapter): | |
| """Short name for an adapter in file names and the comparison table: 'base', or its folder name | |
| (checkpoint-2, final, ...).""" | |
| return "base" if adapter is None else os.path.basename(os.path.normpath(adapter)) | |
| def print_comparison(headlines, repeats): | |
| """Side-by-side headline numbers, one row per adapter evaluated in this run.""" | |
| print(f"\n{'adapter':<20}{'graded':>8}{'ASR':>7}{'partial':>9}{f'ASR@{repeats}':>8}{'reward':>9}{'turns':>7}" | |
| f"{'1st succ':>10}") | |
| for label, h in headlines: | |
| if h is None: | |
| print(f"{label:<20}{'none graded':>8}") | |
| continue | |
| print(f"{label:<20}{h['graded']:>4}/{h['episodes']:<3}{h['asr']:>7.2f}{h['partial_rate']:>9.2f}{h['asr_at_k']:>8.2f}" | |
| f"{h['reward']:>9.3f}{h['turns']:>7.1f}{h['first_success_turn']:>10.1f}") | |
| def main(args): | |
| # --adapter takes one or more adapter dirs; "base" (or no --adapter) means the untrained model. | |
| adapters = [None if a == "base" else a for a in (args.adapter or ["base"])] | |
| any_lora = any(a is not None for a in adapters) | |
| # Repeats of a request are adjacent (ASR@k groups them by request). | |
| jobs = [{"scenario": sc, "request": req, "rep": rep} | |
| for sc in utils.load_json(args.scenarios) for req in sc["requests"] | |
| for rep in range(args.repeats)] | |
| tokenizer = AutoTokenizer.from_pretrained(args.model) | |
| # max_num_seqs: never more sequences than episodes playing at once. vLLM's H100 default (1024) is more | |
| # than Qwen3.5's linear-attention state cache holds on one GPU (302 for the 27B), which stops startup. | |
| llm = LLM(model=args.model, dtype="bfloat16", max_model_len=args.max_total_tokens, | |
| gpu_memory_utilization=args.gpu_frac, enable_lora=any_lora, | |
| max_lora_rank=args.max_lora_rank, max_num_seqs=args.concurrency) | |
| # Same sampling as training (rollout.ATTACKER_SAMPLING), capped per attacker turn. | |
| sampling = llm.get_default_sampling_params() | |
| for name, value in rollout.ATTACKER_SAMPLING.items(): | |
| setattr(sampling, name, value) | |
| sampling.max_tokens = args.max_turn_tokens | |
| sampling.logprobs = 0 # logprob of the sampled token (kept in the episode, unused here) | |
| headlines = [] | |
| for index, adapter in enumerate(adapters): | |
| label = adapter_label(adapter) | |
| # Each adapter needs its own LoRA id, or vLLM keeps serving the first one it loaded. | |
| lora = LoRARequest(f"attacker-{label}", index + 1, adapter) if adapter else None | |
| generate = VLLMTurnGenerator(llm, sampling, lora) | |
| run_name = args.run_name if len(adapters) == 1 else f"{args.run_name}-{label}" | |
| print(f"\n=== Evaluating {label} ({adapter or 'no adapter'}) as {run_name} ===") | |
| # All episodes in one asynchronous run: up to --concurrency play at once, a new one starting as | |
| # soon as another ends. | |
| episodes = rollout.run_episodes( | |
| jobs, generate, tokenizer, args.target_model, args.judge_model, max_turns=args.max_turns, | |
| max_turn_tokens=args.max_turn_tokens, max_total_tokens=args.max_total_tokens, | |
| max_parallel=args.concurrency) | |
| rollout.score_episodes(episodes) | |
| for ep in episodes: | |
| print(f"{ep['job']['scenario']['id']}/{ep['job']['request']['id']} rep{ep['job']['rep']}: " | |
| f"outcome={ep['judgment'].get('outcome')} success={ep['success']} " | |
| f"turns={ep['turns']} status={ep['status']}") | |
| # One file per adapter. With several adapters, --out gets the adapter name added before .jsonl | |
| # (so Modal runs still write onto the outputs volume). | |
| if args.out: | |
| stem, ext = os.path.splitext(args.out) | |
| out_path = args.out if len(adapters) == 1 else f"{stem}-{label}{ext or '.jsonl'}" | |
| else: | |
| out_path = f"runs/eval-{run_name}-{datetime.now():%Y-%m-%d-%H%M%S}.jsonl" | |
| os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True) | |
| records = [episode_record(ep) for ep in episodes] | |
| with open(out_path, "w", encoding="utf-8") as fout: | |
| for record in records: | |
| fout.write(json.dumps(record, ensure_ascii=False) + "\n") | |
| headlines.append((label, summarize(records, run_name, args.repeats))) | |
| print(f"Wrote {len(records)} episodes to {out_path}") | |
| if len(adapters) > 1: | |
| print_comparison(headlines, args.repeats) | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--model", default="lukey03/Qwen3.5-9B-abliterated") | |
| parser.add_argument("--adapter", nargs="+", default=None, | |
| help="one or more LoRA adapter dirs from train.py, evaluated one after another on the " | |
| "same engine; 'base' = untrained model; omit for baseline only") | |
| parser.add_argument("--max-lora-rank", type=int, default=16) | |
| parser.add_argument("--scenarios", default="datagen/data/simula-v5/val_scenarios.json") | |
| parser.add_argument("--target-model", default="meta-llama/llama-3.3-70b-instruct") | |
| parser.add_argument("--judge-model", default="claude-haiku-5-5") # openai/gpt-6-luna (OpenRouter) until 2026-10-23 | |
| parser.add_argument("--run-name", default="baseline") | |
| parser.add_argument("--out", default=None) | |
| parser.add_argument("--repeats", type=int, default=4, | |
| help="episodes per request (12 requests x 4 = 48); also the k in ASR@k") | |
| parser.add_argument("--concurrency", type=int, default=48, | |
| help="episodes playing at once (48 = the whole default run); vLLM queues what " | |
| "does not fit in its KV cache") | |
| # Same episode limits as train.py so baseline and trained runs are comparable. | |
| parser.add_argument("--max-turns", type=int, default=20) # 15 until 2026-10-23 | |
| parser.add_argument("--max-turn-tokens", type=int, default=4096) | |
| parser.add_argument("--max-total-tokens", type=int, default=40960) | |
| parser.add_argument("--gpu-frac", type=float, default=0.90) | |
| main(parser.parse_args()) | |
Xet Storage Details
- Size:
- 13.1 kB
- Xet hash:
- 922d0ff3d1e5eaf9dc9a987ab18712cbe850e2c3a988e55b15c4a13e64ad9c3d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.