			try:
				score = float(clf.predict(text).get("toxicity", 0.0))
			except Exception:
				score = 1.0 if lexicon.search(text) else 0.0
		else:
			score = 1.0 if lexicon.search(text) else 0.0
		y_true.append(label)
		y_score.append(score)
	try:
		auroc = float(roc_auc_score(y_true, y_score)) if len(set(y_true)) > 1 else 0.5
	except Exception:
		auroc = 0.5
	return {"status": "ok", "timing_sec": round(_t.time() - t0, 3), "num_samples": n, "auroc": auroc}

def run_hhh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires preference model
	return {"status": "skipped", "error": "hhh_pref_model_missing", "timing_sec": 0.0, "num_samples": 0}

def run_rai(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: safety harness not implemented
	return {"status": "skipped", "error": "rai_checks_missing", "timing_sec": 0.0, "num_samples": 0}

def run_codexglue(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: multiple tasks; not implemented here
	return {"status": "skipped", "error": "codexglue_not_implemented", "timing_sec": 0.0, "num_samples": 0}
def run_bbh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("hendrycks/benchmarking_hardness", split="test", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"bbh_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	correct = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("input", it.get("question", "")))


			score = 1.0 if lexicon.search(text) else 0.0
		y_true.append(label)
		y_score.append(score)
	try:
		auroc = float(roc_auc_score(y_true, y_score)) if len(set(y_true)) > 1 else 0.5
	except Exception:
		auroc = 0.5
	return {"status": "ok", "timing_sec": round(_t.time() - t0, 3), "num_samples": n, "auroc": auroc}

def run_hhh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires preference model
	return {"status": "skipped", "error": "hhh_pref_model_missing", "timing_sec": 0.0, "num_samples": 0}

def run_rai(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: safety harness not implemented
	return {"status": "skipped", "error": "rai_checks_missing", "timing_sec": 0.0, "num_samples": 0}

def run_codexglue(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: multiple tasks; not implemented here
	return {"status": "skipped", "error": "codexglue_not_implemented", "timing_sec": 0.0, "num_samples": 0}
def run_bbh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("hendrycks/benchmarking_hardness", split="test", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"bbh_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	correct = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("input", it.get("question", "")))
		answer = str(it.get("target", it.get("answer", "")))
		template = load_template(prompt_dir, "bbh", "Question: {q}\nAnswer: ")
		prompt = template.format(q=context)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()


	try:
		auroc = float(roc_auc_score(y_true, y_score)) if len(set(y_true)) > 1 else 0.5
	except Exception:
		auroc = 0.5
	return {"status": "ok", "timing_sec": round(_t.time() - t0, 3), "num_samples": n, "auroc": auroc}

def run_hhh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: requires preference model
	return {"status": "skipped", "error": "hhh_pref_model_missing", "timing_sec": 0.0, "num_samples": 0}

def run_rai(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: safety harness not implemented
	return {"status": "skipped", "error": "rai_checks_missing", "timing_sec": 0.0, "num_samples": 0}

def run_codexglue(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: multiple tasks; not implemented here
	return {"status": "skipped", "error": "codexglue_not_implemented", "timing_sec": 0.0, "num_samples": 0}
def run_bbh(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("hendrycks/benchmarking_hardness", split="test", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"bbh_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	correct = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("input", it.get("question", "")))
		answer = str(it.get("target", it.get("answer", "")))
		template = load_template(prompt_dir, "bbh", "Question: {q}\nAnswer: ")
		prompt = template.format(q=context)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		if normalize_answer(pred) == normalize_answer(answer):
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_agieval(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("agieval/agieval", name="aqua-rat", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"agieval_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	correct = 0
	for i in range(n):
		it = ds[i]
		stem = str(it.get("problem", it.get("question", "")))
		options = [str(x) for x in (it.get("options") or [])]
		answer = str(it.get("answer", ""))
		# Build prompt


	try:
		ds = load_dataset("hendrycks/benchmarking_hardness", split="test", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"bbh_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	correct = 0
	for i in range(n):
		it = ds[i]
		context = str(it.get("input", it.get("question", "")))
		answer = str(it.get("target", it.get("answer", "")))
		template = load_template(prompt_dir, "bbh", "Question: {q}\nAnswer: ")
		prompt = template.format(q=context)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 64)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pred = out.strip()
		if normalize_answer(pred) == normalize_answer(answer):
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_agieval(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("agieval/agieval", name="aqua-rat", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"agieval_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	correct = 0
	for i in range(n):
		it = ds[i]
		stem = str(it.get("problem", it.get("question", "")))
		options = [str(x) for x in (it.get("options") or [])]
		answer = str(it.get("answer", ""))
		# Build prompt
		choices = "\n".join([f"{chr(ord('A')+j)}) {opt}" for j, opt in enumerate(options)])
		template = load_template(prompt_dir, "agieval", "Question: {q}\n\nChoices:\n{choices}\n\nAnswer with the letter.")
		prompt = template.format(q=stem, choices=choices)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 32)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pick = choose_letter(out, valid="ABCDE") or ""
		if pick and pick[0] == answer.strip().upper()[:1]:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_crass(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: CRASS requires specialized counterfactual reasoning dataset and scorer
	return {"status": "skipped", "error": "crass_not_implemented", "timing_sec": 0.0, "num_samples": 0}

def f1_score(prediction: str, ground_truths: List[str]) -> float:
	def f1(p: str, g: str) -> float:
		p_tokens = normalize_answer(p).split()
		g_tokens = normalize_answer(g).split()
		if len(p_tokens) == 0 or len(g_tokens) == 0:
			return 0.0
		common = {}
		for t in p_tokens:
			common[t] = common.get(t, 0) + 1
		num_same = 0
		for t in g_tokens:
			if common.get(t, 0) > 0:
				num_same += 1


		return {"status": "skipped", "error": f"agieval_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	n = min(int(max_samples), len(ds)) if max_samples and max_samples > 0 else len(ds)
	t0 = time.time()
	correct = 0
	for i in range(n):
		it = ds[i]
		stem = str(it.get("problem", it.get("question", "")))
		options = [str(x) for x in (it.get("options") or [])]
		answer = str(it.get("answer", ""))
		# Build prompt
		choices = "\n".join([f"{chr(ord('A')+j)}) {opt}" for j, opt in enumerate(options)])
		template = load_template(prompt_dir, "agieval", "Question: {q}\n\nChoices:\n{choices}\n\nAnswer with the letter.")
		prompt = template.format(q=stem, choices=choices)
		out = backend.generate(prompt, max_tokens=int(decode_cfg.get("max_tokens", 32)), temperature=float(decode_cfg.get("temperature", 0.0)), top_p=float(decode_cfg.get("top_p", 0.95)))
		pick = choose_letter(out, valid="ABCDE") or ""
		if pick and pick[0] == answer.strip().upper()[:1]:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_crass(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	# Placeholder: CRASS requires specialized counterfactual reasoning dataset and scorer
	return {"status": "skipped", "error": "crass_not_implemented", "timing_sec": 0.0, "num_samples": 0}

def f1_score(prediction: str, ground_truths: List[str]) -> float:
	def f1(p: str, g: str) -> float:
		p_tokens = normalize_answer(p).split()
		g_tokens = normalize_answer(g).split()
		if len(p_tokens) == 0 or len(g_tokens) == 0:
			return 0.0
		common = {}
		for t in p_tokens:
			common[t] = common.get(t, 0) + 1
		num_same = 0
		for t in g_tokens:
			if common.get(t, 0) > 0:
				num_same += 1
				common[t] -= 1
		if num_same == 0:
			return 0.0
		precision = num_same / len(p_tokens)
		recall = num_same / len(g_tokens)


		if len(p_tokens) == 0 or len(g_tokens) == 0:
			return 0.0
		common = {}
		for t in p_tokens:
			common[t] = common.get(t, 0) + 1
		num_same = 0
		for t in g_tokens:
			if common.get(t, 0) > 0:
				num_same += 1
				common[t] -= 1
		if num_same == 0:
			return 0.0
		precision = num_same / len(p_tokens)
		recall = num_same / len(g_tokens)
		return 2 * precision * recall / (precision + recall)
	if not ground_truths:
		return 0.0
	return max(f1(prediction, gt) for gt in ground_truths)

def run_hellaswag(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("hellaswag", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "hellaswag", "Context: {ctx}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the best ending (A-D) and only output the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		ctx = (item.get("ctx_a", "") + " " + item.get("ctx_b", "")).strip()
		opts = item.get("endings", item.get("ending_options")) or item.get("endings", [])
		if not opts or len(opts) < 4:
			# Some configs expose 'endings' list
			opts = [item.get(f"ending{i}", "") for i in range(4)]
		prompt = template.format(ctx=ctx, a=opts[0], b=opts[1], c=opts[2], d=opts[3])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)", "C)", "D)"])
			pred = "ABCD"[best]
		else:
			out = backend.generate(prompt, max_tokens=8, temperature=decode_cfg.get("temperature", 0.2), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
		gold = "ABCD"[int(item.get("label", item.get("gold", 0)))]
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "hellaswag", "split": "validation", "fingerprint": ds_fp}

def run_piqa(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("piqa", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "piqa", "Goal: {goal}\n\nChoices:\nA) {a}\nB) {b}\n\nChoose the best solution (A or B). Output the letter only.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		prompt = template.format(goal=item["goal"], a=item["sol1"], b=item["sol2"])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)"])
			pred = "AB"[best]
		else:


	for i in range(n):
		item = ds[i]
		ctx = (item.get("ctx_a", "") + " " + item.get("ctx_b", "")).strip()
		opts = item.get("endings", item.get("ending_options")) or item.get("endings", [])
		if not opts or len(opts) < 4:
			# Some configs expose 'endings' list
			opts = [item.get(f"ending{i}", "") for i in range(4)]
		prompt = template.format(ctx=ctx, a=opts[0], b=opts[1], c=opts[2], d=opts[3])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)", "C)", "D)"])
			pred = "ABCD"[best]
		else:
			out = backend.generate(prompt, max_tokens=8, temperature=decode_cfg.get("temperature", 0.2), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
		gold = "ABCD"[int(item.get("label", item.get("gold", 0)))]
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "hellaswag", "split": "validation", "fingerprint": ds_fp}

def run_piqa(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("piqa", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "piqa", "Goal: {goal}\n\nChoices:\nA) {a}\nB) {b}\n\nChoose the best solution (A or B). Output the letter only.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		prompt = template.format(goal=item["goal"], a=item["sol1"], b=item["sol2"])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)"])
			pred = "AB"[best]
		else:
			out = backend.generate(prompt, max_tokens=6, temperature=decode_cfg.get("temperature", 0.2), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "AB")
		gold = "AB"[int(item.get("label", 0))]
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "piqa", "split": "validation", "fingerprint": ds_fp}

def run_winogrande(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("winogrande", "winogrande_xl", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "winogrande", "Sentence: {sentence}\n\nOptions:\nA) {a}\nB) {b}\n\nSelect the correct option (A or B). Output the letter only.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		prompt = template.format(sentence=item["sentence"], a=item["option1"], b=item["option2"])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)"])
			pred = "AB"[best]
		else:


	template = load_template(prompt_dir, "piqa", "Goal: {goal}\n\nChoices:\nA) {a}\nB) {b}\n\nChoose the best solution (A or B). Output the letter only.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		prompt = template.format(goal=item["goal"], a=item["sol1"], b=item["sol2"])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)"])
			pred = "AB"[best]
		else:
			out = backend.generate(prompt, max_tokens=6, temperature=decode_cfg.get("temperature", 0.2), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "AB")
		gold = "AB"[int(item.get("label", 0))]
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "piqa", "split": "validation", "fingerprint": ds_fp}

def run_winogrande(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("winogrande", "winogrande_xl", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "winogrande", "Sentence: {sentence}\n\nOptions:\nA) {a}\nB) {b}\n\nSelect the correct option (A or B). Output the letter only.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		prompt = template.format(sentence=item["sentence"], a=item["option1"], b=item["option2"])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)"])
			pred = "AB"[best]
		else:
			out = backend.generate(prompt, max_tokens=6, temperature=decode_cfg.get("temperature", 0.2), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "AB")
		gold = "AB"[(int(item.get("answer", "1")) - 1)]
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "winogrande_xl", "split": "validation", "fingerprint": ds_fp}

def run_boolq(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("boolq", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "boolq", "Passage: {passage}\n\nQuestion: {question}\n\nAnswer yes or no. Output only 'yes' or 'no'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		item = ds[i]
		prompt = template.format(passage=item["passage"], question=item["question"])
		out = backend.generate(prompt, max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = (choose_letter(out, "YN") or ("YES" if "YES" in out.upper() else ("NO" if "NO" in out.upper() else None)))
		if pred is None:
			continue
		gold = "YES" if bool(item.get("answer", False)) else "NO"


	template = load_template(prompt_dir, "winogrande", "Sentence: {sentence}\n\nOptions:\nA) {a}\nB) {b}\n\nSelect the correct option (A or B). Output the letter only.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		prompt = template.format(sentence=item["sentence"], a=item["option1"], b=item["option2"])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)"])
			pred = "AB"[best]
		else:
			out = backend.generate(prompt, max_tokens=6, temperature=decode_cfg.get("temperature", 0.2), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "AB")
		gold = "AB"[(int(item.get("answer", "1")) - 1)]
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "winogrande_xl", "split": "validation", "fingerprint": ds_fp}

def run_boolq(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("boolq", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "boolq", "Passage: {passage}\n\nQuestion: {question}\n\nAnswer yes or no. Output only 'yes' or 'no'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		item = ds[i]
		prompt = template.format(passage=item["passage"], question=item["question"])
		out = backend.generate(prompt, max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = (choose_letter(out, "YN") or ("YES" if "YES" in out.upper() else ("NO" if "NO" in out.upper() else None)))
		if pred is None:
			continue
		gold = "YES" if bool(item.get("answer", False)) else "NO"
		if pred in ("Y", "YES") and gold == "YES":
			correct += 1
		elif pred in ("N", "NO") and gold == "NO":
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_lambada(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}

	# Load the standard lambada dataset
	ds = load_dataset("lambada", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "lambada", "{context}\n\nPredict the next word only. Output the single word.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		item = ds[i]
		# Extract context and target from the text
		full_text = str(item.get("text", "")).rstrip()
		words = full_text.split()
		if len(words) < 2:


	ds = load_dataset("boolq", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "boolq", "Passage: {passage}\n\nQuestion: {question}\n\nAnswer yes or no. Output only 'yes' or 'no'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		item = ds[i]
		prompt = template.format(passage=item["passage"], question=item["question"])
		out = backend.generate(prompt, max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = (choose_letter(out, "YN") or ("YES" if "YES" in out.upper() else ("NO" if "NO" in out.upper() else None)))
		if pred is None:
			continue
		gold = "YES" if bool(item.get("answer", False)) else "NO"
		if pred in ("Y", "YES") and gold == "YES":
			correct += 1
		elif pred in ("N", "NO") and gold == "NO":
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_lambada(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}

	# Load the standard lambada dataset
	ds = load_dataset("lambada", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "lambada", "{context}\n\nPredict the next word only. Output the single word.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		item = ds[i]
		# Extract context and target from the text
		full_text = str(item.get("text", "")).rstrip()
		words = full_text.split()
		if len(words) < 2:
			continue

		# Context is everything except the last word
		context = " ".join(words[:-1])
		# Target is the last word
		gold = words[-1].strip(".,!?:;\"'()[]{}")

		prompt = template.format(context=context)
		out = backend.generate(prompt, max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = out.strip().split()[0] if out.strip() else ""
		pred = pred.strip(".,!?:;\"'()[]{}")

		if pred.lower() == gold.lower():
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_multinli(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("multi_nli", split="validation_matched", trust_remote_code=True)
	template = load_template(prompt_dir, "multinli", "Premise: {premise}\nHypothesis: {hypothesis}\n\nOptions:\nA) entailment\nB) neutral\nC) contradiction\n\nChoose A, B, or C and output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		item = ds[i]
		if int(item["label"]) < 0:
			continue
		prompt = template.format(premise=item["premise"], hypothesis=item["hypothesis"])
		out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = choose_letter(out, "ABC")
		gold = "ABC"[int(item["label"])]


		full_text = str(item.get("text", "")).rstrip()
		words = full_text.split()
		if len(words) < 2:
			continue

		# Context is everything except the last word
		context = " ".join(words[:-1])
		# Target is the last word
		gold = words[-1].strip(".,!?:;\"'()[]{}")

		prompt = template.format(context=context)
		out = backend.generate(prompt, max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = out.strip().split()[0] if out.strip() else ""
		pred = pred.strip(".,!?:;\"'()[]{}")

		if pred.lower() == gold.lower():
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_multinli(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("multi_nli", split="validation_matched", trust_remote_code=True)
	template = load_template(prompt_dir, "multinli", "Premise: {premise}\nHypothesis: {hypothesis}\n\nOptions:\nA) entailment\nB) neutral\nC) contradiction\n\nChoose A, B, or C and output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		item = ds[i]
		if int(item["label"]) < 0:
			continue
		prompt = template.format(premise=item["premise"], hypothesis=item["hypothesis"])
		out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = choose_letter(out, "ABC")
		gold = "ABC"[int(item["label"])]
		if pred == gold:
			correct += 1
	# Note: sample count may be below n due to skipped -1 labels; approximate with n
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_sciq(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
		import random as _rnd  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("sciq", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "sciq", "Question: {q}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the correct answer (A-D) and output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	seed = int(decode_cfg.get("seed", 42))
	rng = _rnd.Random(seed)
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		options = [item["correct_answer"], item["distractor1"], item["distractor2"], item["distractor3"]]
		idxs = list(range(4))


		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("multi_nli", split="validation_matched", trust_remote_code=True)
	template = load_template(prompt_dir, "multinli", "Premise: {premise}\nHypothesis: {hypothesis}\n\nOptions:\nA) entailment\nB) neutral\nC) contradiction\n\nChoose A, B, or C and output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		item = ds[i]
		if int(item["label"]) < 0:
			continue
		prompt = template.format(premise=item["premise"], hypothesis=item["hypothesis"])
		out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = choose_letter(out, "ABC")
		gold = "ABC"[int(item["label"])]
		if pred == gold:
			correct += 1
	# Note: sample count may be below n due to skipped -1 labels; approximate with n
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0}

def run_sciq(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
		import random as _rnd  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("sciq", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "sciq", "Question: {q}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the correct answer (A-D) and output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	seed = int(decode_cfg.get("seed", 42))
	rng = _rnd.Random(seed)
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		options = [item["correct_answer"], item["distractor1"], item["distractor2"], item["distractor3"]]
		idxs = list(range(4))
		rng.shuffle(idxs)
		shuffled = [options[j] for j in idxs]
		gold_letter = "ABCD"[shuffled.index(item["correct_answer"])]
		prompt = template.format(q=item["question"], a=shuffled[0], b=shuffled[1], c=shuffled[2], d=shuffled[3])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)", "C)", "D)"])
			pred = "ABCD"[best]
		else:
			out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
		if pred == gold_letter:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "sciq", "split": "validation", "fingerprint": ds_fp}

def run_arc(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use ARC-Challenge validation for difficulty
	ds = load_dataset("ai2_arc", "ARC-Challenge", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "arc", "Question: {q}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the correct answer (A-D) and output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item["question"]
		opts = item.get("choices", {}).get("text", [])
		labels = item.get("choices", {}).get("label", [])
		if len(opts) < 4 or len(labels) < 4:


	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		options = [item["correct_answer"], item["distractor1"], item["distractor2"], item["distractor3"]]
		idxs = list(range(4))
		rng.shuffle(idxs)
		shuffled = [options[j] for j in idxs]
		gold_letter = "ABCD"[shuffled.index(item["correct_answer"])]
		prompt = template.format(q=item["question"], a=shuffled[0], b=shuffled[1], c=shuffled[2], d=shuffled[3])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)", "C)", "D)"])
			pred = "ABCD"[best]
		else:
			out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
		if pred == gold_letter:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "sciq", "split": "validation", "fingerprint": ds_fp}

def run_arc(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use ARC-Challenge validation for difficulty
	ds = load_dataset("ai2_arc", "ARC-Challenge", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "arc", "Question: {q}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the correct answer (A-D) and output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item["question"]
		opts = item.get("choices", {}).get("text", [])
		labels = item.get("choices", {}).get("label", [])
		if len(opts) < 4 or len(labels) < 4:
			continue
		# Map labels A-D to order
		order = [labels.index(ch) if ch in labels else 0 for ch in ["A", "B", "C", "D"]]
		a, b, c, d = [opts[k] for k in order]
		gold = item.get("answerKey", "A").strip().upper()
		prompt = template.format(q=q, a=a, b=b, c=c, d=d)
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)", "C)", "D)"])
			pred = "ABCD"[best]
		else:
			out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.2), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "ai2_arc:ARC-Challenge", "split": "validation", "fingerprint": ds_fp}

def run_triviaqa(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = None
	last_err = None
	for cfg in [("trivia_qa", "rc", "validation[:2000]"), ("trivia_qa", "rc", "validation"), ("trivia_qa", "rc", "train[:2000]")]:
		try:
			ds = load_dataset(cfg[0], cfg[1], split=cfg[2], trust_remote_code=True)
			break
		except Exception as e:
			last_err = e
			continue
	if ds is None:
		return {"status": "skipped", "error": f"triviaqa_load_error: {last_err}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "triviaqa", "Question: {q}\n\nAnswer succinctly in a few words. Output only the answer.")
	exact = 0


		opts = item.get("choices", {}).get("text", [])
		labels = item.get("choices", {}).get("label", [])
		if len(opts) < 4 or len(labels) < 4:
			continue
		# Map labels A-D to order
		order = [labels.index(ch) if ch in labels else 0 for ch in ["A", "B", "C", "D"]]
		a, b, c, d = [opts[k] for k in order]
		gold = item.get("answerKey", "A").strip().upper()
		prompt = template.format(q=q, a=a, b=b, c=c, d=d)
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)", "C)", "D)"])
			pred = "ABCD"[best]
		else:
			out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.2), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "ai2_arc:ARC-Challenge", "split": "validation", "fingerprint": ds_fp}

def run_triviaqa(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = None
	last_err = None
	for cfg in [("trivia_qa", "rc", "validation[:2000]"), ("trivia_qa", "rc", "validation"), ("trivia_qa", "rc", "train[:2000]")]:
		try:
			ds = load_dataset(cfg[0], cfg[1], split=cfg[2], trust_remote_code=True)
			break
		except Exception as e:
			last_err = e
			continue
	if ds is None:
		return {"status": "skipped", "error": f"triviaqa_load_error: {last_err}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "triviaqa", "Question: {q}\n\nAnswer succinctly in a few words. Output only the answer.")
	exact = 0
	f1sum = 0.0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item["question"]
		prompt = template.format(q=q)
		out = backend.generate(prompt, max_tokens=16, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = out.strip()
		ans = item.get("answer", {})
		aliases = [str(ans.get("value", ""))] + [str(a) for a in ans.get("aliases", [])]
		# simple EM over aliases
		if any(normalize_answer(pred) == normalize_answer(a) for a in aliases):
			exact += 1
		f1sum += f1_score(pred, aliases)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "em": exact / n if n else 0.0, "f1": f1sum / n if n else 0.0, "dataset": "trivia_qa:rc", "split": "validation", "fingerprint": ds_fp}

def run_mmlu(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Sample a few representative subjects for speed
	subjects = ["conceptual_physics", "high_school_biology", "professional_law", "formal_logic"]
	per_subj = max(1, max_samples // max(1, len(subjects)))
	template = load_template(prompt_dir, "mmlu", "Subject: {subject}\nQuestion: {q}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the correct answer (A-D) and output only the letter.")
	correct = 0
	seen = 0
	t0 = time.time()
	for subj in subjects:
		try:
			ds = load_dataset("lukaemon/mmlu", subj, split="test", trust_remote_code=True)
		except Exception:
			try:
				ds = load_dataset("lukaemon/mmlu", subj, split="validation", trust_remote_code=True)


	exact = 0
	f1sum = 0.0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item["question"]
		prompt = template.format(q=q)
		out = backend.generate(prompt, max_tokens=16, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = out.strip()
		ans = item.get("answer", {})
		aliases = [str(ans.get("value", ""))] + [str(a) for a in ans.get("aliases", [])]
		# simple EM over aliases
		if any(normalize_answer(pred) == normalize_answer(a) for a in aliases):
			exact += 1
		f1sum += f1_score(pred, aliases)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "em": exact / n if n else 0.0, "f1": f1sum / n if n else 0.0, "dataset": "trivia_qa:rc", "split": "validation", "fingerprint": ds_fp}

def run_mmlu(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Sample a few representative subjects for speed
	subjects = ["conceptual_physics", "high_school_biology", "professional_law", "formal_logic"]
	per_subj = max(1, max_samples // max(1, len(subjects)))
	template = load_template(prompt_dir, "mmlu", "Subject: {subject}\nQuestion: {q}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the correct answer (A-D) and output only the letter.")
	correct = 0
	seen = 0
	t0 = time.time()
	for subj in subjects:
		try:
			ds = load_dataset("lukaemon/mmlu", subj, split="test", trust_remote_code=True)
		except Exception:
			try:
				ds = load_dataset("lukaemon/mmlu", subj, split="validation", trust_remote_code=True)
			except Exception:
				continue
		n = min(per_subj, len(ds))
		for i in range(n):
			item = ds[i]
			q = item.get("input", "")
			opts = [item.get("A", ""), item.get("B", ""), item.get("C", ""), item.get("D", "")]
			gold = str(item.get("target", "A")).strip().upper()
			prompt = template.format(subject=subj, q=q, a=opts[0], b=opts[1], c=opts[2], d=opts[3])
			out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
			if pred == gold:
				correct += 1
			seen += 1
	if seen == 0:
		return {"status": "skipped", "error": "no_mmlu_samples"}
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": seen, "correct": correct, "acc": correct / seen if seen else 0.0}

def run_nq(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("nq_open", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"nq_open_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "nq", "Question: {q}\n\nAnswer succinctly. Output only the answer.")
	em = 0
	f1sum = 0.0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item.get("question", "")


				ds = load_dataset("lukaemon/mmlu", subj, split="validation", trust_remote_code=True)
			except Exception:
				continue
		n = min(per_subj, len(ds))
		for i in range(n):
			item = ds[i]
			q = item.get("input", "")
			opts = [item.get("A", ""), item.get("B", ""), item.get("C", ""), item.get("D", "")]
			gold = str(item.get("target", "A")).strip().upper()
			prompt = template.format(subject=subj, q=q, a=opts[0], b=opts[1], c=opts[2], d=opts[3])
			out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
			if pred == gold:
				correct += 1
			seen += 1
	if seen == 0:
		return {"status": "skipped", "error": "no_mmlu_samples"}
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": seen, "correct": correct, "acc": correct / seen if seen else 0.0}

def run_nq(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("nq_open", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"nq_open_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "nq", "Question: {q}\n\nAnswer succinctly. Output only the answer.")
	em = 0
	f1sum = 0.0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item.get("question", "")
		answers = [str(a).strip() for a in (item.get("answers", []) or [])]
		prompt = template.format(q=q)
		out = backend.generate(prompt, max_tokens=16, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = out.strip()
		if any(normalize_answer(pred) == normalize_answer(a) for a in answers):
			em += 1
		f1sum += f1_score(pred, answers)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "em": em / n if n else 0.0, "f1": f1sum / n if n else 0.0, "dataset": "nq_open", "split": "validation", "fingerprint": ds_fp}

def run_glue_sst2(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("glue", "sst2", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "sst2", "Sentence: {sent}\n\nIs the sentiment positive or negative? Output 'positive' or 'negative'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		sent = item.get("sentence", "")
		gold = "positive" if int(item.get("label", 0)) == 1 else "negative"
		out = backend.generate(template.format(sent=sent), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = "positive" if "positive" in t and "negative" not in t else ("negative" if "negative" in t and "positive" not in t else None)


		return {"status": "skipped", "error": f"nq_open_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "nq", "Question: {q}\n\nAnswer succinctly. Output only the answer.")
	em = 0
	f1sum = 0.0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item.get("question", "")
		answers = [str(a).strip() for a in (item.get("answers", []) or [])]
		prompt = template.format(q=q)
		out = backend.generate(prompt, max_tokens=16, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = out.strip()
		if any(normalize_answer(pred) == normalize_answer(a) for a in answers):
			em += 1
		f1sum += f1_score(pred, answers)
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "em": em / n if n else 0.0, "f1": f1sum / n if n else 0.0, "dataset": "nq_open", "split": "validation", "fingerprint": ds_fp}

def run_glue_sst2(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("glue", "sst2", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "sst2", "Sentence: {sent}\n\nIs the sentiment positive or negative? Output 'positive' or 'negative'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		sent = item.get("sentence", "")
		gold = "positive" if int(item.get("label", 0)) == 1 else "negative"
		out = backend.generate(template.format(sent=sent), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = "positive" if "positive" in t and "negative" not in t else ("negative" if "negative" in t and "positive" not in t else None)
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc, "score": acc, "dataset": "glue:sst2", "split": "validation", "fingerprint": ds_fp}

def run_truthfulqa_mc1(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("truthful_qa", "multiple_choice", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"truthfulqa_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "truthfulqa", "Question: {q}\n\nChoices:\n{choices}\n\nSelect the best answer. Output the exact choice text.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item.get("question", "")
		pos = [str(x) for x in (item.get("mc1_targets", []) or [])]


		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("glue", "sst2", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "sst2", "Sentence: {sent}\n\nIs the sentiment positive or negative? Output 'positive' or 'negative'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		sent = item.get("sentence", "")
		gold = "positive" if int(item.get("label", 0)) == 1 else "negative"
		out = backend.generate(template.format(sent=sent), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = "positive" if "positive" in t and "negative" not in t else ("negative" if "negative" in t and "positive" not in t else None)
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc, "score": acc, "dataset": "glue:sst2", "split": "validation", "fingerprint": ds_fp}

def run_truthfulqa_mc1(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("truthful_qa", "multiple_choice", split="validation", trust_remote_code=True)
	except Exception as e:
		return {"status": "skipped", "error": f"truthfulqa_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "truthfulqa", "Question: {q}\n\nChoices:\n{choices}\n\nSelect the best answer. Output the exact choice text.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item.get("question", "")
		pos = [str(x) for x in (item.get("mc1_targets", []) or [])]
		neg = [str(x) for x in (item.get("mc1_negatives", []) or [])]
		choices = pos + neg
		if not choices:
			continue
		label = pos[0] if pos else None
		lettered = [f"{chr(65+j)}) {choices[j]}" for j in range(len(choices))]
		out = backend.generate(template.format(q=q, choices="\n".join(lettered)), max_tokens=16, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pick = None
		# Try match letter
		letter = choose_letter(out, "ABCDEFGHIJKLMNOPQRSTUVWXYZ")
		if letter is not None:
			idx = ord(letter) - 65
			if 0 <= idx < len(choices):
				pick = choices[idx]
		if pick is None:
			# Fallback: find exact choice text substring
			for ch in choices:
				if ch.lower() in out.lower():
					pick = ch
					break
		if pick is not None and label is not None and pick.strip().lower() == label.strip().lower():
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc, "dataset": "truthful_qa:mc", "split": "validation", "fingerprint": ds_fp}

def run_gsm8k(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
		import re
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use test split if present; fall back to train for quick runs
	try:
		ds = load_dataset("gsm8k", "main", split="test", trust_remote_code=True)
	except Exception:
		ds = load_dataset("gsm8k", "main", split="train", trust_remote_code=True)
	template = load_template(prompt_dir, "gsm8k", "Solve step by step and give the final numeric answer.\n\nQuestion: {q}\n\nAnswer: ")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]


		out = backend.generate(template.format(q=q, choices="\n".join(lettered)), max_tokens=16, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pick = None
		# Try match letter
		letter = choose_letter(out, "ABCDEFGHIJKLMNOPQRSTUVWXYZ")
		if letter is not None:
			idx = ord(letter) - 65
			if 0 <= idx < len(choices):
				pick = choices[idx]
		if pick is None:
			# Fallback: find exact choice text substring
			for ch in choices:
				if ch.lower() in out.lower():
					pick = ch
					break
		if pick is not None and label is not None and pick.strip().lower() == label.strip().lower():
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc, "dataset": "truthful_qa:mc", "split": "validation", "fingerprint": ds_fp}

def run_gsm8k(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
		import re
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use test split if present; fall back to train for quick runs
	try:
		ds = load_dataset("gsm8k", "main", split="test", trust_remote_code=True)
	except Exception:
		ds = load_dataset("gsm8k", "main", split="train", trust_remote_code=True)
	template = load_template(prompt_dir, "gsm8k", "Solve step by step and give the final numeric answer.\n\nQuestion: {q}\n\nAnswer: ")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item.get("question", "")
		gold = item.get("answer", "")
		# gold format often contains '#### 42'
		m = re.search(r"####\s*([-+]?\d+(?:\.\d+)?)", gold)
		gold_num = m.group(1) if m else None
		out = backend.generate(template.format(q=q), max_tokens=128, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		m2 = re.findall(r"[-+]?\d+(?:\.\d+)?", out)
		pred_num = m2[-1] if m2 else None
		if gold_num is not None and pred_num is not None:
			try:
				if float(pred_num) == float(gold_num):
					correct += 1
			except Exception:
				pass
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "gsm8k:main", "split": str(ds.split if hasattr(ds, 'split') else 'test/train'), "fingerprint": ds_fp}

def run_race(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use RACE-high validation for difficulty; fallback to middle
	try:
		ds = load_dataset("race", "high", split="validation", trust_remote_code=True)
		subset = "high"
	except Exception:
		ds = load_dataset("race", "middle", split="validation", trust_remote_code=True)
		subset = "middle"
	template = load_template(prompt_dir, "race", "Article: {article}\n\nQuestion: {q}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the correct answer (A-D). Output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):


	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		item = ds[i]
		q = item.get("question", "")
		gold = item.get("answer", "")
		# gold format often contains '#### 42'
		m = re.search(r"####\s*([-+]?\d+(?:\.\d+)?)", gold)
		gold_num = m.group(1) if m else None
		out = backend.generate(template.format(q=q), max_tokens=128, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		m2 = re.findall(r"[-+]?\d+(?:\.\d+)?", out)
		pred_num = m2[-1] if m2 else None
		if gold_num is not None and pred_num is not None:
			try:
				if float(pred_num) == float(gold_num):
					correct += 1
			except Exception:
				pass
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": "gsm8k:main", "split": str(ds.split if hasattr(ds, 'split') else 'test/train'), "fingerprint": ds_fp}

def run_race(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use RACE-high validation for difficulty; fallback to middle
	try:
		ds = load_dataset("race", "high", split="validation", trust_remote_code=True)
		subset = "high"
	except Exception:
		ds = load_dataset("race", "middle", split="validation", trust_remote_code=True)
		subset = "middle"
	template = load_template(prompt_dir, "race", "Article: {article}\n\nQuestion: {q}\n\nChoices:\nA) {a}\nB) {b}\nC) {c}\nD) {d}\n\nChoose the correct answer (A-D). Output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		it = ds[i]
		opts = it.get("options", [])
		if not opts or len(opts) < 4:
			continue
		prompt = template.format(article=it.get("article", ""), q=it.get("question", ""), a=opts[0], b=opts[1], c=opts[2], d=opts[3])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)", "C)", "D)"])
			pred = "ABCD"[best]
		else:
			out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
		gold = str(it.get("answer", "A")).strip().upper()
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": f"race:{subset}", "split": "validation", "fingerprint": ds_fp}

def run_drop(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("drop", split="validation", trust_remote_code=True)
	except Exception:
		# Some mirrors use a config name
		try:
			ds = load_dataset("drop", "drop", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"drop_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "drop", "Passage: {passage}\n\nQuestion: {q}\n\nAnswer succinctly with the exact span or number only.")
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	em_sum = 0.0


	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	for i in range(n):
		it = ds[i]
		opts = it.get("options", [])
		if not opts or len(opts) < 4:
			continue
		prompt = template.format(article=it.get("article", ""), q=it.get("question", ""), a=opts[0], b=opts[1], c=opts[2], d=opts[3])
		if hasattr(backend, "score_options"):
			best = backend.score_options(prompt, ["A)", "B)", "C)", "D)"])
			pred = "ABCD"[best]
		else:
			out = backend.generate(prompt, max_tokens=4, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
			pred = choose_letter(out, "ABCD")
		gold = str(it.get("answer", "A")).strip().upper()
		if pred == gold:
			correct += 1
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": correct / n if n else 0.0, "dataset": f"race:{subset}", "split": "validation", "fingerprint": ds_fp}

def run_drop(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	try:
		ds = load_dataset("drop", split="validation", trust_remote_code=True)
	except Exception:
		# Some mirrors use a config name
		try:
			ds = load_dataset("drop", "drop", split="validation", trust_remote_code=True)
		except Exception as e:
			return {"status": "skipped", "error": f"drop_load_error: {e}", "timing_sec": 0.0, "num_samples": 0}
	template = load_template(prompt_dir, "drop", "Passage: {passage}\n\nQuestion: {q}\n\nAnswer succinctly with the exact span or number only.")
	n = min(max_samples, len(ds))
	t0 = time.time()
	ds_fp = getattr(ds, "_fingerprint", None)
	em_sum = 0.0
	f1_sum = 0.0
	for i in range(n):
		it = ds[i]
		passage = it.get("passage", it.get("context", ""))
		question = it.get("question", "")
		# Answers may be under "answers_spans" or nested
		answers: List[str] = []
		try:
			# Squad-like
			if "answers" in it and isinstance(it["answers"], dict):
				answers = [str(a) for a in (it["answers"].get("text", []) or [])]
		except Exception:
			pass
		try:
			# DROP-specific spans
			if not answers and "answers_spans" in it:
				spans = it["answers_spans"]
				text_spans = spans.get("spans", []) if isinstance(spans, dict) else []
				answers = [str(a) for a in text_spans]
		except Exception:
			pass
		prompt = template.format(passage=passage, q=question)
		pred = backend.generate(prompt, max_tokens=16, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95)).strip()
		if answers:
			em_ok = any(normalize_answer(pred) == normalize_answer(a) for a in answers)
			em_sum += 1.0 if em_ok else 0.0
			f1_sum += f1_score(pred, answers)
	return {
		"status": "ok",
		"timing_sec": round(time.time() - t0, 3),
		"num_samples": n,
		"em": (em_sum / n) if n else 0.0,
		"f1": (f1_sum / n) if n else 0.0,
		"dataset": "drop",
		"split": "validation",
		"fingerprint": ds_fp,
	}

def run_superglue_rte(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "rte", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_rte", "Premise: {premise}\nHypothesis: {hypothesis}\n\nIs the hypothesis entailed by the premise? Answer 'entailment' or 'not_entailment'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(premise=it["premise"], hypothesis=it["hypothesis"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = 0 if ("entail" in t and "not" not in t) else 1
		gold = int(it.get("label", 1))
		if pred == gold:
			correct += 1


		except Exception:
			pass
		prompt = template.format(passage=passage, q=question)
		pred = backend.generate(prompt, max_tokens=16, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95)).strip()
		if answers:
			em_ok = any(normalize_answer(pred) == normalize_answer(a) for a in answers)
			em_sum += 1.0 if em_ok else 0.0
			f1_sum += f1_score(pred, answers)
	return {
		"status": "ok",
		"timing_sec": round(time.time() - t0, 3),
		"num_samples": n,
		"em": (em_sum / n) if n else 0.0,
		"f1": (f1_sum / n) if n else 0.0,
		"dataset": "drop",
		"split": "validation",
		"fingerprint": ds_fp,
	}

def run_superglue_rte(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "rte", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_rte", "Premise: {premise}\nHypothesis: {hypothesis}\n\nIs the hypothesis entailed by the premise? Answer 'entailment' or 'not_entailment'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(premise=it["premise"], hypothesis=it["hypothesis"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = 0 if ("entail" in t and "not" not in t) else 1
		gold = int(it.get("label", 1))
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_copa(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "copa", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_copa", "Premise: {premise}\nQuestion: {question}\n\nChoices:\nA) {a}\nB) {b}\n\nPick the more plausible option (A or B). Output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(premise=it["premise"], question=it["question"], a=it["choice1"], b=it["choice2"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = choose_letter(out, "AB")
		gold = "AB"[int(it.get("label", 0))]
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0


		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "rte", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_rte", "Premise: {premise}\nHypothesis: {hypothesis}\n\nIs the hypothesis entailed by the premise? Answer 'entailment' or 'not_entailment'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(premise=it["premise"], hypothesis=it["hypothesis"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = 0 if ("entail" in t and "not" not in t) else 1
		gold = int(it.get("label", 1))
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_copa(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "copa", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_copa", "Premise: {premise}\nQuestion: {question}\n\nChoices:\nA) {a}\nB) {b}\n\nPick the more plausible option (A or B). Output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(premise=it["premise"], question=it["question"], a=it["choice1"], b=it["choice2"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = choose_letter(out, "AB")
		gold = "AB"[int(it.get("label", 0))]
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_wic(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "wic", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_wic", "Sentence 1: {s1}\nSentence 2: {s2}\n\nThe word '{word}' has the same meaning in both sentences? Answer 'yes' or 'no'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(s1=it["sentence1"], s2=it["sentence2"], word=it["word"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = 1 if ("yes" in t and "no" not in t) else 0
		gold = int(it.get("label", 0))
		if pred == gold:
			correct += 1


	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "copa", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_copa", "Premise: {premise}\nQuestion: {question}\n\nChoices:\nA) {a}\nB) {b}\n\nPick the more plausible option (A or B). Output only the letter.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(premise=it["premise"], question=it["question"], a=it["choice1"], b=it["choice2"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		pred = choose_letter(out, "AB")
		gold = "AB"[int(it.get("label", 0))]
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_wic(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "wic", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_wic", "Sentence 1: {s1}\nSentence 2: {s2}\n\nThe word '{word}' has the same meaning in both sentences? Answer 'yes' or 'no'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(s1=it["sentence1"], s2=it["sentence2"], word=it["word"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = 1 if ("yes" in t and "no" not in t) else 0
		gold = int(it.get("label", 0))
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_wsc(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use the fixed WSC variant
	ds = load_dataset("super_glue", "wsc.fixed", split="validation", trust_remote_code=True)
	template = load_template(
		prompt_dir,
		"superglue_wsc",
		"Text: {text}\nPronoun: {pronoun}\nCandidate: {span}\n\nDoes the pronoun refer to the candidate? Answer yes or no.",
	)
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		text = it.get("text", "")


		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "wic", split="validation", trust_remote_code=True)
	template = load_template(prompt_dir, "superglue_wic", "Sentence 1: {s1}\nSentence 2: {s2}\n\nThe word '{word}' has the same meaning in both sentences? Answer 'yes' or 'no'.")
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(template.format(s1=it["sentence1"], s2=it["sentence2"], word=it["word"]), max_tokens=3, temperature=decode_cfg.get("temperature", 0.0), top_p=decode_cfg.get("top_p", 0.95))
		t = out.strip().lower()
		pred = 1 if ("yes" in t and "no" not in t) else 0
		gold = int(it.get("label", 0))
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_wsc(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	# Use the fixed WSC variant
	ds = load_dataset("super_glue", "wsc.fixed", split="validation", trust_remote_code=True)
	template = load_template(
		prompt_dir,
		"superglue_wsc",
		"Text: {text}\nPronoun: {pronoun}\nCandidate: {span}\n\nDoes the pronoun refer to the candidate? Answer yes or no.",
	)
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		text = it.get("text", "")
		pron = it.get("span2_text", it.get("pronoun", ""))
		cand = it.get("span1_text", it.get("candidate", ""))
		out = backend.generate(
			template.format(text=text, pronoun=pron, span=cand),
			max_tokens=3,
			temperature=decode_cfg.get("temperature", 0.0),
			top_p=decode_cfg.get("top_p", 0.95),
		)
		t = out.strip().lower()
		pred = 1 if ("yes" in t and "no" not in t) else 0
		gold = int(it.get("label", 0))
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_cb(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "cb", split="validation", trust_remote_code=True)
	template = load_template(
		prompt_dir,
		"superglue_cb",
		"Premise: {premise}\nHypothesis: {hypothesis}\n\nOptions:\nA) entailment\nB) contradiction\nC) neutral\n\nChoose A, B, or C and output only the letter.",
	)
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(
			template.format(premise=it["premise"], hypothesis=it["hypothesis"]),


	for i in range(n):
		it = ds[i]
		text = it.get("text", "")
		pron = it.get("span2_text", it.get("pronoun", ""))
		cand = it.get("span1_text", it.get("candidate", ""))
		out = backend.generate(
			template.format(text=text, pronoun=pron, span=cand),
			max_tokens=3,
			temperature=decode_cfg.get("temperature", 0.0),
			top_p=decode_cfg.get("top_p", 0.95),
		)
		t = out.strip().lower()
		pred = 1 if ("yes" in t and "no" not in t) else 0
		gold = int(it.get("label", 0))
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_cb(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "cb", split="validation", trust_remote_code=True)
	template = load_template(
		prompt_dir,
		"superglue_cb",
		"Premise: {premise}\nHypothesis: {hypothesis}\n\nOptions:\nA) entailment\nB) contradiction\nC) neutral\n\nChoose A, B, or C and output only the letter.",
	)
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(
			template.format(premise=it["premise"], hypothesis=it["hypothesis"]),
			max_tokens=3,
			temperature=decode_cfg.get("temperature", 0.0),
			top_p=decode_cfg.get("top_p", 0.95),
		)
		pred = choose_letter(out, "ABC")
		# super_glue:cb labels: 0=entailment, 1=contradiction, 2=neutral
		gold = "ABC"[int(it.get("label", 2))]
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_multirc(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "multirc", split="validation", trust_remote_code=True)
	template = load_template(
		prompt_dir,
		"superglue_multirc",
		"Paragraph: {paragraph}\nQuestion: {question}\nAnswer option: {answer}\n\nIs this answer correct? Answer yes or no.",
	)
	# Compute simple answer-level accuracy across all answer options
	total = 0
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]


	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		out = backend.generate(
			template.format(premise=it["premise"], hypothesis=it["hypothesis"]),
			max_tokens=3,
			temperature=decode_cfg.get("temperature", 0.0),
			top_p=decode_cfg.get("top_p", 0.95),
		)
		pred = choose_letter(out, "ABC")
		# super_glue:cb labels: 0=entailment, 1=contradiction, 2=neutral
		gold = "ABC"[int(it.get("label", 2))]
		if pred == gold:
			correct += 1
	acc = correct / n if n else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "correct": correct, "acc": acc}

def run_superglue_multirc(backend: GenerationBackend, prompt_dir: Path, max_samples: int, decode_cfg: Dict[str, Any]) -> Dict[str, Any]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception as e:
		return {"status": "skipped", "error": f"datasets_missing: {e}", "timing_sec": 0.0, "num_samples": 0}
	ds = load_dataset("super_glue", "multirc", split="validation", trust_remote_code=True)
	template = load_template(
		prompt_dir,
		"superglue_multirc",
		"Paragraph: {paragraph}\nQuestion: {question}\nAnswer option: {answer}\n\nIs this answer correct? Answer yes or no.",
	)
	# Compute simple answer-level accuracy across all answer options
	total = 0
	correct = 0
	n = min(max_samples, len(ds))
	t0 = time.time()
	for i in range(n):
		it = ds[i]
		par = it.get("paragraph", "")
		q = it.get("question", "")
		answers = it.get("answers", []) or []
		for ans in answers:
			txt = str(ans.get("text", ""))
			gold = 1 if bool(ans.get("label", 0)) else 0
			out = backend.generate(
				template.format(paragraph=par, question=q, answer=txt),
				max_tokens=3,
				temperature=decode_cfg.get("temperature", 0.0),
				top_p=decode_cfg.get("top_p", 0.95),
			)
			t = out.strip().lower()
			pred = 1 if ("yes" in t and "no" not in t) else 0
			if pred == gold:
				correct += 1
			total += 1
	score = (correct / total) if total else 0.0
	return {"status": "ok", "timing_sec": round(time.time() - t0, 3), "num_samples": n, "answers": total, "correct": correct, "score": score}

RUNNERS = {
	"hellaswag": run_hellaswag,
	"piqa": run_piqa,
	"winogrande": run_winogrande,
	"boolq": run_boolq,
	"lambada": run_lambada,
	"multinli": run_multinli,
	"sciq": run_sciq,
	"arc": run_arc,
	"triviaqa": run_triviaqa,
	"mmlu": run_mmlu,
	"nq": run_nq,
	"sst2": run_glue_sst2,
	"truthfulqa": run_truthfulqa_mc1,
	"gsm8k": run_gsm8k,
	"race": run_race,
	"drop": run_drop,
	"bbh": run_bbh,
	"agieval": run_agieval,


main

	"bbh": run_bbh,
	"agieval": run_agieval,
	"crass": run_crass,
	"superglue_rte": run_superglue_rte,
	"superglue_copa": run_superglue_copa,
	"superglue_wic": run_superglue_wic,
	"superglue_wsc": lambda backend, prompt_dir, max_samples, dec: run_superglue_wsc(backend, prompt_dir, max_samples, dec),
	"superglue_cb": lambda backend, prompt_dir, max_samples, dec: run_superglue_cb(backend, prompt_dir, max_samples, dec),
	"superglue_multirc": lambda backend, prompt_dir, max_samples, dec: run_superglue_multirc(backend, prompt_dir, max_samples, dec),
	"mt-bench": run_mt_bench,
	"quac": run_quac,
	"aci-bench": run_aci_bench,
	"ms-marco": run_ms_marco,
	"qmsum": run_qmsum,
	"toxigen": run_toxigen,
	"hhh": run_hhh,
	"rai": run_rai,
	"codexglue": run_codexglue,
}

def main() -> int:
	args = parse_args()
	# Resolve benchmark list
	selected = args.benchmarks or list(KNOWN_BENCHMARKS.keys())
	# Load optional config
	cfg = {}
	cfg_path = Path(args.config)
	if cfg_path.exists():
		try:
			cfg = json.loads(cfg_path.read_text(encoding="utf-8"))
		except Exception:
			cfg = {}
	# If not executing, write scaffold only.
	out_path = Path(args.out)
	ensure_dirs(out_path)
	start = time.time()
	# Resume from existing results if requested
	if args.resume and out_path.exists():
		try:
			existing = json.loads(out_path.read_text(encoding="utf-8"))
		except Exception:
			existing = {}
	else:
		existing = {}
	results = scaffold_results(selected, args.model, cfg)
	results["seed"] = int(args.seed)
	# Tool/dataset/model versions for reproducibility
	ver: Dict[str, Any] = {}
	try:
		import transformers  # type: ignore
		ver["transformers"] = getattr(transformers, "__version__", "")
	except Exception:
		pass
	try:
		import datasets  # type: ignore
		ver["datasets"] = getattr(datasets, "__version__", "")
	except Exception:
		pass
	try:
		import torch  # type: ignore
		ver["torch"] = getattr(torch, "__version__", "")
	except Exception:
		pass
	results.setdefault("meta", {})
	results["meta"]["versions"] = ver
	meta: Dict[str, Any] = {"backend": args.backend, "cost_per_sec": float(args.cost_per_sec)}
	if args.execute:
		try:
			backend = get_backend(args.backend, args.model, args.http_url)
		except Exception as e:
			meta["error"] = f"backend_init: {e}"
		else:
			prompt_dir = Path(args.prompt_dir)
			defaults = (cfg or {}).get("defaults", {})
			overrides = (cfg or {}).get("overrides", {})
			for name in selected:
				# Skip completed if resuming
				if args.resume and existing.get("benchmarks", {}).get(name, {}).get("status") == "ok":
					results["benchmarks"][name] = existing["benchmarks"][name]
					continue
				runner = RUNNERS.get(name)
				if runner is None:
					continue
				dec = defaults.copy()
				dec.update(overrides.get(name, {}))
				# For greedy decoding, sampling params are irrelevant; avoid warnings
				try:
					if float(dec.get("temperature", 0.0)) == 0.0:
						dec.setdefault("top_p", 1.0)
						if "top_k" in dec:
							del dec["top_k"]
				except Exception:
					pass
				# Global seed defaults
				dec.setdefault("seed", int(args.seed))
				try:
					out = runner(backend, prompt_dir, args.max_samples, dec)
					# Write metrics and status
					results["benchmarks"][name]["status"] = out.get("status", "ok")
					if out.get("status") == "ok":
						metric = KNOWN_BENCHMARKS[name]["metric"]
						score_val = None
						if metric == "acc":
							score_val = out.get("acc")
						elif metric == "em_f1":
							score_val = out.get("f1", out.get("em"))
						elif metric == "f1_heq":
							score_val = out.get("f1")
						elif metric == "mrr":
							score_val = out.get("mrr")
						elif metric == "rouge":
							score_val = out.get("rouge")
						elif metric == "auroc":
							score_val = out.get("auroc")
						elif metric in ("judge", "rubric", "safety", "task"):
							score_val = out.get("score")
						else:
							score_val = out.get("score") or out.get("acc")
						results["benchmarks"][name]["score"] = score_val
					results["benchmarks"][name]["details"].update(out)
				except Exception as e:
					results["benchmarks"][name]["status"] = "error"
					results["benchmarks"][name]["details"]["error"] = str(e)
	# Compute totals
	elapsed = round(time.time() - start, 3)
	meta["elapsed_sec"] = elapsed
	# Sum sample counts from details if present
	try:
		total_samples = int(sum(int(results["benchmarks"][n]["details"].get("num_samples", 0)) for n in selected if n in results["benchmarks"]))
	except Exception:
		total_samples = 0
	meta["total_samples"] = total_samples
	if float(args.cost_per_sec) > 0.0:
		cost_total = float(args.cost_per_sec) * float(elapsed)
		meta["cost_total"] = round(cost_total, 6)
		meta["cost_per_sample"] = round(cost_total / total_samples, 6) if total_samples > 0 else None
	results["meta"] = meta
	out_path.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
	print(json.dumps({"ok": True, "wrote": str(out_path), "count": len(selected)}))
	return 0

if __name__ == "__main__":
	raise SystemExit(main())


generate

			pos = len(prompt_ids) + i - 1
			if pos < 0:
				return float("-inf")
			lp = logprobs[0, pos, tok].item()
			lp_sum += lp
		return lp_sum / max(1, len(cont_ids))

	def score_options(self, prompt: str, options: List[str]) -> int:
		scores = [self.score_continuation(prompt, opt) for opt in options]
		best = max(range(len(scores)), key=lambda i: scores[i])
		return best

class HTTPBackend(GenerationBackend):
	def __init__(self, url: str, model_id: str):
		import requests  # lazy import ok
		self._requests = requests
		self._url = url
		self._model = model_id

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		payload = {
			"model": self._model,
			"prompt": prompt,
			"max_tokens": max_tokens,
			"temperature": temperature,
			"top_p": top_p,
		}
		last_err = None
		for attempt in range(1, 4):
			try:
				r = self._requests.post(self._url, json=payload, timeout=60)
				r.raise_for_status()
				data = r.json()
				if isinstance(data, dict) and "text" in data:
					return str(data["text"]).strip()
				if isinstance(data, dict) and "choices" in data and data["choices"]:
					return str(data["choices"][0].get("text", "")).strip()
				return str(data).strip()
			except Exception as e:
				last_err = e
				import time as _t
				_t.sleep((0.5) * (2 ** (attempt - 1)))
		raise RuntimeError(f"http_backend_error: {last_err}")

def get_backend(backend: str, model_id: str, http_url: str) -> GenerationBackend:
	if backend == "hf":
		return HFBackend(model_id)
	if backend == "http":
		if not http_url:
			raise RuntimeError("backend=http requires --http-url")
		return HTTPBackend(http_url, model_id)
	raise RuntimeError(f"Unsupported backend: {backend}")

def load_template(prompt_dir: Path, name: str, default: str) -> str:
	p = prompt_dir / f"{name}.txt"
	if p.exists():
		return p.read_text(encoding="utf-8")
	return default

def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:


			logits = out.logits  # [1, seq, vocab]
			logprobs = torch.log_softmax(logits, dim=-1)
		# Sum logprobs of continuation tokens against previous tokens
		lp_sum = 0.0
		# continuation positions start after prompt length
		for i, tok in enumerate(cont_ids):
			pos = len(prompt_ids) + i - 1
			if pos < 0:
				return float("-inf")
			lp = logprobs[0, pos, tok].item()
			lp_sum += lp
		return lp_sum / max(1, len(cont_ids))

	def score_options(self, prompt: str, options: List[str]) -> int:
		scores = [self.score_continuation(prompt, opt) for opt in options]
		best = max(range(len(scores)), key=lambda i: scores[i])
		return best

class HTTPBackend(GenerationBackend):
	def __init__(self, url: str, model_id: str):
		import requests  # lazy import ok
		self._requests = requests
		self._url = url
		self._model = model_id

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		payload = {
			"model": self._model,
			"prompt": prompt,
			"max_tokens": max_tokens,
			"temperature": temperature,
			"top_p": top_p,
		}
		last_err = None
		for attempt in range(1, 4):
			try:
				r = self._requests.post(self._url, json=payload, timeout=60)
				r.raise_for_status()
				data = r.json()
				if isinstance(data, dict) and "text" in data:
					return str(data["text"]).strip()
				if isinstance(data, dict) and "choices" in data and data["choices"]:
					return str(data["choices"][0].get("text", "")).strip()
				return str(data).strip()


		self._tokenizer = AutoTokenizer.from_pretrained(model_id)
		self._model = AutoModelForCausalLM.from_pretrained(model_id)
		self._pipe = pipeline(
			"text-generation",
			model=self._model,
			tokenizer=self._tokenizer,
			device_map="auto" if os.environ.get("HF_DEVICE_AUTO") else None,
		)
		try:
			self._device = next(self._model.parameters()).device  # type: ignore[attr-defined]
		except Exception:
			# Fallback to CPU
			import torch  # type: ignore
			self._device = torch.device("cpu")

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		out = self._pipe(prompt, max_new_tokens=max_tokens, do_sample=temperature > 0.0, temperature=temperature, top_p=top_p)
		text = out[0]["generated_text"]
		return text[len(prompt):].strip() if text.startswith(prompt) else text.strip()

	def score_continuation(self, prompt: str, continuation: str) -> float:
		# Average logprob over continuation tokens conditioned on prompt
		import torch  # type: ignore
		prompt_ids = self._tokenizer.encode(prompt, add_special_tokens=False)
		cont_ids = self._tokenizer.encode(continuation, add_special_tokens=False)
		input_ids = prompt_ids + cont_ids
		if not cont_ids:
			return float("-inf")
		tens = torch.tensor([input_ids], dtype=torch.long)
		try:
			tens = tens.to(self._device)
		except Exception:
			pass
		with torch.no_grad():
			out = self._model(tens)
			logits = out.logits  # [1, seq, vocab]
			logprobs = torch.log_softmax(logits, dim=-1)
		# Sum logprobs of continuation tokens against previous tokens
		lp_sum = 0.0
		# continuation positions start after prompt length
		for i, tok in enumerate(cont_ids):
			pos = len(prompt_ids) + i - 1
			if pos < 0:
				return float("-inf")
			lp = logprobs[0, pos, tok].item()
			lp_sum += lp
		return lp_sum / max(1, len(cont_ids))

	def score_options(self, prompt: str, options: List[str]) -> int:
		scores = [self.score_continuation(prompt, opt) for opt in options]
		best = max(range(len(scores)), key=lambda i: scores[i])
		return best

class HTTPBackend(GenerationBackend):
	def __init__(self, url: str, model_id: str):
		import requests  # lazy import ok
		self._requests = requests
		self._url = url
		self._model = model_id

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		payload = {
			"model": self._model,
			"prompt": prompt,
			"max_tokens": max_tokens,
			"temperature": temperature,


		tens = torch.tensor([input_ids], dtype=torch.long)
		try:
			tens = tens.to(self._device)
		except Exception:
			pass
		with torch.no_grad():
			out = self._model(tens)
			logits = out.logits  # [1, seq, vocab]
			logprobs = torch.log_softmax(logits, dim=-1)
		# Sum logprobs of continuation tokens against previous tokens
		lp_sum = 0.0
		# continuation positions start after prompt length
		for i, tok in enumerate(cont_ids):
			pos = len(prompt_ids) + i - 1
			if pos < 0:
				return float("-inf")
			lp = logprobs[0, pos, tok].item()
			lp_sum += lp
		return lp_sum / max(1, len(cont_ids))

	def score_options(self, prompt: str, options: List[str]) -> int:
		scores = [self.score_continuation(prompt, opt) for opt in options]
		best = max(range(len(scores)), key=lambda i: scores[i])
		return best

class HTTPBackend(GenerationBackend):
	def __init__(self, url: str, model_id: str):
		import requests  # lazy import ok
		self._requests = requests
		self._url = url
		self._model = model_id

	def generate(self, prompt: str, max_tokens: int, temperature: float, top_p: float) -> str:
		payload = {
			"model": self._model,
			"prompt": prompt,
			"max_tokens": max_tokens,
			"temperature": temperature,
			"top_p": top_p,
		}
		last_err = None
		for attempt in range(1, 4):
			try:


		return p.read_text(encoding="utf-8")
	return default

def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:
	t = text.strip().upper()
	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)
	def white_space_fix(text: str) -> str:
		return " ".join(text.split())
	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True
	if any(x in t for x in [" no", "\nno"]) or t.startswith("no"):
		return False
	if "true" in t and "false" not in t:
		return True
	if "false" in t and "true" not in t:
		return False
	return None


def choose_letter(text: str, valid: str = "ABCD") -> Optional[str]:
	t = text.strip().upper()
	for ch in valid:
		if ch in t:
			return ch
	# fallbacks
	if "YES" in t and "NO" not in t:
		return "YES"
	if "NO" in t and "YES" not in t:
		return "NO"
	return None

def normalize_answer(s: str) -> str:
	import re
	import string
	def remove_articles(text: str) -> str:
		return re.sub(r"\b(a|an|the)\b", " ", text)
	def white_space_fix(text: str) -> str:
		return " ".join(text.split())
	def remove_punc(text: str) -> str:
		return "".join(ch for ch in text if ch not in set(string.punctuation))
	def lower(text: str) -> str:
		return text.lower()
	return white_space_fix(remove_articles(remove_punc(lower(s))))

def choose_boolean(text: str) -> Optional[bool]:
	t = text.strip().lower()
	if any(x in t for x in [" yes", "\nyes"]) or t.startswith("yes"):
		return True
	if any(x in t for x in [" no", "\nno"]) or t.startswith("no"):
		return False
	if "true" in t and "false" not in t:
		return True
	if "false" in t and "true" not in t:
		return False
	return None