File size: 4,649 Bytes
53587e2 770a262 53587e2 770a262 53587e2 770a262 53587e2 770a262 53587e2 770a262 53587e2 770a262 53587e2 770a262 53587e2 770a262 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import httpx
from harness.adapters import Adapter
from harness.verifier import verify
PROMPTS = Path(__file__).resolve().parent / "prompts"
# The arena gives 120 s per request. Long inputs may take up to 75 s end to end,
# so the model call gets 100 s and the rest is left for our own work.
SLM_TIMEOUT_S = 100.0
@dataclass
class HarnessResult:
answer: Any
verified: bool
errors: str = ""
trace: str = ""
steps: int = 0
attempts: list[str] = field(default_factory=list)
def complete(
slm_url: str, prompt: str, max_tokens: int, stop: list[str], seed: int | None = None
) -> tuple[str, str]:
"""One completion call. Returns (text, finish_reason); raises on transport errors."""
body: dict[str, Any] = {
"model": "slm",
"prompt": prompt,
"max_tokens": max_tokens,
"stop": stop,
"temperature": 0,
}
if seed is not None:
body["seed"] = seed
with httpx.Client(timeout=SLM_TIMEOUT_S) as http:
response = http.post(slm_url.rstrip("/") + "/v1/completions", json=body)
response.raise_for_status()
choice = (response.json().get("choices") or [{}])[0]
return str(choice.get("text") or ""), str(choice.get("finish_reason") or "")
def _run_compression(
task: dict[str, Any], *, adapter: Adapter, slm_url: str, config: dict[str, Any], seed: int | None
) -> HarnessResult:
"""ctx_compress: exactly one full model pass, then deterministic repair.
The arena allows two model calls per item (one full pass, one repair pass
on the output only). Deterministic repair is faster, cheaper and never
invents anything, so the second call is not used. About 1 in 7 model
responses are faulty on purpose; any failure falls through to a summary
built from the session itself instead of crashing.
"""
max_tokens = int((config.get("slm") or {}).get("max_new_tokens") or 3072)
prefill = getattr(adapter, "prefill", "")
trace: list[str] = []
draft = ""
try:
prompt = adapter.render(PROMPTS, "task", task)
text, reason = complete(slm_url, prompt, max_tokens, adapter.stop, seed)
draft = prefill + text if prefill and not text.lstrip().startswith("## ") else text
trace.append(f"call 1: {len(text)} chars, finish {reason or 'unknown'}")
except Exception as exc: # timeouts, 5xx, bad JSON: never crash
trace.append(f"call 1 failed: {type(exc).__name__}")
try:
answer, note = adapter.finish(draft, task) # type: ignore[attr-defined]
except Exception as exc:
trace.append(f"finish failed: {type(exc).__name__}")
answer, note = adapter.finish("", task) # type: ignore[attr-defined]
trace.append(f"repair: {note or 'none needed'}")
verified = bool(draft.strip()) and "built from the session" not in note
return HarnessResult(
answer=answer,
verified=verified,
errors=note,
trace="\n".join(trace),
steps=1,
attempts=[draft],
)
def run_harness(
task: dict[str, Any],
*,
adapter: Adapter,
slm_url: str,
tools_url: str,
config: dict[str, Any],
seed: int | None = None,
) -> HarnessResult:
if hasattr(adapter, "finish"):
return _run_compression(task, adapter=adapter, slm_url=slm_url, config=config, seed=seed)
# other arenas: the template's verify and retry loop
max_tokens = int((config.get("slm") or {}).get("max_new_tokens") or 256)
max_steps = int((config.get("harness") or {}).get("max_steps") or 3)
errors = ""
answer = ""
trace: list[str] = []
attempts: list[str] = []
for step in range(1, max_steps + 1):
kind = "task" if not errors else "retry"
prompt = adapter.render(PROMPTS, kind, task, draft=answer, errors=errors, tools_url=tools_url)
try:
answer, _ = complete(slm_url, prompt, max_tokens, adapter.stop, seed)
except Exception as exc:
trace.append(f"step {step}: call failed {type(exc).__name__}")
errors = "the model call failed"
continue
attempts.append(answer)
verdict = verify(adapter, answer, task, tools_url)
trace.append(f"step {step}: {'ok' if verdict.ok else verdict.reason}")
if verdict.ok:
return HarnessResult(verdict.answer, True, trace="\n".join(trace), steps=step, attempts=attempts)
errors = verdict.reason
return HarnessResult(adapter.clean(answer), False, errors, "\n".join(trace), max_steps, attempts)
|