File size: 2,015 Bytes
2afdef9 ba578b8 2afdef9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | """A JevBench adapter for yev: the harness's own question dict goes to ``system_one`` unchanged.
from examples.jevbench_adapter import YevAdapter
from jevbench.adapters.conformance import check_adapter
print(check_adapter(YevAdapter("HallD/Yev-Decision-1")).summary())
"""
import time
from jevbench.adapters.base import DecisionResult, build_question
from yev import Yev
class YevAdapter:
name = "yev_local"
cost_basis = "local_no_provider_tariff"
price_input_per_m = None
price_output_per_m = None
def __init__(self, model="HallD/Yev-Decision-1"):
self.model, self.client = model, Yev(model, lazy=False)
def reserve_estimate(self, task):
return None
def run(self, task):
result = DecisionResult(adapter=self.name, ok=False, probs_source="native", model=self.model)
question = build_question(task)
result.request_body = {"state": task.state, "questions": {"decision": question}}
started = time.perf_counter()
try:
response = self.client.system_one(task.state, {"decision": question}, windows="mean")
except Exception as error: # a failed answer is a failed attempt
result.latency_s = time.perf_counter() - started
result.error = f"{type(error).__name__}: {str(error)[:300]}"
return result
result.latency_s = time.perf_counter() - started
kind = task.question["type"]
if kind == "noul":
p = float(response.nouls["decision"].noul)
result.probs = {"yes": p, "no": 1.0 - p}
else:
answer = response.choices["decision"] if kind == "choice" else response.scores["decision"]
result.probs = {str(k): float(v) for k, v in answer.probabilities.items()}
result.usage = {"input_tokens": response.usage.input_tokens, "output_tokens": 0}
result.ok = True
result.raw = {"runtime": {"probability_origin": "native-softmax", "windows": "mean"}}
return result
|