| """A JevBench adapter for yev: the harness's own question dict goes to ``system_one`` unchanged. |
| |
| from examples.jevbench_adapter import YevAdapter |
| from jevbench.adapters.conformance import check_adapter |
| print(check_adapter(YevAdapter("HallD/Yev-Decision-1")).summary()) |
| """ |
|
|
| import time |
|
|
| from jevbench.adapters.base import DecisionResult, build_question |
|
|
| from yev import Yev |
|
|
|
|
| class YevAdapter: |
| name = "yev_local" |
| cost_basis = "local_no_provider_tariff" |
| price_input_per_m = None |
| price_output_per_m = None |
|
|
| def __init__(self, model="HallD/Yev-Decision-1"): |
| self.model, self.client = model, Yev(model, lazy=False) |
|
|
| def reserve_estimate(self, task): |
| return None |
|
|
| def run(self, task): |
| result = DecisionResult(adapter=self.name, ok=False, probs_source="native", model=self.model) |
| question = build_question(task) |
| result.request_body = {"state": task.state, "questions": {"decision": question}} |
| started = time.perf_counter() |
| try: |
| response = self.client.system_one(task.state, {"decision": question}, windows="mean") |
| except Exception as error: |
| result.latency_s = time.perf_counter() - started |
| result.error = f"{type(error).__name__}: {str(error)[:300]}" |
| return result |
| result.latency_s = time.perf_counter() - started |
| kind = task.question["type"] |
| if kind == "noul": |
| p = float(response.nouls["decision"].noul) |
| result.probs = {"yes": p, "no": 1.0 - p} |
| else: |
| answer = response.choices["decision"] if kind == "choice" else response.scores["decision"] |
| result.probs = {str(k): float(v) for k, v in answer.probabilities.items()} |
| result.usage = {"input_tokens": response.usage.input_tokens, "output_tokens": 0} |
| result.ok = True |
| result.raw = {"runtime": {"probability_origin": "native-softmax", "windows": "mean"}} |
| return result |
|
|
