JevBench adapter: report input token usage
Browse files- README.md +2 -1
- examples/jevbench_adapter.py +1 -0
README.md
CHANGED
|
@@ -63,4 +63,5 @@ asks from the command line.
|
|
| 63 |
|
| 64 |
`examples/jevbench_adapter.py` is an adapter for the
|
| 65 |
[JevBench](https://github.com/fstandhartinger/jevbench) harness: it sends the harness's own question
|
| 66 |
-
to `system_one` and returns the probabilities
|
|
|
|
|
|
| 63 |
|
| 64 |
`examples/jevbench_adapter.py` is an adapter for the
|
| 65 |
[JevBench](https://github.com/fstandhartinger/jevbench) harness: it sends the harness's own question
|
| 66 |
+
to `system_one` and returns the probabilities, with the input tokens the model read as `usage`
|
| 67 |
+
(output tokens are always 0).
|
examples/jevbench_adapter.py
CHANGED
|
@@ -43,6 +43,7 @@ class YevAdapter:
|
|
| 43 |
else:
|
| 44 |
answer = response.choices["decision"] if kind == "choice" else response.scores["decision"]
|
| 45 |
result.probs = {str(k): float(v) for k, v in answer.probabilities.items()}
|
|
|
|
| 46 |
result.ok = True
|
| 47 |
result.raw = {"runtime": {"probability_origin": "native-softmax", "windows": "mean"}}
|
| 48 |
return result
|
|
|
|
| 43 |
else:
|
| 44 |
answer = response.choices["decision"] if kind == "choice" else response.scores["decision"]
|
| 45 |
result.probs = {str(k): float(v) for k, v in answer.probabilities.items()}
|
| 46 |
+
result.usage = {"input_tokens": response.usage.input_tokens, "output_tokens": 0}
|
| 47 |
result.ok = True
|
| 48 |
result.raw = {"runtime": {"probability_origin": "native-softmax", "windows": "mean"}}
|
| 49 |
return result
|