HallD commited on
Commit
ba578b8
·
verified ·
1 Parent(s): 7c6db2d

JevBench adapter: report input token usage

Browse files
Files changed (2) hide show
  1. README.md +2 -1
  2. examples/jevbench_adapter.py +1 -0
README.md CHANGED
@@ -63,4 +63,5 @@ asks from the command line.
63
 
64
  `examples/jevbench_adapter.py` is an adapter for the
65
  [JevBench](https://github.com/fstandhartinger/jevbench) harness: it sends the harness's own question
66
- to `system_one` and returns the probabilities.
 
 
63
 
64
  `examples/jevbench_adapter.py` is an adapter for the
65
  [JevBench](https://github.com/fstandhartinger/jevbench) harness: it sends the harness's own question
66
+ to `system_one` and returns the probabilities, with the input tokens the model read as `usage`
67
+ (output tokens are always 0).
examples/jevbench_adapter.py CHANGED
@@ -43,6 +43,7 @@ class YevAdapter:
43
  else:
44
  answer = response.choices["decision"] if kind == "choice" else response.scores["decision"]
45
  result.probs = {str(k): float(v) for k, v in answer.probabilities.items()}
 
46
  result.ok = True
47
  result.raw = {"runtime": {"probability_origin": "native-softmax", "windows": "mean"}}
48
  return result
 
43
  else:
44
  answer = response.choices["decision"] if kind == "choice" else response.scores["decision"]
45
  result.probs = {str(k): float(v) for k, v in answer.probabilities.items()}
46
+ result.usage = {"input_tokens": response.usage.input_tokens, "output_tokens": 0}
47
  result.ok = True
48
  result.raw = {"runtime": {"probability_origin": "native-softmax", "windows": "mean"}}
49
  return result