ASTRALK commited on
Commit
19fbe2b
·
verified ·
1 Parent(s): 7e67714

Upload benchmark-results.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. benchmark-results.json +176 -0
benchmark-results.json ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accounting": {
3
+ "attempts_reserved": 8778,
4
+ "attempts_settled": 8777,
5
+ "legacy_reported_usd": "2.303337629999998",
6
+ "provider_estimates_are_not_invoices": true,
7
+ "provider_estimates_present": 8777,
8
+ "provider_estimates_usd": "1.170480420000000052057",
9
+ "remaining_shared_headroom_usd": "10.478050722000001955290",
10
+ "shared_inclusive_reserved_upper_usd": "7.521949277999998044710",
11
+ "token_priced_estimates_usd": "1.48640490",
12
+ "unknown_reserves_usd": "0.15831040"
13
+ },
14
+ "artifacts": [
15
+ {
16
+ "path": "scoring/v2-glm/results/baseline-v2-full/snapshots/8f05c2e977d847448d4cc4b9d31647cd.json",
17
+ "sha256": "92f56240a946748bb8f521caec44a95b50ceaaa01fa192441fae0847687756c9"
18
+ },
19
+ {
20
+ "path": "scoring/v2-glm/results/baseline-v2-full/snapshots/7e401c4b4392458a8493a5d3b28d6581.json",
21
+ "sha256": "c263f5bdc7471109c773c4de302a9f002dfb3c9af8403c2d07269b61225a0df9"
22
+ },
23
+ {
24
+ "path": "scoring/v2-glm/results/baseline-v2-full/snapshots/2d6a928ac7a94d19831c87be1c647e73.json",
25
+ "sha256": "bdbc4d9299a868a7e32d4a20395e6718edba4dcd351fc854e609da23a91b55d9"
26
+ },
27
+ {
28
+ "path": "runs/v2/compute/selfaware-5426dad9c4a4/output/selfaware-v2/grades.jsonl",
29
+ "sha256": "50c65780a13cee692a5712bf2de53c551dd48deb6f169dcdc4ecc6105b30b3ee"
30
+ },
31
+ {
32
+ "path": "runs/v2/compute/selfaware-5426dad9c4a4/collected.json",
33
+ "sha256": "305ede842b43db0be6c4f041c82b1e3a2dd743b40b12bc2610cb6be0ccdc8538"
34
+ },
35
+ {
36
+ "path": "runs/v2/tiny/responses.jsonl",
37
+ "sha256": "aa7a20232e7652d857433cc93efaf4dcfbc9f212ca28bb3ab6c6775444d9077f"
38
+ },
39
+ {
40
+ "path": "runs/v2/8b/responses.jsonl",
41
+ "sha256": "10cec296f90e9f77be709a7c7643532acf5750786b16047b2e4d9ba1c22175a8"
42
+ }
43
+ ],
44
+ "complete": true,
45
+ "kind": "aya-four-benchmark-results-v2",
46
+ "metrics": {
47
+ "8b": {
48
+ "abstention_bench": {
49
+ "abstention_graded": 978,
50
+ "behavior_accuracy": 0.7269938650306749,
51
+ "behavior_correct": 711,
52
+ "conditional_correctness_expected": 439,
53
+ "conditional_correctness_missing_or_error": 0,
54
+ "expected": 978,
55
+ "joint_successes": 589,
56
+ "joint_task_success": 0.6022494887525562,
57
+ "missing_or_error": 0
58
+ },
59
+ "hallulens_precisewikiqa": {
60
+ "abstained": 43,
61
+ "correct_answer_rate": 0.147,
62
+ "expected": 1000,
63
+ "false_refusal_rate": 0.043,
64
+ "graded": 1000,
65
+ "hallucination_rate_when_not_abstained": 0.8463949843260188,
66
+ "incorrect_non_abstained": 809,
67
+ "missing_or_error": 0,
68
+ "non_abstained": 957,
69
+ "unverifiable_non_abstained": 1
70
+ },
71
+ "selfaware": {
72
+ "answerable_reference_accuracy": 0.3900769888793841,
73
+ "answerable_reference_correct": 912,
74
+ "expected": 3369,
75
+ "f1": 0.43704197201865425,
76
+ "fn": 703,
77
+ "fp": 142,
78
+ "graded": 3369,
79
+ "missing_or_error": 0,
80
+ "precision": 0.6978723404255319,
81
+ "recall": 0.3181377303588749,
82
+ "tp": 328
83
+ },
84
+ "simpleqa_verified": {
85
+ "accuracy_when_attempted": 0.05981112277019937,
86
+ "answer_coverage": 0.953,
87
+ "correct": 57,
88
+ "correct_answer_yield": 0.057,
89
+ "expected": 1000,
90
+ "graded": 1000,
91
+ "incorrect": 896,
92
+ "missing_or_error": 0,
93
+ "not_attempted": 47
94
+ }
95
+ },
96
+ "tiny": {
97
+ "abstention_bench": {
98
+ "abstention_graded": 978,
99
+ "behavior_accuracy": 0.6830265848670757,
100
+ "behavior_correct": 668,
101
+ "conditional_correctness_expected": 440,
102
+ "conditional_correctness_missing_or_error": 0,
103
+ "expected": 978,
104
+ "joint_successes": 501,
105
+ "joint_task_success": 0.5122699386503068,
106
+ "missing_or_error": 0
107
+ },
108
+ "hallulens_precisewikiqa": {
109
+ "abstained": 32,
110
+ "correct_answer_rate": 0.078,
111
+ "expected": 1000,
112
+ "false_refusal_rate": 0.032,
113
+ "graded": 1000,
114
+ "hallucination_rate_when_not_abstained": 0.9194214876033058,
115
+ "incorrect_non_abstained": 890,
116
+ "missing_or_error": 0,
117
+ "non_abstained": 968,
118
+ "unverifiable_non_abstained": 0
119
+ },
120
+ "selfaware": {
121
+ "answerable_reference_accuracy": 0.3203592814371258,
122
+ "answerable_reference_correct": 749,
123
+ "expected": 3369,
124
+ "f1": 0.34073013600572655,
125
+ "fn": 793,
126
+ "fp": 128,
127
+ "graded": 3369,
128
+ "missing_or_error": 0,
129
+ "precision": 0.6502732240437158,
130
+ "recall": 0.23084384093113483,
131
+ "tp": 238
132
+ },
133
+ "simpleqa_verified": {
134
+ "accuracy_when_attempted": 0.038865546218487396,
135
+ "answer_coverage": 0.952,
136
+ "correct": 37,
137
+ "correct_answer_yield": 0.037,
138
+ "expected": 1000,
139
+ "graded": 1000,
140
+ "incorrect": 915,
141
+ "missing_or_error": 0,
142
+ "not_attempted": 48
143
+ }
144
+ }
145
+ },
146
+ "new_generation_finish_reasons": {
147
+ "8b": {
148
+ "eos": 1979,
149
+ "length": 2390
150
+ },
151
+ "tiny": {
152
+ "eos": 2589,
153
+ "length": 1780
154
+ }
155
+ },
156
+ "no_assistant_factual_adjudication": true,
157
+ "reporting_scope": [
158
+ "simpleqa_verified",
159
+ "abstention_bench",
160
+ "hallulens_precisewikiqa",
161
+ "selfaware"
162
+ ],
163
+ "v2_inputs": {
164
+ "evaluation_sha256": "c69c16f295bd7f2e8ee62e7ab3f537efa87859fec04b01dac4d0bd03288354b3",
165
+ "expected_counts_per_model": {
166
+ "abstention_bench": 978,
167
+ "factscore_bios": 180,
168
+ "hallulens_nonexistentrefusal": 3000,
169
+ "hallulens_precisewikiqa": 1000,
170
+ "simpleqa_verified": 1000
171
+ },
172
+ "manifest_sha256": "f958c5834ce6311c1a66d66eeca90b2adc3bc9ccc26dcdf043ed9cee5f140ffb",
173
+ "reuse_sha256": "bc12dcf726e965533b391f708be4992f9633c226585459330971e2e5c8bc6574"
174
+ },
175
+ "verified_at": "2026-11-12T14:40:48.895220+00:00"
176
+ }