Add HRM KVReuse12 step114274 evaluation answers and 29 scored benchmarks
Browse files- evaluations/hrm-text/20261008-hrm-kvreuse12-step114274/README.md +56 -0
- evaluations/hrm-text/20261008-hrm-kvreuse12-step114274/extended-hrm_kvreuse12_5e191387.tar.gz +3 -0
- evaluations/hrm-text/20261008-hrm-kvreuse12-step114274/manifest.json +25 -0
- evaluations/hrm-text/20261008-hrm-kvreuse12-step114274/official-hrm_kvreuse12_5e191387.tar.gz +3 -0
evaluations/hrm-text/20261008-hrm-kvreuse12-step114274/README.md
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HRM KVReuse12 EMA: step 114274
|
| 2 |
+
|
| 3 |
+
All 112,420 answers generated; 29/31 benchmarks scored. HumanEval+ and MBPP+ pass@1 remain pending isolated EvalPlus execution.
|
| 4 |
+
|
| 5 |
+
## hrm-text-official
|
| 6 |
+
|
| 7 |
+
| Benchmark | Metric | Score (%) |
|
| 8 |
+
|---|---|---:|
|
| 9 |
+
| GSM8k | acc | 45.26 |
|
| 10 |
+
| MATH | acc | 27.70 |
|
| 11 |
+
| DROP | f1 | 27.96 |
|
| 12 |
+
| MMLU | acc | 36.57 |
|
| 13 |
+
| ARC | acc | 33.70 |
|
| 14 |
+
| HellaSwag | acc | 30.09 |
|
| 15 |
+
| Winogrande | acc | 50.51 |
|
| 16 |
+
| BoolQ | acc | 38.00 |
|
| 17 |
+
| BoolQ | macro_f1 | 27.77 |
|
| 18 |
+
|
| 19 |
+
## hrm-text-extended-v1
|
| 20 |
+
|
| 21 |
+
| Benchmark | Metric | Score (%) |
|
| 22 |
+
|---|---|---:|
|
| 23 |
+
| triviaqa | canonical_em | 2.02 |
|
| 24 |
+
| triviaqa | canonical_f1 | 3.80 |
|
| 25 |
+
| nq | canonical_em | 1.77 |
|
| 26 |
+
| nq | canonical_f1 | 4.99 |
|
| 27 |
+
| webq | canonical_em | 8.32 |
|
| 28 |
+
| webq | canonical_f1 | 16.65 |
|
| 29 |
+
| webq | all_answer_em | 13.44 |
|
| 30 |
+
| webq | all_answer_f1 | 22.79 |
|
| 31 |
+
| lambada_openai | acc | 20.36 |
|
| 32 |
+
| tydiqa_goldp | canonical_em | 48.86 |
|
| 33 |
+
| tydiqa_goldp | canonical_f1 | 61.92 |
|
| 34 |
+
| squadv2 | canonical_em | 51.18 |
|
| 35 |
+
| squadv2 | canonical_f1 | 66.74 |
|
| 36 |
+
| coqa | canonical_em | 14.03 |
|
| 37 |
+
| coqa | canonical_f1 | 15.90 |
|
| 38 |
+
| svamp | acc | 58.00 |
|
| 39 |
+
| asdiv | acc | 64.69 |
|
| 40 |
+
| mawps | acc | 79.18 |
|
| 41 |
+
| induction_head | acc | 0.40 |
|
| 42 |
+
| var_assign_d0_math | acc | 97.70 |
|
| 43 |
+
| var_assign_d0_code | acc | 98.70 |
|
| 44 |
+
| var_assign_d1_math | acc | 36.20 |
|
| 45 |
+
| var_assign_d1_code | acc | 27.80 |
|
| 46 |
+
| bigbench_dyck_languages | acc | 0.60 |
|
| 47 |
+
| bigbench_qa_wikidata | canonical_em | 33.80 |
|
| 48 |
+
| bigbench_qa_wikidata | canonical_f1 | 34.21 |
|
| 49 |
+
| arc_easy | acc | 44.78 |
|
| 50 |
+
| bigbench_cs_algorithms | acc | 6.52 |
|
| 51 |
+
| bigbench_cs_algorithms | strict_acc | 5.45 |
|
| 52 |
+
| bbh_cot | acc | 8.72 |
|
| 53 |
+
| math500 | acc | 25.00 |
|
| 54 |
+
| humaneval_plus | pass@1 | pending |
|
| 55 |
+
| mbpp_plus | pass@1 | pending |
|
| 56 |
+
|
evaluations/hrm-text/20261008-hrm-kvreuse12-step114274/extended-hrm_kvreuse12_5e191387.tar.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:33316bef22382cf63c6a07232d218f200be192cdc254de1dfe4539e8cc1a5f97
|
| 3 |
+
size 6544322
|
evaluations/hrm-text/20261008-hrm-kvreuse12-step114274/manifest.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"snapshot": "20261008-hrm-kvreuse12-step114274",
|
| 3 |
+
"checkpoint": "small_search_1epoch/HRM-H4-L3-W768-H11-L8-BP16-KVReuse12-causal-AdamATan2-EMA-seed0-1epoch__20261007-031120_hrm-h4-bp16-kvreuse12-h20x16--5e191387/step_114274",
|
| 4 |
+
"weights": "ema",
|
| 5 |
+
"scored_benchmarks": 29,
|
| 6 |
+
"generated_benchmarks": 31,
|
| 7 |
+
"examples": 112420,
|
| 8 |
+
"files": [
|
| 9 |
+
{
|
| 10 |
+
"path": "README.md",
|
| 11 |
+
"bytes": 1665,
|
| 12 |
+
"sha256": "681f8645136f31cb0e13fa03bffff78855ca8bbfd03eb708b8cde8d0a5c83d2c"
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"path": "extended-hrm_kvreuse12_5e191387.tar.gz",
|
| 16 |
+
"bytes": 6544322,
|
| 17 |
+
"sha256": "33316bef22382cf63c6a07232d218f200be192cdc254de1dfe4539e8cc1a5f97"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"path": "official-hrm_kvreuse12_5e191387.tar.gz",
|
| 21 |
+
"bytes": 3778328,
|
| 22 |
+
"sha256": "7d0024d67c62444a7696db7fbe008761cc64b8dc6216f7487780163fbdc27b04"
|
| 23 |
+
}
|
| 24 |
+
]
|
| 25 |
+
}
|
evaluations/hrm-text/20261008-hrm-kvreuse12-step114274/official-hrm_kvreuse12_5e191387.tar.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7d0024d67c62444a7696db7fbe008761cc64b8dc6216f7487780163fbdc27b04
|
| 3 |
+
size 3778328
|