| { |
| "format": "dwarf_v2_standard_eval_summary_v1", |
| "model": "DWARF-v2 55M Base — FA@L3", |
| "checkpoint_sha256": "a35ee66025cf616a54c215f865263c38230b4ad2aafef466070f3b73e1da0279", |
| "release_model_safetensors_sha256": "cac0db1dbc008e546df7b227952bd71307f9d0b0dc4c909c3d5bbc0aeb8b2c95", |
| "evaluator": { |
| "name": "lm-evaluation-harness", |
| "version": "0.4.12", |
| "model_backend": "hf", |
| "model_args": "pretrained=<staged release>,trust_remote_code=True,dtype=bfloat16", |
| "num_fewshot": 0, |
| "seed": 42, |
| "batch_size_per_process": 4 |
| }, |
| "execution": { |
| "run_id": "standard_0shot_lmeval_0.4.12_20260719T190728Z", |
| "all_task_exit_codes_zero": true, |
| "parallelism": "four independent task processes per GPU; RTX 4090 and RTX 3090", |
| "raw_prediction_level_artifacts": "retained locally in an evaluation-only audit directory and intentionally excluded from the model upload because they contain held-out evaluation labels" |
| }, |
| "results": [ |
| { |
| "benchmark": "ARC-Easy", |
| "harness_task": "arc_easy", |
| "split": "test", |
| "examples": 2376, |
| "primary_metric": "acc_norm", |
| "value": 0.39141414141414144, |
| "stderr": 0.010014917532627792, |
| "secondary_metrics": {"acc": 0.4217171717171717} |
| }, |
| { |
| "benchmark": "ARC-Challenge", |
| "harness_task": "arc_challenge", |
| "split": "test", |
| "examples": 1172, |
| "primary_metric": "acc_norm", |
| "value": 0.22184300341296928, |
| "stderr": 0.012141659068148035, |
| "secondary_metrics": {"acc": 0.189419795221843} |
| }, |
| { |
| "benchmark": "HellaSwag", |
| "harness_task": "hellaswag", |
| "split": "validation", |
| "examples": 10042, |
| "primary_metric": "acc_norm", |
| "value": 0.2740489942242581, |
| "stderr": 0.004451222241494582, |
| "secondary_metrics": {"acc": 0.2700657239593706} |
| }, |
| { |
| "benchmark": "WinoGrande", |
| "harness_task": "winogrande", |
| "split": "validation", |
| "examples": 1267, |
| "primary_metric": "acc", |
| "value": 0.516179952644041, |
| "stderr": 0.014045126130978676 |
| }, |
| { |
| "benchmark": "PIQA", |
| "harness_task": "piqa", |
| "split": "validation", |
| "examples": 1838, |
| "primary_metric": "acc_norm", |
| "value": 0.5875952121871599, |
| "stderr": 0.011485407152743092, |
| "secondary_metrics": {"acc": 0.5914036996735582} |
| }, |
| { |
| "benchmark": "OpenBookQA (main)", |
| "harness_task": "openbookqa_main_local", |
| "split": "test", |
| "examples": 500, |
| "primary_metric": "acc_norm", |
| "value": 0.258, |
| "stderr": 0.019586711785215868, |
| "secondary_metrics": {"acc": 0.158}, |
| "note": "Official main Parquet data; local transport adapter preserved the lm-eval 0.4.12 task prompt, answer-index target, choices, and metrics." |
| }, |
| { |
| "benchmark": "BoolQ", |
| "harness_task": "boolq_local", |
| "split": "validation", |
| "examples": 3270, |
| "primary_metric": "acc", |
| "value": 0.6009174311926605, |
| "stderr": 0.008565077958836848, |
| "note": "Official BoolQ Parquet data; local transport adapter preserved the lm-eval 0.4.12 task prompt, choices, and metric." |
| }, |
| { |
| "benchmark": "LAMBADA (OpenAI)", |
| "harness_task": "lambada_openai", |
| "split": "test", |
| "examples": 5153, |
| "primary_metric": "acc", |
| "value": 0.19367358820104794, |
| "stderr": 0.005505575403793799, |
| "secondary_metrics": {"perplexity": 217.88268117058854, "perplexity_stderr": 9.941434408306826}, |
| "target_token_audit": {"one_token": 3585, "multi_token": 1568, "multi_token_fraction": 0.30428876382689696, "max_target_tokens": 8} |
| } |
| ] |
| } |
|
|
