| { | |
| "harness": "lm-evaluation-harness 0.4.13", | |
| "fewshot": 0, | |
| "limit_per_task": 500, | |
| "device": "cpu", | |
| "nova_model": "plasmova/nova-v3", | |
| "baseline_model": "HuggingFaceTB/SmolLM2-135M", | |
| "comparison": { | |
| "hellaswag": { | |
| "metric": "acc_norm", | |
| "nova": 0.436, | |
| "smollm2": 0.442, | |
| "delta": -0.006000000000000005, | |
| "nova_stderr": 0.0221989546414768, | |
| "smollm2_stderr": 0.02223197069632112 | |
| }, | |
| "winogrande": { | |
| "acc": { | |
| "nova": 0.526, | |
| "smollm2": 0.52, | |
| "delta": 0.006000000000000005, | |
| "nova_stderr": 0.02235279165091416, | |
| "smollm2_stderr": 0.02236516042423134 | |
| }, | |
| "acc_norm": { | |
| "nova": 0.522, | |
| "smollm2": 0.506, | |
| "delta": 0.016000000000000014, | |
| "nova_stderr": 0.02236139673920787, | |
| "smollm2_stderr": 0.022381462412439324 | |
| } | |
| } | |
| }, | |
| "nova_evaluation_seconds": "452.16297420003684", | |
| "baseline_evaluation_seconds": "185.393080800015" | |
| } | |