{
  "format": "dwarf_v2_standard_eval_summary_v1",
  "model": "DWARF-v2 55M Base — FA@L3",
  "checkpoint_sha256": "a35ee66025cf616a54c215f865263c38230b4ad2aafef466070f3b73e1da0279",
  "release_model_safetensors_sha256": "cac0db1dbc008e546df7b227952bd71307f9d0b0dc4c909c3d5bbc0aeb8b2c95",
  "evaluator": {
    "name": "lm-evaluation-harness",
    "version": "0.4.12",
    "model_backend": "hf",
    "model_args": "pretrained=<staged release>,trust_remote_code=True,dtype=bfloat16",
    "num_fewshot": 0,
    "seed": 42,
    "batch_size_per_process": 4
  },
  "execution": {
    "run_id": "standard_0shot_lmeval_0.4.12_20260719T190728Z",
    "all_task_exit_codes_zero": true,
    "parallelism": "four independent task processes per GPU; RTX 4090 and RTX 3090",
    "raw_prediction_level_artifacts": "retained locally in an evaluation-only audit directory and intentionally excluded from the model upload because they contain held-out evaluation labels"
  },
  "results": [
    {
      "benchmark": "ARC-Easy",
      "harness_task": "arc_easy",
      "split": "test",
      "examples": 2376,
      "primary_metric": "acc_norm",
      "value": 0.39141414141414144,
      "stderr": 0.010014917532627792,
      "secondary_metrics": {"acc": 0.4217171717171717}
    },
    {
      "benchmark": "ARC-Challenge",
      "harness_task": "arc_challenge",
      "split": "test",
      "examples": 1172,
      "primary_metric": "acc_norm",
      "value": 0.22184300341296928,
      "stderr": 0.012141659068148035,
      "secondary_metrics": {"acc": 0.189419795221843}
    },
    {
      "benchmark": "HellaSwag",
      "harness_task": "hellaswag",
      "split": "validation",
      "examples": 10042,
      "primary_metric": "acc_norm",
      "value": 0.2740489942242581,
      "stderr": 0.004451222241494582,
      "secondary_metrics": {"acc": 0.2700657239593706}
    },
    {
      "benchmark": "WinoGrande",
      "harness_task": "winogrande",
      "split": "validation",
      "examples": 1267,
      "primary_metric": "acc",
      "value": 0.516179952644041,
      "stderr": 0.014045126130978676
    },
    {
      "benchmark": "PIQA",
      "harness_task": "piqa",
      "split": "validation",
      "examples": 1838,
      "primary_metric": "acc_norm",
      "value": 0.5875952121871599,
      "stderr": 0.011485407152743092,
      "secondary_metrics": {"acc": 0.5914036996735582}
    },
    {
      "benchmark": "OpenBookQA (main)",
      "harness_task": "openbookqa_main_local",
      "split": "test",
      "examples": 500,
      "primary_metric": "acc_norm",
      "value": 0.258,
      "stderr": 0.019586711785215868,
      "secondary_metrics": {"acc": 0.158},
      "note": "Official main Parquet data; local transport adapter preserved the lm-eval 0.4.12 task prompt, answer-index target, choices, and metrics."
    },
    {
      "benchmark": "BoolQ",
      "harness_task": "boolq_local",
      "split": "validation",
      "examples": 3270,
      "primary_metric": "acc",
      "value": 0.6009174311926605,
      "stderr": 0.008565077958836848,
      "note": "Official BoolQ Parquet data; local transport adapter preserved the lm-eval 0.4.12 task prompt, choices, and metric."
    },
    {
      "benchmark": "LAMBADA (OpenAI)",
      "harness_task": "lambada_openai",
      "split": "test",
      "examples": 5153,
      "primary_metric": "acc",
      "value": 0.19367358820104794,
      "stderr": 0.005505575403793799,
      "secondary_metrics": {"perplexity": 217.88268117058854, "perplexity_stderr": 9.941434408306826},
      "target_token_audit": {"one_token": 3585, "multi_token": 1568, "multi_token_fraction": 0.30428876382689696, "max_target_tokens": 8}
    }
  ]
}