{
  "eval_version": "sn-loop-eval-v1",
  "seed": 20261030,
  "tasks": {
    "ifeval": {
      "n": 60,
      "file": "ifeval_input_data.jsonl",
      "sha256": "3e9e9e7bd42ef54862f0c71c9a4e6f0b17a9c259f8f4b2b4e5fbcad917228118",
      "repo": "google/IFEval",
      "rev": "966cd89545d6b6acfd7638bc708b98261ca58e84",
      "split": "train (only split; = the IFEval eval set)",
      "license": "Apache-2.0",
      "eligible_pool": "495/541 prompts with only supported instruction ids",
      "raw_file_sha256": "6a85310ca8ce15eff755aa08a3a4ff931c7e273e7515ebb3c492ea85fd8288f2"
    },
    "gsm8k": {
      "n": 80,
      "file": "main/test-00000-of-00001.parquet",
      "sha256": "5bb91d328c1ffc424d6679fe8edb36ed08152e4c75e75b343303e15cacc2da49",
      "repo": "openai/gsm8k",
      "rev": "740312add88f781978c0658806c59bc2815b9866",
      "split": "main/test",
      "license": "MIT",
      "raw_file_sha256": "ee7b8da9e381df27b9e3f7758a159ab2bdaa4dbaa910546cbbc47e0cb44e4f59"
    },
    "mmlu_pro": {
      "n": 60,
      "file": "data/test-00000-of-00001.parquet",
      "sha256": "e9d8d5717def43a0b995710ab093a13de48b4cfb4db19cd7c66affacfb492ea7",
      "repo": "TIGER-Lab/MMLU-Pro",
      "rev": "b189ec765aa7ed75c8acfea42df31fdae71f97be",
      "split": "test",
      "license": "MIT",
      "raw_file_sha256": "0e24a191921c2f453518a537a8b2117bd137e7714d4ef1565e9ba06c1ecb9ad8"
    },
    "humaneval": {
      "n": 40,
      "file": "openai_humaneval/test-00000-of-00001.parquet",
      "sha256": "1e775e30085b890c19c48a583bfabb8f5f54598b5bf54796839b92e5bafdab9f",
      "repo": "openai/openai_humaneval",
      "rev": "7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544",
      "split": "test",
      "license": "MIT",
      "raw_file_sha256": "2f2871a15fbc95b6c683043359f4ed8e144c5a1c4f24f25f66bc51f598dfcfb6"
    },
    "tools": {
      "n": 20,
      "file": "eval/tool_cases.py",
      "sha256": "037b2e288f4f063e8a2d994799b4a51e4d97740b1cf62da75844243749dbb08a",
      "repo": "(hand-written, eval/tool_cases.py)",
      "rev": "in-repo",
      "split": "n/a",
      "license": "Apache-2.0 (this project); BFCL-style categories, no BFCL text"
    },
    "vqa": {
      "n": 40,
      "file": "data/validation-00000-of-00003.parquet",
      "sha256": "7d47303ecdf3386d04a852ab8b7e8be8d34137a1c4d10143331cc7175ce80d9d",
      "repo": "lmms-lab/textvqa",
      "rev": "9c0699cd19768ac5ab97568f6b3cbac4c0062884",
      "split": "validation (shard 0 of 3, 1667 of 5000 rows)",
      "license": "TextVQA annotations CC-BY-4.0; images from OpenImages (CC-BY-2.0, per-image Flickr attribution)",
      "eligible_pool": "1630/1667 questions with a >=3-annotator consensus answer",
      "raw_file_sha256": "491649b5b8add2c5bde13a1e36707002fcd11898699255d65e2e568ac4fbddff"
    }
  }
}