lm_eval_base.json

{
  "hellaswag": {
    "name": "hellaswag",
    "alias": "hellaswag",
    "sample_len": 10042,
    "acc,none": 0.2808205536745668,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.29416450906193986,
    "acc_norm_stderr,none": "N/A"
  },
  "arc_easy": {
    "name": "arc_easy",
    "alias": "arc_easy",
    "sample_len": 2376,
    "acc,none": 0.45664983164983164,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.4086700336700337,
    "acc_norm_stderr,none": "N/A"
  },
  "arc_challenge": {
    "name": "arc_challenge",
    "alias": "arc_challenge",
    "sample_len": 1172,
    "acc,none": 0.189419795221843,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.22866894197952217,
    "acc_norm_stderr,none": "N/A"
  },
  "piqa": {
    "name": "piqa",
    "alias": "piqa",
    "sample_len": 1838,
    "acc,none": 0.6202393906420022,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.5984766050054406,
    "acc_norm_stderr,none": "N/A"
  },
  "winogrande": {
    "name": "winogrande",
    "alias": "winogrande",
    "sample_len": 1267,
    "acc,none": 0.5035516969218626,
    "acc_stderr,none": "N/A"
  },
  "openbookqa": {
    "name": "openbookqa",
    "alias": "openbookqa",
    "sample_len": 500,
    "acc,none": 0.156,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.318,
    "acc_norm_stderr,none": "N/A"
  },
  "sciq": {
    "name": "sciq",
    "alias": "sciq",
    "sample_len": 1000,
    "acc,none": 0.719,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.59,
    "acc_norm_stderr,none": "N/A"
  },
  "lambada_openai": {
    "name": "lambada_openai",
    "alias": "lambada_openai",
    "sample_len": 5153,
    "perplexity,none": 176.07916349254722,
    "perplexity_stderr,none": "N/A",
    "acc,none": 0.18144770036871724,
    "acc_stderr,none": "N/A"
  }
}