lm_eval_sft.json

{
  "hellaswag": {
    "name": "hellaswag",
    "alias": "hellaswag",
    "sample_len": 10042,
    "acc,none": 0.28251344353714397,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.2954590718980283,
    "acc_norm_stderr,none": "N/A"
  },
  "arc_easy": {
    "name": "arc_easy",
    "alias": "arc_easy",
    "sample_len": 2376,
    "acc,none": 0.4524410774410774,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.4128787878787879,
    "acc_norm_stderr,none": "N/A"
  },
  "arc_challenge": {
    "name": "arc_challenge",
    "alias": "arc_challenge",
    "sample_len": 1172,
    "acc,none": 0.2030716723549488,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.2525597269624573,
    "acc_norm_stderr,none": "N/A"
  },
  "piqa": {
    "name": "piqa",
    "alias": "piqa",
    "sample_len": 1838,
    "acc,none": 0.6169749727965179,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.6011969532100109,
    "acc_norm_stderr,none": "N/A"
  },
  "winogrande": {
    "name": "winogrande",
    "alias": "winogrande",
    "sample_len": 1267,
    "acc,none": 0.5138121546961326,
    "acc_stderr,none": "N/A"
  },
  "openbookqa": {
    "name": "openbookqa",
    "alias": "openbookqa",
    "sample_len": 500,
    "acc,none": 0.19,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.318,
    "acc_norm_stderr,none": "N/A"
  },
  "sciq": {
    "name": "sciq",
    "alias": "sciq",
    "sample_len": 1000,
    "acc,none": 0.662,
    "acc_stderr,none": "N/A",
    "acc_norm,none": 0.576,
    "acc_norm_stderr,none": "N/A"
  },
  "lambada_openai": {
    "name": "lambada_openai",
    "alias": "lambada_openai",
    "sample_len": 5153,
    "perplexity,none": 241.01776041846503,
    "perplexity_stderr,none": "N/A",
    "acc,none": 0.17581991073161266,
    "acc_stderr,none": "N/A"
  }
}