{
  "config": {
    "batch_size": 64,
    "batch_sizes": [],
    "bootstrap_iters": 100000,
    "device": "cuda",
    "fewshot_seed": 1234,
    "gen_kwargs": null,
    "limit": null,
    "model": "MicroLoopHarnessLM",
    "model_args": null,
    "numpy_seed": 1234,
    "random_seed": 0,
    "torch_seed": 1234,
    "use_cache": null
  },
  "configs": {
    "arc_challenge": {
      "dataset_name": "ARC-Challenge",
      "dataset_path": "allenai/ai2_arc",
      "description": "",
      "doc_to_choice": "{{choices.text}}",
      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
      "doc_to_target": "{{choices.label.index(answerKey)}}",
      "doc_to_text": "Question: {{question}}\nAnswer:",
      "fewshot_config": {
        "doc_to_choice": "{{choices.text}}",
        "doc_to_target": "{{choices.label.index(answerKey)}}",
        "doc_to_text": "Question: {{question}}\nAnswer:",
        "fewshot_delimiter": "\n\n",
        "fewshot_indices": null,
        "gen_prefix": null,
        "process_docs": null,
        "sampler": "default",
        "samples": null,
        "split": null,
        "target_delimiter": " "
      },
      "fewshot_delimiter": "\n\n",
      "metadata": {
        "config_source": "D:\\llm\\frost\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\arc\\arc_challenge.yaml",
        "version": 1.0
      },
      "metric_list": [
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc"
        },
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc_norm"
        }
      ],
      "num_fewshot": 0,
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": true,
      "target_delimiter": " ",
      "task": "arc_challenge",
      "test_split": "test",
      "training_split": "train",
      "unsafe_code": false,
      "validation_split": "validation"
    },
    "arc_easy": {
      "dataset_name": "ARC-Easy",
      "dataset_path": "allenai/ai2_arc",
      "description": "",
      "doc_to_choice": "{{choices.text}}",
      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
      "doc_to_target": "{{choices.label.index(answerKey)}}",
      "doc_to_text": "Question: {{question}}\nAnswer:",
      "fewshot_config": {
        "doc_to_choice": "{{choices.text}}",
        "doc_to_target": "{{choices.label.index(answerKey)}}",
        "doc_to_text": "Question: {{question}}\nAnswer:",
        "fewshot_delimiter": "\n\n",
        "fewshot_indices": null,
        "gen_prefix": null,
        "process_docs": null,
        "sampler": "default",
        "samples": null,
        "split": null,
        "target_delimiter": " "
      },
      "fewshot_delimiter": "\n\n",
      "metadata": {
        "config_source": "D:\\llm\\frost\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\arc\\arc_easy.yaml",
        "version": 1.0
      },
      "metric_list": [
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc"
        },
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc_norm"
        }
      ],
      "num_fewshot": 0,
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": true,
      "target_delimiter": " ",
      "task": "arc_easy",
      "test_split": "test",
      "training_split": "train",
      "unsafe_code": false,
      "validation_split": "validation"
    },
    "hellaswag": {
      "dataset_path": "Rowan/hellaswag",
      "description": "",
      "doc_to_choice": "choices",
      "doc_to_target": "{{label}}",
      "doc_to_text": "{{query}}",
      "fewshot_config": {
        "doc_to_choice": "choices",
        "doc_to_target": "{{label}}",
        "doc_to_text": "{{query}}",
        "fewshot_delimiter": "\n\n",
        "fewshot_indices": null,
        "gen_prefix": null,
        "process_docs": "<callable function>",
        "sampler": "default",
        "samples": null,
        "split": null,
        "target_delimiter": " "
      },
      "fewshot_delimiter": "\n\n",
      "metadata": {
        "config_source": "D:\\llm\\frost\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\hellaswag\\hellaswag.yaml",
        "version": 1.0
      },
      "metric_list": [
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc"
        },
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc_norm"
        }
      ],
      "num_fewshot": 0,
      "output_type": "multiple_choice",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
      "repeats": 1,
      "should_decontaminate": false,
      "target_delimiter": " ",
      "task": "hellaswag",
      "training_split": "train",
      "unsafe_code": false,
      "validation_split": "validation"
    },
    "piqa": {
      "dataset_path": "baber/piqa",
      "description": "",
      "doc_to_choice": "{{[sol1, sol2]}}",
      "doc_to_decontamination_query": "goal",
      "doc_to_target": "label",
      "doc_to_text": "Question: {{goal}}\nAnswer:",
      "fewshot_config": {
        "doc_to_choice": "{{[sol1, sol2]}}",
        "doc_to_target": "label",
        "doc_to_text": "Question: {{goal}}\nAnswer:",
        "fewshot_delimiter": "\n\n",
        "fewshot_indices": null,
        "gen_prefix": null,
        "process_docs": null,
        "sampler": "default",
        "samples": null,
        "split": null,
        "target_delimiter": " "
      },
      "fewshot_delimiter": "\n\n",
      "metadata": {
        "config_source": "D:\\llm\\frost\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\piqa\\piqa.yaml",
        "version": 1.0
      },
      "metric_list": [
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc"
        },
        {
          "aggregation": "mean",
          "higher_is_better": true,
          "metric": "acc_norm"
        }
      ],
      "num_fewshot": 0,
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": true,
      "target_delimiter": " ",
      "task": "piqa",
      "training_split": "train",
      "unsafe_code": false,
      "validation_split": "validation"
    }
  },
  "date": 1786715056.057948,
  "frost_evaluation": {
    "batch_size": 64,
    "checkpoint": "D:\\llm\\frost\\artifacts\\runs\\natural20b_pilot_muon_b32\\checkpoint-tokens-0500000000",
    "device": "cuda",
    "num_fewshot": 0,
    "tokenizer": "D:\\llm\\frost\\artifacts\\runs\\e3_tokenizers\\tok_4k_digit"
  },
  "git_hash": "9ac6cbb",
  "group_subtasks": {},
  "higher_is_better": {
    "arc_challenge": {
      "acc": true,
      "acc_norm": true
    },
    "arc_easy": {
      "acc": true,
      "acc_norm": true
    },
    "hellaswag": {
      "acc": true,
      "acc_norm": true
    },
    "piqa": {
      "acc": true,
      "acc_norm": true
    }
  },
  "lm_eval_version": "0.4.12",
  "n-samples": {
    "arc_challenge": {
      "effective": 1172,
      "original": 1172
    },
    "arc_easy": {
      "effective": 2376,
      "original": 2376
    },
    "hellaswag": {
      "effective": 10042,
      "original": 10042
    },
    "piqa": {
      "effective": 1838,
      "original": 1838
    }
  },
  "n-shot": {
    "arc_challenge": 0,
    "arc_easy": 0,
    "hellaswag": 0,
    "piqa": 0
  },
  "pretty_env_info": "the JSON object must be str, bytes or bytearray, not NoneType",
  "results": {
    "arc_challenge": {
      "acc,none": 0.1697952218430034,
      "acc_norm,none": 0.2226962457337884,
      "acc_norm_stderr,none": 0.012158314774829888,
      "acc_stderr,none": 0.010971775157784207,
      "alias": "arc_challenge",
      "name": "arc_challenge",
      "sample_len": 1172
    },
    "arc_easy": {
      "acc,none": 0.3409090909090909,
      "acc_norm,none": 0.3312289562289562,
      "acc_norm_stderr,none": 0.009657641311350737,
      "acc_stderr,none": 0.009726579593423981,
      "alias": "arc_easy",
      "name": "arc_easy",
      "sample_len": 2376
    },
    "hellaswag": {
      "acc,none": 0.2669786895040829,
      "acc_norm,none": 0.2756423023302131,
      "acc_norm_stderr,none": 0.004459241474518529,
      "acc_stderr,none": 0.004414770331224373,
      "alias": "hellaswag",
      "name": "hellaswag",
      "sample_len": 10042
    },
    "piqa": {
      "acc,none": 0.5625680087051143,
      "acc_norm,none": 0.5554951033732318,
      "acc_norm_stderr,none": 0.011593746871584282,
      "acc_stderr,none": 0.011574126069682592,
      "alias": "piqa",
      "name": "piqa",
      "sample_len": 1838
    }
  },
  "transformers_version": "5.14.1",
  "upper_git_hash": null,
  "versions": {
    "arc_challenge": 1.0,
    "arc_easy": 1.0,
    "hellaswag": 1.0,
    "piqa": 1.0
  }
}