| { | |
| "config": { | |
| "batch_size": 64, | |
| "batch_sizes": [], | |
| "bootstrap_iters": 100000, | |
| "device": "cuda", | |
| "fewshot_seed": 1234, | |
| "gen_kwargs": null, | |
| "limit": null, | |
| "model": "MicroLoopHarnessLM", | |
| "model_args": null, | |
| "numpy_seed": 1234, | |
| "random_seed": 0, | |
| "torch_seed": 1234, | |
| "use_cache": null | |
| }, | |
| "configs": { | |
| "arc_challenge": { | |
| "dataset_name": "ARC-Challenge", | |
| "dataset_path": "allenai/ai2_arc", | |
| "description": "", | |
| "doc_to_choice": "{{choices.text}}", | |
| "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", | |
| "doc_to_target": "{{choices.label.index(answerKey)}}", | |
| "doc_to_text": "Question: {{question}}\nAnswer:", | |
| "fewshot_config": { | |
| "doc_to_choice": "{{choices.text}}", | |
| "doc_to_target": "{{choices.label.index(answerKey)}}", | |
| "doc_to_text": "Question: {{question}}\nAnswer:", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_indices": null, | |
| "gen_prefix": null, | |
| "process_docs": null, | |
| "sampler": "default", | |
| "samples": null, | |
| "split": null, | |
| "target_delimiter": " " | |
| }, | |
| "fewshot_delimiter": "\n\n", | |
| "metadata": { | |
| "config_source": "D:\\llm\\frost\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\arc\\arc_challenge.yaml", | |
| "version": 1.0 | |
| }, | |
| "metric_list": [ | |
| { | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "metric": "acc" | |
| }, | |
| { | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "metric": "acc_norm" | |
| } | |
| ], | |
| "num_fewshot": 0, | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": true, | |
| "target_delimiter": " ", | |
| "task": "arc_challenge", | |
| "test_split": "test", | |
| "training_split": "train", | |
| "unsafe_code": false, | |
| "validation_split": "validation" | |
| }, | |
| "arc_easy": { | |
| "dataset_name": "ARC-Easy", | |
| "dataset_path": "allenai/ai2_arc", | |
| "description": "", | |
| "doc_to_choice": "{{choices.text}}", | |
| "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", | |
| "doc_to_target": "{{choices.label.index(answerKey)}}", | |
| "doc_to_text": "Question: {{question}}\nAnswer:", | |
| "fewshot_config": { | |
| "doc_to_choice": "{{choices.text}}", | |
| "doc_to_target": "{{choices.label.index(answerKey)}}", | |
| "doc_to_text": "Question: {{question}}\nAnswer:", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_indices": null, | |
| "gen_prefix": null, | |
| "process_docs": null, | |
| "sampler": "default", | |
| "samples": null, | |
| "split": null, | |
| "target_delimiter": " " | |
| }, | |
| "fewshot_delimiter": "\n\n", | |
| "metadata": { | |
| "config_source": "D:\\llm\\frost\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\arc\\arc_easy.yaml", | |
| "version": 1.0 | |
| }, | |
| "metric_list": [ | |
| { | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "metric": "acc" | |
| }, | |
| { | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "metric": "acc_norm" | |
| } | |
| ], | |
| "num_fewshot": 0, | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": true, | |
| "target_delimiter": " ", | |
| "task": "arc_easy", | |
| "test_split": "test", | |
| "training_split": "train", | |
| "unsafe_code": false, | |
| "validation_split": "validation" | |
| }, | |
| "hellaswag": { | |
| "dataset_path": "Rowan/hellaswag", | |
| "description": "", | |
| "doc_to_choice": "choices", | |
| "doc_to_target": "{{label}}", | |
| "doc_to_text": "{{query}}", | |
| "fewshot_config": { | |
| "doc_to_choice": "choices", | |
| "doc_to_target": "{{label}}", | |
| "doc_to_text": "{{query}}", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_indices": null, | |
| "gen_prefix": null, | |
| "process_docs": "<callable function>", | |
| "sampler": "default", | |
| "samples": null, | |
| "split": null, | |
| "target_delimiter": " " | |
| }, | |
| "fewshot_delimiter": "\n\n", | |
| "metadata": { | |
| "config_source": "D:\\llm\\frost\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\hellaswag\\hellaswag.yaml", | |
| "version": 1.0 | |
| }, | |
| "metric_list": [ | |
| { | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "metric": "acc" | |
| }, | |
| { | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "metric": "acc_norm" | |
| } | |
| ], | |
| "num_fewshot": 0, | |
| "output_type": "multiple_choice", | |
| "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "target_delimiter": " ", | |
| "task": "hellaswag", | |
| "training_split": "train", | |
| "unsafe_code": false, | |
| "validation_split": "validation" | |
| }, | |
| "piqa": { | |
| "dataset_path": "baber/piqa", | |
| "description": "", | |
| "doc_to_choice": "{{[sol1, sol2]}}", | |
| "doc_to_decontamination_query": "goal", | |
| "doc_to_target": "label", | |
| "doc_to_text": "Question: {{goal}}\nAnswer:", | |
| "fewshot_config": { | |
| "doc_to_choice": "{{[sol1, sol2]}}", | |
| "doc_to_target": "label", | |
| "doc_to_text": "Question: {{goal}}\nAnswer:", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_indices": null, | |
| "gen_prefix": null, | |
| "process_docs": null, | |
| "sampler": "default", | |
| "samples": null, | |
| "split": null, | |
| "target_delimiter": " " | |
| }, | |
| "fewshot_delimiter": "\n\n", | |
| "metadata": { | |
| "config_source": "D:\\llm\\frost\\.venv\\Lib\\site-packages\\lm_eval\\tasks\\piqa\\piqa.yaml", | |
| "version": 1.0 | |
| }, | |
| "metric_list": [ | |
| { | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "metric": "acc" | |
| }, | |
| { | |
| "aggregation": "mean", | |
| "higher_is_better": true, | |
| "metric": "acc_norm" | |
| } | |
| ], | |
| "num_fewshot": 0, | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": true, | |
| "target_delimiter": " ", | |
| "task": "piqa", | |
| "training_split": "train", | |
| "unsafe_code": false, | |
| "validation_split": "validation" | |
| } | |
| }, | |
| "date": 1786715056.057948, | |
| "frost_evaluation": { | |
| "batch_size": 64, | |
| "checkpoint": "D:\\llm\\frost\\artifacts\\runs\\natural20b_pilot_muon_b32\\checkpoint-tokens-0500000000", | |
| "device": "cuda", | |
| "num_fewshot": 0, | |
| "tokenizer": "D:\\llm\\frost\\artifacts\\runs\\e3_tokenizers\\tok_4k_digit" | |
| }, | |
| "git_hash": "9ac6cbb", | |
| "group_subtasks": {}, | |
| "higher_is_better": { | |
| "arc_challenge": { | |
| "acc": true, | |
| "acc_norm": true | |
| }, | |
| "arc_easy": { | |
| "acc": true, | |
| "acc_norm": true | |
| }, | |
| "hellaswag": { | |
| "acc": true, | |
| "acc_norm": true | |
| }, | |
| "piqa": { | |
| "acc": true, | |
| "acc_norm": true | |
| } | |
| }, | |
| "lm_eval_version": "0.4.12", | |
| "n-samples": { | |
| "arc_challenge": { | |
| "effective": 1172, | |
| "original": 1172 | |
| }, | |
| "arc_easy": { | |
| "effective": 2376, | |
| "original": 2376 | |
| }, | |
| "hellaswag": { | |
| "effective": 10042, | |
| "original": 10042 | |
| }, | |
| "piqa": { | |
| "effective": 1838, | |
| "original": 1838 | |
| } | |
| }, | |
| "n-shot": { | |
| "arc_challenge": 0, | |
| "arc_easy": 0, | |
| "hellaswag": 0, | |
| "piqa": 0 | |
| }, | |
| "pretty_env_info": "the JSON object must be str, bytes or bytearray, not NoneType", | |
| "results": { | |
| "arc_challenge": { | |
| "acc,none": 0.1697952218430034, | |
| "acc_norm,none": 0.2226962457337884, | |
| "acc_norm_stderr,none": 0.012158314774829888, | |
| "acc_stderr,none": 0.010971775157784207, | |
| "alias": "arc_challenge", | |
| "name": "arc_challenge", | |
| "sample_len": 1172 | |
| }, | |
| "arc_easy": { | |
| "acc,none": 0.3409090909090909, | |
| "acc_norm,none": 0.3312289562289562, | |
| "acc_norm_stderr,none": 0.009657641311350737, | |
| "acc_stderr,none": 0.009726579593423981, | |
| "alias": "arc_easy", | |
| "name": "arc_easy", | |
| "sample_len": 2376 | |
| }, | |
| "hellaswag": { | |
| "acc,none": 0.2669786895040829, | |
| "acc_norm,none": 0.2756423023302131, | |
| "acc_norm_stderr,none": 0.004459241474518529, | |
| "acc_stderr,none": 0.004414770331224373, | |
| "alias": "hellaswag", | |
| "name": "hellaswag", | |
| "sample_len": 10042 | |
| }, | |
| "piqa": { | |
| "acc,none": 0.5625680087051143, | |
| "acc_norm,none": 0.5554951033732318, | |
| "acc_norm_stderr,none": 0.011593746871584282, | |
| "acc_stderr,none": 0.011574126069682592, | |
| "alias": "piqa", | |
| "name": "piqa", | |
| "sample_len": 1838 | |
| } | |
| }, | |
| "transformers_version": "5.14.1", | |
| "upper_git_hash": null, | |
| "versions": { | |
| "arc_challenge": 1.0, | |
| "arc_easy": 1.0, | |
| "hellaswag": 1.0, | |
| "piqa": 1.0 | |
| } | |
| } |