{ "results": { "hellaswag": { "name": "hellaswag", "alias": "hellaswag", "sample_len": 10042, "acc,none": 0.3040231029675364, "acc_stderr,none": 0.00459052357205796, "acc_norm,none": 0.34534953196574386, "acc_norm_stderr,none": 0.004745103543901273 }, "arc_challenge": { "name": "arc_challenge", "alias": "arc_challenge", "sample_len": 1172, "acc,none": 0.18430034129692832, "acc_stderr,none": 0.011330517933037408, "acc_norm,none": 0.2380546075085324, "acc_norm_stderr,none": 0.012445770028026208 } }, "group_subtasks": {}, "configs": { "arc_challenge": { "task": "arc_challenge", "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Challenge", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": "{{choices.text}}", "doc_to_target": "{{choices.label.index(answerKey)}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0, "pretrained": "absltnull/predBor-v0.5", "config_source": "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\arc\\arc_challenge.yaml" } }, "hellaswag": { "task": "hellaswag", "dataset_path": "Rowan/hellaswag", "training_split": "train", "validation_split": "validation", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", "doc_to_text": "{{query}}", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": "", "fewshot_indices": null, "samples": null, "doc_to_text": "{{query}}", "doc_to_choice": "choices", "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "absltnull/predBor-v0.5", "config_source": "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lm_eval\\tasks\\hellaswag\\hellaswag.yaml" } } }, "versions": { "arc_challenge": 1.0, "hellaswag": 1.0 }, "n-shot": { "arc_challenge": 0, "hellaswag": 0 }, "higher_is_better": { "arc_challenge": { "acc": true, "acc_norm": true }, "hellaswag": { "acc": true, "acc_norm": true } }, "n-samples": { "hellaswag": { "original": 10042, "effective": 10042 }, "arc_challenge": { "original": 1172, "effective": 1172 } }, "config": { "model": "hf", "model_args": { "pretrained": "absltnull/predBor-v0.5" }, "model_num_parameters": 779392512, "model_dtype": "torch.float32", "model_revision": "main", "model_sha": "d2f6884ce777628ec6e05aba33b2154c0fc01890", "batch_size": "4", "batch_sizes": [], "device": "cuda:0", "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": {}, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": null, "date": 1782048025.6308908, "pretty_env_info": "PyTorch version: 2.11.0+cu130\nIs debug build: False\nCUDA used to build PyTorch: 13.0\nROCM used to build PyTorch: N/A\n\nOS: Microsoft Windows 11 Pro (10.0.26200 64-bit)\nGCC version: Could not collect\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: N/A\n\nPython version: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 64 bit (AMD64)] (64-bit runtime)\nPython platform: Windows-10-10.0.26200-SP0\nIs CUDA available: True\nCUDA runtime version: 13.0.48\r\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 2050\nNvidia driver version: 591.86\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nName: 12th Gen Intel(R) Core(TM) i5-12450H\nManufacturer: GenuineIntel\nFamily: 205\nArchitecture: 9\nProcessorType: 3\nDeviceID: CPU0\nCurrentClockSpeed: 2000\nMaxClockSpeed: 2000\nL2CacheSize: 7168\nL2CacheSpeed: None\nRevision: None\n\nVersions of relevant libraries:\n[pip3] flash_attn==2.8.3+cu130torch2.11\n[pip3] numpy==2.2.6\n[pip3] optree==0.18.0\n[pip3] rotary-embedding-torch==0.6.4\n[pip3] torch==2.11.0+cu130\n[pip3] torchaudio==2.11.0+cu130\n[pip3] torchcrepe==0.0.20\n[pip3] torchgen==0.0.1\n[pip3] torchvision==0.26.0+cu130\n[conda] Could not collect", "transformers_version": "5.12.0", "lm_eval_version": "0.4.12", "upper_git_hash": null, "tokenizer_pad_token": [ "", "0" ], "tokenizer_eos_token": [ "", "2" ], "tokenizer_bos_token": [ "", "1" ], "eot_token_id": 2, "max_length": 4096, "task_hashes": {}, "model_source": "hf", "model_name": "absltnull/predBor-v0.5", "model_name_sanitized": "absltnull__predBor-v0.5", "system_instruction": null, "system_instruction_sha": null, "fewshot_as_multiturn": null, "chat_template": null, "chat_template_sha": null, "total_evaluation_time_seconds": "2531.7302202" }