benchmarks/eval_code.json

{
 "results": {
  "humaneval": {
   "name": "humaneval",
   "alias": "humaneval",
   "sample_len": 164,
   "pass@1,create_test": 0.0,
   "pass@1_stderr,create_test": 0.0
  }
 },
 "group_subtasks": {},
 "configs": {
  "humaneval": {
   "task": "humaneval",
   "dataset_path": "openai/openai_humaneval",
   "test_split": "test",
   "doc_to_text": "{{prompt}}",
   "doc_to_target": "{{test}}\ncheck({{entry_point}})",
   "unsafe_code": true,
   "description": "",
   "target_delimiter": " ",
   "fewshot_delimiter": "\n\n",
   "fewshot_config": {
    "sampler": "default",
    "split": null,
    "process_docs": null,
    "fewshot_indices": null,
    "samples": null,
    "doc_to_text": "{{prompt}}",
    "doc_to_choice": null,
    "doc_to_target": "{{test}}\ncheck({{entry_point}})",
    "gen_prefix": null,
    "fewshot_delimiter": "\n\n",
    "target_delimiter": " "
   },
   "num_fewshot": 0,
   "metric_list": [
    {
     "metric": "def pass_at_k(references: list[str], predictions: list[list[str]], k: list[int] = None):\n    global compute_\n    assert k is not None\n    if isinstance(k, int):\n        k = [k]\n    res = compute_.compute(\n        references=references,\n        predictions=predictions,\n        k=k,\n    )\n    return res[0]\n",
     "aggregation": "mean",
     "higher_is_better": true,
     "k": [
      1
     ]
    }
   ],
   "output_type": "generate_until",
   "generation_kwargs": {
    "until": [
     "\nclass",
     "\ndef",
     "\n#",
     "\nif",
     "\nprint"
    ],
    "max_gen_toks": 1024,
    "do_sample": false
   },
   "repeats": 1,
   "filter_list": [
    {
     "name": "create_test",
     "filter": [
      {
       "function": "custom",
       "filter_fn": "<function build_predictions at 0x7835491a2f20>"
      }
     ]
    }
   ],
   "should_decontaminate": false,
   "metadata": {
    "version": 1.0,
    "model": "dsv4-hybrid",
    "base_url": "http://localhost:PORT/v1/chat/completions",
    "num_concurrent": 8,
    "max_retries": 3,
    "tokenized_requests": false,
    "tokenizer": "<model_path>",
    "tokenizer_backend": "huggingface",
    "config_source": "<lm_eval>/tasks/humaneval/humaneval.yaml"
   }
  }
 },
 "versions": {
  "humaneval": 1.0
 },
 "n-shot": {
  "humaneval": 0
 },
 "higher_is_better": {
  "humaneval": {
   "pass_at_k": true,
   "pass@1": true
  }
 },
 "n-samples": {
  "humaneval": {
   "original": 164,
   "effective": 164
  }
 },
 "config": {
  "model": "local-chat-completions",
  "model_args": {
   "model": "dsv4-hybrid",
   "base_url": "http://localhost:PORT/v1/chat/completions",
   "num_concurrent": 8,
   "max_retries": 3,
   "tokenized_requests": false,
   "tokenizer": "<model_path>",
   "tokenizer_backend": "huggingface"
  },
  "batch_size": 1,
  "batch_sizes": [],
  "device": "cuda:0",
  "use_cache": null,
  "limit": 164.0,
  "bootstrap_iters": 100000,
  "gen_kwargs": {},
  "random_seed": 0,
  "numpy_seed": 1234,
  "torch_seed": 1234,
  "fewshot_seed": 1234
 },
 "git_hash": null,
 "date": 1786210286.7190146,
 "transformers_version": "5.14.1",
 "lm_eval_version": "0.4.12",
 "upper_git_hash": null,
 "tokenizer_pad_token": [
  "<｜end▁of▁sentence｜>",
  "1"
 ],
 "tokenizer_eos_token": [
  "<｜end▁of▁sentence｜>",
  "1"
 ],
 "tokenizer_bos_token": [
  "<｜begin▁of▁sentence｜>",
  "0"
 ],
 "eot_token_id": 1,
 "max_length": 2047,
 "task_hashes": {},
 "model_source": "local-chat-completions",
 "model_name": "dsv4-hybrid",
 "model_name_sanitized": "dsv4-hybrid",
 "system_instruction": null,
 "system_instruction_sha": null,
 "fewshot_as_multiturn": true,
 "chat_template": null,
 "chat_template_sha": null,
 "total_evaluation_time_seconds": "34.3056967179873",
 "hardware_note": "2x NVIDIA RTX PRO 6000 Blackwell (96GB, SM120), TP=2. Detailed host environment omitted."
}