n10b/eval_grid/software_development/eval_simpletom_behavior-qa/metrics-all.jsonl

{"task_name": "simpletom_behavior-qa:mc", "task_hash": "bb627a959f40590c0a687da5cf76c852", "model_hash": "b721f21c65afe94422ee2fb8ae478fdf", "model_config": {"model": "/tmp/merged_5316915", "revision": null, "trust_remote_code": true, "max_length": 8192, "model_path": null, "model_type": "vllm", "tokenizer": "/home/hice1/gmatlin3/scratch/hf_cache/hub/models--allenai--OLMo-3-1025-7B/snapshots/a81bae42db3975be1671e27b9c9a56da1a9f980f"}, "task_config": {"task_name": "simpletom_behavior-qa:mc", "task_core": "simpletom_behavior-qa", "limit": 10000, "split": "test", "num_shots": 5, "fewshot_seed": 1234, "primary_metric": "acc_per_char", "random_subsample_seed": 1234, "context_kwargs": null, "generation_kwargs": null, "metric_kwargs": {"uncond_docid_offset": 1000000, "extra_metric_names": ["no_answer"]}, "native_id_field": "id", "fewshot_source": null, "dataset_path": "allenai/SimpleToM", "dataset_name": "behavior-qa", "use_chat_format": null, "version": 0, "revision": null, "compute_gold_bpb": false, "external_eval": null, "custom_kwargs": null, "skip_model_judges": null, "model_max_length": null, "metadata": {"regimes": [], "alias": "simpletom_behavior-qa:mc::tom-rebuttal"}}, "compute_config": {"batch_size": 1, "max_batch_size": 32, "output_dir": "/home/hice1/gmatlin3/scratch/n10b/eval_grid/software_development/eval_simpletom_behavior-qa", "num_recorded_inputs": 3, "save_raw_requests": true, "recompute_metrics": false}, "processing_time": 527.8015668392181, "current_date": "2026-05-24 23:20:54 UTC", "num_instances": 1147, "beaker_info": {}, "metrics": {"acc_raw": 0.7994768962510898, "acc_per_token": 0.6521360069747166, "acc_per_char": 0.5893635571054926, "acc_per_byte": 0.5893635571054926, "sum_logits_corr": -20.905842721272723, "logits_per_token_corr": -2.0188347487677736, "logits_per_char_corr": -0.3788240714739546, "bits_per_byte_corr": 0.5465242677132962, "acc_uncond": 0.4664341761115955, "primary_score": 0.5893635571054926, "extra_metrics": {"no_answer": 0.0}}, "task_idx": 0}