n10b/eval_grid/software_development/eval_simpletom_behavior-qa/metrics.json

{"all_primary_scores": ["simpletom_behavior-qa:mc::tom-rebuttal: 0.589364"], "tasks": [{"alias": "simpletom_behavior-qa:mc::tom-rebuttal", "metrics": {"acc_raw": 0.7994768962510898, "acc_per_token": 0.6521360069747166, "acc_per_char": 0.5893635571054926, "acc_per_byte": 0.5893635571054926, "sum_logits_corr": -20.905842721272723, "logits_per_token_corr": -2.0188347487677736, "logits_per_char_corr": -0.3788240714739546, "bits_per_byte_corr": 0.5465242677132962, "acc_uncond": 0.4664341761115955, "primary_score": 0.5893635571054926, "extra_metrics": {"no_answer": 0.0}}, "num_instances": 1147, "processing_time": 527.8015668392181, "task_config": {"task_name": "simpletom_behavior-qa:mc", "task_core": "simpletom_behavior-qa", "limit": 10000, "split": "test", "num_shots": 5, "primary_metric": "acc_per_char", "metric_kwargs": {"uncond_docid_offset": 1000000, "extra_metric_names": ["no_answer"]}, "dataset_path": "allenai/SimpleToM", "dataset_name": "behavior-qa", "version": 0, "metadata": {"regimes": [], "alias": "simpletom_behavior-qa:mc::tom-rebuttal"}}}], "model_config": {"model": "/tmp/merged_5316915", "trust_remote_code": true, "max_length": 8192, "model_type": "vllm", "tokenizer": "/home/hice1/gmatlin3/scratch/hf_cache/hub/models--allenai--OLMo-3-1025-7B/snapshots/a81bae42db3975be1671e27b9c9a56da1a9f980f"}}