n10b/eval_grid/software/eval_simpletom_behavior-qa/metrics.json

{"all_primary_scores": ["simpletom_behavior-qa:mc::tom-rebuttal: 0.586748"], "tasks": [{"alias": "simpletom_behavior-qa:mc::tom-rebuttal", "metrics": {"acc_raw": 0.8064516129032258, "acc_per_token": 0.6713164777680907, "acc_per_char": 0.5867480383609416, "acc_per_byte": 0.5867480383609416, "sum_logits_corr": -18.612338178551383, "logits_per_token_corr": -1.7933694652625625, "logits_per_char_corr": -0.33474413148584997, "bits_per_byte_corr": 0.48292970629673077, "acc_uncond": 0.4115082824760244, "primary_score": 0.5867480383609416, "extra_metrics": {"no_answer": 0.0}}, "num_instances": 1147, "processing_time": 520.8581883907318, "task_config": {"task_name": "simpletom_behavior-qa:mc", "task_core": "simpletom_behavior-qa", "limit": 10000, "split": "test", "num_shots": 5, "primary_metric": "acc_per_char", "metric_kwargs": {"uncond_docid_offset": 1000000, "extra_metric_names": ["no_answer"]}, "dataset_path": "allenai/SimpleToM", "dataset_name": "behavior-qa", "version": 0, "metadata": {"regimes": [], "alias": "simpletom_behavior-qa:mc::tom-rebuttal"}}}], "model_config": {"model": "/tmp/merged_5316908", "trust_remote_code": true, "max_length": 8192, "model_type": "vllm", "tokenizer": "/home/hice1/gmatlin3/scratch/hf_cache/hub/models--allenai--OLMo-3-1025-7B/snapshots/a81bae42db3975be1671e27b9c9a56da1a9f980f"}}