n10b/eval_grid/software_development/eval_simpletom_judgment-qa/metrics.json

{"all_primary_scores": ["simpletom_judgment-qa:mc::tom-rebuttal: 1.0"], "tasks": [{"alias": "simpletom_judgment-qa:mc::tom-rebuttal", "metrics": {"acc_raw": 1.0, "acc_per_token": 1.0, "acc_per_char": 1.0, "acc_per_byte": 1.0, "sum_logits_corr": -0.9803053918094853, "logits_per_token_corr": -0.49015269590474264, "logits_per_char_corr": -0.08911867198268048, "bits_per_byte_corr": 0.12857106612011568, "acc_uncond": 0.998256320836966, "primary_score": 1.0, "extra_metrics": {"no_answer": 0.0}}, "num_instances": 1147, "processing_time": 526.5461225509644, "task_config": {"task_name": "simpletom_judgment-qa:mc", "task_core": "simpletom_judgment-qa", "limit": 10000, "split": "test", "num_shots": 5, "primary_metric": "acc_per_char", "metric_kwargs": {"uncond_docid_offset": 1000000, "extra_metric_names": ["no_answer"]}, "dataset_path": "allenai/SimpleToM", "dataset_name": "judgment-qa", "version": 0, "metadata": {"regimes": [], "alias": "simpletom_judgment-qa:mc::tom-rebuttal"}}}], "model_config": {"model": "/tmp/merged_5316923", "trust_remote_code": true, "max_length": 8192, "model_type": "vllm", "tokenizer": "/home/hice1/gmatlin3/scratch/hf_cache/hub/models--allenai--OLMo-3-1025-7B/snapshots/a81bae42db3975be1671e27b9c9a56da1a9f980f"}}