Spaces:
Running
Running
| { | |
| "accepted_benchmark_count": 7, | |
| "accepted_benchmarks": [ | |
| "aime2025", | |
| "arenahardwriting", | |
| "bfcl", | |
| "gpqamain", | |
| "gsm8k", | |
| "healthbench", | |
| "humaneval" | |
| ], | |
| "accepted_model_count": 4, | |
| "accepted_models": { | |
| "HuggingFaceTB_SmolLM3-3B-Base": "SmolLM3-3B-Base", | |
| "Qwen_Qwen3-1.7B-Base": "Qwen3-1.7B-Base", | |
| "Qwen_Qwen3-4B-Base": "Qwen3-4B-Base", | |
| "google_gemma-3-4b-pt": "Gemma-3-4B-PT" | |
| }, | |
| "cell_counts": { | |
| "aime2025": [ | |
| 47, | |
| 48, | |
| 48, | |
| 48 | |
| ], | |
| "arenahardwriting": [ | |
| 47, | |
| 48, | |
| 50, | |
| 48 | |
| ], | |
| "bfcl": [ | |
| 47, | |
| 48, | |
| 48, | |
| 48 | |
| ], | |
| "gpqamain": [ | |
| 46, | |
| 49, | |
| 47, | |
| 49 | |
| ], | |
| "gsm8k": [ | |
| 47, | |
| 47, | |
| 49, | |
| 48 | |
| ], | |
| "healthbench": [ | |
| 47, | |
| 48, | |
| 48, | |
| 48 | |
| ], | |
| "humaneval": [ | |
| 47, | |
| 47, | |
| 48, | |
| 48 | |
| ] | |
| }, | |
| "duplicate_job_pairs": 25, | |
| "excluded_auxiliary_data": { | |
| "counted_as_task_root": false, | |
| "directory_count": 1, | |
| "file_count": 2397, | |
| "present": true, | |
| "top_level_path": "viewer_data" | |
| }, | |
| "excluded_dirs_count": 0, | |
| "inventory": { | |
| "all_entries_digest": "045ae5c714aa605b4295e345970cdf9a330600f709ef18508e8bef5eb3eec13d", | |
| "dir_count": 14117, | |
| "dir_entries_digest": "d480b9917811f32cfe7e055a029f475bfb2fa0c1cb02d0d08a7de0e200714132", | |
| "file_count": 97209, | |
| "file_entries_digest": "320253d2791e878f6539c58365ddc9ac93baffb53a5c78244910ef153f067ca4", | |
| "page_count": 112, | |
| "rejected_siblings_count": 85883, | |
| "rejected_siblings_digest": "116dc22723f1cc13bf71461ff83dd03479c74a2740957787e6ca642a59628eea", | |
| "total_entries": 111326 | |
| }, | |
| "matrix": [ | |
| { | |
| "benchmark": "aime2025", | |
| "count": 47, | |
| "model": "Qwen3-1.7B-Base" | |
| }, | |
| { | |
| "benchmark": "aime2025", | |
| "count": 48, | |
| "model": "Qwen3-4B-Base" | |
| }, | |
| { | |
| "benchmark": "aime2025", | |
| "count": 48, | |
| "model": "SmolLM3-3B-Base" | |
| }, | |
| { | |
| "benchmark": "aime2025", | |
| "count": 48, | |
| "model": "Gemma-3-4B-PT" | |
| }, | |
| { | |
| "benchmark": "arenahardwriting", | |
| "count": 47, | |
| "model": "Qwen3-1.7B-Base" | |
| }, | |
| { | |
| "benchmark": "arenahardwriting", | |
| "count": 48, | |
| "model": "Qwen3-4B-Base" | |
| }, | |
| { | |
| "benchmark": "arenahardwriting", | |
| "count": 50, | |
| "model": "SmolLM3-3B-Base" | |
| }, | |
| { | |
| "benchmark": "arenahardwriting", | |
| "count": 48, | |
| "model": "Gemma-3-4B-PT" | |
| }, | |
| { | |
| "benchmark": "bfcl", | |
| "count": 47, | |
| "model": "Qwen3-1.7B-Base" | |
| }, | |
| { | |
| "benchmark": "bfcl", | |
| "count": 48, | |
| "model": "Qwen3-4B-Base" | |
| }, | |
| { | |
| "benchmark": "bfcl", | |
| "count": 48, | |
| "model": "SmolLM3-3B-Base" | |
| }, | |
| { | |
| "benchmark": "bfcl", | |
| "count": 48, | |
| "model": "Gemma-3-4B-PT" | |
| }, | |
| { | |
| "benchmark": "gpqamain", | |
| "count": 46, | |
| "model": "Qwen3-1.7B-Base" | |
| }, | |
| { | |
| "benchmark": "gpqamain", | |
| "count": 49, | |
| "model": "Qwen3-4B-Base" | |
| }, | |
| { | |
| "benchmark": "gpqamain", | |
| "count": 47, | |
| "model": "SmolLM3-3B-Base" | |
| }, | |
| { | |
| "benchmark": "gpqamain", | |
| "count": 49, | |
| "model": "Gemma-3-4B-PT" | |
| }, | |
| { | |
| "benchmark": "gsm8k", | |
| "count": 47, | |
| "model": "Qwen3-1.7B-Base" | |
| }, | |
| { | |
| "benchmark": "gsm8k", | |
| "count": 47, | |
| "model": "Qwen3-4B-Base" | |
| }, | |
| { | |
| "benchmark": "gsm8k", | |
| "count": 49, | |
| "model": "SmolLM3-3B-Base" | |
| }, | |
| { | |
| "benchmark": "gsm8k", | |
| "count": 48, | |
| "model": "Gemma-3-4B-PT" | |
| }, | |
| { | |
| "benchmark": "healthbench", | |
| "count": 47, | |
| "model": "Qwen3-1.7B-Base" | |
| }, | |
| { | |
| "benchmark": "healthbench", | |
| "count": 48, | |
| "model": "Qwen3-4B-Base" | |
| }, | |
| { | |
| "benchmark": "healthbench", | |
| "count": 48, | |
| "model": "SmolLM3-3B-Base" | |
| }, | |
| { | |
| "benchmark": "healthbench", | |
| "count": 48, | |
| "model": "Gemma-3-4B-PT" | |
| }, | |
| { | |
| "benchmark": "humaneval", | |
| "count": 47, | |
| "model": "Qwen3-1.7B-Base" | |
| }, | |
| { | |
| "benchmark": "humaneval", | |
| "count": 47, | |
| "model": "Qwen3-4B-Base" | |
| }, | |
| { | |
| "benchmark": "humaneval", | |
| "count": 48, | |
| "model": "SmolLM3-3B-Base" | |
| }, | |
| { | |
| "benchmark": "humaneval", | |
| "count": 48, | |
| "model": "Gemma-3-4B-PT" | |
| } | |
| ], | |
| "missing_root_cell_pairs": 3, | |
| "protocol": { | |
| "commit_sh_analysis": { | |
| "current_benchmarks_in_arrays": [ | |
| "healthbench" | |
| ], | |
| "current_models_in_arrays": [ | |
| "Qwen/Qwen3-4B-Base" | |
| ], | |
| "gpu_counts_found": [ | |
| 8 | |
| ], | |
| "has_metr_branch": true, | |
| "htcondor_branch": { | |
| "active_jobs": 8, | |
| "gpu_spec": "default (single_task.sub num_gpus=1)", | |
| "one_hour_jobs": 1, | |
| "ten_hour_jobs": 7 | |
| }, | |
| "htcondor_mpi_is_branch": { | |
| "active_jobs": 1, | |
| "gpus": 8, | |
| "hours": 100, | |
| "note": "Active 100-hour, eight-GPU METR command" | |
| }, | |
| "num_hours_values": [ | |
| 1, | |
| 10, | |
| 100 | |
| ], | |
| "scheduler_dependent": true | |
| }, | |
| "cuda_device_requirement": "TARGET.CUDADeviceName == \"NVIDIA H100 80GB HBM3\"", | |
| "evaluation_dir_count": 7, | |
| "evaluation_dirs_present": [ | |
| "src/eval/tasks/aime2025", | |
| "src/eval/tasks/arenahardwriting", | |
| "src/eval/tasks/bfcl", | |
| "src/eval/tasks/gpqamain", | |
| "src/eval/tasks/gsm8k", | |
| "src/eval/tasks/healthbench", | |
| "src/eval/tasks/humaneval" | |
| ], | |
| "limitation_five_minute_grace": true, | |
| "limitation_multi_gpu_extension": true, | |
| "num_gpus_default": 1, | |
| "receives_num_hours": true, | |
| "request_gpus_binding": "request_gpus = $(num_gpus)", | |
| "solve_timeout_formula": "NUM_HOURS * 60 + 5", | |
| "source_references": { | |
| "commit_sh_analysis.current_benchmarks_in_arrays": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "3c43144e1186a160f450e747b95861fea6d16747", | |
| "lines": [ | |
| 18 | |
| ], | |
| "path": "src/commit_utils/commit.sh", | |
| "raw_sha256": "663ecb37cc4e6a16dcfcf8135bdbadf325f0b067192fe7a71d2231ba37eaae8e" | |
| }, | |
| "commit_sh_analysis.current_models_in_arrays": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "3c43144e1186a160f450e747b95861fea6d16747", | |
| "lines": [ | |
| 6 | |
| ], | |
| "path": "src/commit_utils/commit.sh", | |
| "raw_sha256": "663ecb37cc4e6a16dcfcf8135bdbadf325f0b067192fe7a71d2231ba37eaae8e" | |
| }, | |
| "commit_sh_analysis.htcondor_branch": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "3c43144e1186a160f450e747b95861fea6d16747", | |
| "lines": [ | |
| 83, | |
| 84, | |
| 85, | |
| 86, | |
| 87, | |
| 88, | |
| 89, | |
| 90 | |
| ], | |
| "path": "src/commit_utils/commit.sh", | |
| "raw_sha256": "663ecb37cc4e6a16dcfcf8135bdbadf325f0b067192fe7a71d2231ba37eaae8e" | |
| }, | |
| "commit_sh_analysis.htcondor_mpi_is_branch": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "3c43144e1186a160f450e747b95861fea6d16747", | |
| "lines": [ | |
| 48 | |
| ], | |
| "path": "src/commit_utils/commit.sh", | |
| "raw_sha256": "663ecb37cc4e6a16dcfcf8135bdbadf325f0b067192fe7a71d2231ba37eaae8e" | |
| }, | |
| "cuda_device_requirement": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "ea7f8790b97301dcdb6f3c104c5555d7ddf4e06a", | |
| "lines": [ | |
| 11 | |
| ], | |
| "path": "src/commit_utils/single_task.sub", | |
| "raw_sha256": "f8ee12da42fdebfc3b4293a22ea8b232c1f8f52cb2f52b103c9f138f0ddc013a" | |
| }, | |
| "evaluation_dirs_present": { | |
| "kind": "git-tree-entries", | |
| "paths": [ | |
| "src/eval/tasks/aime2025", | |
| "src/eval/tasks/arenahardwriting", | |
| "src/eval/tasks/bfcl", | |
| "src/eval/tasks/gpqamain", | |
| "src/eval/tasks/gsm8k", | |
| "src/eval/tasks/healthbench", | |
| "src/eval/tasks/humaneval" | |
| ] | |
| }, | |
| "num_gpus_default": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "ea7f8790b97301dcdb6f3c104c5555d7ddf4e06a", | |
| "lines": [ | |
| 2 | |
| ], | |
| "path": "src/commit_utils/single_task.sub", | |
| "raw_sha256": "f8ee12da42fdebfc3b4293a22ea8b232c1f8f52cb2f52b103c9f138f0ddc013a" | |
| }, | |
| "receives_num_hours": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "0642ec47ee7acd2528cdab7d343ddba11cbc84db", | |
| "lines": [ | |
| 7, | |
| 24, | |
| 59, | |
| 62, | |
| 122, | |
| 162 | |
| ], | |
| "path": "src/run_task.sh", | |
| "raw_sha256": "10b0238018f202209c06f12ff05d021a0ca03b98d42c86a65e80cacd4fbe7033" | |
| }, | |
| "request_gpus_binding": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "ea7f8790b97301dcdb6f3c104c5555d7ddf4e06a", | |
| "lines": [ | |
| 10 | |
| ], | |
| "path": "src/commit_utils/single_task.sub", | |
| "raw_sha256": "f8ee12da42fdebfc3b4293a22ea8b232c1f8f52cb2f52b103c9f138f0ddc013a" | |
| }, | |
| "solve_timeout_formula": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "0642ec47ee7acd2528cdab7d343ddba11cbc84db", | |
| "lines": [ | |
| 122 | |
| ], | |
| "path": "src/run_task.sh", | |
| "raw_sha256": "10b0238018f202209c06f12ff05d021a0ca03b98d42c86a65e80cacd4fbe7033" | |
| }, | |
| "timeout_grace_minutes": { | |
| "commit": "d3496fa7d5788a007d6cd143167471ccdfc688d0", | |
| "git_object_sha1": "0642ec47ee7acd2528cdab7d343ddba11cbc84db", | |
| "lines": [ | |
| 122 | |
| ], | |
| "path": "src/run_task.sh", | |
| "raw_sha256": "10b0238018f202209c06f12ff05d021a0ca03b98d42c86a65e80cacd4fbe7033" | |
| } | |
| }, | |
| "task_dir_10h_suffix": true, | |
| "timeout_formula_found": true, | |
| "timeout_grace_minutes": 5 | |
| }, | |
| "recognized_root_cell_pairs": 1313, | |
| "recognized_root_count": 47, | |
| "recognized_task_count": 1338, | |
| "unrecognized_dirs_count": 0 | |
| } | |