Commit ·
d48d59d
1
Parent(s): 6944b86
Add Nemotron-120B Claude Code results for all 3 benchmarks
Browse files
results/nemotron-120b-rh-swe-bench-claude-code.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"benchmark": {
|
| 3 |
+
"name": "RH SWE-Bench",
|
| 4 |
+
"repo": "rounakbende10/rh-swe-bench",
|
| 5 |
+
"num_tasks": 357,
|
| 6 |
+
"url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
|
| 7 |
+
},
|
| 8 |
+
"harness": {
|
| 9 |
+
"name": "Claude Code",
|
| 10 |
+
"skills": [],
|
| 11 |
+
"is_oss": false,
|
| 12 |
+
"url": "https://github.com/anthropics/claude-code"
|
| 13 |
+
},
|
| 14 |
+
"model": {
|
| 15 |
+
"name": "Nemotron-3-Super-120B-NVFP4",
|
| 16 |
+
"repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
|
| 17 |
+
"is_oss": true,
|
| 18 |
+
"num_params": 120,
|
| 19 |
+
"precision": "nvfp4",
|
| 20 |
+
"url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
| 21 |
+
},
|
| 22 |
+
"environment": {
|
| 23 |
+
"name": "harbor",
|
| 24 |
+
"config": {
|
| 25 |
+
"name": "rh-swe-bench"
|
| 26 |
+
},
|
| 27 |
+
"url": "https://github.com/harbor-framework/harbor"
|
| 28 |
+
},
|
| 29 |
+
"metrics": {
|
| 30 |
+
"n_tasks": 357,
|
| 31 |
+
"n_errors": 14,
|
| 32 |
+
"score": 0.224,
|
| 33 |
+
"n_input_tokens": 783670817,
|
| 34 |
+
"n_cache_tokens": 0,
|
| 35 |
+
"n_output_tokens": 5510291,
|
| 36 |
+
"n_total_tokens": 789181108,
|
| 37 |
+
"agent_time_seconds": 144352,
|
| 38 |
+
"total_time_seconds": 355122,
|
| 39 |
+
"cost_usd": 38.03,
|
| 40 |
+
"mean_input_tokens_per_task": 2195156,
|
| 41 |
+
"mean_cache_tokens_per_task": 0,
|
| 42 |
+
"mean_output_tokens_per_task": 15434,
|
| 43 |
+
"mean_tokens_per_task": 2210591,
|
| 44 |
+
"mean_cost_usd_per_task": 0.11,
|
| 45 |
+
"mean_total_time_seconds_per_task": 994,
|
| 46 |
+
"mean_agent_time_seconds_per_task": 404
|
| 47 |
+
}
|
| 48 |
+
}
|
results/nemotron-120b-swebench-pro-ansible-claude-code.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"benchmark": {
|
| 3 |
+
"name": "SWE-Bench Pro -- Ansible",
|
| 4 |
+
"repo": "ScaleAI/SWE-bench_Pro",
|
| 5 |
+
"num_tasks": 96,
|
| 6 |
+
"url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
|
| 7 |
+
},
|
| 8 |
+
"harness": {
|
| 9 |
+
"name": "Claude Code",
|
| 10 |
+
"skills": [],
|
| 11 |
+
"is_oss": false,
|
| 12 |
+
"url": "https://github.com/anthropics/claude-code"
|
| 13 |
+
},
|
| 14 |
+
"model": {
|
| 15 |
+
"name": "Nemotron-3-Super-120B-NVFP4",
|
| 16 |
+
"repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
|
| 17 |
+
"is_oss": true,
|
| 18 |
+
"num_params": 120,
|
| 19 |
+
"precision": "nvfp4",
|
| 20 |
+
"url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
| 21 |
+
},
|
| 22 |
+
"environment": {
|
| 23 |
+
"name": "harbor",
|
| 24 |
+
"config": {
|
| 25 |
+
"name": "swe-bench-pro-ansible"
|
| 26 |
+
},
|
| 27 |
+
"url": "https://github.com/harbor-framework/harbor"
|
| 28 |
+
},
|
| 29 |
+
"metrics": {
|
| 30 |
+
"n_tasks": 96,
|
| 31 |
+
"n_errors": 0,
|
| 32 |
+
"score": 0.432,
|
| 33 |
+
"n_input_tokens": 406026150,
|
| 34 |
+
"n_cache_tokens": 0,
|
| 35 |
+
"n_output_tokens": 2839236,
|
| 36 |
+
"n_total_tokens": 408865386,
|
| 37 |
+
"agent_time_seconds": 100355,
|
| 38 |
+
"total_time_seconds": 115220,
|
| 39 |
+
"cost_usd": 26.44,
|
| 40 |
+
"mean_input_tokens_per_task": 4229439,
|
| 41 |
+
"mean_cache_tokens_per_task": 0,
|
| 42 |
+
"mean_output_tokens_per_task": 29575,
|
| 43 |
+
"mean_tokens_per_task": 4259014,
|
| 44 |
+
"mean_cost_usd_per_task": 0.28,
|
| 45 |
+
"mean_total_time_seconds_per_task": 1200,
|
| 46 |
+
"mean_agent_time_seconds_per_task": 1045
|
| 47 |
+
}
|
| 48 |
+
}
|
results/nemotron-120b-swebench-verified-claude-code.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"benchmark": {
|
| 3 |
+
"name": "SWE-Bench Verified",
|
| 4 |
+
"repo": "SWE-bench/SWE-bench_Verified",
|
| 5 |
+
"num_tasks": 500,
|
| 6 |
+
"url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
|
| 7 |
+
},
|
| 8 |
+
"harness": {
|
| 9 |
+
"name": "Claude Code",
|
| 10 |
+
"skills": [],
|
| 11 |
+
"is_oss": false,
|
| 12 |
+
"url": "https://github.com/anthropics/claude-code"
|
| 13 |
+
},
|
| 14 |
+
"model": {
|
| 15 |
+
"name": "Nemotron-3-Super-120B-NVFP4",
|
| 16 |
+
"repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
|
| 17 |
+
"is_oss": true,
|
| 18 |
+
"num_params": 120,
|
| 19 |
+
"precision": "nvfp4",
|
| 20 |
+
"url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
| 21 |
+
},
|
| 22 |
+
"environment": {
|
| 23 |
+
"name": "harbor",
|
| 24 |
+
"config": {
|
| 25 |
+
"name": "swe-bench-verified"
|
| 26 |
+
},
|
| 27 |
+
"url": "https://github.com/harbor-framework/harbor"
|
| 28 |
+
},
|
| 29 |
+
"metrics": {
|
| 30 |
+
"n_tasks": 500,
|
| 31 |
+
"n_errors": 0,
|
| 32 |
+
"score": 0.472,
|
| 33 |
+
"n_input_tokens": 882330271,
|
| 34 |
+
"n_cache_tokens": 0,
|
| 35 |
+
"n_output_tokens": 6689285,
|
| 36 |
+
"n_total_tokens": 889019556,
|
| 37 |
+
"agent_time_seconds": 174288,
|
| 38 |
+
"total_time_seconds": 410828,
|
| 39 |
+
"cost_usd": 45.91,
|
| 40 |
+
"mean_input_tokens_per_task": 1764660,
|
| 41 |
+
"mean_cache_tokens_per_task": 0,
|
| 42 |
+
"mean_output_tokens_per_task": 13378,
|
| 43 |
+
"mean_tokens_per_task": 1778039,
|
| 44 |
+
"mean_cost_usd_per_task": 0.09,
|
| 45 |
+
"mean_total_time_seconds_per_task": 821,
|
| 46 |
+
"mean_agent_time_seconds_per_task": 348
|
| 47 |
+
}
|
| 48 |
+
}
|