rounakbende commited on
Commit
d48d59d
·
1 Parent(s): 6944b86

Add Nemotron-120B Claude Code results for all 3 benchmarks

Browse files
results/nemotron-120b-rh-swe-bench-claude-code.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "RH SWE-Bench",
4
+ "repo": "rounakbende10/rh-swe-bench",
5
+ "num_tasks": 357,
6
+ "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
7
+ },
8
+ "harness": {
9
+ "name": "Claude Code",
10
+ "skills": [],
11
+ "is_oss": false,
12
+ "url": "https://github.com/anthropics/claude-code"
13
+ },
14
+ "model": {
15
+ "name": "Nemotron-3-Super-120B-NVFP4",
16
+ "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
17
+ "is_oss": true,
18
+ "num_params": 120,
19
+ "precision": "nvfp4",
20
+ "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "name": "rh-swe-bench"
26
+ },
27
+ "url": "https://github.com/harbor-framework/harbor"
28
+ },
29
+ "metrics": {
30
+ "n_tasks": 357,
31
+ "n_errors": 14,
32
+ "score": 0.224,
33
+ "n_input_tokens": 783670817,
34
+ "n_cache_tokens": 0,
35
+ "n_output_tokens": 5510291,
36
+ "n_total_tokens": 789181108,
37
+ "agent_time_seconds": 144352,
38
+ "total_time_seconds": 355122,
39
+ "cost_usd": 38.03,
40
+ "mean_input_tokens_per_task": 2195156,
41
+ "mean_cache_tokens_per_task": 0,
42
+ "mean_output_tokens_per_task": 15434,
43
+ "mean_tokens_per_task": 2210591,
44
+ "mean_cost_usd_per_task": 0.11,
45
+ "mean_total_time_seconds_per_task": 994,
46
+ "mean_agent_time_seconds_per_task": 404
47
+ }
48
+ }
results/nemotron-120b-swebench-pro-ansible-claude-code.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "SWE-Bench Pro -- Ansible",
4
+ "repo": "ScaleAI/SWE-bench_Pro",
5
+ "num_tasks": 96,
6
+ "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
7
+ },
8
+ "harness": {
9
+ "name": "Claude Code",
10
+ "skills": [],
11
+ "is_oss": false,
12
+ "url": "https://github.com/anthropics/claude-code"
13
+ },
14
+ "model": {
15
+ "name": "Nemotron-3-Super-120B-NVFP4",
16
+ "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
17
+ "is_oss": true,
18
+ "num_params": 120,
19
+ "precision": "nvfp4",
20
+ "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "name": "swe-bench-pro-ansible"
26
+ },
27
+ "url": "https://github.com/harbor-framework/harbor"
28
+ },
29
+ "metrics": {
30
+ "n_tasks": 96,
31
+ "n_errors": 0,
32
+ "score": 0.432,
33
+ "n_input_tokens": 406026150,
34
+ "n_cache_tokens": 0,
35
+ "n_output_tokens": 2839236,
36
+ "n_total_tokens": 408865386,
37
+ "agent_time_seconds": 100355,
38
+ "total_time_seconds": 115220,
39
+ "cost_usd": 26.44,
40
+ "mean_input_tokens_per_task": 4229439,
41
+ "mean_cache_tokens_per_task": 0,
42
+ "mean_output_tokens_per_task": 29575,
43
+ "mean_tokens_per_task": 4259014,
44
+ "mean_cost_usd_per_task": 0.28,
45
+ "mean_total_time_seconds_per_task": 1200,
46
+ "mean_agent_time_seconds_per_task": 1045
47
+ }
48
+ }
results/nemotron-120b-swebench-verified-claude-code.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "SWE-Bench Verified",
4
+ "repo": "SWE-bench/SWE-bench_Verified",
5
+ "num_tasks": 500,
6
+ "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
7
+ },
8
+ "harness": {
9
+ "name": "Claude Code",
10
+ "skills": [],
11
+ "is_oss": false,
12
+ "url": "https://github.com/anthropics/claude-code"
13
+ },
14
+ "model": {
15
+ "name": "Nemotron-3-Super-120B-NVFP4",
16
+ "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
17
+ "is_oss": true,
18
+ "num_params": 120,
19
+ "precision": "nvfp4",
20
+ "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "name": "swe-bench-verified"
26
+ },
27
+ "url": "https://github.com/harbor-framework/harbor"
28
+ },
29
+ "metrics": {
30
+ "n_tasks": 500,
31
+ "n_errors": 0,
32
+ "score": 0.472,
33
+ "n_input_tokens": 882330271,
34
+ "n_cache_tokens": 0,
35
+ "n_output_tokens": 6689285,
36
+ "n_total_tokens": 889019556,
37
+ "agent_time_seconds": 174288,
38
+ "total_time_seconds": 410828,
39
+ "cost_usd": 45.91,
40
+ "mean_input_tokens_per_task": 1764660,
41
+ "mean_cache_tokens_per_task": 0,
42
+ "mean_output_tokens_per_task": 13378,
43
+ "mean_tokens_per_task": 1778039,
44
+ "mean_cost_usd_per_task": 0.09,
45
+ "mean_total_time_seconds_per_task": 821,
46
+ "mean_agent_time_seconds_per_task": 348
47
+ }
48
+ }