Hugging Face's logo

Spaces:
taagarwa
/
coding-agent-leaderboard
Running

App Files Community
4
hemajv commited on
Commit
20bd53c
·
1 Parent(s): 977205a

Add Shellbench results

Browse files
Files changed (5) hide show
  1. results/gemma4-31b-fp8-shellbench-openclaw.json +47 -0
  2. results/gpt-oss-120b-shellbench-openclaw.json +48 -0
  3. results/mistral4-119b-fp4-shellbench-openclaw.json +48 -0
  4. results/nemotron-120b-shellbench-openclaw.json +48 -0
  5. results/qwen3-6-27b-fp8-shellbench-openclaw.json +47 -0
results/gemma4-31b-fp8-shellbench-openclaw.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "Shellbench",
4
+ "repo": "ShellBench/public-tasks",
5
+ "num_tasks": 115,
6
+ "url": "https://github.com/ShellBench/public-tasks"
7
+ },
8
+ "harness": {
9
+ "name": "OpenClaw",
10
+ "skills": [],
11
+ "is_oss": true,
12
+ "url": "https://github.com/OpenClaw/OpenClaw"
13
+ },
14
+ "model": {
15
+ "name": "Gemma4-31B-FP8",
16
+ "repo": "RedHatAI/gemma-4-31B-it-FP8-block",
17
+ "is_oss": true,
18
+ "num_params": 31,
19
+ "precision": "fp8",
20
+ "url": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
26
+ },
27
+ "url": "https://github.com/harbor-framework/harbor"
28
+ },
29
+ "metrics": {
30
+ "n_tasks": 115,
31
+ "n_errors": 6,
32
+ "score": 0.0,
33
+ "n_input_tokens": 0,
34
+ "n_cache_tokens": 0,
35
+ "n_output_tokens": 0,
36
+ "n_total_tokens": 0,
37
+ "agent_time_seconds": 29895,
38
+ "total_time_seconds": 36503,
39
+ "cost_usd": 8.3,
40
+ "mean_input_tokens_per_task": 0,
41
+ "mean_cache_tokens_per_task": 0,
42
+ "mean_output_tokens_per_task": 0,
43
+ "mean_tokens_per_task": 0,
44
+ "mean_cost_usd_per_task": 0.07,
45
+ "mean_total_time_seconds_per_task": 317,
46
+ "mean_agent_time_seconds_per_task": 259
47
+ }
results/gpt-oss-120b-shellbench-openclaw.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "Shellbench",
4
+ "repo": "ShellBench/public-tasks",
5
+ "num_tasks": 115,
6
+ "url": "https://github.com/ShellBench/public-tasks"
7
+ },
8
+ "harness": {
9
+ "name": "OpenClaw",
10
+ "skills": [],
11
+ "is_oss": true,
12
+ "url": "https://github.com/OpenClaw/OpenClaw"
13
+ },
14
+ "model": {
15
+ "name": "gpt-oss-120b",
16
+ "repo": "RedHatAI/gpt-oss-120b",
17
+ "is_oss": true,
18
+ "num_params": 120,
19
+ "precision": "",
20
+ "url": "https://huggingface.co/RedHatAI/gpt-oss-120b"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
26
+ },
27
+ "url": "https://github.com/harbor-framework/harbor"
28
+ },
29
+ "metrics": {
30
+ "n_tasks": 115,
31
+ "n_errors": 1,
32
+ "score": 0.0,
33
+ "n_input_tokens": 0,
34
+ "n_cache_tokens": 0,
35
+ "n_output_tokens": 0,
36
+ "n_total_tokens": 0,
37
+ "agent_time_seconds": 8170,
38
+ "total_time_seconds": 13296,
39
+ "cost_usd": 2.27,
40
+ "mean_input_tokens_per_task": 0,
41
+ "mean_cache_tokens_per_task": 0,
42
+ "mean_output_tokens_per_task": 0,
43
+ "mean_tokens_per_task": 0,
44
+ "mean_cost_usd_per_task": 0.02,
45
+ "mean_total_time_seconds_per_task": 115,
46
+ "mean_agent_time_seconds_per_task": 71
47
+ }
48
+ }
results/mistral4-119b-fp4-shellbench-openclaw.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "Shellbench",
4
+ "repo": "ShellBench/public-tasks",
5
+ "num_tasks": 115,
6
+ "url": "https://github.com/ShellBench/public-tasks"
7
+ },
8
+ "harness": {
9
+ "name": "OpenClaw",
10
+ "skills": [],
11
+ "is_oss": true,
12
+ "url": "https://github.com/OpenClaw/OpenClaw"
13
+ },
14
+ "model": {
15
+ "name": "Mistral-Small-4-119B-2603-NVFP4",
16
+ "repo": "RedHatAI/Mistral-Small-4-119B-2603-NVFP4",
17
+ "is_oss": true,
18
+ "num_params": 119,
19
+ "precision": "fp4",
20
+ "url": "https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
26
+ },
27
+ "url": "https://github.com/harbor-framework/harbor"
28
+ },
29
+ "metrics": {
30
+ "n_tasks": 115,
31
+ "n_errors": 2,
32
+ "score": 0.009,
33
+ "n_input_tokens": 0,
34
+ "n_cache_tokens": 0,
35
+ "n_output_tokens": 0,
36
+ "n_total_tokens": 0,
37
+ "agent_time_seconds": 4846,
38
+ "total_time_seconds": 9862,
39
+ "cost_usd": 1.35,
40
+ "mean_input_tokens_per_task": 0,
41
+ "mean_cache_tokens_per_task": 0,
42
+ "mean_output_tokens_per_task": 0,
43
+ "mean_tokens_per_task": 0,
44
+ "mean_cost_usd_per_task": 0.01,
45
+ "mean_total_time_seconds_per_task": 85,
46
+ "mean_agent_time_seconds_per_task": 42
47
+ }
48
+ }
results/nemotron-120b-shellbench-openclaw.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "Shellbench",
4
+ "repo": "ShellBench/public-tasks",
5
+ "num_tasks": 115,
6
+ "url": "https://github.com/ShellBench/public-tasks"
7
+ },
8
+ "harness": {
9
+ "name": "OpenClaw",
10
+ "skills": [],
11
+ "is_oss": true,
12
+ "url": "https://github.com/OpenClaw/OpenClaw"
13
+ },
14
+ "model": {
15
+ "name": "Nemotron-3-Super-120B-NVFP4",
16
+ "repo": "RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
17
+ "is_oss": true,
18
+ "num_params": 120,
19
+ "precision": "fp4",
20
+ "url": "https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
26
+ },
27
+ "url": "https://github.com/harbor-framework/harbor"
28
+ },
29
+ "metrics": {
30
+ "n_tasks": 115,
31
+ "n_errors": 1,
32
+ "score": 0.014,
33
+ "n_input_tokens": 0,
34
+ "n_cache_tokens": 0,
35
+ "n_output_tokens": 0,
36
+ "n_total_tokens": 0,
37
+ "agent_time_seconds": 21575,
38
+ "total_time_seconds": 26392,
39
+ "cost_usd": 5.99,
40
+ "mean_input_tokens_per_task": 0,
41
+ "mean_cache_tokens_per_task": 0,
42
+ "mean_output_tokens_per_task": 0,
43
+ "mean_tokens_per_task": 0,
44
+ "mean_cost_usd_per_task": 0.05,
45
+ "mean_total_time_seconds_per_task": 229,
46
+ "mean_agent_time_seconds_per_task": 187
47
+ }
48
+ }
results/qwen3-6-27b-fp8-shellbench-openclaw.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "Shellbench",
4
+ "repo": "ShellBench/public-tasks",
5
+ "num_tasks": 115,
6
+ "url": "https://github.com/ShellBench/public-tasks"
7
+ },
8
+ "harness": {
9
+ "name": "OpenClaw",
10
+ "skills": [],
11
+ "is_oss": true,
12
+ "url": "https://github.com/OpenClaw/OpenClaw"
13
+ },
14
+ "model": {
15
+ "name": "Qwen3.6-27B-FP8",
16
+ "repo": "RedHatAI/Qwen3.6-27B-FP8",
17
+ "is_oss": true,
18
+ "num_params": 27,
19
+ "precision": "fp8",
20
+ "url": "https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "path": "/Users/hveeradh/public-tasks/tasks/115-tasks"
26
+ },
27
+ "url": "https://github.com/harbor-framework/harbor"
28
+ },
29
+ "metrics": {
30
+ "n_tasks": 115,
31
+ "n_errors": 1,
32
+ "score": 0.128,
33
+ "n_input_tokens": 0,
34
+ "n_cache_tokens": 0,
35
+ "n_output_tokens": 0,
36
+ "n_total_tokens": 0,
37
+ "agent_time_seconds": 10812,
38
+ "total_time_seconds": 16949,
39
+ "cost_usd": 6.01,
40
+ "mean_input_tokens_per_task": 0,
41
+ "mean_cache_tokens_per_task": 0,
42
+ "mean_output_tokens_per_task": 0,
43
+ "mean_tokens_per_task": 0,
44
+ "mean_cost_usd_per_task": 0.05,
45
+ "mean_total_time_seconds_per_task": 147,
46
+ "mean_agent_time_seconds_per_task": 94
47
+ }