rounakbende commited on
Commit
9ed9b6e
·
1 Parent(s): cc1d87b

Add RH SWE-Bench results for Claude Opus 4.6 and Sonnet 4.6

Browse files

- Opus 4.6 + Claude Code: 63.3% (226/357), $434.54
- Sonnet 4.6 + Claude Code: 55.7% (199/357), $349.77
- 357 tasks, 25 repos, Go/Python/C
- Dataset: https://huggingface.co/datasets/rounakbende/rh-swe-bench

results/rh-swe-bench-claude-opus-4-6-claude-code.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "RH SWE-Bench",
4
+ "repo": "rounakbende10/rh-swe-bench",
5
+ "num_tasks": 357,
6
+ "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
7
+ },
8
+ "harness": {
9
+ "name": "Claude Code",
10
+ "skills": [],
11
+ "is_oss": false,
12
+ "url": "https://github.com/anthropics/claude-code"
13
+ },
14
+ "model": {
15
+ "name": "Opus 4.6",
16
+ "repo": "Opus 4.6",
17
+ "is_oss": false,
18
+ "num_params": 1000,
19
+ "precision": "bf16",
20
+ "url": "https://www.anthropic.com/news/claude-opus-4-6"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "path": null,
26
+ "name": "rh-swe-bench",
27
+ "version": null,
28
+ "ref": null,
29
+ "registry_url": null,
30
+ "registry_path": null,
31
+ "overwrite": false,
32
+ "download_dir": null,
33
+ "task_names": null,
34
+ "exclude_task_names": null,
35
+ "n_tasks": null
36
+ },
37
+ "url": "https://github.com/harbor-framework/harbor"
38
+ },
39
+ "metrics": {
40
+ "n_tasks": 357,
41
+ "n_errors": 0,
42
+ "score": 0.633,
43
+ "n_input_tokens": 438501505,
44
+ "n_cache_tokens": 419608736,
45
+ "n_output_tokens": 2823496,
46
+ "n_total_tokens": 860933737,
47
+ "agent_time_seconds": 126529,
48
+ "total_time_seconds": 243310,
49
+ "cost_usd": 434.54,
50
+ "mean_input_tokens_per_task": 1228295,
51
+ "mean_cache_tokens_per_task": 1175374,
52
+ "mean_output_tokens_per_task": 7908,
53
+ "mean_tokens_per_task": 2411579,
54
+ "mean_cost_usd_per_task": 1.22,
55
+ "mean_total_time_seconds_per_task": 681,
56
+ "mean_agent_time_seconds_per_task": 354
57
+ }
58
+ }
results/rh-swe-bench-claude-sonnet-4-6-claude-code.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "benchmark": {
3
+ "name": "RH SWE-Bench",
4
+ "repo": "rounakbende10/rh-swe-bench",
5
+ "num_tasks": 357,
6
+ "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
7
+ },
8
+ "harness": {
9
+ "name": "Claude Code",
10
+ "skills": [],
11
+ "is_oss": false,
12
+ "url": "https://github.com/anthropics/claude-code"
13
+ },
14
+ "model": {
15
+ "name": "Sonnet 4.6",
16
+ "repo": "Sonnet 4.6",
17
+ "is_oss": false,
18
+ "num_params": 1000,
19
+ "precision": "bf16",
20
+ "url": "https://www.anthropic.com/news/claude-sonnet-4-6"
21
+ },
22
+ "environment": {
23
+ "name": "harbor",
24
+ "config": {
25
+ "path": null,
26
+ "name": "rh-swe-bench",
27
+ "version": null,
28
+ "ref": null,
29
+ "registry_url": null,
30
+ "registry_path": null,
31
+ "overwrite": false,
32
+ "download_dir": null,
33
+ "task_names": null,
34
+ "exclude_task_names": null,
35
+ "n_tasks": null
36
+ },
37
+ "url": "https://github.com/harbor-framework/harbor"
38
+ },
39
+ "metrics": {
40
+ "n_tasks": 357,
41
+ "n_errors": 0,
42
+ "score": 0.557,
43
+ "n_input_tokens": 500304168,
44
+ "n_cache_tokens": 479035103,
45
+ "n_output_tokens": 4937782,
46
+ "n_total_tokens": 984277053,
47
+ "agent_time_seconds": 150652,
48
+ "total_time_seconds": 269008,
49
+ "cost_usd": 349.77,
50
+ "mean_input_tokens_per_task": 1401412,
51
+ "mean_cache_tokens_per_task": 1341835,
52
+ "mean_output_tokens_per_task": 13831,
53
+ "mean_tokens_per_task": 2757078,
54
+ "mean_cost_usd_per_task": 0.98,
55
+ "mean_total_time_seconds_per_task": 753,
56
+ "mean_agent_time_seconds_per_task": 421
57
+ }
58
+ }