Hugging Face's logo

cuber12
/
agent-harness

code-agents
software-engineering
llm-evaluation
reproducibility
Model card Files
xet
 Community
agent-harness
File size: 1,546 Bytes
d61821a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
"""Metric registry for paper-ready aggregation."""

METRIC_GROUPS: dict[str, tuple[str, ...]] = {
    "correctness": (
        "resolved_at_1",
        "fail_to_pass_rate",
        "pass_to_pass_rate",
        "regression_count",
    ),
    "localization": (
        "file_recall_at_1",
        "file_recall_at_5",
        "file_recall_at_10",
        "function_recall_at_10",
        "mrr",
        "ndcg_at_10",
        "all_gold_within_token_budget",
    ),
    "navigation": (
        "tokens_to_first_gold",
        "tool_calls_to_first_gold",
        "seconds_to_first_gold",
        "irrelevant_files_before_first_gold",
        "repeated_read_count",
        "context_churn_tokens",
    ),
    "editing": (
        "patch_applies",
        "syntax_valid",
        "lint_passes",
        "build_passes",
        "files_changed",
        "hunks_changed",
        "lines_changed",
    ),
    "efficiency": (
        "input_tokens",
        "output_tokens",
        "cached_tokens",
        "tool_result_tokens",
        "model_calls",
        "tool_calls",
        "test_runs",
        "wall_time_seconds",
        "tokens_per_second",
        "time_to_first_token_seconds",
    ),
    "index": (
        "index_build_seconds",
        "index_update_seconds",
        "query_p50_ms",
        "query_p95_ms",
        "index_ram_bytes",
        "index_disk_bytes",
    ),
    "reliability": (
        "timed_out",
        "tool_failure_count",
        "parser_failure_count",
        "malformed_edit_count",
        "failure_stage",
    ),
}