Hugging Face's logo

Spaces:
burtenshaw
/
beam-pi-programbench
Running

App Files Community
beam-pi-programbench
File size: 5,674 Bytes
5741b22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
{
  "study_id": "beam-pi-programbench-20261009",
  "model": "Beam-501B-A23B",
  "space_id": "burtenshaw/beam-pi-programbench",
  "project": "beam-pi-programbench-20261009",
  "global_api_token_cap": 130000000,
  "category_caps": {
    "benchmark": 120000000,
    "validation": 10000000
  },
  "daily_headroom_tokens": 20000000,
  "max_inflight_requests": 5,
  "episode_wall_seconds": 7200,
  "snapshot_seconds": [
    30,
    60,
    120,
    300,
    600,
    1200,
    2400,
    4800,
    7200
  ],
  "reference_prefill_tps": 10000,
  "reference_decode_tps": 265,
  "context_accounting": "estimated fresh input; full provider prompt at window transitions; actual completion tokens including reasoning",
  "tasks": [
    {
      "instance_id": "burntsushi__ripgrep.3b7fd44"
    },
    {
      "instance_id": "jqlang__jq.b33a763"
    },
    {
      "instance_id": "burntsushi__xsv.f430466"
    },
    {
      "instance_id": "cmatsuoka__figlet.202a0a8"
    },
    {
      "instance_id": "sharkdp__hexyl.2e26437"
    }
  ],
  "runs": [
    {
      "run_id": "xsv-single-r1",
      "task": "burntsushi__xsv.f430466",
      "condition": "single",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 1000000
    },
    {
      "run_id": "xsv-peers-r1",
      "task": "burntsushi__xsv.f430466",
      "condition": "peers",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "xsv-async-r1",
      "task": "burntsushi__xsv.f430466",
      "condition": "async",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "hexyl-peers-r1",
      "task": "sharkdp__hexyl.2e26437",
      "condition": "peers",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "hexyl-async-r1",
      "task": "sharkdp__hexyl.2e26437",
      "condition": "async",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "hexyl-single-r1",
      "task": "sharkdp__hexyl.2e26437",
      "condition": "single",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 1000000
    },
    {
      "run_id": "ripgrep-async-r1",
      "task": "burntsushi__ripgrep.3b7fd44",
      "condition": "async",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "ripgrep-single-r1",
      "task": "burntsushi__ripgrep.3b7fd44",
      "condition": "single",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 1000000
    },
    {
      "run_id": "ripgrep-peers-r1",
      "task": "burntsushi__ripgrep.3b7fd44",
      "condition": "peers",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "figlet-single-r1",
      "task": "cmatsuoka__figlet.202a0a8",
      "condition": "single",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 1000000
    },
    {
      "run_id": "figlet-peers-r1",
      "task": "cmatsuoka__figlet.202a0a8",
      "condition": "peers",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "figlet-async-r1",
      "task": "cmatsuoka__figlet.202a0a8",
      "condition": "async",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "jq-peers-r1",
      "task": "jqlang__jq.b33a763",
      "condition": "peers",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "jq-async-r1",
      "task": "jqlang__jq.b33a763",
      "condition": "async",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 200000
    },
    {
      "run_id": "jq-single-r1",
      "task": "jqlang__jq.b33a763",
      "condition": "single",
      "category": "benchmark",
      "api_token_cap": 8000000,
      "context_token_cap": 1000000,
      "agent_context_token_cap": 1000000
    },
    {
      "run_id": "validation-single",
      "task": "synthetic-tool-and-git-validation",
      "condition": "single",
      "category": "validation",
      "api_token_cap": 3000000,
      "context_token_cap": 100000,
      "agent_context_token_cap": 100000
    },
    {
      "run_id": "validation-peers",
      "task": "synthetic-tool-and-git-validation",
      "condition": "peers",
      "category": "validation",
      "api_token_cap": 3000000,
      "context_token_cap": 100000,
      "agent_context_token_cap": 50000
    },
    {
      "run_id": "validation-async",
      "task": "synthetic-tool-and-git-validation",
      "condition": "async",
      "category": "validation",
      "api_token_cap": 3000000,
      "context_token_cap": 100000,
      "agent_context_token_cap": 50000
    }
  ]
}