Spaces:
Sleeping
Sleeping
I766446 commited on
Commit ·
e512403
1
Parent(s): aa17113
Fix: grader field must be plain string in openenv.yaml and GET /tasks
Browse files- openenv.yaml +3 -15
- server.py +3 -8
openenv.yaml
CHANGED
|
@@ -149,11 +149,7 @@ tasks:
|
|
| 149 |
straightforward for a competent agent.
|
| 150 |
dataset: tasks/easy/scenarios.json
|
| 151 |
n_scenarios: 5
|
| 152 |
-
grader:
|
| 153 |
-
enabled: true
|
| 154 |
-
class: tasks.easy.grader.grade
|
| 155 |
-
fn: tasks.easy.grader.grade
|
| 156 |
-
score_range: [0.0, 1.0]
|
| 157 |
weights: { decision: 0.50, evidence: 0.25, reasoning: 0.15, efficiency: 0.10 }
|
| 158 |
ideal_max_steps: 8
|
| 159 |
ground_truth_distribution: { allow: 1, flag: 1, remove: 3 }
|
|
@@ -166,11 +162,7 @@ tasks:
|
|
| 166 |
and evidence weighing.
|
| 167 |
dataset: tasks/medium/scenarios.json
|
| 168 |
n_scenarios: 5
|
| 169 |
-
grader:
|
| 170 |
-
enabled: true
|
| 171 |
-
class: tasks.medium.grader.grade
|
| 172 |
-
fn: tasks.medium.grader.grade
|
| 173 |
-
score_range: [0.0, 1.0]
|
| 174 |
weights: { decision: 0.35, evidence: 0.35, reasoning: 0.20, efficiency: 0.10 }
|
| 175 |
ideal_max_steps: 12
|
| 176 |
ground_truth_distribution: { allow: 1, flag: 4, remove: 0 }
|
|
@@ -183,11 +175,7 @@ tasks:
|
|
| 183 |
tolerance for ambiguity. All four scoring dimensions count equally.
|
| 184 |
dataset: tasks/hard/scenarios.json
|
| 185 |
n_scenarios: 5
|
| 186 |
-
grader:
|
| 187 |
-
enabled: true
|
| 188 |
-
class: tasks.hard.grader.grade
|
| 189 |
-
fn: tasks.hard.grader.grade
|
| 190 |
-
score_range: [0.0, 1.0]
|
| 191 |
weights: { decision: 0.30, evidence: 0.30, reasoning: 0.30, efficiency: 0.10 }
|
| 192 |
ideal_max_steps: 18
|
| 193 |
ground_truth_distribution: { allow: 0, flag: 5, remove: 0 }
|
|
|
|
| 149 |
straightforward for a competent agent.
|
| 150 |
dataset: tasks/easy/scenarios.json
|
| 151 |
n_scenarios: 5
|
| 152 |
+
grader: tasks.easy.grader.grade
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
weights: { decision: 0.50, evidence: 0.25, reasoning: 0.15, efficiency: 0.10 }
|
| 154 |
ideal_max_steps: 8
|
| 155 |
ground_truth_distribution: { allow: 1, flag: 1, remove: 3 }
|
|
|
|
| 162 |
and evidence weighing.
|
| 163 |
dataset: tasks/medium/scenarios.json
|
| 164 |
n_scenarios: 5
|
| 165 |
+
grader: tasks.medium.grader.grade
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
weights: { decision: 0.35, evidence: 0.35, reasoning: 0.20, efficiency: 0.10 }
|
| 167 |
ideal_max_steps: 12
|
| 168 |
ground_truth_distribution: { allow: 1, flag: 4, remove: 0 }
|
|
|
|
| 175 |
tolerance for ambiguity. All four scoring dimensions count equally.
|
| 176 |
dataset: tasks/hard/scenarios.json
|
| 177 |
n_scenarios: 5
|
| 178 |
+
grader: tasks.hard.grader.grade
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
weights: { decision: 0.30, evidence: 0.30, reasoning: 0.30, efficiency: 0.10 }
|
| 180 |
ideal_max_steps: 18
|
| 181 |
ground_truth_distribution: { allow: 0, flag: 5, remove: 0 }
|
server.py
CHANGED
|
@@ -310,14 +310,9 @@ async def list_tasks():
|
|
| 310 |
),
|
| 311 |
"n_scenarios": len(task),
|
| 312 |
"scenario_ids": [s["id"] for s in task.get_all_scenarios()],
|
| 313 |
-
"grader":
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
"fn": f"tasks.{difficulty}.grader.grade",
|
| 317 |
-
"score_range": [0.0, 1.0],
|
| 318 |
-
},
|
| 319 |
-
"grader_fn": f"tasks.{difficulty}.grader.grade",
|
| 320 |
-
"score_range": [0.0, 1.0],
|
| 321 |
"grader_weights": {
|
| 322 |
"decision": grader.W_DECISION,
|
| 323 |
"evidence": grader.W_EVIDENCE,
|
|
|
|
| 310 |
),
|
| 311 |
"n_scenarios": len(task),
|
| 312 |
"scenario_ids": [s["id"] for s in task.get_all_scenarios()],
|
| 313 |
+
"grader": f"tasks.{difficulty}.grader.grade",
|
| 314 |
+
"grader_fn": f"tasks.{difficulty}.grader.grade",
|
| 315 |
+
"score_range": [0.0, 1.0],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
"grader_weights": {
|
| 317 |
"decision": grader.W_DECISION,
|
| 318 |
"evidence": grader.W_EVIDENCE,
|