Files changed (1) hide show
  1. openenv.yaml +12 -9
openenv.yaml CHANGED
@@ -61,25 +61,28 @@ tasks:
61
  name: "Easy Email Sorting"
62
  description: "Sort obviously spam emails like prize winning scams"
63
  difficulty: easy
64
- score_range: [0.01, 0.99]
65
- grader: grade_easy_sorting
66
- grader_endpoint: /graders/easy_sorting
 
67
 
68
  - id: medium_sorting
69
  name: "Medium Email Sorting"
70
  description: "Distinguish between newsletters, promotions, and important emails"
71
  difficulty: medium
72
- score_range: [0.01, 0.99]
73
- grader: grade_medium_sorting
74
- grader_endpoint: /graders/medium_sorting
 
75
 
76
  - id: hard_sorting
77
  name: "Hard Email Sorting"
78
  description: "Detect phishing emails disguised as banks and trusted services"
79
  difficulty: hard
80
- score_range: [0.01, 0.99]
81
- grader: grade_hard_sorting
82
- grader_endpoint: /graders/hard_sorting
 
83
 
84
  episode:
85
  max_steps: 10
 
61
  name: "Easy Email Sorting"
62
  description: "Sort obviously spam emails like prize winning scams"
63
  difficulty: easy
64
+ max_steps: 10
65
+ grader:
66
+ type: llm
67
+ prompt_template: "Score the agent's email classification performance from 0.0 to 1.0 based on accuracy. The agent must correctly classify obvious spam emails (prize scams, fake giveaways) vs important emails. A score of 1.0 means perfect classification, 0.0 means all wrong."
68
 
69
  - id: medium_sorting
70
  name: "Medium Email Sorting"
71
  description: "Distinguish between newsletters, promotions, and important emails"
72
  difficulty: medium
73
+ max_steps: 10
74
+ grader:
75
+ type: llm
76
+ prompt_template: "Score the agent's email classification performance from 0.0 to 1.0. The agent must distinguish between spam, promotion, and important emails across 3 categories. A score of 1.0 means perfect classification, 0.0 means all wrong."
77
 
78
  - id: hard_sorting
79
  name: "Hard Email Sorting"
80
  description: "Detect phishing emails disguised as banks and trusted services"
81
  difficulty: hard
82
+ max_steps: 10
83
+ grader:
84
+ type: llm
85
+ prompt_template: "Score the agent's email classification performance from 0.0 to 1.0. The agent must detect subtle phishing emails disguised as legitimate bank and service communications. A score of 1.0 means perfect classification, 0.0 means all wrong."
86
 
87
  episode:
88
  max_steps: 10