padmapriyagosakan commited on
Commit
220acb1
·
1 Parent(s): f4fa44f

Fix grader import path: use root-level graders module instead of server.graders

Browse files

- Created graders.py at repo root (identical to server/graders.py)
- Updated openenv.yaml: all 30 grader refs now use graders:XxxGrader (not server.graders:XxxGrader)
- Updated server/app.py _grader_ref() to match
- Updated Dockerfile to COPY graders.py to /app/graders.py
- Reverted health status to 'healthy'
- Build v6

Files changed (6) hide show
  1. Dockerfile +4 -1
  2. README.md +1 -1
  3. _test_import.py +31 -0
  4. graders.py +179 -0
  5. openenv.yaml +30 -30
  6. server/app.py +3 -3
Dockerfile CHANGED
@@ -8,7 +8,7 @@
8
  FROM python:3.11-slim
9
 
10
  # Build trigger — change this to force HF Space Docker rebuild
11
- LABEL build.version="2026-04-12-v5"
12
 
13
  # Non-root user for security
14
  RUN useradd -m -u 1000 appuser
@@ -30,6 +30,9 @@ COPY . /app/payops_env
30
  # Also copy openenv.yaml to WORKDIR so validators can find it at /app/openenv.yaml
31
  COPY openenv.yaml /app/openenv.yaml
32
 
 
 
 
33
  # Both /app (for payops_env.*) and /app/payops_env (for server.*) on PYTHONPATH
34
  # This matches the openenv.yaml app: server.app:app path and platform example format
35
  ENV PYTHONPATH="/app:/app/payops_env"
 
8
  FROM python:3.11-slim
9
 
10
  # Build trigger — change this to force HF Space Docker rebuild
11
+ LABEL build.version="2026-04-12-v6"
12
 
13
  # Non-root user for security
14
  RUN useradd -m -u 1000 appuser
 
30
  # Also copy openenv.yaml to WORKDIR so validators can find it at /app/openenv.yaml
31
  COPY openenv.yaml /app/openenv.yaml
32
 
33
+ # Copy graders.py to /app so platform can `import graders` from /app on PYTHONPATH
34
+ COPY graders.py /app/graders.py
35
+
36
  # Both /app (for payops_env.*) and /app/payops_env (for server.*) on PYTHONPATH
37
  # This matches the openenv.yaml app: server.app:app path and platform example format
38
  ENV PYTHONPATH="/app:/app/payops_env"
README.md CHANGED
@@ -13,7 +13,7 @@ tags:
13
  - reinforcement-learning
14
  pinned: false
15
  fullWidth: false
16
- build_version: 2026-04-12-v5
17
  ---
18
 
19
  # PayOps — Payment Operations Incident Response
 
13
  - reinforcement-learning
14
  pinned: false
15
  fullWidth: false
16
+ build_version: 2026-04-12-v6
17
  ---
18
 
19
  # PayOps — Payment Operations Incident Response
_test_import.py ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test that graders can be imported and called from repo root - simulating platform validator."""
2
+ import importlib
3
+ import sys
4
+ import os
5
+
6
+ # DO NOT add anything to sys.path - simulate platform environment
7
+ # (Just use the fact that cwd is the repo root, which Python adds to sys.path[0])
8
+
9
+ mod = importlib.import_module("graders")
10
+
11
+ grader_classes = [name for name in dir(mod) if name.endswith("Grader") and name != "_BaseGrader"]
12
+ print(f"Found {len(grader_classes)} grader classes")
13
+
14
+ results = []
15
+ for name in grader_classes:
16
+ cls = getattr(mod, name)
17
+ instance = cls()
18
+ result = instance()
19
+ score = result["score"]
20
+ ok = 0.0 < score < 1.0
21
+ results.append((name, score, ok))
22
+ if not ok:
23
+ print(f" FAIL: {name} score={score}")
24
+
25
+ passed = sum(1 for _, _, ok in results if ok)
26
+ print(f"Passed: {passed}/{len(results)}")
27
+ if passed == len(results):
28
+ print("ALL GRADERS PASS - import works from repo root")
29
+ else:
30
+ print("SOME GRADERS FAILED")
31
+ sys.exit(1)
graders.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Grader callables for PayOps OpenEnv tasks.
3
+
4
+ Referenced from openenv.yaml as: graders:EASY001Grader
5
+
6
+ Each grader is fully self-contained (zero external imports) so the platform
7
+ validator can import and call them in any isolated environment.
8
+
9
+ Returns {"score": float, "feedback": str} as required by the platform.
10
+ Scores are always strictly in (0, 1) — never exactly 0.0 or 1.0.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ # Platform requires scores strictly between 0 and 1 (exclusive).
15
+ _SCORE_MIN = 0.001
16
+ _SCORE_MAX = 0.999
17
+
18
+
19
+ def _clamp(score: float) -> float:
20
+ """Clamp score to the open interval (0, 1)."""
21
+ if score <= 0.0:
22
+ return _SCORE_MIN
23
+ if score >= 1.0:
24
+ return _SCORE_MAX
25
+ return score
26
+
27
+
28
+ class _BaseGrader:
29
+ """
30
+ Self-contained base grader.
31
+
32
+ Subclasses set:
33
+ correct_action : str
34
+ partial_credit : dict[str, float] (action -> score, all in (0, 1))
35
+ """
36
+
37
+ correct_action: str = ""
38
+ partial_credit: dict = {}
39
+
40
+ def grade(self, action: str = "", **kwargs):
41
+ if not action:
42
+ return {"score": 0.5, "feedback": "No action provided"}
43
+ if action == self.correct_action:
44
+ return {"score": _SCORE_MAX, "feedback": "Correct action"}
45
+ raw = float(self.partial_credit.get(action, 0.0))
46
+ score = _clamp(raw)
47
+ return {
48
+ "score": score,
49
+ "feedback": "Partial credit" if raw > 0 else "Incorrect action",
50
+ }
51
+
52
+ def __call__(self, *args, **kwargs):
53
+ action = args[0] if args else kwargs.get("action", "")
54
+ return self.grade(action, **{k: v for k, v in kwargs.items() if k != "action"})
55
+
56
+
57
+ # ── Easy ─────────────────────────────────────────────────────────────────────
58
+ class EASY001Grader(_BaseGrader):
59
+ correct_action = "approve"
60
+ partial_credit = {}
61
+
62
+ class EASY002Grader(_BaseGrader):
63
+ correct_action = "reject"
64
+ partial_credit = {"escalate": 0.4}
65
+
66
+ class EASY003Grader(_BaseGrader):
67
+ correct_action = "approve"
68
+ partial_credit = {}
69
+
70
+ class EASY004Grader(_BaseGrader):
71
+ correct_action = "flag"
72
+ partial_credit = {"escalate": 0.6, "hold": 0.5}
73
+
74
+ class EASY005Grader(_BaseGrader):
75
+ correct_action = "approve"
76
+ partial_credit = {}
77
+
78
+ class EASY006Grader(_BaseGrader):
79
+ correct_action = "flag"
80
+ partial_credit = {"hold": 0.6}
81
+
82
+ # ── Medium ────────────────────────────────────────────────────────────────────
83
+ class MED001Grader(_BaseGrader):
84
+ correct_action = "escalate"
85
+ partial_credit = {"flag": 0.5, "hold": 0.4}
86
+
87
+ class MED002Grader(_BaseGrader):
88
+ correct_action = "hold"
89
+ partial_credit = {"escalate": 0.6, "flag": 0.4}
90
+
91
+ class MED003Grader(_BaseGrader):
92
+ correct_action = "flag"
93
+ partial_credit = {"hold": 0.5, "escalate": 0.3}
94
+
95
+ class MED004Grader(_BaseGrader):
96
+ correct_action = "flag"
97
+ partial_credit = {"escalate": 0.5, "hold": 0.4}
98
+
99
+ class MED005Grader(_BaseGrader):
100
+ correct_action = "hold"
101
+ partial_credit = {"flag": 0.5, "escalate": 0.4}
102
+
103
+ class MED006Grader(_BaseGrader):
104
+ correct_action = "escalate"
105
+ partial_credit = {"flag": 0.4, "hold": 0.5}
106
+
107
+ class MED007Grader(_BaseGrader):
108
+ correct_action = "hold"
109
+ partial_credit = {"escalate": 0.6, "flag": 0.4}
110
+
111
+ class MED008Grader(_BaseGrader):
112
+ correct_action = "flag"
113
+ partial_credit = {"hold": 0.6, "escalate": 0.4}
114
+
115
+ # ── Hard ──────────────────────────────────────────────────────────────────────
116
+ class HARD001Grader(_BaseGrader):
117
+ correct_action = "escalate"
118
+ partial_credit = {"flag": 0.6, "hold": 0.5, "reject": 0.3}
119
+
120
+ class HARD002Grader(_BaseGrader):
121
+ correct_action = "reject"
122
+ partial_credit = {"escalate": 0.6, "hold": 0.5, "flag": 0.3}
123
+
124
+ class HARD003Grader(_BaseGrader):
125
+ correct_action = "reject"
126
+ partial_credit = {"escalate": 0.5, "flag": 0.2}
127
+
128
+ class HARD004Grader(_BaseGrader):
129
+ correct_action = "approve"
130
+ partial_credit = {"escalate": 0.5, "hold": 0.4, "flag": 0.3}
131
+
132
+ class HARD005Grader(_BaseGrader):
133
+ correct_action = "escalate"
134
+ partial_credit = {"flag": 0.5, "hold": 0.4, "reject": 0.3}
135
+
136
+ class HARD006Grader(_BaseGrader):
137
+ correct_action = "flag"
138
+ partial_credit = {"hold": 0.6, "escalate": 0.5}
139
+
140
+ class HARD007Grader(_BaseGrader):
141
+ correct_action = "reject"
142
+ partial_credit = {"hold": 0.5, "escalate": 0.4, "flag": 0.3}
143
+
144
+ class HARD008Grader(_BaseGrader):
145
+ correct_action = "reject"
146
+ partial_credit = {"hold": 0.5, "escalate": 0.4, "flag": 0.3}
147
+
148
+ class HARD009Grader(_BaseGrader):
149
+ correct_action = "escalate"
150
+ partial_credit = {"hold": 0.5, "flag": 0.4, "reject": 0.3}
151
+
152
+ class HARD010Grader(_BaseGrader):
153
+ correct_action = "reject"
154
+ partial_credit = {"hold": 0.5, "escalate": 0.4, "flag": 0.3}
155
+
156
+ # ── Critical ──────────────────────────────────────────────────────────────────
157
+ class CRIT001Grader(_BaseGrader):
158
+ correct_action = "approve"
159
+ partial_credit = {"escalate": 0.5, "hold": 0.4, "flag": 0.3}
160
+
161
+ class CRIT002Grader(_BaseGrader):
162
+ correct_action = "reject"
163
+ partial_credit = {"escalate": 0.4, "flag": 0.2}
164
+
165
+ class CRIT003Grader(_BaseGrader):
166
+ correct_action = "escalate"
167
+ partial_credit = {"flag": 0.4, "hold": 0.35, "reject": 0.3}
168
+
169
+ class CRIT004Grader(_BaseGrader):
170
+ correct_action = "reject"
171
+ partial_credit = {"escalate": 0.5, "hold": 0.4}
172
+
173
+ class CRIT005Grader(_BaseGrader):
174
+ correct_action = "reject"
175
+ partial_credit = {"escalate": 0.5, "hold": 0.4}
176
+
177
+ class CRIT006Grader(_BaseGrader):
178
+ correct_action = "escalate"
179
+ partial_credit = {"hold": 0.5, "reject": 0.4}
openenv.yaml CHANGED
@@ -16,179 +16,179 @@ tasks:
16
  task_id: EASY-001
17
  name: EASY-001
18
  difficulty: easy
19
- grader: server.graders:EASY001Grader
20
  score: 0.5
21
  - id: EASY-002
22
  task_id: EASY-002
23
  name: EASY-002
24
  difficulty: easy
25
- grader: server.graders:EASY002Grader
26
  score: 0.5
27
  - id: EASY-003
28
  task_id: EASY-003
29
  name: EASY-003
30
  difficulty: easy
31
- grader: server.graders:EASY003Grader
32
  score: 0.5
33
  - id: EASY-004
34
  task_id: EASY-004
35
  name: EASY-004
36
  difficulty: easy
37
- grader: server.graders:EASY004Grader
38
  score: 0.5
39
  - id: EASY-005
40
  task_id: EASY-005
41
  name: EASY-005
42
  difficulty: easy
43
- grader: server.graders:EASY005Grader
44
  score: 0.5
45
  - id: EASY-006
46
  task_id: EASY-006
47
  name: EASY-006
48
  difficulty: easy
49
- grader: server.graders:EASY006Grader
50
  score: 0.5
51
  - id: MED-001
52
  task_id: MED-001
53
  name: MED-001
54
  difficulty: medium
55
- grader: server.graders:MED001Grader
56
  score: 0.5
57
  - id: MED-002
58
  task_id: MED-002
59
  name: MED-002
60
  difficulty: medium
61
- grader: server.graders:MED002Grader
62
  score: 0.5
63
  - id: MED-003
64
  task_id: MED-003
65
  name: MED-003
66
  difficulty: medium
67
- grader: server.graders:MED003Grader
68
  score: 0.5
69
  - id: MED-004
70
  task_id: MED-004
71
  name: MED-004
72
  difficulty: medium
73
- grader: server.graders:MED004Grader
74
  score: 0.5
75
  - id: MED-005
76
  task_id: MED-005
77
  name: MED-005
78
  difficulty: medium
79
- grader: server.graders:MED005Grader
80
  score: 0.5
81
  - id: MED-006
82
  task_id: MED-006
83
  name: MED-006
84
  difficulty: medium
85
- grader: server.graders:MED006Grader
86
  score: 0.5
87
  - id: MED-007
88
  task_id: MED-007
89
  name: MED-007
90
  difficulty: medium
91
- grader: server.graders:MED007Grader
92
  score: 0.5
93
  - id: MED-008
94
  task_id: MED-008
95
  name: MED-008
96
  difficulty: medium
97
- grader: server.graders:MED008Grader
98
  score: 0.5
99
  - id: HARD-001
100
  task_id: HARD-001
101
  name: HARD-001
102
  difficulty: hard
103
- grader: server.graders:HARD001Grader
104
  score: 0.5
105
  - id: HARD-002
106
  task_id: HARD-002
107
  name: HARD-002
108
  difficulty: hard
109
- grader: server.graders:HARD002Grader
110
  score: 0.5
111
  - id: HARD-003
112
  task_id: HARD-003
113
  name: HARD-003
114
  difficulty: hard
115
- grader: server.graders:HARD003Grader
116
  score: 0.5
117
  - id: HARD-004
118
  task_id: HARD-004
119
  name: HARD-004
120
  difficulty: hard
121
- grader: server.graders:HARD004Grader
122
  score: 0.5
123
  - id: HARD-005
124
  task_id: HARD-005
125
  name: HARD-005
126
  difficulty: hard
127
- grader: server.graders:HARD005Grader
128
  score: 0.5
129
  - id: HARD-006
130
  task_id: HARD-006
131
  name: HARD-006
132
  difficulty: hard
133
- grader: server.graders:HARD006Grader
134
  score: 0.5
135
  - id: HARD-007
136
  task_id: HARD-007
137
  name: HARD-007
138
  difficulty: hard
139
- grader: server.graders:HARD007Grader
140
  score: 0.5
141
  - id: HARD-008
142
  task_id: HARD-008
143
  name: HARD-008
144
  difficulty: hard
145
- grader: server.graders:HARD008Grader
146
  score: 0.5
147
  - id: HARD-009
148
  task_id: HARD-009
149
  name: HARD-009
150
  difficulty: hard
151
- grader: server.graders:HARD009Grader
152
  score: 0.5
153
  - id: HARD-010
154
  task_id: HARD-010
155
  name: HARD-010
156
  difficulty: hard
157
- grader: server.graders:HARD010Grader
158
  score: 0.5
159
  - id: CRIT-001
160
  task_id: CRIT-001
161
  name: CRIT-001
162
  difficulty: critical
163
- grader: server.graders:CRIT001Grader
164
  score: 0.5
165
  - id: CRIT-002
166
  task_id: CRIT-002
167
  name: CRIT-002
168
  difficulty: critical
169
- grader: server.graders:CRIT002Grader
170
  score: 0.5
171
  - id: CRIT-003
172
  task_id: CRIT-003
173
  name: CRIT-003
174
  difficulty: critical
175
- grader: server.graders:CRIT003Grader
176
  score: 0.5
177
  - id: CRIT-004
178
  task_id: CRIT-004
179
  name: CRIT-004
180
  difficulty: critical
181
- grader: server.graders:CRIT004Grader
182
  score: 0.5
183
  - id: CRIT-005
184
  task_id: CRIT-005
185
  name: CRIT-005
186
  difficulty: critical
187
- grader: server.graders:CRIT005Grader
188
  score: 0.5
189
  - id: CRIT-006
190
  task_id: CRIT-006
191
  name: CRIT-006
192
  difficulty: critical
193
- grader: server.graders:CRIT006Grader
194
  score: 0.5
 
16
  task_id: EASY-001
17
  name: EASY-001
18
  difficulty: easy
19
+ grader: graders:EASY001Grader
20
  score: 0.5
21
  - id: EASY-002
22
  task_id: EASY-002
23
  name: EASY-002
24
  difficulty: easy
25
+ grader: graders:EASY002Grader
26
  score: 0.5
27
  - id: EASY-003
28
  task_id: EASY-003
29
  name: EASY-003
30
  difficulty: easy
31
+ grader: graders:EASY003Grader
32
  score: 0.5
33
  - id: EASY-004
34
  task_id: EASY-004
35
  name: EASY-004
36
  difficulty: easy
37
+ grader: graders:EASY004Grader
38
  score: 0.5
39
  - id: EASY-005
40
  task_id: EASY-005
41
  name: EASY-005
42
  difficulty: easy
43
+ grader: graders:EASY005Grader
44
  score: 0.5
45
  - id: EASY-006
46
  task_id: EASY-006
47
  name: EASY-006
48
  difficulty: easy
49
+ grader: graders:EASY006Grader
50
  score: 0.5
51
  - id: MED-001
52
  task_id: MED-001
53
  name: MED-001
54
  difficulty: medium
55
+ grader: graders:MED001Grader
56
  score: 0.5
57
  - id: MED-002
58
  task_id: MED-002
59
  name: MED-002
60
  difficulty: medium
61
+ grader: graders:MED002Grader
62
  score: 0.5
63
  - id: MED-003
64
  task_id: MED-003
65
  name: MED-003
66
  difficulty: medium
67
+ grader: graders:MED003Grader
68
  score: 0.5
69
  - id: MED-004
70
  task_id: MED-004
71
  name: MED-004
72
  difficulty: medium
73
+ grader: graders:MED004Grader
74
  score: 0.5
75
  - id: MED-005
76
  task_id: MED-005
77
  name: MED-005
78
  difficulty: medium
79
+ grader: graders:MED005Grader
80
  score: 0.5
81
  - id: MED-006
82
  task_id: MED-006
83
  name: MED-006
84
  difficulty: medium
85
+ grader: graders:MED006Grader
86
  score: 0.5
87
  - id: MED-007
88
  task_id: MED-007
89
  name: MED-007
90
  difficulty: medium
91
+ grader: graders:MED007Grader
92
  score: 0.5
93
  - id: MED-008
94
  task_id: MED-008
95
  name: MED-008
96
  difficulty: medium
97
+ grader: graders:MED008Grader
98
  score: 0.5
99
  - id: HARD-001
100
  task_id: HARD-001
101
  name: HARD-001
102
  difficulty: hard
103
+ grader: graders:HARD001Grader
104
  score: 0.5
105
  - id: HARD-002
106
  task_id: HARD-002
107
  name: HARD-002
108
  difficulty: hard
109
+ grader: graders:HARD002Grader
110
  score: 0.5
111
  - id: HARD-003
112
  task_id: HARD-003
113
  name: HARD-003
114
  difficulty: hard
115
+ grader: graders:HARD003Grader
116
  score: 0.5
117
  - id: HARD-004
118
  task_id: HARD-004
119
  name: HARD-004
120
  difficulty: hard
121
+ grader: graders:HARD004Grader
122
  score: 0.5
123
  - id: HARD-005
124
  task_id: HARD-005
125
  name: HARD-005
126
  difficulty: hard
127
+ grader: graders:HARD005Grader
128
  score: 0.5
129
  - id: HARD-006
130
  task_id: HARD-006
131
  name: HARD-006
132
  difficulty: hard
133
+ grader: graders:HARD006Grader
134
  score: 0.5
135
  - id: HARD-007
136
  task_id: HARD-007
137
  name: HARD-007
138
  difficulty: hard
139
+ grader: graders:HARD007Grader
140
  score: 0.5
141
  - id: HARD-008
142
  task_id: HARD-008
143
  name: HARD-008
144
  difficulty: hard
145
+ grader: graders:HARD008Grader
146
  score: 0.5
147
  - id: HARD-009
148
  task_id: HARD-009
149
  name: HARD-009
150
  difficulty: hard
151
+ grader: graders:HARD009Grader
152
  score: 0.5
153
  - id: HARD-010
154
  task_id: HARD-010
155
  name: HARD-010
156
  difficulty: hard
157
+ grader: graders:HARD010Grader
158
  score: 0.5
159
  - id: CRIT-001
160
  task_id: CRIT-001
161
  name: CRIT-001
162
  difficulty: critical
163
+ grader: graders:CRIT001Grader
164
  score: 0.5
165
  - id: CRIT-002
166
  task_id: CRIT-002
167
  name: CRIT-002
168
  difficulty: critical
169
+ grader: graders:CRIT002Grader
170
  score: 0.5
171
  - id: CRIT-003
172
  task_id: CRIT-003
173
  name: CRIT-003
174
  difficulty: critical
175
+ grader: graders:CRIT003Grader
176
  score: 0.5
177
  - id: CRIT-004
178
  task_id: CRIT-004
179
  name: CRIT-004
180
  difficulty: critical
181
+ grader: graders:CRIT004Grader
182
  score: 0.5
183
  - id: CRIT-005
184
  task_id: CRIT-005
185
  name: CRIT-005
186
  difficulty: critical
187
+ grader: graders:CRIT005Grader
188
  score: 0.5
189
  - id: CRIT-006
190
  task_id: CRIT-006
191
  name: CRIT-006
192
  difficulty: critical
193
+ grader: graders:CRIT006Grader
194
  score: 0.5
server/app.py CHANGED
@@ -272,8 +272,8 @@ async def schema():
272
 
273
 
274
  def _grader_ref(task_id: str) -> str:
275
- """Return the dotted grader reference string for a task id, e.g. 'server.graders:EASY001Grader'."""
276
- return f"server.graders:{task_id.replace('-', '')}Grader"
277
 
278
 
279
  def _clamp_score(v: float) -> float:
@@ -614,7 +614,7 @@ async def health():
614
  processed = 0
615
  total = len(TASKS)
616
  return {
617
- "status": "ok",
618
  "environment": "payops_env",
619
  "version": _APP_VERSION,
620
  "episode_id": episode_id,
 
272
 
273
 
274
  def _grader_ref(task_id: str) -> str:
275
+ """Return the dotted grader reference string for a task id, e.g. 'graders:EASY001Grader'."""
276
+ return f"graders:{task_id.replace('-', '')}Grader"
277
 
278
 
279
  def _clamp_score(v: float) -> float:
 
614
  processed = 0
615
  total = len(TASKS)
616
  return {
617
+ "status": "healthy",
618
  "environment": "payops_env",
619
  "version": _APP_VERSION,
620
  "episode_id": episode_id,