junaid0600 commited on
Commit
d4b572f
Β·
1 Parent(s): 59746b9

again fixed graders

Browse files
Files changed (1) hide show
  1. env/graders.py +193 -167
env/graders.py CHANGED
@@ -1,31 +1,51 @@
1
  import re
 
2
  from env.models import Action, DifficultyLevel
3
  from env.tasks import task_manager
4
 
5
- # HELPERS
 
 
 
 
6
 
7
- SCORE_MIN = 0.001
8
- SCORE_MAX = 0.999
9
 
10
- def _clamp(value: float) -> float:
11
- """Clamp a score to strictly (0, 1) β€” never 0.0 or 1.0."""
12
- return round(max(min(float(value), SCORE_MAX), SCORE_MIN), 4)
 
 
 
 
 
 
 
 
 
 
 
13
 
 
 
 
 
 
 
 
14
 
15
  def _normalize(text: str) -> str:
16
- """Normalize SQL for comparison β€” lowercase, strip whitespace, collapse spaces."""
17
  if not isinstance(text, str):
18
  return ""
19
  return re.sub(r"\s+", " ", text.strip().lower())
20
 
 
21
  def _safe_get(payload: dict, key: str, default=None):
22
- """Safe dict access β€” never KeyError."""
23
  if not isinstance(payload, dict):
24
  return default
25
  return payload.get(key, default)
26
 
 
27
  def _score_explanation(explanation: str) -> float:
28
- """Score explanation quality by length and keyword richness."""
29
  if not explanation or not isinstance(explanation, str):
30
  return SCORE_MIN
31
  explanation = explanation.strip()
@@ -37,31 +57,25 @@ def _score_explanation(explanation: str) -> float:
37
  return 0.10
38
  return 0.15
39
 
 
40
  def _score_confidence(confidence) -> float:
41
- """Give partial credit for providing a valid confidence score."""
42
  try:
43
  c = float(confidence)
44
- if 0.0 <= c <= 1.0:
45
  return 0.05
46
  except (TypeError, ValueError):
47
  pass
48
  return SCORE_MIN
49
 
 
50
  def _query_similarity(submitted: str, expected: str) -> float:
51
- """
52
- Multi-level SQL similarity check.
53
- Returns SCORE_MIN - SCORE_MAX based on how close the submitted query is to expected.
54
- Handles case, whitespace, and keyword-level matching.
55
- """
56
  s = _normalize(submitted)
57
  e = _normalize(expected)
58
 
59
- # Exact match after normalization
60
- # NOTE: max similarity is SCORE_MAX (0.999), so threshold must be <= SCORE_MAX
61
  if s == e:
 
62
  return SCORE_MAX
63
 
64
- # Tokenize and check keyword overlap
65
  s_tokens = set(s.split())
66
  e_tokens = set(e.split())
67
 
@@ -70,89 +84,85 @@ def _query_similarity(submitted: str, expected: str) -> float:
70
 
71
  overlap = len(s_tokens & e_tokens) / len(e_tokens)
72
 
73
- # Check critical keywords present
74
  critical_keywords = _extract_critical_keywords(e)
75
  critical_found = sum(1 for kw in critical_keywords if kw in s)
76
- critical_score = critical_found / len(critical_keywords) if critical_keywords else 0.0
 
 
 
 
77
 
78
- # Weighted combination
79
- similarity = round((overlap * 0.4) + (critical_score * 0.6), 4)
80
- return _clamp(similarity)
81
 
82
- def _extract_critical_keywords(query: str) -> list[str]:
83
- """Extract SQL keywords that are critical to correctness."""
84
  keywords = [
85
  "left join", "inner join", "right join",
86
  "group by", "order by", "having",
87
  "partition by", "coalesce", "distinct",
88
  "where", "on", "and", "or", "not",
89
  "count", "sum", "avg", "max", "min",
90
- "select", "from", "join"
91
  ]
92
- found = []
93
  q = query.lower()
94
- for kw in keywords:
95
- if kw in q:
96
- found.append(kw)
97
- return found
98
 
99
  def _score_error_type(submitted_type: str, expected_type: str) -> float:
100
- """Score for correctly identifying the error type."""
101
  if not submitted_type:
102
  return SCORE_MIN
103
  s = submitted_type.strip().lower()
104
  e = expected_type.strip().lower()
105
  if s == e:
106
  return 0.10
107
- # Partial: performance ↔ optimization are related
108
  related = {
109
  "performance": ["optimization", "slow", "index", "scan"],
110
  "logic": ["semantic", "incorrect", "wrong"],
111
- "syntax": ["parse", "grammar", "token"]
112
  }
113
  for canonical, aliases in related.items():
114
  if e == canonical and any(alias in s for alias in aliases):
115
  return 0.05
116
  return SCORE_MIN
117
 
118
- def _score_error_location(submitted_location: str, expected_location: str) -> float:
119
- """Score for correctly identifying WHERE in the query the error is."""
 
120
  if not submitted_location or not expected_location:
121
  return SCORE_MIN
122
  s = submitted_location.strip().lower()
123
  e = expected_location.strip().lower()
124
  if s == e:
125
  return 0.15
126
- # Partial: check if key location words overlap
127
  e_words = set(e.split())
128
  s_words = set(s.split())
129
- overlap = len(e_words & s_words) / len(e_words) if e_words else 0.0
 
 
130
  return _clamp(overlap * 0.10)
131
 
132
 
133
- # GRADERS PER DIFFICULTY
 
 
134
 
135
- def grade_easy(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
136
  """
137
- Easy task grader β€” syntax errors.
138
- Max score: SCORE_MAX
139
- Partial credit across: fix correctness, error location, error type, explanation, confidence.
140
- DETERMINISTIC: same input always returns same score.
141
  """
142
  if action is None or action.payload is None:
143
  return SCORE_MIN, {"error": "null_action"}, "No action provided."
144
 
145
- payload = action.payload
146
- score = 0.0
147
- breakdown = {}
148
  feedback_parts = []
149
 
150
- # ── 1. Query fix correctness (0.50) ──────────────────────────
151
- submitted_query = _safe_get(payload, "fixed_query", "") or _safe_get(payload, "optimized_query", "")
 
152
  expected_query = ground_truth.get("fixed_query", "")
153
  similarity = _query_similarity(submitted_query, expected_query)
154
 
155
- # Threshold uses SCORE_MAX (0.999) since that is the exact-match ceiling
156
  if similarity >= SCORE_MAX:
157
  fix_score = 0.50
158
  feedback_parts.append("Correct fix applied.")
@@ -166,63 +176,65 @@ def grade_easy(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
166
  fix_score = 0.0
167
  feedback_parts.append("Fix is incorrect or not provided.")
168
 
 
169
  score += fix_score
170
- breakdown["fix_correctness"] = _clamp(fix_score) if fix_score > 0 else SCORE_MIN
171
 
172
- # ── 2. Error location (0.15) ─────────────────────────────────
173
- submitted_location = _safe_get(payload, "error_location", "")
174
- expected_location = ground_truth.get("error_location", "")
175
- loc_score = _score_error_location(str(submitted_location), expected_location)
176
- score += loc_score
177
  breakdown["error_location"] = _clamp(loc_score)
 
178
  if loc_score > SCORE_MIN:
179
  feedback_parts.append("Correctly identified error location.")
180
 
181
- # ── 3. Error type (0.10) ─────────────────────────────────────
182
- submitted_type = _safe_get(payload, "error_type", "")
183
- expected_type = ground_truth.get("error_type", "syntax")
184
- type_score = _score_error_type(str(submitted_type), expected_type)
185
- score += type_score
186
  breakdown["error_type"] = _clamp(type_score)
 
187
  if type_score > SCORE_MIN:
188
  feedback_parts.append("Correctly identified error type.")
189
 
190
- # ── 4. Explanation quality (0.15) ─────────────────────────���──
191
- explanation = _safe_get(payload, "explanation", "") or _safe_get(payload, "change_made", "")
192
- expl_score = _score_explanation(str(explanation) if explanation else "")
193
- score += expl_score
194
  breakdown["explanation"] = _clamp(expl_score)
 
195
  if expl_score > SCORE_MIN:
196
  feedback_parts.append("Explanation provided.")
197
 
198
- # ── 5. Confidence (0.05) ─────────────────────────────────────
199
- confidence = _safe_get(payload, "confidence", None)
200
- conf_score = _score_confidence(confidence)
201
- score += conf_score
202
  breakdown["confidence"] = _clamp(conf_score)
 
203
 
204
  final_score = _clamp(score)
205
- feedback = " ".join(feedback_parts) if feedback_parts else "No valid response provided."
206
  return final_score, breakdown, feedback
207
 
208
 
209
- def grade_medium(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
210
  """
211
- Medium task grader β€” logic errors (wrong JOINs, wrong aggregations, etc).
212
- Max score: SCORE_MAX
213
- Higher bar: must correctly identify the logic flaw, not just syntax.
214
- DETERMINISTIC: same input always returns same score.
215
  """
216
  if action is None or action.payload is None:
217
  return SCORE_MIN, {"error": "null_action"}, "No action provided."
218
 
219
- payload = action.payload
220
- score = 0.0
221
- breakdown = {}
222
  feedback_parts = []
223
 
224
- # ── 1. Query fix correctness (0.40) ──────────────────────────
225
- submitted_query = _safe_get(payload, "fixed_query", "") or _safe_get(payload, "optimized_query", "")
 
226
  expected_query = ground_truth.get("fixed_query", "")
227
  similarity = _query_similarity(submitted_query, expected_query)
228
 
@@ -242,85 +254,82 @@ def grade_medium(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
242
  fix_score = 0.0
243
  feedback_parts.append("Fix is incorrect or missing.")
244
 
 
245
  score += fix_score
246
- breakdown["fix_correctness"] = _clamp(fix_score) if fix_score > 0 else SCORE_MIN
247
 
248
- # ── 2. Identifies the logic flaw (0.20) ──────────────────────
249
- explanation = str(_safe_get(payload, "explanation", "") or _safe_get(payload, "change_made", "") or "")
 
250
  error_type = ground_truth.get("error_type", "logic")
251
 
252
  logic_keywords = {
253
- "logic": ["join", "left join", "inner join", "having", "where", "group by",
254
- "aggregate", "subquery", "correlation", "distinct", "count"],
255
- "performance": ["index", "scan", "n+1", "correlated", "cartesian", "window"]
 
 
256
  }
257
-
258
  keywords_to_check = logic_keywords.get(error_type, logic_keywords["logic"])
259
  expl_lower = explanation.lower()
260
  keyword_hits = sum(1 for kw in keywords_to_check if kw in expl_lower)
261
  logic_score = _clamp(min(keyword_hits * 0.05, 0.20))
262
- score += logic_score
263
  breakdown["logic_flaw_identification"] = _clamp(logic_score)
 
264
  if logic_score > SCORE_MIN:
265
  feedback_parts.append("Shows understanding of the logic flaw.")
266
 
267
- # ── 3. Error location (0.15) ─────────────────────────────────
268
- submitted_location = _safe_get(payload, "error_location", "")
269
- expected_location = ground_truth.get("error_location", "")
270
- loc_score = _score_error_location(str(submitted_location), expected_location)
271
- score += loc_score
272
  breakdown["error_location"] = _clamp(loc_score)
 
273
 
274
- # ── 4. Explanation quality (0.15) ────────────────────────────
275
  expl_score = _score_explanation(explanation)
276
- score += expl_score
277
  breakdown["explanation"] = _clamp(expl_score)
 
278
 
279
- # ── 5. Confidence (0.05) ─────────────────────────────────────
280
- confidence = _safe_get(payload, "confidence", None)
281
- conf_score = _score_confidence(confidence)
282
- score += conf_score
283
  breakdown["confidence"] = _clamp(conf_score)
 
284
 
285
- # ── 6. Impact analysis bonus (0.05) ──────────────────────────
286
  impact = str(_safe_get(payload, "impact", "") or "")
287
- if len(impact.strip()) > 20:
288
- score += 0.05
289
- breakdown["impact_analysis"] = 0.05
 
290
  feedback_parts.append("Impact analysis provided.")
291
- else:
292
- breakdown["impact_analysis"] = SCORE_MIN
293
 
294
  final_score = _clamp(score)
295
- feedback = " ".join(feedback_parts) if feedback_parts else "No valid response provided."
296
  return final_score, breakdown, feedback
297
 
298
 
299
- def grade_hard(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
300
  """
301
- Hard task grader β€” performance issues (N+1, missing index, cartesian, etc).
302
- Max score: SCORE_MAX but frontier models expected ~0.10-0.20.
303
- Extremely strict β€” requires deep understanding of performance concepts.
304
- DETERMINISTIC: same input always returns same score.
305
  """
306
  if action is None or action.payload is None:
307
  return SCORE_MIN, {"error": "null_action"}, "No action provided."
308
 
309
- # ── FIX: initialize all variables before use ──────────────────
310
  payload = action.payload
311
  score = 0.0
312
  breakdown = {}
313
  feedback_parts = []
314
- rubric = ground_truth.get("scoring_rubric", {})
315
 
316
- # ── 1. Query correctness (0.30) ──────────────────────────────
317
- submitted_query = (
318
- _safe_get(payload, "optimized_query", "")
319
- or _safe_get(payload, "fixed_query", "")
320
- or ""
321
- )
322
- expected_query = ground_truth.get("fixed_query", "")
323
- similarity = _query_similarity(submitted_query, expected_query)
324
 
325
  if similarity >= SCORE_MAX:
326
  fix_score = 0.30
@@ -338,69 +347,75 @@ def grade_hard(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
338
  fix_score = 0.0
339
  feedback_parts.append("Query does not address the performance issue.")
340
 
 
341
  score += fix_score
342
- breakdown["query_correctness"] = _clamp(fix_score) if fix_score > 0 else SCORE_MIN
343
 
344
- # ── 2. Performance concept identification (0.30) ──────────────
345
- explanation = str(_safe_get(payload, "explanation", "") or _safe_get(payload, "change_made", "") or "")
346
- optimization = str(_safe_get(payload, "optimization_type", "") or "")
347
- combined_text = (explanation + " " + optimization).lower()
348
- perf_issue = ground_truth.get("performance_issue", {})
349
- issue_type = perf_issue.get("type", "").lower()
 
 
350
 
351
  performance_concept_map = {
352
- "n+1": ["n+1", "correlated subquery", "subquery per row", "multiple queries", "join instead"],
353
- "full table scan": ["full table scan", "index not used", "function on column", "sargable", "range scan", "seek"],
354
- "cartesian product": ["cartesian", "cross join", "missing join condition", "implicit join", "comma join"],
355
- "select *": ["select *", "over-fetch", "covering index", "column projection", "unnecessary columns"],
356
- "window function": ["window function", "partition by", "row_number", "subquery filter", "where clause window"]
 
 
 
 
 
 
 
357
  }
358
 
359
- concept_score = SCORE_MIN
360
  for concept, keywords in performance_concept_map.items():
361
- if any(concept_part in issue_type for concept_part in concept.split()):
362
  hits = sum(1 for kw in keywords if kw in combined_text)
363
- concept_score = _clamp(min(hits * 0.06, 0.30))
364
  break
365
 
366
- score += concept_score
367
  breakdown["performance_concept"] = _clamp(concept_score)
368
- if concept_score > SCORE_MIN:
 
369
  feedback_parts.append("Demonstrates understanding of the performance issue.")
370
 
371
- # ── 3. Explanation depth (0.15) ───────────────────────────────
372
  expl_score = _score_explanation(explanation)
373
  if len(explanation.strip()) > 150:
374
  expl_score = min(expl_score + 0.05, 0.15)
375
- score += expl_score
376
  breakdown["explanation_depth"] = _clamp(expl_score)
 
377
 
378
- # ── 4. Root cause analysis (0.10) ─────────────────────────────
379
  root_cause = str(_safe_get(payload, "root_cause", "") or "")
380
- if len(root_cause.strip()) > 30:
381
- score += 0.10
382
- breakdown["root_cause_analysis"] = 0.10
 
383
  feedback_parts.append("Root cause analysis provided.")
384
- else:
385
- breakdown["root_cause_analysis"] = SCORE_MIN
386
 
387
- # ── 5. Expected improvement (0.10) ────────────────────────────
388
  improvement = str(_safe_get(payload, "expected_improvement", "") or "")
389
- if len(improvement.strip()) > 20:
390
- score += 0.10
391
- breakdown["expected_improvement"] = 0.10
 
392
  feedback_parts.append("Performance improvement estimate provided.")
393
- else:
394
- breakdown["expected_improvement"] = SCORE_MIN
395
 
396
- # ── 6. Confidence (0.05) ──────────────────────────────────────
397
- confidence = _safe_get(payload, "confidence", None)
398
- conf_score = _score_confidence(confidence)
399
- score += conf_score
400
  breakdown["confidence"] = _clamp(conf_score)
 
401
 
402
  final_score = _clamp(score)
403
- feedback = " ".join(feedback_parts) if feedback_parts else "Performance issue not identified."
404
  return final_score, breakdown, feedback
405
 
406
 
@@ -408,11 +423,10 @@ def grade_hard(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
408
  # MAIN GRADER DISPATCHER
409
  # ─────────────────────────────────────────────
410
 
411
- def grade(action: Action, task_id: str) -> tuple[float, dict, str]:
412
  """
413
- Main grader entry point.
414
- Looks up ground truth, dispatches to correct grader by difficulty.
415
- ALWAYS returns (float, dict, str) β€” never crashes.
416
  """
417
  if action is None:
418
  return SCORE_MIN, {"error": "null_action"}, "No action provided."
@@ -425,12 +439,24 @@ def grade(action: Action, task_id: str) -> tuple[float, dict, str]:
425
 
426
  try:
427
  if difficulty == "easy":
428
- return grade_easy(action, ground_truth)
429
  elif difficulty == "medium":
430
- return grade_medium(action, ground_truth)
431
  elif difficulty == "hard":
432
- return grade_hard(action, ground_truth)
433
  else:
434
- return SCORE_MIN, {"error": "unknown_difficulty"}, f"Unknown difficulty: {difficulty}"
 
 
 
 
 
 
 
 
 
 
 
 
435
  except Exception as e:
436
- return SCORE_MIN, {"error": str(e)}, f"Grader error: {str(e)}"
 
1
  import re
2
+ import math
3
  from env.models import Action, DifficultyLevel
4
  from env.tasks import task_manager
5
 
6
+ # ─────────────────────────────────────────────
7
+ # SCORE BOUNDS (strictly between 0 and 1)
8
+ # ─────────────────────────────────────────────
9
+ SCORE_MIN = 0.001 # 0 < SCORE_MIN < 1
10
+ SCORE_MAX = 0.999 # 0 < SCORE_MAX < 1
11
 
 
 
12
 
13
+ def _clamp(value) -> float:
14
+ """
15
+ Guarantee the returned float is strictly inside (0, 1).
16
+ Handles NaN, Inf, None, strings, and any numeric type safely.
17
+ The round() call is applied AFTER the clamp, never before.
18
+ """
19
+ try:
20
+ v = float(value)
21
+ except (TypeError, ValueError):
22
+ return SCORE_MIN
23
+
24
+ # Guard against NaN and Β±Inf before any comparison
25
+ if not math.isfinite(v):
26
+ return SCORE_MIN
27
 
28
+ clamped = max(min(v, SCORE_MAX), SCORE_MIN)
29
+ return round(clamped, 4)
30
+
31
+
32
+ # ─────────────────────────────────────────────
33
+ # HELPERS
34
+ # ─────────────────────────────────────────────
35
 
36
  def _normalize(text: str) -> str:
 
37
  if not isinstance(text, str):
38
  return ""
39
  return re.sub(r"\s+", " ", text.strip().lower())
40
 
41
+
42
  def _safe_get(payload: dict, key: str, default=None):
 
43
  if not isinstance(payload, dict):
44
  return default
45
  return payload.get(key, default)
46
 
47
+
48
  def _score_explanation(explanation: str) -> float:
 
49
  if not explanation or not isinstance(explanation, str):
50
  return SCORE_MIN
51
  explanation = explanation.strip()
 
57
  return 0.10
58
  return 0.15
59
 
60
+
61
  def _score_confidence(confidence) -> float:
 
62
  try:
63
  c = float(confidence)
64
+ if math.isfinite(c) and 0.0 <= c <= 1.0:
65
  return 0.05
66
  except (TypeError, ValueError):
67
  pass
68
  return SCORE_MIN
69
 
70
+
71
  def _query_similarity(submitted: str, expected: str) -> float:
 
 
 
 
 
72
  s = _normalize(submitted)
73
  e = _normalize(expected)
74
 
 
 
75
  if s == e:
76
+ # Exact match β€” return SCORE_MAX, NOT 1.0
77
  return SCORE_MAX
78
 
 
79
  s_tokens = set(s.split())
80
  e_tokens = set(e.split())
81
 
 
84
 
85
  overlap = len(s_tokens & e_tokens) / len(e_tokens)
86
 
 
87
  critical_keywords = _extract_critical_keywords(e)
88
  critical_found = sum(1 for kw in critical_keywords if kw in s)
89
+ critical_score = (critical_found / len(critical_keywords)
90
+ if critical_keywords else 0.0)
91
+
92
+ raw = (overlap * 0.4) + (critical_score * 0.6)
93
+ return _clamp(raw)
94
 
 
 
 
95
 
96
+ def _extract_critical_keywords(query: str) -> list:
 
97
  keywords = [
98
  "left join", "inner join", "right join",
99
  "group by", "order by", "having",
100
  "partition by", "coalesce", "distinct",
101
  "where", "on", "and", "or", "not",
102
  "count", "sum", "avg", "max", "min",
103
+ "select", "from", "join",
104
  ]
 
105
  q = query.lower()
106
+ return [kw for kw in keywords if kw in q]
107
+
 
 
108
 
109
  def _score_error_type(submitted_type: str, expected_type: str) -> float:
 
110
  if not submitted_type:
111
  return SCORE_MIN
112
  s = submitted_type.strip().lower()
113
  e = expected_type.strip().lower()
114
  if s == e:
115
  return 0.10
 
116
  related = {
117
  "performance": ["optimization", "slow", "index", "scan"],
118
  "logic": ["semantic", "incorrect", "wrong"],
119
+ "syntax": ["parse", "grammar", "token"],
120
  }
121
  for canonical, aliases in related.items():
122
  if e == canonical and any(alias in s for alias in aliases):
123
  return 0.05
124
  return SCORE_MIN
125
 
126
+
127
+ def _score_error_location(submitted_location: str,
128
+ expected_location: str) -> float:
129
  if not submitted_location or not expected_location:
130
  return SCORE_MIN
131
  s = submitted_location.strip().lower()
132
  e = expected_location.strip().lower()
133
  if s == e:
134
  return 0.15
 
135
  e_words = set(e.split())
136
  s_words = set(s.split())
137
+ if not e_words:
138
+ return SCORE_MIN
139
+ overlap = len(e_words & s_words) / len(e_words)
140
  return _clamp(overlap * 0.10)
141
 
142
 
143
+ # ─────────────────────────────────────────────
144
+ # GRADERS
145
+ # ─────────────────────────────────────────────
146
 
147
+ def grade_easy(action: Action, ground_truth: dict) -> tuple:
148
  """
149
+ Easy β€” syntax errors.
150
+ Scoring budget: fix(0.50) + loc(0.15) + type(0.10) + expl(0.15) + conf(0.05) = 0.95
 
 
151
  """
152
  if action is None or action.payload is None:
153
  return SCORE_MIN, {"error": "null_action"}, "No action provided."
154
 
155
+ payload = action.payload
156
+ score = 0.0
157
+ breakdown = {}
158
  feedback_parts = []
159
 
160
+ # 1. Fix correctness (0.50)
161
+ submitted_query = (_safe_get(payload, "fixed_query", "")
162
+ or _safe_get(payload, "optimized_query", "") or "")
163
  expected_query = ground_truth.get("fixed_query", "")
164
  similarity = _query_similarity(submitted_query, expected_query)
165
 
 
166
  if similarity >= SCORE_MAX:
167
  fix_score = 0.50
168
  feedback_parts.append("Correct fix applied.")
 
176
  fix_score = 0.0
177
  feedback_parts.append("Fix is incorrect or not provided.")
178
 
179
+ breakdown["fix_correctness"] = _clamp(fix_score)
180
  score += fix_score
 
181
 
182
+ # 2. Error location (0.15)
183
+ loc_score = _score_error_location(
184
+ str(_safe_get(payload, "error_location", "") or ""),
185
+ ground_truth.get("error_location", ""),
186
+ )
187
  breakdown["error_location"] = _clamp(loc_score)
188
+ score += loc_score
189
  if loc_score > SCORE_MIN:
190
  feedback_parts.append("Correctly identified error location.")
191
 
192
+ # 3. Error type (0.10)
193
+ type_score = _score_error_type(
194
+ str(_safe_get(payload, "error_type", "") or ""),
195
+ ground_truth.get("error_type", "syntax"),
196
+ )
197
  breakdown["error_type"] = _clamp(type_score)
198
+ score += type_score
199
  if type_score > SCORE_MIN:
200
  feedback_parts.append("Correctly identified error type.")
201
 
202
+ # 4. Explanation quality (0.15)
203
+ explanation = (_safe_get(payload, "explanation", "")
204
+ or _safe_get(payload, "change_made", "") or "")
205
+ expl_score = _score_explanation(str(explanation))
206
  breakdown["explanation"] = _clamp(expl_score)
207
+ score += expl_score
208
  if expl_score > SCORE_MIN:
209
  feedback_parts.append("Explanation provided.")
210
 
211
+ # 5. Confidence (0.05)
212
+ conf_score = _score_confidence(_safe_get(payload, "confidence", None))
 
 
213
  breakdown["confidence"] = _clamp(conf_score)
214
+ score += conf_score
215
 
216
  final_score = _clamp(score)
217
+ feedback = " ".join(feedback_parts) or "No valid response provided."
218
  return final_score, breakdown, feedback
219
 
220
 
221
+ def grade_medium(action: Action, ground_truth: dict) -> tuple:
222
  """
223
+ Medium β€” logic errors.
224
+ Scoring budget: fix(0.40) + logic(0.20) + loc(0.15) + expl(0.15)
225
+ + conf(0.05) + impact(0.05) = 1.00 -> clamped to SCORE_MAX
 
226
  """
227
  if action is None or action.payload is None:
228
  return SCORE_MIN, {"error": "null_action"}, "No action provided."
229
 
230
+ payload = action.payload
231
+ score = 0.0
232
+ breakdown = {}
233
  feedback_parts = []
234
 
235
+ # 1. Fix correctness (0.40)
236
+ submitted_query = (_safe_get(payload, "fixed_query", "")
237
+ or _safe_get(payload, "optimized_query", "") or "")
238
  expected_query = ground_truth.get("fixed_query", "")
239
  similarity = _query_similarity(submitted_query, expected_query)
240
 
 
254
  fix_score = 0.0
255
  feedback_parts.append("Fix is incorrect or missing.")
256
 
257
+ breakdown["fix_correctness"] = _clamp(fix_score)
258
  score += fix_score
 
259
 
260
+ # 2. Logic flaw identification (0.20)
261
+ explanation = str(_safe_get(payload, "explanation", "")
262
+ or _safe_get(payload, "change_made", "") or "")
263
  error_type = ground_truth.get("error_type", "logic")
264
 
265
  logic_keywords = {
266
+ "logic": ["join", "left join", "inner join", "having", "where",
267
+ "group by", "aggregate", "subquery", "correlation",
268
+ "distinct", "count"],
269
+ "performance": ["index", "scan", "n+1", "correlated",
270
+ "cartesian", "window"],
271
  }
 
272
  keywords_to_check = logic_keywords.get(error_type, logic_keywords["logic"])
273
  expl_lower = explanation.lower()
274
  keyword_hits = sum(1 for kw in keywords_to_check if kw in expl_lower)
275
  logic_score = _clamp(min(keyword_hits * 0.05, 0.20))
 
276
  breakdown["logic_flaw_identification"] = _clamp(logic_score)
277
+ score += logic_score
278
  if logic_score > SCORE_MIN:
279
  feedback_parts.append("Shows understanding of the logic flaw.")
280
 
281
+ # 3. Error location (0.15)
282
+ loc_score = _score_error_location(
283
+ str(_safe_get(payload, "error_location", "") or ""),
284
+ ground_truth.get("error_location", ""),
285
+ )
286
  breakdown["error_location"] = _clamp(loc_score)
287
+ score += loc_score
288
 
289
+ # 4. Explanation quality (0.15)
290
  expl_score = _score_explanation(explanation)
 
291
  breakdown["explanation"] = _clamp(expl_score)
292
+ score += expl_score
293
 
294
+ # 5. Confidence (0.05)
295
+ conf_score = _score_confidence(_safe_get(payload, "confidence", None))
 
 
296
  breakdown["confidence"] = _clamp(conf_score)
297
+ score += conf_score
298
 
299
+ # 6. Impact analysis bonus (0.05)
300
  impact = str(_safe_get(payload, "impact", "") or "")
301
+ impact_score = 0.05 if len(impact.strip()) > 20 else 0.0
302
+ breakdown["impact_analysis"] = _clamp(impact_score)
303
+ score += impact_score
304
+ if impact_score > 0:
305
  feedback_parts.append("Impact analysis provided.")
 
 
306
 
307
  final_score = _clamp(score)
308
+ feedback = " ".join(feedback_parts) or "No valid response provided."
309
  return final_score, breakdown, feedback
310
 
311
 
312
+ def grade_hard(action: Action, ground_truth: dict) -> tuple:
313
  """
314
+ Hard β€” performance issues (N+1, missing index, cartesian, etc).
315
+ Scoring budget: query(0.30) + concept(0.30) + expl(0.15)
316
+ + root(0.10) + improvement(0.10) + conf(0.05) = 1.00 -> clamped
 
317
  """
318
  if action is None or action.payload is None:
319
  return SCORE_MIN, {"error": "null_action"}, "No action provided."
320
 
321
+ # All variables initialised before first use
322
  payload = action.payload
323
  score = 0.0
324
  breakdown = {}
325
  feedback_parts = []
326
+ _rubric = ground_truth.get("scoring_rubric", {}) # reserved for future use
327
 
328
+ # 1. Query correctness (0.30)
329
+ submitted_query = (_safe_get(payload, "optimized_query", "")
330
+ or _safe_get(payload, "fixed_query", "") or "")
331
+ expected_query = ground_truth.get("fixed_query", "")
332
+ similarity = _query_similarity(submitted_query, expected_query)
 
 
 
333
 
334
  if similarity >= SCORE_MAX:
335
  fix_score = 0.30
 
347
  fix_score = 0.0
348
  feedback_parts.append("Query does not address the performance issue.")
349
 
350
+ breakdown["query_correctness"] = _clamp(fix_score)
351
  score += fix_score
 
352
 
353
+ # 2. Performance concept identification (0.30)
354
+ explanation = str(_safe_get(payload, "explanation", "")
355
+ or _safe_get(payload, "change_made", "") or "")
356
+ optimization = str(_safe_get(payload, "optimization_type", "") or "")
357
+ combined_text = (explanation + " " + optimization).lower()
358
+ perf_issue = ground_truth.get("performance_issue", {})
359
+ issue_type = (perf_issue.get("type", "").lower()
360
+ if isinstance(perf_issue, dict) else "")
361
 
362
  performance_concept_map = {
363
+ "n+1": ["n+1", "correlated subquery", "subquery per row",
364
+ "multiple queries", "join instead"],
365
+ "full table scan": ["full table scan", "index not used",
366
+ "function on column", "sargable",
367
+ "range scan", "seek"],
368
+ "cartesian product": ["cartesian", "cross join",
369
+ "missing join condition",
370
+ "implicit join", "comma join"],
371
+ "select *": ["select *", "over-fetch", "covering index",
372
+ "column projection", "unnecessary columns"],
373
+ "window function": ["window function", "partition by", "row_number",
374
+ "subquery filter", "where clause window"],
375
  }
376
 
377
+ concept_score = 0.0
378
  for concept, keywords in performance_concept_map.items():
379
+ if any(part in issue_type for part in concept.split()):
380
  hits = sum(1 for kw in keywords if kw in combined_text)
381
+ concept_score = min(hits * 0.06, 0.30)
382
  break
383
 
 
384
  breakdown["performance_concept"] = _clamp(concept_score)
385
+ score += concept_score
386
+ if concept_score > 0:
387
  feedback_parts.append("Demonstrates understanding of the performance issue.")
388
 
389
+ # 3. Explanation depth (0.15)
390
  expl_score = _score_explanation(explanation)
391
  if len(explanation.strip()) > 150:
392
  expl_score = min(expl_score + 0.05, 0.15)
 
393
  breakdown["explanation_depth"] = _clamp(expl_score)
394
+ score += expl_score
395
 
396
+ # 4. Root cause analysis (0.10)
397
  root_cause = str(_safe_get(payload, "root_cause", "") or "")
398
+ root_score = 0.10 if len(root_cause.strip()) > 30 else 0.0
399
+ breakdown["root_cause_analysis"] = _clamp(root_score)
400
+ score += root_score
401
+ if root_score > 0:
402
  feedback_parts.append("Root cause analysis provided.")
 
 
403
 
404
+ # 5. Expected improvement (0.10)
405
  improvement = str(_safe_get(payload, "expected_improvement", "") or "")
406
+ imp_score = 0.10 if len(improvement.strip()) > 20 else 0.0
407
+ breakdown["expected_improvement"] = _clamp(imp_score)
408
+ score += imp_score
409
+ if imp_score > 0:
410
  feedback_parts.append("Performance improvement estimate provided.")
 
 
411
 
412
+ # 6. Confidence (0.05)
413
+ conf_score = _score_confidence(_safe_get(payload, "confidence", None))
 
 
414
  breakdown["confidence"] = _clamp(conf_score)
415
+ score += conf_score
416
 
417
  final_score = _clamp(score)
418
+ feedback = " ".join(feedback_parts) or "Performance issue not identified."
419
  return final_score, breakdown, feedback
420
 
421
 
 
423
  # MAIN GRADER DISPATCHER
424
  # ─────────────────────────────────────────────
425
 
426
+ def grade(action: Action, task_id: str) -> tuple:
427
  """
428
+ Main entry point. Always returns (float, dict, str) β€” never crashes.
429
+ The returned float is always strictly inside (0, 1).
 
430
  """
431
  if action is None:
432
  return SCORE_MIN, {"error": "null_action"}, "No action provided."
 
439
 
440
  try:
441
  if difficulty == "easy":
442
+ result = grade_easy(action, ground_truth)
443
  elif difficulty == "medium":
444
+ result = grade_medium(action, ground_truth)
445
  elif difficulty == "hard":
446
+ result = grade_hard(action, ground_truth)
447
  else:
448
+ return (SCORE_MIN,
449
+ {"error": "unknown_difficulty"},
450
+ f"Unknown difficulty: {difficulty}")
451
+
452
+ # Final safety net: re-clamp the returned score and every breakdown value
453
+ final_score, breakdown, feedback = result
454
+ safe_score = _clamp(final_score)
455
+ safe_breakdown = {
456
+ k: _clamp(v) if isinstance(v, (int, float)) else v
457
+ for k, v in breakdown.items()
458
+ }
459
+ return safe_score, safe_breakdown, feedback
460
+
461
  except Exception as e:
462
+ return SCORE_MIN, {"error": str(e)}, f"Grader error: {str(e)}"