Update graders.py

graders.py

@@ -1,83 +1,147 @@
-import os
-import json
-import urllib.request
+"""
+graders.py — Email Sorting OpenEnv
+3 grader tasks: easy, medium, hard. Scores 0.0–1.0.
+"""
 
-SPACE_URL = os.environ.get("SPACE_URL", "http://localhost:7860")
+from env import EMAILS
 
-def call_reset():
-    req = urllib.request.Request(f"{SPACE_URL}/reset", method="POST")
-    req.add_header("Content-Type", "application/json")
-    with urllib.request.urlopen(req, data=b"{}") as r:
-        return json.loads(r.read())
-
-def call_step(action):
-    payload = json.dumps({"action": action}).encode()
-    req = urllib.request.Request(f"{SPACE_URL}/step", method="POST")
-    req.add_header("Content-Type", "application/json")
-    with urllib.request.urlopen(req, data=payload) as r:
-        return json.loads(r.read())
+# ============================================
+# TASK 1 — EASY: Obvious spam detection
+# ============================================
 
 EASY_EMAILS = [
-    {"subject": "You won $1,000,000!", "body": "Click here to claim your prize.", "label": "spam"},
-    {"subject": "FREE iPhone giveaway", "body": "You have been selected. Claim now!", "label": "spam"},
-    {"subject": "Meeting at 3pm today", "body": "Reminder about our team sync.", "label": "important"},
-    {"subject": "Your invoice is ready", "body": "Your monthly invoice is attached.", "label": "important"},
-    {"subject": "50% off this weekend", "body": "Flash sale on all items this weekend.", "label": "promotion"},
+    {"subject": "You won $1,000,000!", "body": "Click here to claim your prize now.", "label": "spam"},
+    {"subject": "FREE iPhone giveaway", "body": "You have been selected. Claim before midnight!", "label": "spam"},
+    {"subject": "Congratulations! You're a winner", "body": "Send your details to collect your reward.", "label": "spam"},
+    {"subject": "Meeting at 3pm today", "body": "Hi, reminder about our team sync at 3pm.", "label": "important"},
+    {"subject": "Your invoice is ready", "body": "Please find your monthly invoice attached.", "label": "important"},
 ]
 
+def grade_easy_sorting(agent_fn=None):
+    """
+    Easy task: classify obvious spam vs important emails.
+    agent_fn(email) -> 'spam' | 'important' | 'promotion'
+    Returns score strictly in (0.0, 1.0)
+    """
+    if agent_fn is None:
+        agent_fn = baseline_agent
+    correct = 0
+    for email in EASY_EMAILS:
+        prediction = agent_fn(email)
+        if prediction == email["label"]:
+            correct += 1
+    raw = correct / len(EASY_EMAILS)
+    score = round(min(0.99, max(0.01, raw)), 2)
+    return {"task": "easy_sorting", "correct": correct, "total": len(EASY_EMAILS), "score": score}
+
+
+# ============================================
+# TASK 2 — MEDIUM: Spam + Promotion + Important
+# ============================================
+
 MEDIUM_EMAILS = [
-    {"subject": "Exclusive member offer", "body": "As a valued member enjoy 20% off.", "label": "promotion"},
-    {"subject": "Action required: password expiry", "body": "Your password expires in 3 days.", "label": "important"},
-    {"subject": "Urgent: verify your account", "body": "Your account will be suspended click to verify.", "label": "spam"},
+    {"subject": "50% off this weekend only!", "body": "Flash sale on all items. Use code SAVE50.", "label": "promotion"},
     {"subject": "New arrivals just for you", "body": "Check out our latest summer collection.", "label": "promotion"},
-    {"subject": "RE: Project update", "body": "Thanks for the update lets connect tomorrow.", "label": "important"},
-    {"subject": "Congratulations you are selected", "body": "Send your details to collect reward.", "label": "spam"},
+    {"subject": "Exclusive member offer inside", "body": "As a valued member, enjoy 20% off.", "label": "promotion"},
+    {"subject": "Action required: password expiry", "body": "Your password expires in 3 days. Reset it now.", "label": "important"},
+    {"subject": "RE: Project update", "body": "Thanks for the update. Let's connect tomorrow.", "label": "important"},
+    {"subject": "Urgent: verify your account", "body": "Your account will be suspended. Click to verify.", "label": "spam"},
 ]
 
+def grade_medium_sorting(agent_fn=None):
+    """
+    Medium task: classify emails across all 3 categories.
+    Returns score strictly in (0.0, 1.0)
+    """
+    if agent_fn is None:
+        agent_fn = baseline_agent
+    correct = 0
+    for email in MEDIUM_EMAILS:
+        prediction = agent_fn(email)
+        if prediction == email["label"]:
+            correct += 1
+    raw = correct / len(MEDIUM_EMAILS)
+    score = round(min(0.99, max(0.01, raw)), 2)
+    return {"task": "medium_sorting", "correct": correct, "total": len(MEDIUM_EMAILS), "score": score}
+
+
+# ============================================
+# TASK 3 — HARD: Subtle/tricky emails
+# ============================================
+
 HARD_EMAILS = [
     {"subject": "Your account statement", "body": "Your monthly statement from XYZ Bank is ready.", "label": "important"},
-    {"subject": "Limited time upgrade your plan", "body": "Switch to premium and save 30% this month.", "label": "promotion"},
-    {"subject": "Security alert", "body": "A new login was detected from unknown device.", "label": "important"},
+    {"subject": "Limited time: upgrade your plan", "body": "Switch to premium and save 30% this month only.", "label": "promotion"},
+    {"subject": "Security alert", "body": "A new login was detected from an unknown device.", "label": "important"},
     {"subject": "You have unclaimed rewards", "body": "Collect your loyalty points before they expire.", "label": "promotion"},
-    {"subject": "Final notice payment overdue", "body": "Send money now to avoid service interruption.", "label": "spam"},
-    {"subject": "Team offsite next Friday", "body": "Please confirm attendance for the offsite.", "label": "important"},
-    {"subject": "Claim your free trial", "body": "Start your 30 day free trial no credit card needed.", "label": "promotion"},
-    {"subject": "You have been pre-approved", "body": "You qualify for a loan apply now.", "label": "spam"},
+    {"subject": "Final notice: payment overdue", "body": "Send $500 to avoid service interruption.", "label": "spam"},
+    {"subject": "Team offsite next Friday", "body": "Please confirm your attendance for the offsite.", "label": "important"},
+    {"subject": "Claim your free trial", "body": "Start your 30-day free trial — no credit card needed.", "label": "promotion"},
+    {"subject": "You've been pre-approved!", "body": "You qualify for a $50,000 loan. Apply now.", "label": "spam"},
 ]
 
-def baseline_agent(email):
+def grade_hard_sorting(agent_fn=None):
+    """
+    Hard task: subtle emails that are easy to misclassify.
+    Returns score strictly in (0.0, 1.0)
+    """
+    if agent_fn is None:
+        agent_fn = baseline_agent
+    correct = 0
+    for email in HARD_EMAILS:
+        prediction = agent_fn(email)
+        if prediction == email["label"]:
+            correct += 1
+    raw = correct / len(HARD_EMAILS)
+    score = round(min(0.99, max(0.01, raw)), 2)
+    return {"task": "hard_sorting", "correct": correct, "total": len(HARD_EMAILS), "score": score}
+
+
+# ============================================
+# SIMPLE RULE-BASED AGENT (for baseline)
+# ============================================
+
+def baseline_agent(email: dict) -> str:
     text = (email.get("subject", "") + " " + email.get("body", "")).lower()
-    spam_kw  = ["won", "free", "prize", "claim", "urgent", "congratulations", "selected", "suspended", "overdue", "approved"]
-    promo_kw = ["off", "sale", "deal", "discount", "offer", "save", "upgrade", "trial", "rewards", "loyalty"]
-    if sum(1 for k in spam_kw  if k in text) >= 2: return "spam"
-    if sum(1 for k in promo_kw if k in text) >= 1: return "promotion"
+    spam_kw  = ["won", "free", "prize", "claim", "urgent", "congratulations",
+                "selected", "suspended", "verify", "overdue", "pre-approved"]
+    promo_kw = ["off", "sale", "deal", "discount", "offer", "save",
+                "shop", "upgrade", "trial", "rewards", "loyalty"]
+    spam_score  = sum(1 for k in spam_kw  if k in text)
+    promo_score = sum(1 for k in promo_kw if k in text)
+    if spam_score >= 2:   return "spam"
+    if promo_score >= 1:  return "promotion"
     return "important"
 
-def clamp(score):
-    """Ensure score is strictly between 0 and 1."""
-    return max(0.01, min(0.99, score))
 
-def grade_task(emails, task_name):
-    correct = 0
-    for email in emails:
-        prediction = baseline_agent(email)
-        if prediction == email["label"]:
-            correct += 1
-    raw_score = correct / len(emails)
-    score = clamp(round(raw_score, 2))
-    print(f"[GRADER] task={task_name} score={score} correct={correct}/{len(emails)}", flush=True)
-    return {"task": task_name, "score": score, "correct": correct, "total": len(emails)}
+# ============================================
+# RUN ALL GRADERS
+# ============================================
 
 def run_all_graders(agent_fn=None):
+    if agent_fn is None:
+        agent_fn = baseline_agent
+
     results = []
-    results.append(grade_task(EASY_EMAILS,   "easy"))
-    results.append(grade_task(MEDIUM_EMAILS, "medium"))
-    results.append(grade_task(HARD_EMAILS,   "hard"))
-    avg = clamp(round(sum(r["score"] for r in results) / len(results), 4))
-    return {"tasks": results, "average_score": avg, "all_passed": True}
+    results.append(grade_easy_sorting(agent_fn))
+    results.append(grade_medium_sorting(agent_fn))
+    results.append(grade_hard_sorting(agent_fn))
+
+    avg_score = round(sum(r["score"] for r in results) / len(results), 4)
+    all_passed = all(r["score"] >= 0.0 for r in results)
+
+    print(f"[GRADER] easy={results[0]['score']} medium={results[1]['score']} hard={results[2]['score']} avg={avg_score}", flush=True)
+
+    return {
+        "tasks": results,
+        "average_score": avg_score,
+        "all_passed": all_passed
+    }
+
 
 if __name__ == "__main__":
-    r = run_all_graders()
-    for t in r["tasks"]:
-        print(f"{t['task']}: {t['score']}")
-    print(f"Average: {r['average_score']}")
+    print("Running all graders with baseline agent...\n")
+    results = run_all_graders()
+    for r in results["tasks"]:
+        print(f"Task: {r['task']:8s} | Score: {r['score']} | {r['correct']}/{r['total']} correct")
+    print(f"\nAverage Score: {results['average_score']}")