Upload baseline.py with huggingface_hub

baseline.py

@@ -0,0 +1,232 @@
+"""Baseline inference script for the Scheduling Optimisation Environment.
+
+Runs GPT-4o-mini (or falls back to deterministic mock responses) against all
+three tasks and prints a structured score report.
+
+Usage:
+    OPENAI_API_KEY=sk-... python baseline.py
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import Any
+
+from environment import INSTANCE_BANK
+from graders.grader_classification import ConflictGrader
+from graders.grader_detection import FeasibilityGrader
+from graders.grader_fix import RepairGrader
+from models import Action
+
+
+def _get_openai_client():
+    """Return an OpenAI client, or None if unavailable."""
+    api_key = os.environ.get("OPENAI_API_KEY", "")
+    if not api_key:
+        return None
+    try:
+        from openai import OpenAI
+        return OpenAI(api_key=api_key)
+    except Exception:
+        return None
+
+
+def _llm_response(client, system_prompt: str, user_prompt: str) -> str:
+    """Call GPT-4o-mini and return the response text."""
+    try:
+        resp = client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            max_tokens=1024,
+            temperature=0.0,
+        )
+        return resp.choices[0].message.content.strip()
+    except Exception as e:
+        print(f"  [LLM error: {e}]")
+        return ""
+
+
+# ---------------------------------------------------------------------------
+# Mock fallback responses (used when no API key is available)
+# ---------------------------------------------------------------------------
+
+# Ground-truth feasibility labels — index aligns with INSTANCE_BANK
+_MOCK_FEASIBILITY: dict[int, str] = {
+    0: "infeasible", 1: "infeasible", 2: "infeasible", 3: "infeasible",
+    4: "infeasible", 5: "infeasible", 6: "infeasible", 7: "infeasible",
+    8: "infeasible", 9: "infeasible", 10: "feasible", 11: "feasible",
+}
+
+# Ground-truth violation types for infeasible instances
+_MOCK_CLASSIFICATION: dict[int, str] = {
+    0: "resource_overload",
+    1: "deadline_violation",
+    2: "precedence_violation",
+    3: "availability_conflict",
+    4: "capacity_exceeded",
+    5: "resource_overload",
+    6: "deadline_violation",
+    7: "precedence_violation",
+    8: "availability_conflict",
+    9: "capacity_exceeded",
+}
+
+
+def _mock_repair(instance_idx: int) -> str:
+    """Return the known optimal schedule JSON for mock mode."""
+    entry = INSTANCE_BANK[instance_idx]
+    optimal = entry.get("optimal_schedule", {})
+    if not optimal:
+        # Return the proposed schedule unchanged as a safe fallback
+        optimal = entry["instance"].get("proposed_schedule", {})
+    return json.dumps(optimal)
+
+
+# ---------------------------------------------------------------------------
+# Baseline runner
+# ---------------------------------------------------------------------------
+
+
+def run_baseline() -> dict[str, Any]:
+    """Execute the baseline across all three tasks and return scores."""
+    client = _get_openai_client()
+    use_llm = client is not None
+    mode = "GPT-4o-mini" if use_llm else "mock (no API key — oracle responses)"
+    print(f"\n{'='*65}")
+    print(f"  SchedulingOptEnv — Baseline Evaluation ({mode})")
+    print(f"{'='*65}\n")
+
+    results: dict[str, Any] = {"mode": mode, "tasks": {}}
+
+    # ----- Task 1: Feasibility Check -----
+    feas_grader = FeasibilityGrader()
+    feas_scores: list[float] = []
+    print("Task 1: Feasibility Check (easy)")
+    for i, entry in enumerate(INSTANCE_BANK):
+        instance_str = json.dumps(entry["instance"], indent=2)
+        if use_llm:
+            resp = _llm_response(
+                client,
+                (
+                    "You are a scheduling expert. Determine if the proposed schedule "
+                    "satisfies all constraints. Reply with ONLY 'feasible' or 'infeasible'."
+                ),
+                instance_str,
+            )
+        else:
+            resp = _MOCK_FEASIBILITY.get(i, "infeasible")
+        action = Action(response=resp, task_id="feasibility_check")
+        score = feas_grader.grade(action, entry)
+        feas_scores.append(score)
+        status = "CORRECT" if score >= 0.95 else "wrong"
+        expected = "feasible" if entry["is_feasible"] else "infeasible"
+        print(
+            f"  Instance {i:2d}: {status:7s} (score={score:.2f})  "
+            f"expected={expected}  [{entry['description'][:45]}]"
+        )
+
+    avg_feas = sum(feas_scores) / len(feas_scores) if feas_scores else 0.0
+    results["tasks"]["feasibility_check"] = {
+        "average_score": round(avg_feas, 4),
+        "num_instances": len(feas_scores),
+        "scores": feas_scores,
+    }
+    print(f"  >> Average: {avg_feas:.3f}\n")
+
+    # ----- Task 2: Conflict Classification -----
+    conf_grader = ConflictGrader()
+    conf_scores: list[float] = []
+    infeasible_entries = [(i, e) for i, e in enumerate(INSTANCE_BANK) if not e["is_feasible"]]
+    print("Task 2: Conflict Classification (medium)")
+    for i, entry in infeasible_entries:
+        instance_str = json.dumps(entry["instance"], indent=2)
+        if use_llm:
+            resp = _llm_response(
+                client,
+                (
+                    "You are a scheduling expert. Identify the constraint violation type. "
+                    "Reply with ONLY one of: resource_overload, deadline_violation, "
+                    "precedence_violation, availability_conflict, capacity_exceeded."
+                ),
+                instance_str,
+            )
+        else:
+            resp = _MOCK_CLASSIFICATION.get(i, "resource_overload")
+        action = Action(response=resp, task_id="conflict_classification")
+        score = conf_grader.grade(action, entry)
+        conf_scores.append(score)
+        status = "EXACT" if score >= 0.95 else ("partial" if score >= 0.45 else "wrong")
+        print(
+            f"  Instance {i:2d}: {status:7s} (score={score:.2f})  "
+            f"expected={entry['violation_type']}"
+        )
+
+    avg_conf = sum(conf_scores) / len(conf_scores) if conf_scores else 0.0
+    results["tasks"]["conflict_classification"] = {
+        "average_score": round(avg_conf, 4),
+        "num_instances": len(conf_scores),
+        "scores": conf_scores,
+    }
+    print(f"  >> Average: {avg_conf:.3f}\n")
+
+    # ----- Task 3: Schedule Repair -----
+    repair_grader = RepairGrader()
+    repair_scores: list[float] = []
+    repairable = [
+        (i, e) for i, e in enumerate(INSTANCE_BANK)
+        if not e["is_feasible"] and e.get("optimal_schedule")
+    ]
+    print("Task 3: Schedule Repair (hard)")
+    for i, entry in repairable:
+        instance_str = json.dumps(entry["instance"], indent=2)
+        if use_llm:
+            resp = _llm_response(
+                client,
+                (
+                    "You are a scheduling expert. Repair the infeasible schedule by "
+                    "returning a JSON object with key 'assignments': a list of "
+                    '{"job_id", "machine_id", "start_time"} dicts that satisfies all '
+                    "constraints and minimises makespan. Return ONLY valid JSON."
+                ),
+                instance_str,
+            )
+        else:
+            resp = _mock_repair(i)
+        action = Action(response=resp, task_id="schedule_repair")
+        score = repair_grader.grade(action, entry)
+        repair_scores.append(score)
+        print(
+            f"  Instance {i:2d}: score={score:.2f}  "
+            f"optimal_makespan={entry['optimal_makespan']}  "
+            f"[{entry['description'][:45]}]"
+        )
+
+    avg_repair = sum(repair_scores) / len(repair_scores) if repair_scores else 0.0
+    results["tasks"]["schedule_repair"] = {
+        "average_score": round(avg_repair, 4),
+        "num_instances": len(repair_scores),
+        "scores": repair_scores,
+    }
+    print(f"  >> Average: {avg_repair:.3f}\n")
+
+    # ----- Summary -----
+    overall = (avg_feas + avg_conf + avg_repair) / 3
+    results["overall_average"] = round(overall, 4)
+    print(f"{'='*65}")
+    print(f"  Overall Average Score: {overall:.3f}")
+    print(f"{'='*65}\n")
+
+    return results
+
+
+if __name__ == "__main__":
+    try:
+        run_baseline()
+    except Exception as e:
+        print(f"Baseline failed: {e}", file=sys.stderr)
+        sys.exit(1)