new rewards

models.py

@@ -99,7 +99,7 @@ class SupergamesObservation(Observation):
 
 class StepResult(BaseModel):
     observation: SupergamesObservation
-    reward: float = Field(..., ge = 0.0, le=1.0)
+    reward: float = Field(..., ge=-100.0, le=100.0)
     done: bool
     info: Optional[Dict[str, Any]] = None
 

server/environment.py

@@ -32,6 +32,8 @@ class SupergamesEnvironment(Environment):
     """Environment for staffing and sprint allocation across Supergames tasks."""
 
     SUPPORTS_CONCURRENT_SESSIONS: bool = True
+    MIN_REWARD: float = -100.0
+    MAX_REWARD: float = 100.0
 
     def __init__(self):
         self.stateData = State(episode_id=str(uuid4()), step_count=0)
@@ -107,18 +109,65 @@ class SupergamesEnvironment(Environment):
 
         return summary
 
-    def computeReward(self, sprintRevenue: float) -> float:
-        if not self.done:
-            averageOptimal = self.estimatedOptimalRevenue / self.totalSteps
-            return round(min(1.0, max(0.0, sprintRevenue / averageOptimal)), 4)
+    def _clampReward(self, reward: float) -> float:
+        return round(min(self.MAX_REWARD, max(self.MIN_REWARD, reward)), 2)
 
-        grader = TASKS[self.taskId]["grade"]
-        if self.taskId == 1:
-            agentImpact = sum(item.revenueImpact for item in self.completedItems)
-            return float(grader(agentImpact, self.initialWorkQueue, self.initialStaffPool))
-        if self.taskId in (2, 3):
-            return float(grader(self.cumulativeRevenue, self.estimatedOptimalRevenue))
-        return float(grader(self.cumulativeRevenue, self.estimatedOptimalRevenue, self.crisisResolved))
+    @staticmethod
+    def _delayImpactPoints(item: WorkItem) -> float:
+        if item.impactDelay == 0:
+            return 45.0
+        if item.impactDelay == 1:
+            return 25.0
+        return 10.0
+
+    @staticmethod
+    def _completedItemPoints(item: WorkItem) -> float:
+        severityPoints = int(item.severity) * 3.0
+        revenuePoints = min(20.0, item.revenueImpact / 20.0)
+        churnPoints = item.churnReduction * 40.0
+        return (
+            SupergamesEnvironment._delayImpactPoints(item)
+            + severityPoints
+            + revenuePoints
+            + churnPoints
+        )
+
+    def computeReward(
+        self,
+        sprintRevenue: float,
+        completedItems: List[WorkItem] | None = None,
+        message: str = "ok",
+    ) -> float:
+        completedItems = completedItems or []
+
+        if message.startswith("Overallocation"):
+            return -70.0
+        if message.startswith("Unknown work item"):
+            return -60.0
+
+        reward = min(35.0, sprintRevenue / 100_000.0)
+        reward += sum(self._completedItemPoints(item) for item in completedItems)
+
+        unresolvedPenalty = 0.0
+        for item in self.workQueue:
+            if item.workType.value != "bug":
+                continue
+            if int(item.severity) == 5:
+                unresolvedPenalty += 18.0
+            elif int(item.severity) == 4:
+                unresolvedPenalty += 10.0
+        reward -= unresolvedPenalty
+
+        churnPenalty = sum(max(0.0, game.churnMult - 1.0) * 40.0 for game in self.games)
+        reward -= churnPenalty
+
+        if self.done:
+            if not any(item.workType.value == "bug" and int(item.severity) >= 4 for item in self.workQueue):
+                reward += 15.0
+            if self.taskId == 4:
+                reward += 20.0 if self.crisisResolved else -40.0
+
+        return self._clampReward(reward)
 
     def reset(self, task_id: int = 1, seed: int = 42) -> SupergamesObservation:
         if task_id not in TASKS:
@@ -159,7 +208,7 @@ class SupergamesEnvironment(Environment):
 
     def step(self, action: SupergamesAction) -> SupergamesObservation:
         if self.done:
-            return self.buildObservation(reward=self.computeReward(0.0))
+            return self.buildObservation(reward=0.0)
 
         self.stateData.step_count += 1
         reasoning = action.reasoning.strip()
@@ -190,7 +239,7 @@ class SupergamesEnvironment(Environment):
             self.crisisResolved = True
 
         self.done = self.stateData.step_count >= self.totalSteps
-        reward = self.computeReward(sprintRevenue)
+        reward = self.computeReward(sprintRevenue, completedItems, message)
 
         obs = self.buildObservation(reward=reward)
         obs.metadata.update(

train_grpo_supergames.ipynb

@@ -319,24 +319,29 @@
         "    return actions, matched_plan_shape\n",
         "\n",
         "\n",
-        "def estimate_revenue_upper_bound(\n",
-        "    games: List[Game],\n",
-        "    work_queue: List[WorkItem],\n",
-        "    total_steps: int,\n",
-        ") -> float:\n",
-        "    base_revenue = sum(game.monthlyRevenue for game in games) * total_steps\n",
-        "    impact_bound = 0.0\n",
-        "    for item in work_queue:\n",
-        "        active_sprints = max(0, total_steps - item.impactDelay)\n",
-        "        impact_bound += item.revenueImpact * (active_sprints / max(1, total_steps))\n",
-        "    return max(1.0, round(base_revenue + impact_bound, 2))\n",
+        "def clamp_reward(reward: float) -> float:\n",
+        "    return round(min(100.0, max(-100.0, reward)), 2)\n",
+        "\n",
+        "\n",
+        "def delay_impact_points(item: WorkItem) -> float:\n",
+        "    if item.impactDelay == 0:\n",
+        "        return 45.0\n",
+        "    if item.impactDelay == 1:\n",
+        "        return 25.0\n",
+        "    return 10.0\n",
+        "\n",
+        "\n",
+        "def completed_item_points(item: WorkItem) -> float:\n",
+        "    severity_points = int(item.severity) * 3.0\n",
+        "    revenue_points = min(20.0, item.revenueImpact / 20.0)\n",
+        "    churn_points = item.churnReduction * 40.0\n",
+        "    return delay_impact_points(item) + severity_points + revenue_points + churn_points\n",
         "\n",
         "\n",
         "def rollout_plan(completion: Any, difficulty: str, task_seed: int) -> Dict[str, Any]:\n",
         "    games, work_queue, staff_pool, total_steps, _goal = generate_task(difficulty, seed=int(task_seed))\n",
-        "    all_items = copy.deepcopy(work_queue)\n",
-        "    pending_crisis_items = [item for item in all_items if item.crisis]\n",
-        "    active_work_queue = [item for item in all_items if not item.crisis]\n",
+        "    pending_crisis_items = [copy.deepcopy(item) for item in work_queue if item.crisis]\n",
+        "    active_work_queue = [copy.deepcopy(item) for item in work_queue if not item.crisis]\n",
         "    games = copy.deepcopy(games)\n",
         "    staff_pool = copy.deepcopy(staff_pool)\n",
         "\n",
@@ -344,7 +349,7 @@
         "        actions, matched_plan_shape = parse_sprint_actions(completion, total_steps)\n",
         "    except Exception:\n",
         "        return {\n",
-        "            \"reward\": -0.5,\n",
+        "            \"reward\": -100.0,\n",
         "            \"valid_json\": False,\n",
         "            \"matched_plan_shape\": False,\n",
         "            \"cumulative_revenue\": 0.0,\n",
@@ -353,7 +358,8 @@
         "\n",
         "    cumulative_revenue = 0.0\n",
         "    messages = []\n",
-        "    completed_count = 0\n",
+        "    reward = 10.0 if matched_plan_shape else 0.0\n",
+        "    crisis_resolved = False\n",
         "\n",
         "    for sprint_idx in range(1, total_steps + 1):\n",
         "        if sprint_idx >= 2 and pending_crisis_items:\n",
@@ -368,20 +374,35 @@
         "            sprint_idx,\n",
         "        )\n",
         "        cumulative_revenue += sprint_revenue\n",
-        "        completed_count += len(completed_items)\n",
         "        messages.append(message)\n",
         "\n",
-        "    upper_bound = estimate_revenue_upper_bound(games, all_items, total_steps)\n",
-        "    revenue_score = max(0.0, min(1.0, cumulative_revenue / upper_bound))\n",
-        "    format_bonus = 0.05 if matched_plan_shape else 0.0\n",
-        "    completion_bonus = min(0.10, completed_count * 0.01)\n",
-        "    reward = revenue_score + format_bonus + completion_bonus\n",
+        "        if message.startswith(\"Overallocation\"):\n",
+        "            reward -= 70.0\n",
+        "            continue\n",
+        "        if message.startswith(\"Unknown work item\"):\n",
+        "            reward -= 60.0\n",
+        "            continue\n",
+        "\n",
+        "        reward += min(35.0, sprint_revenue / 100_000.0)\n",
+        "        reward += sum(completed_item_points(item) for item in completed_items)\n",
+        "        crisis_resolved = crisis_resolved or any(item.crisis for item in completed_items)\n",
         "\n",
-        "    if any(message.startswith(\"Overallocation\") or message.startswith(\"Unknown work item\") for message in messages):\n",
-        "        reward -= 0.25\n",
+        "        for item in active_work_queue:\n",
+        "            if item.workType.value != \"bug\":\n",
+        "                continue\n",
+        "            if int(item.severity) == 5:\n",
+        "                reward -= 18.0\n",
+        "            elif int(item.severity) == 4:\n",
+        "                reward -= 10.0\n",
+        "        reward -= sum(max(0.0, game.churnMult - 1.0) * 40.0 for game in games)\n",
+        "\n",
+        "    if not any(item.workType.value == \"bug\" and int(item.severity) >= 4 for item in active_work_queue):\n",
+        "        reward += 15.0\n",
+        "    if any(item.crisis for item in work_queue):\n",
+        "        reward += 20.0 if crisis_resolved else -40.0\n",
         "\n",
         "    return {\n",
-        "        \"reward\": float(max(-0.5, min(1.2, reward))),\n",
+        "        \"reward\": float(clamp_reward(reward)),\n",
         "        \"valid_json\": True,\n",
         "        \"matched_plan_shape\": matched_plan_shape,\n",
         "        \"cumulative_revenue\": cumulative_revenue,\n",