feat: add GRPO training notebook for GridMind-RL environment

scripts/gridmind_grpo_colab.ipynb

@@ -1007,30 +1007,39 @@
     "    except:\n",
     "        return 0.0\n",
     "\n",
-    "print(\"Evaluating trained model (2 episodes per task)...\")\n",
-    "trained_scores = {}\n",
-    "for task_id in [1, 2, 3, 4]:\n",
-    "    scores = []\n",
-    "    for ep in range(2):\n",
-    "        score = run_llm_episode(task_id=task_id)\n",
-    "        scores.append(score)\n",
-    "        print(f\"  Task {task_id} Episode {ep+1}: {score:.3f}\")\n",
-    "    trained_scores[task_id] = sum(scores) / len(scores)\n",
-    "\n",
-    "print(f\"\\nTrained Model Scores:\")\n",
-    "for task_id, avg in trained_scores.items():\n",
-    "    baseline = baseline_scores[task_id]\n",
-    "    improvement = ((avg - baseline) / baseline * 100) if baseline > 0 else 0\n",
-    "    print(f\"  Task {task_id}: {avg:.3f} (baseline: {baseline:.3f}, {improvement:+.1f}%)\")\n",
+    "RUN_EVALUATION = False\n",
+    "EVAL_EPISODES_PER_TASK = 1\n",
     "\n",
-    "trained_avg = sum(trained_scores.values()) / len(trained_scores)\n",
+    "trained_scores = {}\n",
     "baseline_avg = sum(baseline_scores.values()) / len(baseline_scores)\n",
-    "overall_improvement = ((trained_avg - baseline_avg) / baseline_avg * 100) if baseline_avg > 0 else 0\n",
-    "\n",
-    "print(f\"\\nOverall Scores:\")\n",
-    "print(f\"  Heuristic baseline: {baseline_avg:.3f}\")\n",
-    "print(f\"  Trained LLM:        {trained_avg:.3f}\")\n",
-    "print(f\"  Improvement:        {overall_improvement:+.1f}%\")"
+    "trained_avg = None\n",
+    "overall_improvement = None\n",
+    "\n",
+    "if RUN_EVALUATION:\n",
+    "    print(f\"Evaluating trained model ({EVAL_EPISODES_PER_TASK} episode(s) per task)...\")\n",
+    "    for task_id in [1, 2, 3, 4]:\n",
+    "        scores = []\n",
+    "        for ep in range(EVAL_EPISODES_PER_TASK):\n",
+    "            score = run_llm_episode(task_id=task_id)\n",
+    "            scores.append(score)\n",
+    "            print(f\"  Task {task_id} Episode {ep+1}: {score:.3f}\")\n",
+    "        trained_scores[task_id] = sum(scores) / len(scores)\n",
+    "\n",
+    "    print(f\"\\nTrained Model Scores:\")\n",
+    "    for task_id, avg in trained_scores.items():\n",
+    "        baseline = baseline_scores[task_id]\n",
+    "        improvement = ((avg - baseline) / baseline * 100) if baseline > 0 else 0\n",
+    "        print(f\"  Task {task_id}: {avg:.3f} (baseline: {baseline:.3f}, {improvement:+.1f}%)\")\n",
+    "\n",
+    "    trained_avg = sum(trained_scores.values()) / len(trained_scores)\n",
+    "    overall_improvement = ((trained_avg - baseline_avg) / baseline_avg * 100) if baseline_avg > 0 else 0\n",
+    "\n",
+    "    print(f\"\\nOverall Scores:\")\n",
+    "    print(f\"  Heuristic baseline: {baseline_avg:.3f}\")\n",
+    "    print(f\"  Trained LLM:        {trained_avg:.3f}\")\n",
+    "    print(f\"  Improvement:        {overall_improvement:+.1f}%\")\n",
+    "else:\n",
+    "    print(\"Skipping trained-model evaluation. Set RUN_EVALUATION = True to generate trained_scores and improvement metrics.\")"
    ]
   },
   {
@@ -1084,16 +1093,17 @@
     "\n",
     "random_by_task = {1: 0.35, 2: 0.28, 3: 0.21, 4: 0.25}\n",
     "heuristic_by_task = baseline_scores\n",
-    "trained_by_task = trained_scores\n",
+    "trained_by_task = trained_scores if trained_scores else {}\n",
     "\n",
     "random_vals = [random_by_task.get(t, 0.3) for t in tasks]\n",
     "heuristic_vals = [heuristic_by_task.get(t, 0.5) for t in tasks]\n",
-    "trained_vals = [trained_by_task.get(t, 0.5) for t in tasks]\n",
+    "trained_vals = [trained_by_task.get(t, np.nan) for t in tasks]\n",
     "\n",
     "baseline_avg = sum(heuristic_vals) / len(heuristic_vals)\n",
-    "trained_avg = sum(trained_vals) / len(trained_vals)\n",
+    "valid_trained_vals = [v for v in trained_vals if not np.isnan(v)]\n",
+    "trained_avg = (sum(valid_trained_vals) / len(valid_trained_vals)) if valid_trained_vals else None\n",
     "random_avg = sum(random_vals) / len(random_vals)\n",
-    "overall_improvement = ((trained_avg - baseline_avg) / baseline_avg * 100) if baseline_avg > 0 else 0\n",
+    "overall_improvement = ((trained_avg - baseline_avg) / baseline_avg * 100) if (trained_avg is not None and baseline_avg > 0) else None\n",
     "\n",
     "def smooth(values, window=5):\n",
     "    if not values or len(values) < 2:\n",
@@ -1193,7 +1203,9 @@
     "x = np.arange(len(tasks))\n",
     "w = 0.35\n",
     "ax2.bar(x - w/2, heuristic_vals, w, label='Heuristic Baseline', color=\"#58a6ff\", alpha=0.9)\n",
-    "ax2.bar(x + w/2, trained_vals, w, label='Trained LLM (GRPO)', color=\"#3fb950\", alpha=0.9)\n",
+    "if valid_trained_vals:\n",
+    "    trained_plot_vals = [0.0 if np.isnan(v) else v for v in trained_vals]\n",
+    "    ax2.bar(x + w/2, trained_plot_vals, w, label='Trained LLM (GRPO)', color=\"#3fb950\", alpha=0.9)\n",
     "ax2.set_xticks(x)\n",
     "ax2.set_xticklabels(task_labels)\n",
     "ax2.set_ylim(0, 1.05)\n",
@@ -1203,14 +1215,20 @@
     "ax2.grid(axis='y', alpha=0.3)\n",
     "fig2.tight_layout()\n",
     "comparison_path = 'results/gridmind_before_after_comparison.png'\n",
-    "fig2.savefig(comparison_path, dpi=100)\n",
+    "if valid_trained_vals:\n",
+    "    fig2.savefig(comparison_path, dpi=100)\n",
+    "else:\n",
+    "    comparison_path = None\n",
     "plt.close(fig2)\n",
     "\n",
     "print(f\"Saved training reward curve to {reward_curve_path}\")\n",
     "print(f\"Saved simple reward curve to {simple_reward_curve_path}\")\n",
     "if simple_loss_curve_path:\n",
     "    print(f\"Saved simple loss curve to {simple_loss_curve_path}\")\n",
-    "print(f\"Saved before/after graph to {comparison_path}\")\n",
+    "if comparison_path:\n",
+    "    print(f\"Saved before/after graph to {comparison_path}\")\n",
+    "else:\n",
+    "    print(\"Skipped before/after graph because RUN_EVALUATION is False.\")\n",
     "\n",
     "results = {\n",
     "    \"heuristic_baseline\": {\n",
@@ -1218,7 +1236,7 @@
     "        \"average\": baseline_avg\n",
     "    },\n",
     "    \"trained_llm\": {\n",
-    "        \"scores_by_task\": {str(k): v for k, v in trained_scores.items()},\n",
+    "        \"scores_by_task\": {str(k): v for k, v in trained_scores.items()} if trained_scores else {},\n",
     "        \"average\": trained_avg\n",
     "    },\n",
     "    \"improvement_percent\": overall_improvement,\n",
@@ -1248,8 +1266,12 @@
     "print(f\"  Model: {MODEL_NAME}\")\n",
     "print(f\"  Themes: {results['themes_covered']}\")\n",
     "print(f\"  Heuristic baseline: {baseline_avg:.3f}\")\n",
-    "print(f\"  Trained LLM: {trained_avg:.3f}\")\n",
-    "print(f\"  Improvement: {overall_improvement:+.1f}%\")"
+    "if trained_avg is not None:\n",
+    "    print(f\"  Trained LLM: {trained_avg:.3f}\")\n",
+    "if overall_improvement is not None:\n",
+    "    print(f\"  Improvement: {overall_improvement:+.1f}%\")\n",
+    "else:\n",
+    "    print(\"  Improvement: evaluation skipped\")"
    ]
   }
  ],