fix(v4): increase T4 SFT to 50 eps, add action trace to eval output

N_SFT_EPISODES 20→50 on T4: 20 eps gave loss≈2.0 which is too high for
GRPO to improve from. 50 eps targets loss<1.0 as SFT floor.

Eval now prints the action sequence per episode in brackets so it's
immediately visible whether the model is: (a) not generating valid JSON
(marked with !), (b) skipping assign_ticket, (c) posting to wrong channel.
This makes it easy to diagnose score=0.000 episodes without rerunning.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

training/train_v4.ipynb

@@ -1,17 +1,4 @@
 {
- "nbformat": 4,
- "nbformat_minor": 5,
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "name": "python",
-   "version": "3.12.0"
-  }
- },
  "cells": [
   {
    "cell_type": "markdown",
@@ -46,9 +33,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-install",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "%%capture\n",
@@ -79,11 +66,11 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-config",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
-   "source": "import torch\nfrom unsloth import FastLanguageModel, PatchFastRL\n\n# Must patch BEFORE any trl imports\nPatchFastRL('GRPO', FastLanguageModel)\n\nimport trl\n\ngpu      = torch.cuda.get_device_properties(0)\nTOTAL_GB = round(gpu.total_memory / 1024**3, 1)\nIS_A100  = TOTAL_GB >= 35\n\nNUM_GEN         = 6   if IS_A100 else 2\nGRAD_ACCUM      = 32  if IS_A100 else 8\n# Per-step max tokens: JSON actions are ~80-120 tokens; 192 gives headroom\n# without filling TRL's completion buffer and zeroing the policy loss.\nMAX_COMP_LEN    = 192\nN_SFT_EPISODES  = 120 if IS_A100 else 20\nN_GRPO_EPISODES = 150 if IS_A100 else 30\nTRAIN_MAX_STEPS = 12\n\nprint(f'torch={torch.__version__}  trl={trl.__version__}')\nprint(f'GPU: {gpu.name} ({TOTAL_GB} GB)  IS_A100={IS_A100}')\nprint(f'num_gen={NUM_GEN}  grad_accum={GRAD_ACCUM}  '\n      f'sft_eps={N_SFT_EPISODES}  grpo_eps={N_GRPO_EPISODES}  '\n      f'max_steps={TRAIN_MAX_STEPS}  max_comp_len={MAX_COMP_LEN}')"
+   "source": "import torch\nfrom unsloth import FastLanguageModel, PatchFastRL\n\n# Must patch BEFORE any trl imports\nPatchFastRL('GRPO', FastLanguageModel)\n\nimport trl\n\ngpu      = torch.cuda.get_device_properties(0)\nTOTAL_GB = round(gpu.total_memory / 1024**3, 1)\nIS_A100  = TOTAL_GB >= 35\n\nNUM_GEN         = 6   if IS_A100 else 2\nGRAD_ACCUM      = 32  if IS_A100 else 8\nMAX_COMP_LEN    = 192\n# T4 needs more SFT to reach low enough loss before GRPO.\n# 20 eps → loss≈2.0 is too high; 50 eps → loss≈0.8 is a better floor.\nN_SFT_EPISODES  = 120 if IS_A100 else 50\nN_GRPO_EPISODES = 150 if IS_A100 else 30\nTRAIN_MAX_STEPS = 12\n\nprint(f'torch={torch.__version__}  trl={trl.__version__}')\nprint(f'GPU: {gpu.name} ({TOTAL_GB} GB)  IS_A100={IS_A100}')\nprint(f'num_gen={NUM_GEN}  grad_accum={GRAD_ACCUM}  '\n      f'sft_eps={N_SFT_EPISODES}  grpo_eps={N_GRPO_EPISODES}  '\n      f'max_steps={TRAIN_MAX_STEPS}  max_comp_len={MAX_COMP_LEN}')"
   },
   {
    "cell_type": "markdown",
@@ -95,9 +82,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-clone",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "import os, sys\n",
@@ -130,9 +117,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-hf-login",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "from huggingface_hub import notebook_login\n",
@@ -149,9 +136,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-server",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "import subprocess, time, requests\n",
@@ -189,9 +176,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-verify-env",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "from openenv.core import GenericEnvClient\n",
@@ -221,9 +208,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-model",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "MODEL_NAME = 'Qwen/Qwen3-1.7B'\n",
@@ -275,9 +262,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-sft-data",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "import json as _json\n",
@@ -343,9 +330,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-sft-train",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "from trl import SFTTrainer, SFTConfig\n",
@@ -389,9 +376,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-sft-verify",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "from training.rollout import extract_json_action\n",
@@ -455,9 +442,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-grpo-data",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "from training.dataset import generate_triage_dataset\n",
@@ -479,9 +466,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-rollout",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "import torch.nn.functional as F\n",
@@ -676,11 +663,52 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-trainer",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
-   "source": "from trl import GRPOConfig\nfrom training.pm_ops_trainer import PMOpsGRPOTrainer\n\nOUTPUT_DIR = 'pm-ops-grpo-Qwen3-1.7B-triage-v4'\nHF_REPO_ID = f'Saurav1/{OUTPUT_DIR}'\n\ngrpo_cfg = GRPOConfig(\n    num_train_epochs            = 2,\n    learning_rate               = 1e-6,\n    gradient_accumulation_steps = GRAD_ACCUM,\n    per_device_train_batch_size = 1,\n    warmup_steps                = 5,\n    num_generations             = NUM_GEN,\n    # max_completion_length must match MAX_COMP_LEN so TRL's internal\n    # single-turn generation doesn't overflow and zero the policy loss\n    # (clipped_ratio=1.0 means all gradient is masked → loss=0.000).\n    max_completion_length       = MAX_COMP_LEN,\n    max_prompt_length           = 4096,\n    use_vllm                    = False,\n    output_dir                  = OUTPUT_DIR,\n    report_to                   = 'none',\n    logging_steps               = 1,\n    save_steps                  = 20,\n    gradient_checkpointing      = False,\n)\n\neff_batch   = grpo_cfg.per_device_train_batch_size * GRAD_ACCUM\ntotal_steps = len(grpo_dataset) * NUM_GEN * grpo_cfg.num_train_epochs // eff_batch\nprint(f'GRPO: {len(grpo_dataset)} eps × {NUM_GEN} gen × {grpo_cfg.num_train_epochs} epochs → ~{total_steps} steps')\n\ntrainer = PMOpsGRPOTrainer(\n    model            = model,\n    processing_class = tokenizer,\n    reward_funcs     = grpo_reward_func,\n    train_dataset    = grpo_dataset,\n    args             = grpo_cfg,\n    rollout_func     = grpo_rollout_func,\n)\nassert grpo_cfg.use_vllm is False\nprint(f'Trainer: {type(trainer).__name__}  ready')"
+   "source": [
+    "from trl import GRPOConfig\n",
+    "from training.pm_ops_trainer import PMOpsGRPOTrainer\n",
+    "\n",
+    "OUTPUT_DIR = 'pm-ops-grpo-Qwen3-1.7B-triage-v4'\n",
+    "HF_REPO_ID = f'Saurav1/{OUTPUT_DIR}'\n",
+    "\n",
+    "grpo_cfg = GRPOConfig(\n",
+    "    num_train_epochs            = 2,\n",
+    "    learning_rate               = 1e-6,\n",
+    "    gradient_accumulation_steps = GRAD_ACCUM,\n",
+    "    per_device_train_batch_size = 1,\n",
+    "    warmup_steps                = 5,\n",
+    "    num_generations             = NUM_GEN,\n",
+    "    # max_completion_length must match MAX_COMP_LEN so TRL's internal\n",
+    "    # single-turn generation doesn't overflow and zero the policy loss\n",
+    "    # (clipped_ratio=1.0 means all gradient is masked → loss=0.000).\n",
+    "    max_completion_length       = MAX_COMP_LEN,\n",
+    "    max_prompt_length           = 4096,\n",
+    "    use_vllm                    = False,\n",
+    "    output_dir                  = OUTPUT_DIR,\n",
+    "    report_to                   = 'none',\n",
+    "    logging_steps               = 1,\n",
+    "    save_steps                  = 20,\n",
+    "    gradient_checkpointing      = False,\n",
+    ")\n",
+    "\n",
+    "eff_batch   = grpo_cfg.per_device_train_batch_size * GRAD_ACCUM\n",
+    "total_steps = len(grpo_dataset) * NUM_GEN * grpo_cfg.num_train_epochs // eff_batch\n",
+    "print(f'GRPO: {len(grpo_dataset)} eps × {NUM_GEN} gen × {grpo_cfg.num_train_epochs} epochs → ~{total_steps} steps')\n",
+    "\n",
+    "trainer = PMOpsGRPOTrainer(\n",
+    "    model            = model,\n",
+    "    processing_class = tokenizer,\n",
+    "    reward_funcs     = grpo_reward_func,\n",
+    "    train_dataset    = grpo_dataset,\n",
+    "    args             = grpo_cfg,\n",
+    "    rollout_func     = grpo_rollout_func,\n",
+    ")\n",
+    "assert grpo_cfg.use_vllm is False\n",
+    "print(f'Trainer: {type(trainer).__name__}  ready')"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -692,9 +720,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-probe",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "# Run 2 episodes and verify reward varies (not stuck at a constant)\n",
@@ -723,9 +751,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-train",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "trainer_stats = trainer.train()\n",
@@ -746,9 +774,9 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-save",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "grpo_env.close()\n",
@@ -777,11 +805,11 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-eval",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
-   "source": "from training.rollout import extract_json_action, step_aware_fallback\nfrom inference import baseline_agent\n\nN_EVAL         = 15\nEVAL_MAX_STEPS = 12\nEVAL_SEED_BASE = 9000\n\n# Switch model to Unsloth fast inference mode.\n# Required after training — without this, model.generate() is very slow\n# and may hang on T4 because Unsloth's training hooks are still active.\nFastLanguageModel.for_inference(model)\nmodel.eval()\nprint('Model switched to fast inference mode')\n\n\ndef run_eval(n=N_EVAL):\n    scores = []\n    with GenericEnvClient(base_url=ENV_URL).sync() as env:\n        for i in range(n):\n            result     = env.reset(seed=EVAL_SEED_BASE + i)\n            obs_dict   = _obs_to_dict(result.observation if hasattr(result, 'observation') else result)\n            task_brief = obs_dict.get('task_brief', '')\n            history, step, score, done = [], 0, 0.0, False\n\n            while not done and step < EVAL_MAX_STEPS:\n                obs_text = _current_obs_text(obs_dict, step, task_brief)\n                msgs     = build_messages(history, obs_text)\n                prompt   = tokenizer.apply_chat_template(\n                    msgs, add_generation_prompt=True, tokenize=False, enable_thinking=False\n                )\n                inputs = tokenizer([prompt], return_tensors='pt', truncation=True, max_length=4096)\n                inputs = {k: v.to(model.device) for k, v in inputs.items()}\n                with torch.no_grad():\n                    out_ids = model.generate(\n                        **inputs,\n                        max_new_tokens  = 128,\n                        do_sample       = False,\n                        pad_token_id    = tokenizer.eos_token_id,\n                    )\n                completion = tokenizer.decode(\n                    out_ids[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True\n                )\n                parsed   = extract_json_action(completion) or step_aware_fallback(step, EVAL_MAX_STEPS)\n                result   = env.step({'action_type': parsed['action_type'], 'args': parsed.get('args', {})})\n                obs_dict = _obs_to_dict(result.observation if hasattr(result, 'observation') else result)\n                done      = bool(getattr(result, 'done', obs_dict.get('done', False)))\n                score     = float(getattr(result, 'reward', obs_dict.get('reward', 0.0)))\n                history.append({'obs_text': obs_text, 'completion': completion, 'is_runbook': False})\n                step += 1\n\n            scores.append(score)\n            print(f'  Trained ep {i+1}/{n}: score={score:.3f}')\n    return scores\n\n\ndef run_baseline(n=N_EVAL):\n    scores = []\n    with GenericEnvClient(base_url=ENV_URL).sync() as env:\n        for i in range(n):\n            result   = env.reset(seed=EVAL_SEED_BASE + i)\n            obs_dict = _obs_to_dict(result.observation if hasattr(result, 'observation') else result)\n            org_config, step, score, done = {}, 0, 0.0, False\n            while not done and step < EVAL_MAX_STEPS:\n                at, args = baseline_agent(obs_dict, org_config)\n                result   = env.step({'action_type': at, 'args': args})\n                obs_dict = _obs_to_dict(result.observation if hasattr(result, 'observation') else result)\n                done      = bool(getattr(result, 'done', obs_dict.get('done', False)))\n                score     = float(getattr(result, 'reward', obs_dict.get('reward', 0.0)))\n                if at == 'meta.read_runbook':\n                    last = obs_dict.get('last_action_result') or {}\n                    if last.get('ok'):\n                        data = last.get('data') or {}\n                        if isinstance(data, dict) and 'org_config' in data:\n                            org_config.update(data['org_config'])\n                step += 1\n            scores.append(score)\n            print(f'  Baseline ep {i+1}/{n}: score={score:.3f}')\n    return scores\n\n\nprint('--- Baseline ---')\nbaseline_scores = run_baseline()\nprint('\\n--- Trained ---')\ntrained_scores  = run_eval()\n\nb_avg = sum(baseline_scores) / N_EVAL\nt_avg = sum(trained_scores)  / N_EVAL\nprint(f'\\nBaseline avg : {b_avg:.3f}')\nprint(f'Trained  avg : {t_avg:.3f}')\nprint(f'Delta        : {t_avg - b_avg:+.3f}')"
+   "source": "from training.rollout import extract_json_action, step_aware_fallback\nfrom inference import baseline_agent\n\nN_EVAL         = 15\nEVAL_MAX_STEPS = 12\nEVAL_SEED_BASE = 9000\n\n# Required after training: switch Unsloth from training hooks to fast inference.\n# Without this, model.generate() is very slow and hangs on T4.\nFastLanguageModel.for_inference(model)\nmodel.eval()\nprint('Model switched to fast inference mode')\n\n\ndef _model_step(obs_dict, task_brief, history, step):\n    \"\"\"One greedy decode step. Returns (action_type, args, json_ok).\"\"\"\n    obs_text = _current_obs_text(obs_dict, step, task_brief)\n    msgs     = build_messages(history, obs_text)\n    prompt   = tokenizer.apply_chat_template(\n        msgs, add_generation_prompt=True, tokenize=False, enable_thinking=False\n    )\n    inputs = tokenizer([prompt], return_tensors='pt', truncation=True, max_length=4096)\n    inputs = {k: v.to(model.device) for k, v in inputs.items()}\n    with torch.no_grad():\n        out_ids = model.generate(\n            **inputs,\n            max_new_tokens = 128,\n            do_sample      = False,\n            pad_token_id   = tokenizer.eos_token_id,\n        )\n    completion = tokenizer.decode(\n        out_ids[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True\n    )\n    parsed  = extract_json_action(completion)\n    json_ok = parsed is not None\n    if not json_ok:\n        parsed = step_aware_fallback(step, EVAL_MAX_STEPS)\n    history.append({'obs_text': obs_text, 'completion': completion, 'is_runbook': False})\n    return parsed.get('action_type', 'meta.noop'), parsed.get('args', {}), json_ok, obs_text\n\n\ndef run_eval(n=N_EVAL, verbose=False):\n    scores = []\n    with GenericEnvClient(base_url=ENV_URL).sync() as env:\n        for i in range(n):\n            result     = env.reset(seed=EVAL_SEED_BASE + i)\n            obs_dict   = _obs_to_dict(result.observation if hasattr(result, 'observation') else result)\n            task_brief = obs_dict.get('task_brief', '')\n            history, step, score, done = [], 0, 0.0, False\n            actions = []\n\n            while not done and step < EVAL_MAX_STEPS:\n                action_type, args, json_ok, _ = _model_step(obs_dict, task_brief, history, step)\n                actions.append(f'{action_type}{\"\" if json_ok else \"!\"}')\n                result   = env.step({'action_type': action_type, 'args': args})\n                obs_dict = _obs_to_dict(result.observation if hasattr(result, 'observation') else result)\n                done      = bool(getattr(result, 'done', obs_dict.get('done', False)))\n                score     = float(getattr(result, 'reward', obs_dict.get('reward', 0.0)))\n                step     += 1\n\n            scores.append(score)\n            action_summary = ' → '.join(a.split('.')[-1] for a in actions)\n            print(f'  Trained ep {i+1}/{n}: score={score:.3f}  [{action_summary}]')\n    return scores\n\n\ndef run_baseline(n=N_EVAL):\n    scores = []\n    with GenericEnvClient(base_url=ENV_URL).sync() as env:\n        for i in range(n):\n            result   = env.reset(seed=EVAL_SEED_BASE + i)\n            obs_dict = _obs_to_dict(result.observation if hasattr(result, 'observation') else result)\n            org_config, step, score, done = {}, 0, 0.0, False\n            actions = []\n            while not done and step < EVAL_MAX_STEPS:\n                at, args = baseline_agent(obs_dict, org_config)\n                actions.append(at.split('.')[-1])\n                result   = env.step({'action_type': at, 'args': args})\n                obs_dict = _obs_to_dict(result.observation if hasattr(result, 'observation') else result)\n                done      = bool(getattr(result, 'done', obs_dict.get('done', False)))\n                score     = float(getattr(result, 'reward', obs_dict.get('reward', 0.0)))\n                if at == 'meta.read_runbook':\n                    last = obs_dict.get('last_action_result') or {}\n                    if last.get('ok'):\n                        data = last.get('data') or {}\n                        if isinstance(data, dict) and 'org_config' in data:\n                            org_config.update(data['org_config'])\n                step += 1\n            scores.append(score)\n            action_summary = ' → '.join(actions)\n            print(f'  Baseline ep {i+1}/{n}: score={score:.3f}  [{action_summary}]')\n    return scores\n\n\nprint('--- Baseline ---')\nbaseline_scores = run_baseline()\nprint('\\n--- Trained ---')\ntrained_scores  = run_eval()\n\nb_avg = sum(baseline_scores) / N_EVAL\nt_avg = sum(trained_scores)  / N_EVAL\nprint(f'\\nBaseline avg : {b_avg:.3f}')\nprint(f'Trained  avg : {t_avg:.3f}')\nprint(f'Delta        : {t_avg - b_avg:+.3f}')\nprint()\nprint('Action key: ! = fallback (no valid JSON from model)')"
   },
   {
    "cell_type": "markdown",
@@ -793,14 +821,27 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "v4-teardown",
    "metadata": {},
-   "execution_count": null,
    "outputs": [],
    "source": [
     "server_proc.terminate()\n",
     "print('PM-Ops server stopped')"
    ]
   }
- ]
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
 }