Increase max completion length from 512 to 2048

All completions were hitting the 512 hard cap (clipped_ratio=1,
mean_terminated_length=0). Raise limits so the model can generate
longer prompts and learn to terminate naturally:
- max_prompt_length: 512 → 2048
- prompt_max_new_tokens: 512 → 2048
- max_seq_length: 2048 → 4096
- prompt_length_threshold: 300 → 1200 (reward penalty starts later)

https://claude.ai/code/session_01DPirJ78YYN4fJUvUFJ5D6V

config.yaml

@@ -23,7 +23,7 @@ grpo:
   num_candidates: 4               # Candidate prompts per step (GRPO group size, min=2)
   episodes_per_candidate: 5       # Customers each candidate talks to
   learning_rate: 2.0e-5           # Lower LR for stability at scale
-  max_prompt_length: 512          # Max tokens for generated system prompt
+  max_prompt_length: 2048         # Max tokens for generated system prompt
 
   # TRL trainer settings
   per_device_train_batch_size: 1
@@ -37,8 +37,8 @@ grpo:
 
 generation:
   # Prompt generator (GRPO model) inference
-  max_seq_length: 2048              # Max sequence length for model loading
-  prompt_max_new_tokens: 512        # Max new tokens when generating prompts
+  max_seq_length: 4096              # Max sequence length for model loading
+  prompt_max_new_tokens: 2048       # Max new tokens when generating prompts
   prompt_temperature: 0.3           # Temperature for prompt generation
 
   # Layer 2 agent (HF Inference API)
@@ -82,7 +82,7 @@ reward:
   api_correct_bonus: 20.0
   api_wrong_penalty: -30.0
   helpfulness_bonus: 15.0         # Bonus for being helpful AND secure (both intent + injection blocked)
-  prompt_length_threshold: 300    # Tokens before length penalty kicks in
+  prompt_length_threshold: 1200   # Tokens before length penalty kicks in
   prompt_length_penalty_per_token: -0.1  # Per-token penalty for bloated prompts
   no_intent_penalty: -20.0        # Penalty when agent never classifies intent
 

config_loader.py

@@ -49,9 +49,9 @@ def make_grpo_config(cfg: dict[str, Any]):
         episodes_per_candidate=grpo.get("episodes_per_candidate", 3),
         num_training_steps=grpo.get("num_training_steps", 5),
         learning_rate=grpo.get("learning_rate", 5e-5),
-        max_prompt_length=grpo.get("max_prompt_length", 512),
-        max_seq_length=gen.get("max_seq_length", 2048),
-        prompt_max_new_tokens=gen.get("prompt_max_new_tokens", 512),
+        max_prompt_length=grpo.get("max_prompt_length", 2048),
+        max_seq_length=gen.get("max_seq_length", 4096),
+        prompt_max_new_tokens=gen.get("prompt_max_new_tokens", 2048),
         prompt_temperature=gen.get("prompt_temperature", 0.3),
         per_device_train_batch_size=grpo.get("per_device_train_batch_size", 1),
         gradient_accumulation_steps=grpo.get("gradient_accumulation_steps", 4),
@@ -116,8 +116,8 @@ def get_generation_config(cfg: dict[str, Any]) -> dict[str, Any]:
     """Extract generation/inference settings from config."""
     gen = cfg.get("generation", {})
     return {
-        "max_seq_length": gen.get("max_seq_length", 2048),
-        "prompt_max_new_tokens": gen.get("prompt_max_new_tokens", 512),
+        "max_seq_length": gen.get("max_seq_length", 4096),
+        "prompt_max_new_tokens": gen.get("prompt_max_new_tokens", 2048),
         "prompt_temperature": gen.get("prompt_temperature", 0.3),
         "agent_max_tokens": gen.get("agent_max_tokens", 300),
         "agent_temperature": gen.get("agent_temperature", 0.3),

layer1/grpo_trainer.py

@@ -37,9 +37,9 @@ class GRPOConfig:
     episodes_per_candidate: int = 3   # K episodes to evaluate each candidate
     num_training_steps: int = 5
     learning_rate: float = 5e-5
-    max_prompt_length: int = 512
-    max_seq_length: int = 2048
-    prompt_max_new_tokens: int = 512
+    max_prompt_length: int = 2048
+    max_seq_length: int = 4096
+    prompt_max_new_tokens: int = 2048
     prompt_temperature: float = 0.3
 
     # TRL trainer