Centralize all training params in config.yaml (single source of truth)

- Add config.yaml with all GRPO, environment, reward, and report params
- Add config_loader.py to parse YAML into GRPOConfig/EnvConfig/RewardConfig
- Move hardcoded TRL trainer values (batch_size, grad_accum, save_steps)
into GRPOConfig and config.yaml
- train.py now loads from config.yaml, CLI flags override YAML values
- Config banner prints all parameters at startup
- Add pyyaml to dependencies

https://claude.ai/code/session_01DPirJ78YYN4fJUvUFJ5D6V

Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 
 COPY . .
 
-RUN pip install --no-cache-dir gradio huggingface-hub requests pydantic matplotlib python-dotenv
+RUN pip install --no-cache-dir gradio huggingface-hub requests pydantic matplotlib python-dotenv pyyaml
 
 EXPOSE 7860
 

config.yaml

@@ -0,0 +1,76 @@
+# ============================================================
+# Training Configuration — Single source of truth
+# ============================================================
+# All training parameters are defined here. CLI flags override
+# these values. To change defaults, edit this file.
+# ============================================================
+
+# --- Layer 1: GRPO RL Training ---
+# Qwen2.5-3B generates candidate system prompts, which are
+# evaluated by having Llama 3.1 8B use them as agent instructions.
+
+grpo:
+  # Prompt generator model (trained via RL)
+  model_name: "unsloth/Qwen2.5-3B-Instruct"
+
+  # LoRA adapter settings
+  lora_r: 16
+  lora_alpha: 16
+  lora_dropout: 0.0
+
+  # GRPO training loop
+  num_training_steps: 10          # Number of policy updates (GRPO iterations)
+  num_candidates: 4               # Candidate prompts per step (GRPO group size)
+  episodes_per_candidate: 7       # Customers each candidate talks to
+  learning_rate: 5.0e-5
+  max_prompt_length: 512          # Max tokens for generated system prompt
+
+  # TRL trainer settings
+  per_device_train_batch_size: 1
+  gradient_accumulation_steps: 4
+  logging_steps: 1
+  save_steps: 10
+
+
+# --- Layer 2: Conversation Environment ---
+# The simulated customer support environment.
+
+environment:
+  domain: "banking"
+  intents:
+    - "transfer"
+    - "check_balance"
+    - "block_card"
+  max_turns: 10                   # Max conversation turns before forced termination
+
+
+# --- Layer 0: Reward Function ---
+# Weights for the reward signal that drives GRPO.
+
+reward:
+  intent_correct_bonus: 50.0
+  intent_wrong_penalty: -50.0
+  fast_bonus: 20.0                # Bonus for <= 3 turns
+  medium_bonus: 10.0              # Bonus for <= 5 turns
+  slow_penalty_per_turn: -5.0     # Per turn beyond 8
+  injection_caught_bonus: 40.0
+  injection_succeeded_penalty: -100.0
+  api_correct_bonus: 20.0
+  api_wrong_penalty: -30.0
+
+
+# --- Report Generation ---
+# Settings for the post-training evaluation report.
+
+report:
+  enabled: true
+  output_dir: "./reports"
+  eval_episodes: 5                # Episodes per checkpoint evaluation
+  example_customers: 3            # Example conversations in report
+
+
+# --- Paths ---
+
+paths:
+  output_dir: "./grpo_output"
+  log_dir: "./logs"

config_loader.py

@@ -0,0 +1,104 @@
+"""
+Loads training configuration from config.yaml.
+
+Single source of truth for all training parameters.
+CLI arguments override values from the YAML file.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from layer0.reward import RewardConfig
+from layer2.environment import EnvConfig
+
+
+_ROOT = Path(__file__).resolve().parent
+_DEFAULT_CONFIG_PATH = _ROOT / "config.yaml"
+
+
+def load_config(config_path: str | Path | None = None) -> dict[str, Any]:
+    """Load the raw YAML config as a dict."""
+    path = Path(config_path) if config_path else _DEFAULT_CONFIG_PATH
+    if not path.exists():
+        raise FileNotFoundError(f"Config file not found: {path}")
+    with open(path) as f:
+        return yaml.safe_load(f)
+
+
+def make_grpo_config(cfg: dict[str, Any]):
+    """Build a GRPOConfig from the loaded YAML dict."""
+    # Import here to avoid circular imports
+    from layer1.grpo_trainer import GRPOConfig
+
+    grpo = cfg.get("grpo", {})
+    env = cfg.get("environment", {})
+    paths = cfg.get("paths", {})
+
+    return GRPOConfig(
+        model_name=grpo.get("model_name", "unsloth/Qwen2.5-3B-Instruct"),
+        lora_r=grpo.get("lora_r", 16),
+        lora_alpha=grpo.get("lora_alpha", 16),
+        lora_dropout=grpo.get("lora_dropout", 0.0),
+        num_candidates=grpo.get("num_candidates", 4),
+        episodes_per_candidate=grpo.get("episodes_per_candidate", 7),
+        num_training_steps=grpo.get("num_training_steps", 10),
+        learning_rate=grpo.get("learning_rate", 5e-5),
+        max_prompt_length=grpo.get("max_prompt_length", 512),
+        per_device_train_batch_size=grpo.get("per_device_train_batch_size", 1),
+        gradient_accumulation_steps=grpo.get("gradient_accumulation_steps", 4),
+        logging_steps=grpo.get("logging_steps", 1),
+        save_steps=grpo.get("save_steps", 10),
+        domain=env.get("domain", "banking"),
+        intents=env.get("intents", ["transfer", "check_balance", "block_card"]),
+        output_dir=paths.get("output_dir", "./grpo_output"),
+    )
+
+
+def make_env_config(cfg: dict[str, Any]) -> EnvConfig:
+    """Build an EnvConfig from the loaded YAML dict."""
+    env = cfg.get("environment", {})
+    reward = cfg.get("reward", {})
+
+    reward_config = RewardConfig(
+        intent_correct_bonus=reward.get("intent_correct_bonus", 50.0),
+        intent_wrong_penalty=reward.get("intent_wrong_penalty", -50.0),
+        fast_bonus=reward.get("fast_bonus", 20.0),
+        medium_bonus=reward.get("medium_bonus", 10.0),
+        slow_penalty_per_turn=reward.get("slow_penalty_per_turn", -5.0),
+        injection_caught_bonus=reward.get("injection_caught_bonus", 40.0),
+        injection_succeeded_penalty=reward.get("injection_succeeded_penalty", -100.0),
+        api_correct_bonus=reward.get("api_correct_bonus", 20.0),
+        api_wrong_penalty=reward.get("api_wrong_penalty", -30.0),
+    )
+
+    return EnvConfig(
+        domain=env.get("domain", "banking"),
+        intents=env.get("intents", ["transfer", "check_balance", "block_card"]),
+        max_turns=env.get("max_turns", 10),
+        reward_config=reward_config,
+    )
+
+
+def get_report_config(cfg: dict[str, Any]) -> dict[str, Any]:
+    """Extract report settings from config."""
+    report = cfg.get("report", {})
+    return {
+        "enabled": report.get("enabled", True),
+        "output_dir": report.get("output_dir", "./reports"),
+        "eval_episodes": report.get("eval_episodes", 5),
+        "example_customers": report.get("example_customers", 3),
+    }
+
+
+def get_paths(cfg: dict[str, Any]) -> dict[str, str]:
+    """Extract path settings from config."""
+    paths = cfg.get("paths", {})
+    return {
+        "output_dir": paths.get("output_dir", "./grpo_output"),
+        "log_dir": paths.get("log_dir", "./logs"),
+    }

layer1/grpo_trainer.py

@@ -39,6 +39,12 @@ class GRPOConfig:
     learning_rate: float = 5e-5
     max_prompt_length: int = 512
 
+    # TRL trainer
+    per_device_train_batch_size: int = 1
+    gradient_accumulation_steps: int = 4
+    logging_steps: int = 1
+    save_steps: int = 10
+
     # Environment
     domain: str = "banking"
     intents: list[str] = field(default_factory=lambda: list(BANKING_INTENTS))
@@ -258,13 +264,13 @@ class GRPOPromptTrainer:
         training_args = TRLGRPOConfig(
             output_dir=self.config.output_dir,
             num_train_epochs=1,
-            per_device_train_batch_size=1,
-            gradient_accumulation_steps=4,
+            per_device_train_batch_size=self.config.per_device_train_batch_size,
+            gradient_accumulation_steps=self.config.gradient_accumulation_steps,
             learning_rate=self.config.learning_rate,
             num_generations=self.config.num_candidates,
             max_completion_length=self.config.max_prompt_length,
-            logging_steps=1,
-            save_steps=10,
+            logging_steps=self.config.logging_steps,
+            save_steps=self.config.save_steps,
         )
 
         trainer = GRPOTrainer(

layer1/train.py

@@ -1,9 +1,15 @@
 """
 Layer 1 — GRPO training script for prompt optimization.
 
+All parameters are loaded from config.yaml (single source of truth).
+CLI flags override config.yaml values.
+
 Usage:
-    # GRPO training (requires GPU + train deps)
-    python -m layer1.train --steps 10
+    # Train with defaults from config.yaml
+    python -m layer1.train
+
+    # Override specific params
+    python -m layer1.train --steps 20 --episodes 10
 
     # Evaluate a single prompt
     python -m layer1.train --mode eval --prompt "You are a helpful agent."
@@ -23,12 +29,8 @@ load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file_
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from layer1.grpo_trainer import (
-    GRPOConfig,
-    GRPOPromptTrainer,
-    PromptEvaluator,
-    build_meta_prompt,
-)
+from config_loader import load_config, make_grpo_config, make_env_config, get_report_config, get_paths
+from layer1.grpo_trainer import GRPOConfig, GRPOPromptTrainer, PromptEvaluator
 from layer1.training_logger import TrainingLogger, ReportGenerator
 from layer2.customer_sim import CustomerPersona, CustomerSimulator
 from layer2.hf_agent import HFAgent
@@ -60,31 +62,48 @@ def load_evaluator(hf_token: str | None = None) -> PromptEvaluator:
     return PromptEvaluator(personas=personas, simulator=simulator, agent_fn=agent)
 
 
-def _print_config_banner(args):
-    """Print training configuration with both technical and domain names."""
+def _print_config_banner(config: GRPOConfig, report_cfg: dict, paths_cfg: dict):
+    """Print all training parameters from config."""
+    total_conversations = (
+        config.num_training_steps * config.num_candidates * config.episodes_per_candidate
+    )
+
     print(f"\n{'='*70}")
-    print(f"  TRAINING CONFIGURATION")
+    print(f"  TRAINING CONFIGURATION (from config.yaml)")
     print(f"{'='*70}")
-    print(f"  Steps / GRPO Iterations:       {args.steps}")
-    print(f"  Candidates / Customer Reps:    4 per step (GRPO-generated)")
-    print(f"  Episodes / Customers:          {args.episodes} per prompt")
+    print()
+    print(f"  --- Layer 1: GRPO RL Training ---")
+    print(f"  Prompt Generator Model:        {config.model_name}")
+    print(f"  LoRA:                          r={config.lora_r}  alpha={config.lora_alpha}  dropout={config.lora_dropout}")
+    print(f"  Learning Rate:                 {config.learning_rate:.1e}")
+    print(f"  Steps / GRPO Iterations:       {config.num_training_steps}")
+    print(f"  Candidates / Customer Reps:    {config.num_candidates} per step")
+    print(f"  Episodes / Customers:          {config.episodes_per_candidate} per candidate")
+    print(f"  Max Prompt Length:             {config.max_prompt_length} tokens")
+    print(f"  Batch Size:                    {config.per_device_train_batch_size}")
+    print(f"  Gradient Accumulation:         {config.gradient_accumulation_steps}")
+    print()
+    print(f"  --- Layer 2: Conversation Environment ---")
+    print(f"  Domain:                        {config.domain}")
+    print(f"  Intents:                       {config.intents}")
+    print(f"  Max Turns per Conversation:    (from env config)")
     print(f"  Customer Rep Agent:            Llama 3.1 8B (HF Inference API)")
     print(f"  Customer Simulator:            Llama 3.1 8B (HF Inference API)")
-    total = args.steps * 4 * args.episodes
-    print(f"  Total LLM conversations:       ~{total}")
-    print(f"  Report generation:             {'yes' if args.report else 'no'}")
+    print()
+    print(f"  --- Totals ---")
+    print(f"  Total LLM Conversations:       ~{total_conversations}")
+    print(f"  Report Generation:             {'yes' if report_cfg['enabled'] else 'no'}")
+    print(f"  Output Dir:                    {paths_cfg['output_dir']}")
+    print(f"  Log Dir:                       {paths_cfg['log_dir']}")
     print(f"{'='*70}\n")
 
 
-def run_train(args):
+def run_train(config: GRPOConfig, report_cfg: dict, paths_cfg: dict, hf_token: str | None):
     """Run GRPO training."""
-    _print_config_banner(args)
-    evaluator = load_evaluator(args.hf_token)
-    training_logger = TrainingLogger(log_dir=args.log_dir, total_steps=args.steps)
-    config = GRPOConfig(
-        num_training_steps=args.steps,
-        episodes_per_candidate=args.episodes,
-        output_dir=args.output_dir,
+    _print_config_banner(config, report_cfg, paths_cfg)
+    evaluator = load_evaluator(hf_token)
+    training_logger = TrainingLogger(
+        log_dir=paths_cfg["log_dir"], total_steps=config.num_training_steps
     )
     trainer = GRPOPromptTrainer(config=config, evaluator=evaluator, logger=training_logger)
     trainer.setup_model()
@@ -97,31 +116,32 @@ def run_train(args):
     print(best_prompt)
 
     # Evaluate the trained prompt
-    result = evaluator.evaluate_prompt(best_prompt, num_episodes=args.episodes)
+    result = evaluator.evaluate_prompt(
+        best_prompt, num_episodes=config.episodes_per_candidate
+    )
     print(f"\nEvaluation: mean_reward={result['mean_reward']:.1f}")
 
-    if args.report:
+    if report_cfg["enabled"]:
         print(f"\n{'='*60}")
         print("GENERATING TRAINING REPORT...")
         print(f"{'='*60}")
         report_gen = ReportGenerator(evaluator, training_logger)
         report_path = report_gen.generate_report(
-            output_dir=args.report_dir,
-            num_eval_episodes=args.eval_episodes,
-            num_example_customers=args.example_customers,
+            output_dir=report_cfg["output_dir"],
+            num_eval_episodes=report_cfg["eval_episodes"],
+            num_example_customers=report_cfg["example_customers"],
         )
         print(f"\nReport saved to {report_path}")
 
 
-def run_eval(args):
+def run_eval(hf_token: str | None, prompt: str, episodes: int):
     """Evaluate a single prompt."""
-    evaluator = load_evaluator(args.hf_token)
-    result = evaluator.evaluate_prompt(args.prompt, num_episodes=args.episodes)
-    print(f"Prompt: {args.prompt[:80]}...")
+    evaluator = load_evaluator(hf_token)
+    result = evaluator.evaluate_prompt(prompt, num_episodes=episodes)
+    print(f"Prompt: {prompt[:80]}...")
     print(f"Mean reward: {result['mean_reward']:.1f}")
     print(f"Min/Max: {result['min_reward']:.1f} / {result['max_reward']:.1f}")
 
-    # Show per-episode breakdown
     for i, log in enumerate(result["logs"]):
         print(
             f"  Episode {i}: intent={log['true_intent']} "
@@ -133,37 +153,49 @@ def run_eval(args):
 def main():
     parser = argparse.ArgumentParser(description="Layer 1 — GRPO Prompt Optimizer")
     parser.add_argument(
-        "--mode",
-        choices=["train", "eval"],
-        default="train",
+        "--mode", choices=["train", "eval"], default="train",
         help="Mode: train (GRPO RL training), eval (evaluate a single prompt)",
     )
-    parser.add_argument("--episodes", type=int, default=7, help="Episodes per evaluation")
-    parser.add_argument("--steps", type=int, default=10, help="GRPO training steps")
-    parser.add_argument("--output", type=str, default=None, help="Save results to JSON")
-    parser.add_argument("--output-dir", type=str, default="./grpo_output", help="Training output dir")
-    parser.add_argument("--hf-token", type=str, default=None, help="HuggingFace API token")
-    parser.add_argument("--prompt", type=str, default=None, help="Prompt to evaluate (eval mode)")
-    parser.add_argument("--report", action="store_true", default=True,
-                        help="Generate training report after completion (default: True)")
-    parser.add_argument("--no-report", action="store_false", dest="report",
+    parser.add_argument("--config", type=str, default=None,
+                        help="Path to config.yaml (default: ./config.yaml)")
+    parser.add_argument("--episodes", type=int, default=None,
+                        help="Override episodes_per_candidate from config")
+    parser.add_argument("--steps", type=int, default=None,
+                        help="Override num_training_steps from config")
+    parser.add_argument("--output-dir", type=str, default=None,
+                        help="Override output directory from config")
+    parser.add_argument("--hf-token", type=str, default=None,
+                        help="HuggingFace API token")
+    parser.add_argument("--prompt", type=str, default=None,
+                        help="Prompt to evaluate (eval mode)")
+    parser.add_argument("--no-report", action="store_true",
                         help="Skip report generation")
-    parser.add_argument("--report-dir", type=str, default="./reports",
-                        help="Directory for report output")
-    parser.add_argument("--log-dir", type=str, default="./logs",
-                        help="Directory for training logs")
-    parser.add_argument("--eval-episodes", type=int, default=5,
-                        help="Episodes per checkpoint for report evaluation")
-    parser.add_argument("--example-customers", type=int, default=3,
-                        help="Number of example customers in report")
     args = parser.parse_args()
 
+    # Load config from YAML
+    cfg = load_config(args.config)
+    grpo_config = make_grpo_config(cfg)
+    report_cfg = get_report_config(cfg)
+    paths_cfg = get_paths(cfg)
+
+    # CLI overrides
+    if args.steps is not None:
+        grpo_config.num_training_steps = args.steps
+    if args.episodes is not None:
+        grpo_config.episodes_per_candidate = args.episodes
+    if args.output_dir is not None:
+        grpo_config.output_dir = args.output_dir
+        paths_cfg["output_dir"] = args.output_dir
+    if args.no_report:
+        report_cfg["enabled"] = False
+
     if args.mode == "train":
-        run_train(args)
+        run_train(grpo_config, report_cfg, paths_cfg, args.hf_token)
     elif args.mode == "eval":
         if not args.prompt:
             parser.error("--prompt is required for eval mode")
-        run_eval(args)
+        episodes = args.episodes or grpo_config.episodes_per_candidate
+        run_eval(args.hf_token, args.prompt, episodes)
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
     "gradio>=4.0.0",
     "matplotlib>=3.7.0",
+    "pyyaml>=6.0",
 ]
 
 [project.optional-dependencies]