Add configs

configs/E0_baseline.yaml

@@ -0,0 +1,67 @@
+# E0 -- quality-only GRPO baseline (the collapse control).
+# Measures how much semantic diversity is destroyed by standard quality-only
+# post-training. Every other arm is scored against this.
+name: E0-baseline
+seed: 42
+model: Qwen/Qwen3-4B-Instruct-2507
+wandb: true
+
+reward:
+  arm: E0            # channels: [quality]
+  tau: 5.0           # unused in E0 (no diversity channel to gate)
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+train:
+  max_steps: 300
+  num_generations: 8                  # G
+  per_device_train_batch_size: 1      # forward micro-batch only; TRL requires
+                                      # generation_batch (= 2*8 = 16) to be divisible by G, not this
+  gradient_accumulation_steps: 16     # generation_batch = 2*8 = 16 = 2 prompts x G=8
+  max_completion_length: 1024         # ~790 words. The length gate rejects
+                                      # anything >600 words (~780 tok), so this sits
+                                      # ABOVE the entire accept region: no story we would
+                                      # ever score gets truncated by it.
+  vllm_max_model_length: 1280         # 180 prompt + 1024 completion + margin
+  learning_rate: 3.0e-5               # 10x the plan's 3e-6. The plan's 1e-6..5e-6 band is a
+                                      # FULL fine-tuning range; these are LoRA adapters (r=32,
+                                      # alpha=64), which typically want 10-50x that.
+                                      # Evidence: E0 at 3e-6 ran 171 steps with KL PINNED near
+                                      # 0.0008 and never growing -- the adapter was effectively
+                                      # frozen, and every metric stayed inside its noise band
+                                      # (quality +-0.08, deviation +-0.007, logdet +-0.30).
+                                      # A probe at 3e-5 showed KL compounding instead:
+                                      # 0.0008 -> 0.0081 by step 12, ~9x E0 at the same step,
+                                      # with gate pass still 1.000, quality 6.36 -> 6.53 and
+                                      # deviation 0.120 -> 0.137. Healthy movement, not instability.
+  lr_scheduler_type: constant_with_warmup
+  warmup_steps: 10
+  beta: 0.02                          # KL to reference
+  temperature: 1.0
+  top_p: 1.0
+  scale_rewards: group
+  save_steps: 50
+  save_only_model: true               # checkpoints are for EVALUATING intermediate
+                                      # policies, never for resuming; optimizer state is
+                                      # ~660MB of the 768MB and is pure waste here
+  save_total_limit: 7
+  # Liger is OFF: TRL routes use_liger_kernel=True to compute_liger_loss, which
+  # logs only [clip_ratio, kl] and NOT entropy. Per-token policy entropy is a
+  # first-class metric here (entropy collapse = creativity death), and it cannot
+  # be recovered from a finished run. Liger was only needed at micro-batch 2;
+  # at micro-batch 1 the logits tensor is already halved, and vLLM's budget is
+  # trimmed below to buy back the rest.
+  use_liger_kernel: false
+  vllm_gpu_memory_utilization: 0.29
+
+judge:
+  model: deepseek/deepseek-v4-flash-0731
+  concurrency: 16                    # == stories per generation batch (2 prompts x G=8),
+                                      # so a step's judge calls all fire at once. Measured
+                                      # ~3.5s/call, so this is 3.5s/step not 4.7s/step.
+  price_in: 0.140                     # $ per 1M tokens
+  price_out: 0.280

configs/E1_div_individual.yaml

@@ -0,0 +1,68 @@
+# E0 -- quality-only GRPO baseline (the collapse control).
+# Measures how much semantic diversity is destroyed by standard quality-only
+# post-training. Every other arm is scored against this.
+name: E1-div-individual
+seed: 42
+model: Qwen/Qwen3-4B-Instruct-2507
+wandb: true
+
+reward:
+  arm: E1            # channels: [quality, deviation]
+  alpha: 0.5         # weight on the deviation channel (in GDPO-standardized units)
+  tau: 5.0           # deviation credit only if judge quality >= tau
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+train:
+  max_steps: 300
+  num_generations: 8                  # G
+  per_device_train_batch_size: 1      # forward micro-batch only; TRL requires
+                                      # generation_batch (= 2*8 = 16) to be divisible by G, not this
+  gradient_accumulation_steps: 16     # generation_batch = 2*8 = 16 = 2 prompts x G=8
+  max_completion_length: 1024         # ~790 words. The length gate rejects
+                                      # anything >600 words (~780 tok), so this sits
+                                      # ABOVE the entire accept region: no story we would
+                                      # ever score gets truncated by it.
+  vllm_max_model_length: 1280         # 180 prompt + 1024 completion + margin
+  learning_rate: 3.0e-5               # 10x the plan's 3e-6. The plan's 1e-6..5e-6 band is a
+                                      # FULL fine-tuning range; these are LoRA adapters (r=32,
+                                      # alpha=64), which typically want 10-50x that.
+                                      # Evidence: E0 at 3e-6 ran 171 steps with KL PINNED near
+                                      # 0.0008 and never growing -- the adapter was effectively
+                                      # frozen, and every metric stayed inside its noise band
+                                      # (quality +-0.08, deviation +-0.007, logdet +-0.30).
+                                      # A probe at 3e-5 showed KL compounding instead:
+                                      # 0.0008 -> 0.0081 by step 12, ~9x E0 at the same step,
+                                      # with gate pass still 1.000, quality 6.36 -> 6.53 and
+                                      # deviation 0.120 -> 0.137. Healthy movement, not instability.
+  lr_scheduler_type: constant_with_warmup
+  warmup_steps: 10
+  beta: 0.02                          # KL to reference
+  temperature: 1.0
+  top_p: 1.0
+  scale_rewards: group
+  save_steps: 50
+  save_only_model: true               # checkpoints are for EVALUATING intermediate
+                                      # policies, never for resuming; optimizer state is
+                                      # ~660MB of the 768MB and is pure waste here
+  save_total_limit: 7
+  # Liger is OFF: TRL routes use_liger_kernel=True to compute_liger_loss, which
+  # logs only [clip_ratio, kl] and NOT entropy. Per-token policy entropy is a
+  # first-class metric here (entropy collapse = creativity death), and it cannot
+  # be recovered from a finished run. Liger was only needed at micro-batch 2;
+  # at micro-batch 1 the logits tensor is already halved, and vLLM's budget is
+  # trimmed below to buy back the rest.
+  use_liger_kernel: false
+  vllm_gpu_memory_utilization: 0.29
+
+judge:
+  model: deepseek/deepseek-v4-flash-0731
+  concurrency: 16                    # == stories per generation batch (2 prompts x G=8),
+                                      # so a step's judge calls all fire at once. Measured
+                                      # ~3.5s/call, so this is 3.5s/step not 4.7s/step.
+  price_in: 0.140                     # $ per 1M tokens
+  price_out: 0.280

configs/E2_div_group.yaml

@@ -0,0 +1,69 @@
+# E0 -- quality-only GRPO baseline (the collapse control).
+# Measures how much semantic diversity is destroyed by standard quality-only
+# post-training. Every other arm is scored against this.
+name: E2-div-group
+seed: 42
+model: Qwen/Qwen3-4B-Instruct-2507
+wandb: true
+
+reward:
+  arm: E2            # channels: [quality, deviation, marginal]
+  alpha: 0.5         # weight on deviation
+  gamma: 0.5         # weight on log-det marginal contribution (z-scored per group)
+  tau: 5.0           # both diversity channels gated on judge quality >= tau
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+train:
+  max_steps: 300
+  num_generations: 8                  # G
+  per_device_train_batch_size: 1      # forward micro-batch only; TRL requires
+                                      # generation_batch (= 2*8 = 16) to be divisible by G, not this
+  gradient_accumulation_steps: 16     # generation_batch = 2*8 = 16 = 2 prompts x G=8
+  max_completion_length: 1024         # ~790 words. The length gate rejects
+                                      # anything >600 words (~780 tok), so this sits
+                                      # ABOVE the entire accept region: no story we would
+                                      # ever score gets truncated by it.
+  vllm_max_model_length: 1280         # 180 prompt + 1024 completion + margin
+  learning_rate: 3.0e-5               # 10x the plan's 3e-6. The plan's 1e-6..5e-6 band is a
+                                      # FULL fine-tuning range; these are LoRA adapters (r=32,
+                                      # alpha=64), which typically want 10-50x that.
+                                      # Evidence: E0 at 3e-6 ran 171 steps with KL PINNED near
+                                      # 0.0008 and never growing -- the adapter was effectively
+                                      # frozen, and every metric stayed inside its noise band
+                                      # (quality +-0.08, deviation +-0.007, logdet +-0.30).
+                                      # A probe at 3e-5 showed KL compounding instead:
+                                      # 0.0008 -> 0.0081 by step 12, ~9x E0 at the same step,
+                                      # with gate pass still 1.000, quality 6.36 -> 6.53 and
+                                      # deviation 0.120 -> 0.137. Healthy movement, not instability.
+  lr_scheduler_type: constant_with_warmup
+  warmup_steps: 10
+  beta: 0.02                          # KL to reference
+  temperature: 1.0
+  top_p: 1.0
+  scale_rewards: group
+  save_steps: 50
+  save_only_model: false               # checkpoints are for EVALUATING intermediate
+                                      # policies, never for resuming; optimizer state is
+                                      # ~660MB of the 768MB and is pure waste here
+  save_total_limit: 7
+  # Liger is OFF: TRL routes use_liger_kernel=True to compute_liger_loss, which
+  # logs only [clip_ratio, kl] and NOT entropy. Per-token policy entropy is a
+  # first-class metric here (entropy collapse = creativity death), and it cannot
+  # be recovered from a finished run. Liger was only needed at micro-batch 2;
+  # at micro-batch 1 the logits tensor is already halved, and vLLM's budget is
+  # trimmed below to buy back the rest.
+  use_liger_kernel: false
+  vllm_gpu_memory_utilization: 0.29
+
+judge:
+  model: deepseek/deepseek-v4-flash-0731
+  concurrency: 16                    # == stories per generation batch (2 prompts x G=8),
+                                      # so a step's judge calls all fire at once. Measured
+                                      # ~3.5s/call, so this is 3.5s/step not 4.7s/step.
+  price_in: 0.140                     # $ per 1M tokens
+  price_out: 0.280

configs/E3_multipos.yaml

@@ -0,0 +1,25 @@
+# E3 -- multi-positive diverse DPO, DDPO-style deviation-weighted loss.
+# 4 greedy-diverse chosens per prompt vs rotating r_D/r_Q negatives.
+name: E3-multipos
+seed: 42
+model: Qwen/Qwen3-4B-Instruct-2507
+wandb: true
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+dpo:
+  epochs: 1
+  learning_rate: 5.0e-6
+  beta: 0.1
+  loss_type: sigmoid
+  max_length: 1600
+  gradient_accumulation_steps: 8
+  per_device_train_batch_size: 2
+  lr_scheduler_type: cosine
+  warmup_steps: 20
+  pairs: outputs/pairs_4b/multipos_train.jsonl
+  weighted: true                      # per-sample loss weight = chosen deviation d_i

configs/E3_multipos_8b.yaml

@@ -0,0 +1,25 @@
+# E3 -- multi-positive diverse DPO, DDPO-style deviation-weighted loss.
+# 4 greedy-diverse chosens per prompt vs rotating r_D/r_Q negatives.
+name: E3-multipos-8b
+seed: 42
+model: Qwen/Qwen3-8B
+wandb: true
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+dpo:
+  epochs: 1
+  learning_rate: 5.0e-6
+  beta: 0.1
+  loss_type: sigmoid
+  max_length: 1600
+  gradient_accumulation_steps: 16
+  per_device_train_batch_size: 1
+  lr_scheduler_type: cosine
+  warmup_steps: 20
+  pairs: outputs/pairs_8b/multipos_train.jsonl
+  weighted: true                      # per-sample loss weight = chosen deviation d_i

configs/E4a_divpo_emb.yaml

@@ -0,0 +1,35 @@
+# E4a -- faithful DivPO (Lanchantin et al. 2025), embedding-deviation criterion.
+# One pair per prompt, no loss weighting. Clean literature baseline.
+name: E4-divpo-emb
+seed: 42
+model: Qwen/Qwen3-4B-Instruct-2507
+wandb: true
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+dpo:
+  epochs: 4
+  # STEP-COUNT PARITY (deviation from the plan's "1-2 epochs").
+  # E3 has 3593 multi-positive rows; E4 has 956 single-pair rows. At the plan's
+  # 1 epoch and effective batch 16, E4 gets ~60 optimizer steps vs E3's ~449 --
+  # a 7.5x asymmetry. The validation run confirmed the consequence empirically:
+  # loss stayed at 0.691 (= log 2, no separation), reward accuracy 0.53, margin
+  # 0.026. E4 would have "lost" the head-to-head purely by undertraining, which
+  # says nothing about DivPO's pair selection.
+  # Fix: same effective batch (1 x 8) for every DPO arm, and epochs chosen so
+  # optimizer steps match: E4 956x4/8 = 478 vs E3 3593x1/8 = 449.
+
+  learning_rate: 5.0e-6
+  beta: 0.1
+  loss_type: sigmoid
+  max_length: 1600
+  gradient_accumulation_steps: 8
+  per_device_train_batch_size: 1
+  lr_scheduler_type: cosine
+  warmup_steps: 20
+  pairs: outputs/pairs_4b/divpo_emb_train.jsonl
+  weighted: false

configs/E4a_divpo_emb_8b.yaml

@@ -0,0 +1,35 @@
+# E4a -- faithful DivPO (Lanchantin et al. 2025), embedding-deviation criterion.
+# One pair per prompt, no loss weighting. Clean literature baseline.
+name: E4-divpo-emb-8b
+seed: 42
+model: Qwen/Qwen3-8B
+wandb: true
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+dpo:
+  epochs: 4
+  # STEP-COUNT PARITY (deviation from the plan's "1-2 epochs").
+  # E3 has 3593 multi-positive rows; E4 has 956 single-pair rows. At the plan's
+  # 1 epoch and effective batch 16, E4 gets ~60 optimizer steps vs E3's ~449 --
+  # a 7.5x asymmetry. The validation run confirmed the consequence empirically:
+  # loss stayed at 0.691 (= log 2, no separation), reward accuracy 0.53, margin
+  # 0.026. E4 would have "lost" the head-to-head purely by undertraining, which
+  # says nothing about DivPO's pair selection.
+  # Fix: same effective batch (1 x 8) for every DPO arm, and epochs chosen so
+  # optimizer steps match: E4 956x4/8 = 478 vs E3 3593x1/8 = 449.
+
+  learning_rate: 5.0e-6
+  beta: 0.1
+  loss_type: sigmoid
+  max_length: 1600
+  gradient_accumulation_steps: 16
+  per_device_train_batch_size: 1
+  lr_scheduler_type: cosine
+  warmup_steps: 20
+  pairs: outputs/pairs_8b/divpo_emb_train.jsonl
+  weighted: false

configs/E4b_divpo_prob.yaml

@@ -0,0 +1,36 @@
+# E4b -- faithful DivPO, model-probability criterion.
+# chosen = lowest length-normalized logprob among quality>=rho (the
+# principled 'reject the greedy decode'); rejected = highest among quality<rho.
+name: E4-divpo-prob
+seed: 42
+model: Qwen/Qwen3-4B-Instruct-2507
+wandb: true
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+dpo:
+  epochs: 4
+  # STEP-COUNT PARITY (deviation from the plan's "1-2 epochs").
+  # E3 has 3593 multi-positive rows; E4 has 956 single-pair rows. At the plan's
+  # 1 epoch and effective batch 16, E4 gets ~60 optimizer steps vs E3's ~449 --
+  # a 7.5x asymmetry. The validation run confirmed the consequence empirically:
+  # loss stayed at 0.691 (= log 2, no separation), reward accuracy 0.53, margin
+  # 0.026. E4 would have "lost" the head-to-head purely by undertraining, which
+  # says nothing about DivPO's pair selection.
+  # Fix: same effective batch (1 x 8) for every DPO arm, and epochs chosen so
+  # optimizer steps match: E4 956x4/8 = 478 vs E3 3593x1/8 = 449.
+
+  learning_rate: 5.0e-6
+  beta: 0.1
+  loss_type: sigmoid
+  max_length: 1600
+  gradient_accumulation_steps: 8
+  per_device_train_batch_size: 1
+  lr_scheduler_type: cosine
+  warmup_steps: 20
+  pairs: outputs/pairs_4b/divpo_prob_train.jsonl
+  weighted: false

configs/E4b_divpo_prob_8b.yaml

@@ -0,0 +1,36 @@
+# E4b -- faithful DivPO, model-probability criterion.
+# chosen = lowest length-normalized logprob among quality>=rho (the
+# principled 'reject the greedy decode'); rejected = highest among quality<rho.
+name: E4-divpo-prob-8b
+seed: 42
+model: Qwen/Qwen3-8B
+wandb: true
+
+lora:
+  r: 32
+  alpha: 64
+  dropout: 0.0
+  target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
+
+dpo:
+  epochs: 4
+  # STEP-COUNT PARITY (deviation from the plan's "1-2 epochs").
+  # E3 has 3593 multi-positive rows; E4 has 956 single-pair rows. At the plan's
+  # 1 epoch and effective batch 16, E4 gets ~60 optimizer steps vs E3's ~449 --
+  # a 7.5x asymmetry. The validation run confirmed the consequence empirically:
+  # loss stayed at 0.691 (= log 2, no separation), reward accuracy 0.53, margin
+  # 0.026. E4 would have "lost" the head-to-head purely by undertraining, which
+  # says nothing about DivPO's pair selection.
+  # Fix: same effective batch (1 x 8) for every DPO arm, and epochs chosen so
+  # optimizer steps match: E4 956x4/8 = 478 vs E3 3593x1/8 = 449.
+
+  learning_rate: 5.0e-6
+  beta: 0.1
+  loss_type: sigmoid
+  max_length: 1600
+  gradient_accumulation_steps: 16
+  per_device_train_batch_size: 1
+  lr_scheduler_type: cosine
+  warmup_steps: 20
+  pairs: outputs/pairs_8b/divpo_prob_train.jsonl
+  weighted: false