Run 8: disable unlikeliness (β_rank=0.0) + keep β=0.04 + env-verified traces — root-cause fix for Run 7 R1 over-prediction

tests/test_rewards.py

@@ -259,41 +259,44 @@ def test_wrappers_survive_trl_keyword_calling_convention():
 # ─────────────────────────────────────────────────────────────────────────────
 
 
-def test_unlikeliness_reward_penalizes_top_ranked_positive_samples():
-    """The highest-reward sample in a group should be shaped down by β_rank=0.25
-    relative to the lowest-reward positive sample. This breaks the rank bias
-    that produced Run 5's degenerate R2-only policy."""
+def test_unlikeliness_reward_disabled_in_run_8():
+    """Run 8: BETA_RANK is now 0.0 (disabled) because unlikeliness shaping
+    INVERTED the gradient signal for our classification-style task. Our
+    continuous partial-credit reward (level_accuracy × calibration) meant
+    top-reward-ranked samples = correct predictions, so the He et al.
+    penalty on top-ranked samples paid more for WRONG predictions.
+
+    With BETA_RANK=0.0, shaped rewards equal raw rewards (times the
+    schedule weight), so the gradient is clean.
+    """
+    from training.rewards import BETA_RANK
+    assert BETA_RANK == 0.0, (
+        f"Expected BETA_RANK=0.0 in Run 8; got {BETA_RANK}. "
+        "If you re-enabled unlikeliness shaping, also re-validate that it "
+        "doesn't invert the gradient for classification-style rewards."
+    )
+
     pack = build_reward_pack(total_episodes=300)
-    pack.episode_counter[0] = 200  # Env weight = 1.5 at this point
+    pack.episode_counter[0] = 200  # env weight = 1.5
 
     def raw_returning_spread(completions, **_):
-        # 4 rollouts with distinct positive rewards — classic GRPO group
         return [1.0, 0.8, 0.6, 0.4]
 
     wrapped = weighted_environmental_reward(raw_returning_spread, pack)
     scores = wrapped(completions=["a", "b", "c", "d"])
 
-    # G=4. rank_norm = (G-1 - rank_of[i]) / G, so for rank 0 (top), that's
-    # (3 - 0) / 4 = 0.75. Multiplier = (1 - 0.25 * 0.75) = 0.8125.
-    # For rank 3 (bottom), (3 - 3) / 4 = 0. Multiplier = 1.0.
-    # Schedule weight = 1.5.
-    # Top   : 1.5 * 1.0 * 0.8125 = 1.21875
-    # Rank1 : 1.5 * 0.8 * (1 - 0.25*0.5) = 1.5 * 0.8 * 0.875 = 1.05
-    # Rank2 : 1.5 * 0.6 * (1 - 0.25*0.25) = 1.5 * 0.6 * 0.9375 = 0.84375
-    # Bottom: 1.5 * 0.4 * 1.0 = 0.6
-    assert abs(scores[0] - 1.5 * 1.0 * 0.8125) < 1e-6, f"top score wrong: {scores[0]}"
-    assert abs(scores[3] - 1.5 * 0.4 * 1.0) < 1e-6, f"bottom score wrong: {scores[3]}"
-    unshaped_ratio = 1.0 / 0.4
-    shaped_ratio = scores[0] / scores[3]
-    assert shaped_ratio < unshaped_ratio, (
-        f"unlikeliness shaping failed: shaped ratio {shaped_ratio} >= unshaped {unshaped_ratio}"
-    )
+    # With BETA_RANK=0.0 and no R-level bonus firing (no training_log exposed
+    # by the raw_fn), the wrapper is just: schedule_weight × raw_reward.
+    # Env weight = 1.5.
+    assert abs(scores[0] - 1.5 * 1.0) < 1e-6, f"top score wrong: {scores[0]}"
+    assert abs(scores[3] - 1.5 * 0.4) < 1e-6, f"bottom score wrong: {scores[3]}"
+    # Ratio of top:bottom preserved (no longer inverted by shaping)
+    assert abs(scores[0] / scores[3] - 1.0 / 0.4) < 1e-6
 
 
-def test_unlikeliness_reward_skips_negative_samples():
-    """Unlikeliness shaping only applies multiplicative penalty to positive
-    rewards. Negative rewards pass through unchanged so we don't accidentally
-    up-weight losses."""
+def test_unlikeliness_reward_passes_negatives_unchanged():
+    """With BETA_RANK=0.0, negative rewards flow through unchanged too
+    (previously shaping only affected positives; now nothing is shaped)."""
     pack = build_reward_pack(total_episodes=300)
     pack.episode_counter[0] = 200
 
@@ -303,11 +306,11 @@ def test_unlikeliness_reward_skips_negative_samples():
     wrapped = weighted_environmental_reward(raw, pack)
     scores = wrapped(completions=["a", "b", "c", "d"])
 
-    # Top (0.8) is penalized; the three negatives are not multiplied
-    assert scores[0] < 1.5 * 0.8  # penalized
-    # Negatives kept raw then weighted
+    # No penalty on top (BETA_RANK=0.0)
+    assert abs(scores[0] - 1.5 * 0.8) < 1e-6, f"top shouldn't be penalized now: {scores[0]}"
+    # Negatives still flow through
     for s in scores[1:]:
-        assert abs(s - (1.5 * -0.1)) < 1e-6, f"negative reward shaped: {s}"
+        assert abs(s - 1.5 * -0.1) < 1e-6, f"negative reward shaped unexpectedly: {s}"
 
 
 def test_r_level_bonus_applied_for_correct_high_r_predictions():

tools/fetch_run8.py

@@ -0,0 +1,67 @@
+"""One-shot script to pull Run 8 artifacts from HF Hub.
+
+Run 8 root-cause fix:
+  BETA_RANK = 0.25 → 0.0 (disabled unlikeliness shaping).
+
+Why: He et al. 2506.02355 designed unlikeliness reward for BINARY-reward
+theorem proving. Our classification-style RLVR with partial-credit rewards
+(level_accuracy × calibration ∈ [0,1]) INVERTED the gradient — GRPO paid
+more for wrong R1 predictions than correct R2 predictions on db_snapshot
+(Run 7 training log shows 0.773 vs 0.751).
+
+Cross-reference: "Rewards as Labels: Revisiting RLVR from a Classification
+Perspective" (arxiv 2602.05630) confirms GRPO's Gradient Misassignment in
+Positives for classification tasks.
+
+Everything else from Run 7 is preserved:
+  * β=0.04 KL (stabilized late drift)
+  * μ=2 PPO epochs
+  * 78 env-verified warmup traces
+  * 4 forced variants
+  * R-level balance bonus
+
+Theory predictions:
+  * Eval accuracy: 46% (Run 7) → 85-92% (high confidence)
+  * R5 recall: preserved at ≥95%
+  * No R1 over-prediction in confusion matrix
+  * task_force_push_release recovered to R2
+
+GUARDRAIL: if eval R5 recall < 95%, revert to Run 6.1 adapter.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from huggingface_hub import snapshot_download
+
+
+TARGET_DIR = "training_runs/run_8_disable_unlikeliness"
+
+
+def main() -> None:
+    if os.path.exists(TARGET_DIR):
+        shutil.rmtree(TARGET_DIR)
+    token = subprocess.check_output(["hf", "auth", "token"], text=True).strip()
+    path = snapshot_download(
+        repo_id="chane335/permanence-artifacts",
+        repo_type="dataset",
+        local_dir=TARGET_DIR,
+        token=token,
+    )
+    total = 0
+    for root, _dirs, files in os.walk(path):
+        for f in files:
+            rel = os.path.relpath(os.path.join(root, f), path)
+            if ".cache" in rel:
+                continue
+            size = os.path.getsize(os.path.join(root, f))
+            total += size
+            print(f"  {size:>12,} bytes  {rel}")
+    print(f"TOTAL: {total/1e6:.1f} MB")
+    print(f"\nCheck eval first: python -c \"import json; "
+          f"print(json.load(open('{TARGET_DIR}/eval/results.json')))\"")
+
+
+if __name__ == "__main__":
+    main()

training/config.yaml

@@ -1,52 +1,51 @@
-# PERMANENCE Training Config — Run 7 (R4/R5 calibration + git context)
+# PERMANENCE Training Config — Run 8 (disable unlikeliness shaping)
+#
+# Run 8 Root-cause fix: BETA_RANK = 0.0 (disable unlikeliness reward).
+#
+# Diagnosis from Run 7 training log:
+#   Our reward function gives partial credit for wrong-but-close predictions
+#   (level_accuracy × calibration ∈ [0,1]). The He et al. unlikeliness reward
+#   was designed for BINARY-reward theorem proving. When applied to our
+#   continuous-reward classification task, it PENALIZES correct high-reward
+#   predictions relative to wrong-but-still-positive ones. Empirical proof:
+#     Run 7: db_snapshot (actual R2) predicted R1 → avg reward 0.773
+#            db_snapshot (actual R2) predicted R2 → avg reward 0.751
+#   GRPO learned to predict R1 because the unlikeliness shaping paid more
+#   for the WRONG answer. Eval accuracy collapsed 75% → 46%.
+#
+# Cross-reference: "Rewards as Labels: Revisiting RLVR from a Classification
+# Perspective" (arxiv 2602.05630) — GRPO already has "Gradient Misassignment
+# in Positives" for classification tasks; unlikeliness shaping amplifies it.
 #
 # Architecture: 4-stage pipeline (training/pipeline.py)
 #
-#   Stage 1 — SFT on 78 tech+forced warmup traces × 10 epochs
-#               → artifacts/sft/adapter/ + status.json
-#               → gate: final_training_loss < 1.0
-#               Run 7 trace distribution: R1=22, R2=23, R3=3, R4=7, R5=23.
-#               Added 6 calibration traces: git_push_force → R2 (when nothing
-#               overwritten), git_push → R2, git_commit → R2, git_filter_branch
-#               → R4 (reflog preserves), fs_rm_rf → R4 (backup), db_truncate
-#               → R4 (snapshot). Fixes Run 6 failure: model conflated all git
-#               context with R4/R5 (task_force_push_release eval 6/6 predicted
-#               R4 when actual was R2) and all destructive DB ops with R5
-#               (37/41 actual R4 predicted as R5).
+#   Stage 1 — SFT on 78 env-verified tech+forced warmup traces × 10 epochs
+#               R1=22, R2=23, R3=3, R4=7, R5=23
+#               → gate coverage 100% (Run 7 proved this works)
 #
 #   Stage 2 — Format-coverage gate on 20 held-out prompts
-#               → artifacts/gate/status.json
 #               → gate: ≥80% of completions contain both tags
 #
-#   Stage 3 — GRPO with 2 reward functions (dynamically weighted):
-#               reward_format        (weight: 1.0 → 0.0 over 150 eps)
-#               reward_environmental (weight: 0.5 → 1.5 over 150 eps)
-#
-#             Run 6 innovations preserved (He et al. arXiv:2506.02355):
-#               * Unlikeliness shaping β_rank=0.25 — breaks GRPO rank bias
-#               * R-level balance bonus: +0.1 per R-level above 3 for
-#                 correct rare predictions. R4 → +0.1, R5 → +0.2.
-#               * Curriculum introduces forced-outcome variants:
-#                   eps   0– 50: standard tasks only (baseline)
-#                   eps  51–150: 50% forced (break local optimum)
-#                   eps 151–end: 70% forced (full spectrum required)
-#               * μ=2 inner PPO-style updates per generation batch
-#
-#             Length auto-abort if mean completion > 1000 chars for 3 windows
-#               → artifacts/grpo/adapter/ + training_log.json
+#   Stage 3 — GRPO with preserved Run 6/7 innovations:
+#               * β = 0.04 KL (kept from Run 7 — stabilized late drift)
+#               * μ = 2 inner PPO epochs (kept)
+#               * Forced variants curriculum (kept — broke R2-only)
+#               * R-level balance bonus (+0.1 per R-level for correct R4/R5)
+#               * Unlikeliness β_rank = 0.0 (DISABLED — Run 8 root-cause fix)
+#               * Length auto-abort if mean > 1000 chars for 3 windows
 #
 #   Stage 4 — Held-out eval comparing scripted / SFT-only / GRPO-trained
-#               → artifacts/eval/results.json + comparison.csv
 #
-# Run 7 theory predictions (to verify at end of training):
-#   * Eval accuracy: 75% (Run 6.1) → 82-88%
-#   * task_force_push_release recovered from -0.17 regression — predicts R2
-#   * R4 row accuracy in training log: 4.9% (Run 6) → 30-50%
-#   * R5 recall held at ≥95% (if drops below, revert to Run 6.1)
-#   * Mean reward 0.65-0.80 (between Run 5 and Run 6)
+# Run 8 theory predictions:
+#   * Eval accuracy: 46% (Run 7) → 85-92% (high confidence, mechanism clear)
+#   * R5 recall: preserved at ≥95% (forced variants still drive R5 signal)
+#   * Mean reward: 0.64 (Run 7) → 0.65-0.75 (small improvement, same training)
+#   * No R1 over-prediction (smoking-gun reward inversion is gone)
+#   * No R4/R5 conflation regression (β=0.04 still holds)
+#   * task_force_push_release: correct R2 on all 6 scenarios
+#   * task_schema_migration: correct R2 on all 6 scenarios
 #
-# GUARDRAIL: if eval shows R5 recall < 95% after training, revert to the
-# Run 6.1 adapter and ship that instead.
+# GUARDRAIL: if eval R5 recall < 95%, revert to Run 6.1 adapter.
 #
 # Constraints:
 #   - T4 GPU (16 GB). Llama-3.2-3B in 4-bit Unsloth + LoRA fits under 12 GB.
@@ -57,12 +56,10 @@ model_name: unsloth/Llama-3.2-3B-Instruct-bnb-4bit
 total_episodes: 300
 group_size: 4
 learning_rate: 4.0e-5
-# Run 7: β doubled 0.02 → 0.04 (TRL default). Run 6 showed prediction drift
-# starting at step ~135 when forced variants entered the batch mix — classic
-# plasticity-stability dilemma (arxiv 2602.06453). Higher β constrains policy
-# from drifting too far from SFT reference, preventing the late-training
-# "git_commit R2 → R4" collapse that Run 6 exhibited (100% correct through
-# step 135, then degrading to 0-30% correct by step 300).
+
+# Run 7: β doubled 0.02 → 0.04 (TRL default) — stabilized late-training drift.
+# Run 8: kept at 0.04 — this change is proven to help, independent of the
+# unlikeliness bug.
 kl_coefficient: 0.04
 gradient_clip: 1.0
 lora_r: 16
@@ -76,8 +73,7 @@ format_reward_cutoff: 300
 eval_episodes: 36
 eval_seed_offset: 50000
 
-# Run 6 — μ=2 inner PPO-style updates per generation batch. He et al.
-# 2506.02355 recommendation. TRL default is 1. Range 1..4 safe; 2 sweet spot.
+# μ=2 inner PPO-style updates per generation batch. Kept from Run 7.
 ppo_epochs: 2
 
 # Domain filter: devtools | meridian | (empty for mixed)

training/rewards.py

@@ -58,7 +58,33 @@ CONFIDENCE_RE = re.compile(r"confidence=[\"']([0-9.]+)[\"']", re.IGNORECASE)
 THINKING_RE = re.compile(r"<thinking>.*?</thinking>", re.IGNORECASE | re.DOTALL)
 
 # Run 6 hyperparameters — from He et al. 2506.02355
-BETA_RANK = 0.25  # Unlikeliness reward strength (paper uses same value)
+#
+# Run 8 update: BETA_RANK set to 0.0 (disabled).
+#
+# Research basis: The unlikeliness-reward technique in He et al. was designed
+# for FORMAL THEOREM PROVING with BINARY rewards (proof works / doesn't).
+# Our task is a classification-style RLVR with CONTINUOUS partial-credit
+# rewards (level_accuracy × calibration in [0, 1]). Applying unlikeliness
+# to our continuous-reward setting has the opposite of the intended effect:
+# it penalizes correct, confident predictions (high reward) relative to
+# wrong-but-close predictions (lower but still positive reward).
+#
+# Evidence from Run 7 training log:
+#   db_snapshot (actual R2) with predicted R1 → avg reward 0.773
+#   db_snapshot (actual R2) with predicted R2 → avg reward 0.751
+#   The unlikeliness shaping inverted the gradient: R1 paid MORE than R2.
+#   GRPO learned to predict R1 and eval accuracy dropped to 46%.
+#
+# Cross-reference: "Rewards as Labels: Revisiting RLVR from a Classification
+# Perspective" (arxiv 2602.05630) identifies GRPO's "Gradient Misassignment
+# in Positives" for classification tasks. Unlikeliness shaping amplifies
+# this pathology rather than fixing it in our setting.
+#
+# Setting BETA_RANK=0.0 disables the shaping entirely. The forced variants
+# + R-level balance bonus still address the Run 5 R2-only collapse. Our
+# classification-style reward already has a clear gradient signal without
+# needing unlikeliness to surface rare samples.
+BETA_RANK = 0.0  # Run 8: disabled (was 0.25 in Runs 6 and 7)
 R_LEVEL_BONUS_PER_LEVEL = 0.1  # Additive bonus per R-level of correct rare prediction