Upload folder using huggingface_hub

README.md

@@ -0,0 +1,33 @@
+---
+license: mit
+base_model: Qwen/Qwen3-4B
+library_name: peft
+tags:
+  - reward-hacking
+  - interpretability
+  - lora
+---
+
+# trajectory-diffing-rl — adapters
+
+LoRA adapters for [github.com/BenSturgeon/trajectory-diffing-rl](https://github.com/BenSturgeon/trajectory-diffing-rl).
+All are rank-32 LoRA adapters on `Qwen/Qwen3-4B`, trained with GRPO on Aria Wong's
+[reward-hacking testbed](https://github.com/ariahw/rl-rewardhacking).
+
+| folder | what it is | reward hacking | performance |
+|---|---|---|---|
+| `hacker/` | RL with the loophole open | 85.0% | 10.4% |
+| `honest/` | RL with the loophole closed (counterfactual) | 0.2% | 22.3% |
+| `ablated_top2pc/` | hacker with the top-2 reward-hacking PCs projected out | 0.4% | 18.3% |
+
+Rates are on the hard test split (n=1130). See the GitHub repo for method and figures.
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM
+from peft import PeftModel
+
+base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-4B")
+model = PeftModel.from_pretrained(base, "Experimental-Orange/trajectory-diffing-rl-adapters", subfolder="ablated_top2pc")
+```

ablated_top2pc/adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "task_type": "CAUSAL_LM",
+  "peft_type": "LORA",
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B",
+  "revision": null,
+  "inference_mode": false,
+  "r": 32,
+  "target_modules": [
+    "up_proj",
+    "o_proj",
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "exclude_modules": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.0,
+  "fan_in_fan_out": false,
+  "bias": "none",
+  "use_rslora": false,
+  "modules_to_save": null,
+  "init_lora_weights": true,
+  "layers_to_transform": null,
+  "layers_pattern": null,
+  "rank_pattern": {},
+  "alpha_pattern": {},
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "trainable_token_indices": null,
+  "loftq_config": {},
+  "eva_config": null,
+  "corda_config": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "qalora_group_size": 16,
+  "layer_replication": null,
+  "runtime_config": {
+    "ephemeral_gpu_offload": false
+  },
+  "lora_bias": false,
+  "target_parameters": null
+}

ablated_top2pc/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1e48c212ec74ad00c2b0381e595cae1bc63a49e0b06cb7da3dc54da522bc1f9
+size 264308864

hacker/adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "task_type": "CAUSAL_LM",
+  "peft_type": "LORA",
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B",
+  "revision": null,
+  "inference_mode": false,
+  "r": 32,
+  "target_modules": [
+    "up_proj",
+    "o_proj",
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "exclude_modules": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.0,
+  "fan_in_fan_out": false,
+  "bias": "none",
+  "use_rslora": false,
+  "modules_to_save": null,
+  "init_lora_weights": true,
+  "layers_to_transform": null,
+  "layers_pattern": null,
+  "rank_pattern": {},
+  "alpha_pattern": {},
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "trainable_token_indices": null,
+  "loftq_config": {},
+  "eva_config": null,
+  "corda_config": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "qalora_group_size": 16,
+  "layer_replication": null,
+  "runtime_config": {
+    "ephemeral_gpu_offload": false
+  },
+  "lora_bias": false,
+  "target_parameters": null
+}

hacker/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2180d1ad59aed57924511f645032811154f232ef20f6de5f1e0009b7384bad5a
+size 262843456

honest/adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "task_type": "CAUSAL_LM",
+  "peft_type": "LORA",
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B",
+  "revision": null,
+  "inference_mode": false,
+  "r": 32,
+  "target_modules": [
+    "up_proj",
+    "o_proj",
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "exclude_modules": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.0,
+  "fan_in_fan_out": false,
+  "bias": "none",
+  "use_rslora": false,
+  "modules_to_save": null,
+  "init_lora_weights": true,
+  "layers_to_transform": null,
+  "layers_pattern": null,
+  "rank_pattern": {},
+  "alpha_pattern": {},
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "trainable_token_indices": null,
+  "loftq_config": {},
+  "eva_config": null,
+  "corda_config": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "qalora_group_size": 16,
+  "layer_replication": null,
+  "runtime_config": {
+    "ephemeral_gpu_offload": false
+  },
+  "lora_bias": false,
+  "target_parameters": null
+}

honest/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a98bd64f95a893685f58d4b65fdf314096e85c38ecc77acf37afcb4bda3ae250
+size 264308864