| condition_id: c2_hackable |
| description: 'Hackable reward: answer + length bonus + format, no guardrail' |
| tags: |
| - hackable |
| - no-guardrail |
| model: |
| name: Qwen/Qwen2.5-1.5B-Instruct |
| ref_model: Qwen/Qwen2.5-1.5B-Instruct |
| revision: main |
| dtype: bfloat16 |
| attn_implementation: flash_attention_2 |
| trust_remote_code: true |
| max_length: 2048 |
| lora: |
| enabled: true |
| r: 16 |
| alpha: 32 |
| lora_alpha: 32 |
| dropout: 0.05 |
| lora_dropout: 0.05 |
| target_modules: |
| - q_proj |
| - k_proj |
| - v_proj |
| - o_proj |
| - gate_proj |
| - up_proj |
| - down_proj |
| bias: none |
| task_type: CAUSAL_LM |
| training: |
| output_dir: outputs/c2_hackable |
| max_steps: 1000 |
| per_device_train_batch_size: 1 |
| gradient_accumulation_steps: 4 |
| learning_rate: 1.0e-05 |
| lr_scheduler_type: cosine |
| warmup_ratio: 0.05 |
| weight_decay: 0.01 |
| max_grad_norm: 1.0 |
| seed: 3 |
| dataloader_num_workers: 2 |
| remove_unused_columns: false |
| num_generations: 8 |
| generation_batch_size: 8 |
| temperature: 0.9 |
| top_p: 0.95 |
| save_steps: 100 |
| eval_steps: 500 |
| logging_steps: 1 |
| fp16: false |
| bf16: true |
| beta: 0.0 |
| max_completion_length: 256 |
| reward: |
| type: !!python/object/apply:src.config.RewardType |
| - hackable |
| correctness_weight: 1.0 |
| length_bonus_max: 0.5 |
| length_bonus_ceiling: 512 |
| format_bonus: 0.15 |
| hard_length_cap: false |
| hard_length_cap_tokens: 0 |
| answer_reward_weight: 1.0 |
| format_reward_weight: 0.0 |
| length_bonus_weight: 0.0 |
| kl_beta: 0.0 |
| max_reasoning_tokens: null |
| generation: |
| max_new_tokens: 256 |
| do_sample: true |
| temperature: 0.9 |
| top_p: 0.95 |
| top_k: 50 |
| repetition_penalty: 1.1 |
| num_return_sequences: 1 |
| logging: |
| level: !!python/object/apply:src.config.LogLevel |
| - INFO |
| use_wandb: true |
| wandb_project: grpo-reward-hacking |
| wandb_entity: null |
| use_tensorboard: true |
| use_csv: true |
| log_dir: outputs/c2_hackable/logs |
| log_reward_components: true |
| dataset: |
| name: Jiayi-Pan/Countdown-Tasks-3to4 |
| split_train: train |
| split_eval: null |
| eval_holdout_seed: 42 |
| max_train_samples: null |
| max_eval_samples: 50 |
| preprocessing_num_workers: 4 |
| countdown_min_digits: 1 |
| countdown_max_digits: 4 |
| countdown_num_numbers: 6 |
|
|
