condition_id: c2_hackable
description: 'Hackable reward: answer + length bonus + format, no guardrail'
tags:
- hackable
- no-guardrail
model:
  name: Qwen/Qwen2.5-1.5B-Instruct
  ref_model: Qwen/Qwen2.5-1.5B-Instruct
  revision: main
  dtype: bfloat16
  attn_implementation: flash_attention_2
  trust_remote_code: true
  max_length: 2048
lora:
  enabled: true
  r: 16
  alpha: 32
  lora_alpha: 32
  dropout: 0.05
  lora_dropout: 0.05
  target_modules:
  - q_proj
  - k_proj
  - v_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj
  bias: none
  task_type: CAUSAL_LM
training:
  output_dir: outputs/c2_hackable
  max_steps: 1000
  per_device_train_batch_size: 1
  gradient_accumulation_steps: 4
  learning_rate: 1.0e-05
  lr_scheduler_type: cosine
  warmup_ratio: 0.05
  weight_decay: 0.01
  max_grad_norm: 1.0
  seed: 3
  dataloader_num_workers: 2
  remove_unused_columns: false
  num_generations: 8
  generation_batch_size: 8
  temperature: 0.9
  top_p: 0.95
  save_steps: 100
  eval_steps: 500
  logging_steps: 1
  fp16: false
  bf16: true
  beta: 0.0
  max_completion_length: 256
reward:
  type: !!python/object/apply:src.config.RewardType
  - hackable
  correctness_weight: 1.0
  length_bonus_max: 0.5
  length_bonus_ceiling: 512
  format_bonus: 0.15
  hard_length_cap: false
  hard_length_cap_tokens: 0
  answer_reward_weight: 1.0
  format_reward_weight: 0.0
  length_bonus_weight: 0.0
  kl_beta: 0.0
  max_reasoning_tokens: null
generation:
  max_new_tokens: 256
  do_sample: true
  temperature: 0.9
  top_p: 0.95
  top_k: 50
  repetition_penalty: 1.1
  num_return_sequences: 1
logging:
  level: !!python/object/apply:src.config.LogLevel
  - INFO
  use_wandb: true
  wandb_project: grpo-reward-hacking
  wandb_entity: null
  use_tensorboard: true
  use_csv: true
  log_dir: outputs/c2_hackable/logs
  log_reward_components: true
dataset:
  name: Jiayi-Pan/Countdown-Tasks-3to4
  split_train: train
  split_eval: null
  eval_holdout_seed: 42
  max_train_samples: null
  max_eval_samples: 50
  preprocessing_num_workers: 4
  countdown_min_digits: 1
  countdown_max_digits: 4
  countdown_num_numbers: 6