training_config.json

{
  "base_model": "xlangai/OpenCUA-7B",
  "trust_remote_code": true,
  "dataset_repo": "ageppert/world-model-transitions",
  "train_file": "transition_train.jsonl",
  "val_file": "transition_val.jsonl",
  "lora_rank": 16,
  "lora_alpha": 32,
  "lora_dropout": 0.05,
  "target_modules": [
    "q_proj",
    "k_proj",
    "v_proj",
    "o_proj"
  ],
  "epochs": 3,
  "lr": 0.0002,
  "per_device_batch_size": 2,
  "gradient_accumulation_steps": 8,
  "max_seq_length": 2048,
  "warmup_ratio": 0.05,
  "weight_decay": 0.01,
  "bf16": true,
  "gradient_checkpointing": true,
  "save_steps": 500,
  "eval_steps": 500,
  "logging_steps": 10,
  "output_dir": "./world_model_output",
  "final_model_dir": "./world_model_final",
  "push_to_hub": true,
  "hub_model_id": "ageppert/world-model-7b-lora",
  "smoke_test": false
}