Hugging Face's logo

kaitooooo
/
human_assisted_preference_optimization

Safetensors
Model card Files
xet
 Community
human_assisted_preference_optimization
File size: 2,402 Bytes
f7d71e7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
local_run_dir: ckpts
exp_name: hapo_stack_three_d0
cache_dir: .cache
seed: 1
warmup_steps: 10
cache_reference_logprobs: false
lr: 5.0e-05
correct_ratio: 0.5
interaction_ratio: 0.25
wandb:
  enabled: true
  entity: null
  project: hapo
minimum_log_interval_secs: 1.0
use_lora: true
batch_size: 8
eval_batch_size: 8
n_epochs: 10
grad_accumulation_steps: 1
save_steps: 500
action_space: 7
do_first_eval: false
eval_every: 2000
debug: false
intermediate_checkpoints: true
model:
  name_or_path: /mnt/bn/xwk-robotlearning/workspace/openvla/ckpts/base_policy/stack_three_d0_300/adapter_dir
  base_path: /mnt/bn/xwk-robotlearning/.cache/huggingface/hub/models--openvla--openvla-7b/snapshots/31f090d05236101ebfc381b61c674dd4746d4ce0
  tokenizer_name_or_path: null
  load_from: null
  from_checkpoint: null
  block_name: LlamaDecoderLayer
  policy_dtype: bfloat16
  reference_dtype: bfloat16
  max_grad_norm: 5.0
  v_head_max_grad_norm: 0.1
  max_length: 1024
  max_prompt_length: 512
  activation_checkpointing: true
  batch_size: 8
  gradient_accumulation_steps: 1
  eval_batch_size: 8
  attn_implementation: flash_attention_2
  use_peft: true
  load_lora_from: null
  peft:
    lora_r: 64
    lora_alpha: 256
    lora_dropout: 0.05
    target_modules: all-linear
  reward_model:
    path: null
    model_class: AutoModelForBradleyTerry
    dtype: float32
    attn_implementation: flash_attention_2
loss:
  beta: 0.1
  trainer: KTOTrainer
  dataloader: UnpairedPreferenceDataLoader
  desirable_weight: 1
  undesirable_weight: 1.0
  bc_weight: 1
  kl_max: 5
  kl_min: -5
  token_distance: 1
trainer:
  _target_: trainer.ddp_robotic_trainer.HAPOTrainer
dataset:
  _target_: dataset.balance_hapo_dataset.BalanceHAPODataset
  kwargs:
    data_root_dir: robotic_data/
    interaction_root_dir: robotic_data/mimicgen_interaction_data/mimicgen_stack_three_d0_interaction
    data_name: mimicgen
    data_list_names: stack_three_d0.json
    interaction_list_names: interaction_data_list.json
    demo_number: 50
    image_shape: 224
    previous_K: 10
eval_dataset:
  _target_: dataset.eval_mimicgen.EvalMimicgen
  kwargs:
    data_root_dir: robotic_data/
    interaction_root_dir: robotic_data/mimicgen_interaction_data/mimicgen_stack_three_d0_interaction
    data_name: mimicgen
    data_list_names: stack_three_d0.json
    interaction_list_names: interaction_data_list.json
    demo_number: 0
    image_shape: 224