File size: 806 Bytes
4917d64 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | batch_size: 8
ckpt_freq: 500
data:
eval_data: ''
shuffle: true
train_data: /data/train.jsonl
do_ckpt: true
do_eval: false
duration_sec: 80.0
eval_freq: 300
first_codebook_weight_multiplier: 100.0
full_finetuning: false
gradient_checkpointing: true
log_freq: 10
lora:
enable: true
ft_embed: false
rank: 64
scaling: 2.0
max_norm: 1.0
max_steps: 3000
moshi_paths:
config_path: null
hf_repo_id: kyutai/moshiko-pytorch-bf16
mimi_path: null
moshi_path: null
tokenizer_path: null
num_ckpt_keep: 3
num_microbatches: 1
optim:
lr: 2.0e-06
pct_start: 0.05
weight_decay: 0.1
overwrite_run_dir: true
param_dtype: bfloat16
run_dir: /checkpoints/run_001
save_adapters: true
seed: 42
text_padding_weight: 0.5
wandb:
key: null
offline: false
project: null
run_name: null
world_size: 1
|