YAML

config.yaml

@@ -20,41 +20,22 @@ hyperparameters:
   batch_size: 32
   gradient_accumulation_steps: 4
   gradient_clipping: 100
-  
+
   optimizer: AdamW
   learning_rate: 1.0e-6
-  
-  scheduler_type: ""
-  #scheduler_type: OneCycle
-  #scheduler_params:
-  #  cycle_first_step_size: 10_000
-  #  cycle_first_stair_count: 10_000
-
-  #  cycle_second_step_size: 15_000
-  #  cycle_second_stair_count: 15_000
-
-  #  decay_step_size: 5_000
-
-  #  cycle_min_lr: 2.5e-4 # 1.0e-5
-  #  cycle_max_lr: 2.5e-4 # 1.0e-4
-  #  decay_lr_rate: 0.0
-
-  #  cycle_min_mom: 0.90
-  #  cycle_max_mom: 0.99
-  #  decay_mom_rate: 0.0
 
 evaluation:
   batch_size: 64
   frequency: 500
   size: 64
-  
+
   steps: 300
   ar_temperature: 0.95
   nar_temperature: 0.25
 
 trainer:
   iterations: 1_000_000
-  
+
   save_tag: step
   save_on_oom: True
   save_on_quit: True
@@ -66,17 +47,14 @@ trainer:
 
   load_state_dict: True
   strict_loading: False
-  #load_tag: "9500"
-  #load_states: False
-  #restart_step_count: True
-  
+
   gc_mode: None # "global_step"
 
-  weight_dtype: bfloat16
+  weight_dtype: float32
 
-  backend: deepspeed
+  backend: local
   deepspeed:
-    zero_optimization_level: 2
+    zero_optimization_level: 0
     use_compression_training: True
 
 inference:
@@ -93,7 +71,7 @@ bitsandbytes:
 
 dataset:
   speaker_name_getter: "lambda p: f'{p.parts[-3]}_{p.parts[-2]}'"
-  
+
   use_hdf5: True
   hdf5_flag: r
   validate: True
@@ -3229,4 +3207,4 @@ dataset:
   ]
   noise: [
     "/data/Other/noise",
-  ]
+  ]