diff --git a/swin_default_LR1e-2_addEOSToken/config__2024_01_17--11_16_51.toml b/swin_default_LR1e-2_addEOSToken/config__2024_01_17--11_16_51.toml new file mode 100644 index 0000000000000000000000000000000000000000..3ca9568f08cf2b1331a30833d0d743f9bbba2c72 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/config__2024_01_17--11_16_51.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-2_addEOSToken" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-2_addEOSToken.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.0001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-2_addEOSToken/config__2024_01_17--12_14_37.toml b/swin_default_LR1e-2_addEOSToken/config__2024_01_17--12_14_37.toml new file mode 100644 index 0000000000000000000000000000000000000000..3ca9568f08cf2b1331a30833d0d743f9bbba2c72 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/config__2024_01_17--12_14_37.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-2_addEOSToken" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-2_addEOSToken.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.0001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-2_addEOSToken/config__2024_01_17--17_16_44.toml b/swin_default_LR1e-2_addEOSToken/config__2024_01_17--17_16_44.toml new file mode 100644 index 0000000000000000000000000000000000000000..3ca9568f08cf2b1331a30833d0d743f9bbba2c72 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/config__2024_01_17--17_16_44.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-2_addEOSToken" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-2_addEOSToken.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.0001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--11_16_21.log b/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--11_16_21.log new file mode 100644 index 0000000000000000000000000000000000000000..3b27a67c00b1161da2fc7d61d1943c1b4366e611 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--11_16_21.log @@ -0,0 +1,142 @@ +01-17 11:16:21 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 11:16:21 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 11:16:21 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 11:16:21 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 11:16:57 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/config__2024_01_17--11_16_51.toml. +01-17 11:16:57 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 503.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-17 11:16:57 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─ModuleList: 2-2 -- +│ │ └─TokenEmbedding: 3-4 1,049,600 +│ │ └─TokenEmbedding: 3-5 1,049,600 +│ │ └─TokenEmbedding: 3-6 1,049,600 +│ │ └─TokenEmbedding: 3-7 1,049,600 +│ │ └─TokenEmbedding: 3-8 1,049,600 +│ │ └─TokenEmbedding: 3-9 1,049,600 +│ │ └─TokenEmbedding: 3-10 1,049,600 +│ │ └─TokenEmbedding: 3-11 1,049,600 +│ └─PreNet: 2-3 -- +│ │ └─Sequential: 3-12 591,360 +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-13 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-14 201,535,488 +│ │ └─AdaptiveLayerNorm: 3-15 2,101,248 +│ └─ModuleList: 2-6 -- +│ │ └─Linear: 3-16 1,049,600 +│ │ └─Linear: 3-17 1,049,600 +│ │ └─Linear: 3-18 1,049,600 +│ │ └─Linear: 3-19 1,049,600 +│ │ └─Linear: 3-20 1,049,600 +│ │ └─Linear: 3-21 1,049,600 +│ │ └─Linear: 3-22 1,049,600 +│ │ └─Linear: 3-23 1,049,600 +│ └─ModuleList: 2-7 -- +│ │ └─TokenEmbedding: 3-24 1,024 +│ │ └─TokenEmbedding: 3-25 1,024 +│ │ └─TokenEmbedding: 3-26 1,024 +│ │ └─TokenEmbedding: 3-27 1,024 +│ │ └─TokenEmbedding: 3-28 1,024 +│ │ └─TokenEmbedding: 3-29 1,024 +│ │ └─TokenEmbedding: 3-30 1,024 +│ │ └─TokenEmbedding: 3-31 1,024 +│ └─MulticlassAccuracy: 2-8 -- +=============================================================================================== +Total params: 235,881,699 +Trainable params: 221,029,888 +Non-trainable params: 14,851,811 +=============================================================================================== +01-17 11:16:57 INFO [logging.py:61]: Training control variables: +01-17 11:16:57 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-17 11:16:57 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-17 11:16:57 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-17 11:16:57 INFO [logging.py:61]: `max_steps`: 500000 +01-17 11:16:57 INFO [logging.py:61]: `max_epochs`: 1000 +01-17 11:16:57 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-17 11:16:57 INFO [logging.py:61]: ========= Epoch 1 out of 1000 ========= +01-17 11:16:57 INFO [logging.py:61]: Begin training... +01-17 11:34:44 INFO [logging.py:61]: Loss 'loss' on epoch 1: 5.376262664794922 +01-17 11:34:44 INFO [logging.py:61]: Loss 'acc' on epoch 1: 0.29953059554100037 +01-17 11:34:44 INFO [logging.py:61]: ========= Epoch 2 out of 1000 ========= +01-17 11:34:44 INFO [logging.py:61]: Begin training... +01-17 11:52:25 INFO [logging.py:61]: Loss 'loss' on epoch 2: 4.830622673034668 +01-17 11:52:25 INFO [logging.py:61]: Loss 'acc' on epoch 2: 0.35284850001335144 +01-17 11:52:25 INFO [logging.py:61]: ========= Epoch 3 out of 1000 ========= +01-17 11:52:25 INFO [logging.py:61]: Begin training... +01-17 12:10:07 INFO [logging.py:61]: Loss 'loss' on epoch 3: 4.957332134246826 +01-17 12:10:07 INFO [logging.py:61]: Loss 'acc' on epoch 3: 0.3474164605140686 +01-17 12:10:07 INFO [logging.py:61]: ========= Epoch 4 out of 1000 ========= +01-17 12:10:07 INFO [logging.py:61]: Begin training... +01-17 12:27:48 INFO [logging.py:61]: Loss 'loss' on epoch 4: 4.950679779052734 +01-17 12:27:48 INFO [logging.py:61]: Loss 'acc' on epoch 4: 0.3592258095741272 +01-17 12:27:48 INFO [logging.py:61]: ========= Epoch 5 out of 1000 ========= +01-17 12:27:48 INFO [logging.py:61]: Begin training... +01-17 12:45:29 INFO [logging.py:61]: Loss 'loss' on epoch 5: 4.960457801818848 +01-17 12:45:29 INFO [logging.py:61]: Loss 'acc' on epoch 5: 0.3581017851829529 +01-17 12:45:29 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005 +01-17 12:45:30 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/pytorch_model.bin +01-17 12:45:33 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/optimizer.bin +01-17 12:45:33 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/scheduler.bin +01-17 12:45:33 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/sampler.bin +01-17 12:45:33 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/sampler_1.bin +01-17 12:45:33 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/random_states_0.pkl +01-17 12:45:33 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/custom_checkpoint_0.pkl +01-17 12:45:33 INFO [logging.py:61]: ========= Epoch 6 out of 1000 ========= +01-17 12:45:33 INFO [logging.py:61]: Begin training... +01-17 13:03:13 INFO [logging.py:61]: Loss 'loss' on epoch 6: 5.089817523956299 +01-17 13:03:13 INFO [logging.py:61]: Loss 'acc' on epoch 6: 0.35066384077072144 +01-17 13:03:13 INFO [logging.py:61]: ========= Epoch 7 out of 1000 ========= +01-17 13:03:13 INFO [logging.py:61]: Begin training... +01-17 13:20:54 INFO [logging.py:61]: Loss 'loss' on epoch 7: 5.060431480407715 +01-17 13:20:54 INFO [logging.py:61]: Loss 'acc' on epoch 7: 0.3525404930114746 +01-17 13:20:54 INFO [logging.py:61]: ========= Epoch 8 out of 1000 ========= +01-17 13:20:54 INFO [logging.py:61]: Begin training... +01-17 13:38:35 INFO [logging.py:61]: Loss 'loss' on epoch 8: 5.10891056060791 +01-17 13:38:35 INFO [logging.py:61]: Loss 'acc' on epoch 8: 0.349361389875412 +01-17 13:38:35 INFO [logging.py:61]: ========= Epoch 9 out of 1000 ========= +01-17 13:38:35 INFO [logging.py:61]: Begin training... +01-17 13:56:16 INFO [logging.py:61]: Loss 'loss' on epoch 9: 5.178280830383301 +01-17 13:56:16 INFO [logging.py:61]: Loss 'acc' on epoch 9: 0.34400177001953125 +01-17 13:56:16 INFO [logging.py:61]: ========= Epoch 10 out of 1000 ========= +01-17 13:56:16 INFO [logging.py:61]: Begin training... +01-17 14:13:57 INFO [logging.py:61]: Loss 'loss' on epoch 10: 5.152740001678467 +01-17 14:13:57 INFO [logging.py:61]: Loss 'acc' on epoch 10: 0.34986352920532227 +01-17 14:13:57 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010 +01-17 14:13:58 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/pytorch_model.bin +01-17 14:14:00 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/optimizer.bin +01-17 14:14:00 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/scheduler.bin +01-17 14:14:00 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/sampler.bin +01-17 14:14:00 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/sampler_1.bin +01-17 14:14:00 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/random_states_0.pkl +01-17 14:14:00 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/custom_checkpoint_0.pkl +01-17 14:14:01 INFO [logging.py:61]: ========= Epoch 11 out of 1000 ========= +01-17 14:14:01 INFO [logging.py:61]: Begin training... +01-17 14:31:41 INFO [logging.py:61]: Loss 'loss' on epoch 11: 5.189244270324707 +01-17 14:31:41 INFO [logging.py:61]: Loss 'acc' on epoch 11: 0.3463723957538605 +01-17 14:31:41 INFO [logging.py:61]: ========= Epoch 12 out of 1000 ========= +01-17 14:31:41 INFO [logging.py:61]: Begin training... +01-17 14:49:22 INFO [logging.py:61]: Loss 'loss' on epoch 12: 5.161355495452881 +01-17 14:49:22 INFO [logging.py:61]: Loss 'acc' on epoch 12: 0.347272664308548 +01-17 14:49:22 INFO [logging.py:61]: ========= Epoch 13 out of 1000 ========= +01-17 14:49:22 INFO [logging.py:61]: Begin training... +01-17 15:07:03 INFO [logging.py:61]: Loss 'loss' on epoch 13: 5.166537761688232 +01-17 15:07:03 INFO [logging.py:61]: Loss 'acc' on epoch 13: 0.34788310527801514 +01-17 15:07:03 INFO [logging.py:61]: ========= Epoch 14 out of 1000 ========= +01-17 15:07:03 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--12_14_33.log b/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--12_14_33.log new file mode 100644 index 0000000000000000000000000000000000000000..de84a6399e415fa0bfa8dea98de712f65fbe6933 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--12_14_33.log @@ -0,0 +1,130 @@ +01-17 12:14:33 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 12:14:33 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 12:14:33 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 12:14:33 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 12:14:38 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/config__2024_01_17--12_14_37.toml. +01-17 12:14:38 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 1007.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-17 12:14:38 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─ModuleList: 2-2 -- +│ │ └─TokenEmbedding: 3-4 1,049,600 +│ │ └─TokenEmbedding: 3-5 1,049,600 +│ │ └─TokenEmbedding: 3-6 1,049,600 +│ │ └─TokenEmbedding: 3-7 1,049,600 +│ │ └─TokenEmbedding: 3-8 1,049,600 +│ │ └─TokenEmbedding: 3-9 1,049,600 +│ │ └─TokenEmbedding: 3-10 1,049,600 +│ │ └─TokenEmbedding: 3-11 1,049,600 +│ └─PreNet: 2-3 -- +│ │ └─Sequential: 3-12 591,360 +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-13 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-14 201,535,488 +│ │ └─AdaptiveLayerNorm: 3-15 2,101,248 +│ └─ModuleList: 2-6 -- +│ │ └─Linear: 3-16 1,049,600 +│ │ └─Linear: 3-17 1,049,600 +│ │ └─Linear: 3-18 1,049,600 +│ │ └─Linear: 3-19 1,049,600 +│ │ └─Linear: 3-20 1,049,600 +│ │ └─Linear: 3-21 1,049,600 +│ │ └─Linear: 3-22 1,049,600 +│ │ └─Linear: 3-23 1,049,600 +│ └─ModuleList: 2-7 -- +│ │ └─TokenEmbedding: 3-24 1,024 +│ │ └─TokenEmbedding: 3-25 1,024 +│ │ └─TokenEmbedding: 3-26 1,024 +│ │ └─TokenEmbedding: 3-27 1,024 +│ │ └─TokenEmbedding: 3-28 1,024 +│ │ └─TokenEmbedding: 3-29 1,024 +│ │ └─TokenEmbedding: 3-30 1,024 +│ │ └─TokenEmbedding: 3-31 1,024 +│ └─MulticlassAccuracy: 2-8 -- +=============================================================================================== +Total params: 235,881,699 +Trainable params: 221,029,888 +Non-trainable params: 14,851,811 +=============================================================================================== +01-17 12:14:38 INFO [logging.py:61]: Training control variables: +01-17 12:14:38 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-17 12:14:38 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-17 12:14:38 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-17 12:14:38 INFO [logging.py:61]: `max_steps`: 500000 +01-17 12:14:38 INFO [logging.py:61]: `max_epochs`: 1000 +01-17 12:14:38 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-17 12:14:38 INFO [logging.py:61]: ========= Epoch 1 out of 1000 ========= +01-17 12:14:38 INFO [logging.py:61]: Begin training... +01-17 12:32:23 INFO [logging.py:61]: Loss 'loss' on epoch 1: 5.352734088897705 +01-17 12:32:23 INFO [logging.py:61]: Loss 'acc' on epoch 1: 0.30075228214263916 +01-17 12:32:23 INFO [logging.py:61]: ========= Epoch 2 out of 1000 ========= +01-17 12:32:23 INFO [logging.py:61]: Begin training... +01-17 12:50:06 INFO [logging.py:61]: Loss 'loss' on epoch 2: 4.763156414031982 +01-17 12:50:06 INFO [logging.py:61]: Loss 'acc' on epoch 2: 0.35356950759887695 +01-17 12:50:06 INFO [logging.py:61]: ========= Epoch 3 out of 1000 ========= +01-17 12:50:06 INFO [logging.py:61]: Begin training... +01-17 13:07:51 INFO [logging.py:61]: Loss 'loss' on epoch 3: 4.902694225311279 +01-17 13:07:51 INFO [logging.py:61]: Loss 'acc' on epoch 3: 0.35302937030792236 +01-17 13:07:51 INFO [logging.py:61]: ========= Epoch 4 out of 1000 ========= +01-17 13:07:51 INFO [logging.py:61]: Begin training... +01-17 13:28:14 INFO [logging.py:61]: Loss 'loss' on epoch 4: 4.946804046630859 +01-17 13:28:14 INFO [logging.py:61]: Loss 'acc' on epoch 4: 0.3474726378917694 +01-17 13:28:14 INFO [logging.py:61]: ========= Epoch 5 out of 1000 ========= +01-17 13:28:14 INFO [logging.py:61]: Begin training... +01-17 13:46:01 INFO [logging.py:61]: Loss 'loss' on epoch 5: 5.042134761810303 +01-17 13:46:01 INFO [logging.py:61]: Loss 'acc' on epoch 5: 0.3472442626953125 +01-17 13:46:01 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005 +01-17 13:46:02 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/pytorch_model.bin +01-17 13:46:05 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/optimizer.bin +01-17 13:46:05 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/scheduler.bin +01-17 13:46:05 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/sampler.bin +01-17 13:46:05 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/sampler_1.bin +01-17 13:46:05 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/random_states_0.pkl +01-17 13:46:05 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0005/custom_checkpoint_0.pkl +01-17 13:46:05 INFO [logging.py:61]: ========= Epoch 6 out of 1000 ========= +01-17 13:46:05 INFO [logging.py:61]: Begin training... +01-17 14:03:48 INFO [logging.py:61]: Loss 'loss' on epoch 6: 5.154253959655762 +01-17 14:03:48 INFO [logging.py:61]: Loss 'acc' on epoch 6: 0.3447455167770386 +01-17 14:03:48 INFO [logging.py:61]: ========= Epoch 7 out of 1000 ========= +01-17 14:03:48 INFO [logging.py:61]: Begin training... +01-17 14:21:30 INFO [logging.py:61]: Loss 'loss' on epoch 7: 5.2775559425354 +01-17 14:21:30 INFO [logging.py:61]: Loss 'acc' on epoch 7: 0.3431953191757202 +01-17 14:21:30 INFO [logging.py:61]: ========= Epoch 8 out of 1000 ========= +01-17 14:21:30 INFO [logging.py:61]: Begin training... +01-17 14:39:10 INFO [logging.py:61]: Loss 'loss' on epoch 8: 5.302144527435303 +01-17 14:39:10 INFO [logging.py:61]: Loss 'acc' on epoch 8: 0.3417191207408905 +01-17 14:39:10 INFO [logging.py:61]: ========= Epoch 9 out of 1000 ========= +01-17 14:39:10 INFO [logging.py:61]: Begin training... +01-17 14:56:49 INFO [logging.py:61]: Loss 'loss' on epoch 9: 5.384851932525635 +01-17 14:56:49 INFO [logging.py:61]: Loss 'acc' on epoch 9: 0.33888909220695496 +01-17 14:56:49 INFO [logging.py:61]: ========= Epoch 10 out of 1000 ========= +01-17 14:56:49 INFO [logging.py:61]: Begin training... +01-17 15:14:29 INFO [logging.py:61]: Loss 'loss' on epoch 10: 5.374612808227539 +01-17 15:14:29 INFO [logging.py:61]: Loss 'acc' on epoch 10: 0.3413293659687042 +01-17 15:14:29 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010 +01-17 15:14:30 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/pytorch_model.bin +01-17 15:14:32 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/optimizer.bin +01-17 15:14:32 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/scheduler.bin +01-17 15:14:32 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/sampler.bin +01-17 15:14:32 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/sampler_1.bin +01-17 15:14:32 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/random_states_0.pkl +01-17 15:14:32 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/custom_checkpoint_0.pkl +01-17 15:14:32 INFO [logging.py:61]: ========= Epoch 11 out of 1000 ========= +01-17 15:14:32 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--17_15_22.log b/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--17_15_22.log new file mode 100644 index 0000000000000000000000000000000000000000..2a83c6ac1ced88f292f87b1ad0b60515b90293ee --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/swin_default_LR1e-2_addEOSToken_2024_01_17--17_15_22.log @@ -0,0 +1,203 @@ +01-17 17:15:22 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 17:15:22 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 17:15:22 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 17:15:22 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken. +01-17 17:17:03 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/config__2024_01_17--17_16_44.toml. +01-17 17:17:03 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 503.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-17 17:17:03 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─ModuleList: 2-2 -- +│ │ └─TokenEmbedding: 3-4 1,049,600 +│ │ └─TokenEmbedding: 3-5 1,049,600 +│ │ └─TokenEmbedding: 3-6 1,049,600 +│ │ └─TokenEmbedding: 3-7 1,049,600 +│ │ └─TokenEmbedding: 3-8 1,049,600 +│ │ └─TokenEmbedding: 3-9 1,049,600 +│ │ └─TokenEmbedding: 3-10 1,049,600 +│ │ └─TokenEmbedding: 3-11 1,049,600 +│ └─PreNet: 2-3 -- +│ │ └─Sequential: 3-12 591,360 +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-13 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-14 201,535,488 +│ │ └─AdaptiveLayerNorm: 3-15 2,101,248 +│ └─ModuleList: 2-6 -- +│ │ └─Linear: 3-16 1,049,600 +│ │ └─Linear: 3-17 1,049,600 +│ │ └─Linear: 3-18 1,049,600 +│ │ └─Linear: 3-19 1,049,600 +│ │ └─Linear: 3-20 1,049,600 +│ │ └─Linear: 3-21 1,049,600 +│ │ └─Linear: 3-22 1,049,600 +│ │ └─Linear: 3-23 1,049,600 +│ └─ModuleList: 2-7 -- +│ │ └─TokenEmbedding: 3-24 1,024 +│ │ └─TokenEmbedding: 3-25 1,024 +│ │ └─TokenEmbedding: 3-26 1,024 +│ │ └─TokenEmbedding: 3-27 1,024 +│ │ └─TokenEmbedding: 3-28 1,024 +│ │ └─TokenEmbedding: 3-29 1,024 +│ │ └─TokenEmbedding: 3-30 1,024 +│ │ └─TokenEmbedding: 3-31 1,024 +│ └─MulticlassAccuracy: 2-8 -- +=============================================================================================== +Total params: 235,881,699 +Trainable params: 221,029,888 +Non-trainable params: 14,851,811 +=============================================================================================== +01-17 17:17:03 INFO [logging.py:61]: Training control variables: +01-17 17:17:03 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-17 17:17:03 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-17 17:17:03 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-17 17:17:03 INFO [logging.py:61]: `max_steps`: 500000 +01-17 17:17:03 INFO [logging.py:61]: `max_epochs`: 1000 +01-17 17:17:03 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-17 17:17:03 INFO [logging.py:61]: Loading states from /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010 +01-17 17:17:07 INFO [logging.py:61]: All model weights loaded successfully +01-17 17:17:13 INFO [logging.py:61]: All optimizer states loaded successfully +01-17 17:17:13 INFO [logging.py:61]: All scheduler states loaded successfully +01-17 17:17:13 INFO [logging.py:61]: All dataloader sampler states loaded successfully +01-17 17:17:13 INFO [logging.py:61]: All random states loaded successfully +01-17 17:17:13 INFO [logging.py:61]: Loading in 1 custom states +01-17 17:17:13 INFO [logging.py:61]: Loading the state of TrainerState from /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0010/custom_checkpoint_0.pkl +01-17 17:17:13 INFO [logging.py:61]: Checkpoint on epoch 10 is loaded. +01-17 17:17:13 INFO [logging.py:61]: ========= Epoch 11 out of 1000 ========= +01-17 17:17:13 INFO [logging.py:61]: Begin training... +01-17 17:34:58 INFO [logging.py:61]: Loss 'loss' on epoch 11: 5.385993003845215 +01-17 17:34:58 INFO [logging.py:61]: Loss 'acc' on epoch 11: 0.339417040348053 +01-17 17:34:58 INFO [logging.py:61]: ========= Epoch 12 out of 1000 ========= +01-17 17:34:58 INFO [logging.py:61]: Begin training... +01-17 17:53:05 INFO [logging.py:61]: Loss 'loss' on epoch 12: 5.376588344573975 +01-17 17:53:06 INFO [logging.py:61]: Loss 'acc' on epoch 12: 0.33914294838905334 +01-17 17:53:06 INFO [logging.py:61]: ========= Epoch 13 out of 1000 ========= +01-17 17:53:06 INFO [logging.py:61]: Begin training... +01-17 18:11:23 INFO [logging.py:61]: Loss 'loss' on epoch 13: 5.388041973114014 +01-17 18:11:24 INFO [logging.py:61]: Loss 'acc' on epoch 13: 0.339685320854187 +01-17 18:11:24 INFO [logging.py:61]: ========= Epoch 14 out of 1000 ========= +01-17 18:11:24 INFO [logging.py:61]: Begin training... +01-17 18:30:04 INFO [logging.py:61]: Loss 'loss' on epoch 14: 5.3795671463012695 +01-17 18:30:04 INFO [logging.py:61]: Loss 'acc' on epoch 14: 0.3413822054862976 +01-17 18:30:04 INFO [logging.py:61]: ========= Epoch 15 out of 1000 ========= +01-17 18:30:04 INFO [logging.py:61]: Begin training... +01-17 18:48:00 INFO [logging.py:61]: Loss 'loss' on epoch 15: 5.390796661376953 +01-17 18:48:00 INFO [logging.py:61]: Loss 'acc' on epoch 15: 0.33877530694007874 +01-17 18:48:00 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0015 +01-17 18:48:02 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0015/pytorch_model.bin +01-17 18:48:05 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0015/optimizer.bin +01-17 18:48:05 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0015/scheduler.bin +01-17 18:48:05 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0015/sampler.bin +01-17 18:48:05 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0015/sampler_1.bin +01-17 18:48:05 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0015/random_states_0.pkl +01-17 18:48:05 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0015/custom_checkpoint_0.pkl +01-17 18:48:05 INFO [logging.py:61]: ========= Epoch 16 out of 1000 ========= +01-17 18:48:05 INFO [logging.py:61]: Begin training... +01-17 19:06:17 INFO [logging.py:61]: Loss 'loss' on epoch 16: 5.389437198638916 +01-17 19:06:17 INFO [logging.py:61]: Loss 'acc' on epoch 16: 0.33847111463546753 +01-17 19:06:17 INFO [logging.py:61]: ========= Epoch 17 out of 1000 ========= +01-17 19:06:17 INFO [logging.py:61]: Begin training... +01-17 19:24:08 INFO [logging.py:61]: Loss 'loss' on epoch 17: 5.380032539367676 +01-17 19:24:08 INFO [logging.py:61]: Loss 'acc' on epoch 17: 0.339590847492218 +01-17 19:24:08 INFO [logging.py:61]: ========= Epoch 18 out of 1000 ========= +01-17 19:24:08 INFO [logging.py:61]: Begin training... +01-17 19:52:50 INFO [logging.py:61]: Loss 'loss' on epoch 18: 5.37690544128418 +01-17 19:52:50 INFO [logging.py:61]: Loss 'acc' on epoch 18: 0.33943551778793335 +01-17 19:52:50 INFO [logging.py:61]: ========= Epoch 19 out of 1000 ========= +01-17 19:52:50 INFO [logging.py:61]: Begin training... +01-17 20:10:43 INFO [logging.py:61]: Loss 'loss' on epoch 19: 5.3855366706848145 +01-17 20:10:43 INFO [logging.py:61]: Loss 'acc' on epoch 19: 0.33942481875419617 +01-17 20:10:43 INFO [logging.py:61]: ========= Epoch 20 out of 1000 ========= +01-17 20:10:43 INFO [logging.py:61]: Begin training... +01-17 20:29:32 INFO [logging.py:61]: Loss 'loss' on epoch 20: 5.374676704406738 +01-17 20:29:32 INFO [logging.py:61]: Loss 'acc' on epoch 20: 0.33983924984931946 +01-17 20:29:32 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0020 +01-17 20:29:33 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0020/pytorch_model.bin +01-17 20:29:36 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0020/optimizer.bin +01-17 20:29:36 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0020/scheduler.bin +01-17 20:29:36 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0020/sampler.bin +01-17 20:29:36 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0020/sampler_1.bin +01-17 20:29:36 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0020/random_states_0.pkl +01-17 20:29:36 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0020/custom_checkpoint_0.pkl +01-17 20:29:36 INFO [logging.py:61]: ========= Epoch 21 out of 1000 ========= +01-17 20:29:36 INFO [logging.py:61]: Begin training... +01-17 20:47:57 INFO [logging.py:61]: Loss 'loss' on epoch 21: 5.386965274810791 +01-17 20:47:57 INFO [logging.py:61]: Loss 'acc' on epoch 21: 0.3395291566848755 +01-17 20:47:57 INFO [logging.py:61]: ========= Epoch 22 out of 1000 ========= +01-17 20:47:57 INFO [logging.py:61]: Begin training... +01-17 21:05:45 INFO [logging.py:61]: Loss 'loss' on epoch 22: 5.365380764007568 +01-17 21:05:45 INFO [logging.py:61]: Loss 'acc' on epoch 22: 0.34075096249580383 +01-17 21:05:45 INFO [logging.py:61]: ========= Epoch 23 out of 1000 ========= +01-17 21:05:45 INFO [logging.py:61]: Begin training... +01-17 21:23:33 INFO [logging.py:61]: Loss 'loss' on epoch 23: 5.38892936706543 +01-17 21:23:33 INFO [logging.py:61]: Loss 'acc' on epoch 23: 0.33941057324409485 +01-17 21:23:33 INFO [logging.py:61]: ========= Epoch 24 out of 1000 ========= +01-17 21:23:33 INFO [logging.py:61]: Begin training... +01-17 21:41:21 INFO [logging.py:61]: Loss 'loss' on epoch 24: 5.370124340057373 +01-17 21:41:21 INFO [logging.py:61]: Loss 'acc' on epoch 24: 0.3396085798740387 +01-17 21:41:21 INFO [logging.py:61]: ========= Epoch 25 out of 1000 ========= +01-17 21:41:21 INFO [logging.py:61]: Begin training... +01-17 21:59:09 INFO [logging.py:61]: Loss 'loss' on epoch 25: 5.369869232177734 +01-17 21:59:09 INFO [logging.py:61]: Loss 'acc' on epoch 25: 0.3406093716621399 +01-17 21:59:09 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0025 +01-17 21:59:11 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0025/pytorch_model.bin +01-17 21:59:14 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0025/optimizer.bin +01-17 21:59:14 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0025/scheduler.bin +01-17 21:59:14 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0025/sampler.bin +01-17 21:59:14 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0025/sampler_1.bin +01-17 21:59:14 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0025/random_states_0.pkl +01-17 21:59:14 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0025/custom_checkpoint_0.pkl +01-17 21:59:16 INFO [logging.py:61]: ========= Epoch 26 out of 1000 ========= +01-17 21:59:16 INFO [logging.py:61]: Begin training... +01-17 22:17:25 INFO [logging.py:61]: Loss 'loss' on epoch 26: 5.375258445739746 +01-17 22:17:25 INFO [logging.py:61]: Loss 'acc' on epoch 26: 0.33873483538627625 +01-17 22:17:25 INFO [logging.py:61]: ========= Epoch 27 out of 1000 ========= +01-17 22:17:25 INFO [logging.py:61]: Begin training... +01-17 22:35:12 INFO [logging.py:61]: Loss 'loss' on epoch 27: 5.3775506019592285 +01-17 22:35:12 INFO [logging.py:61]: Loss 'acc' on epoch 27: 0.33919477462768555 +01-17 22:35:12 INFO [logging.py:61]: ========= Epoch 28 out of 1000 ========= +01-17 22:35:12 INFO [logging.py:61]: Begin training... +01-17 22:53:00 INFO [logging.py:61]: Loss 'loss' on epoch 28: 5.390237331390381 +01-17 22:53:00 INFO [logging.py:61]: Loss 'acc' on epoch 28: 0.33835938572883606 +01-17 22:53:00 INFO [logging.py:61]: ========= Epoch 29 out of 1000 ========= +01-17 22:53:00 INFO [logging.py:61]: Begin training... +01-17 23:10:48 INFO [logging.py:61]: Loss 'loss' on epoch 29: 5.371942520141602 +01-17 23:10:48 INFO [logging.py:61]: Loss 'acc' on epoch 29: 0.3405751883983612 +01-17 23:10:48 INFO [logging.py:61]: ========= Epoch 30 out of 1000 ========= +01-17 23:10:48 INFO [logging.py:61]: Begin training... +01-17 23:28:35 INFO [logging.py:61]: Loss 'loss' on epoch 30: 5.369765281677246 +01-17 23:28:35 INFO [logging.py:61]: Loss 'acc' on epoch 30: 0.34033069014549255 +01-17 23:28:35 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0030 +01-17 23:28:36 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0030/pytorch_model.bin +01-17 23:28:45 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0030/optimizer.bin +01-17 23:28:45 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0030/scheduler.bin +01-17 23:28:45 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0030/sampler.bin +01-17 23:28:45 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0030/sampler_1.bin +01-17 23:28:45 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0030/random_states_0.pkl +01-17 23:28:45 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_addEOSToken/checkpoints/epoch_0030/custom_checkpoint_0.pkl +01-17 23:28:45 INFO [logging.py:61]: ========= Epoch 31 out of 1000 ========= +01-17 23:28:45 INFO [logging.py:61]: Begin training... +01-17 23:47:05 INFO [logging.py:61]: Loss 'loss' on epoch 31: 5.372854232788086 +01-17 23:47:05 INFO [logging.py:61]: Loss 'acc' on epoch 31: 0.3400099575519562 +01-17 23:47:05 INFO [logging.py:61]: ========= Epoch 32 out of 1000 ========= +01-17 23:47:05 INFO [logging.py:61]: Begin training... +01-18 00:04:54 INFO [logging.py:61]: Loss 'loss' on epoch 32: 5.364051342010498 +01-18 00:04:54 INFO [logging.py:61]: Loss 'acc' on epoch 32: 0.3399272561073303 +01-18 00:04:54 INFO [logging.py:61]: ========= Epoch 33 out of 1000 ========= +01-18 00:04:54 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498943.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498943.0 new file mode 100644 index 0000000000000000000000000000000000000000..6d630390ee1af5e5ac2b2020e007f2f363be66b2 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498943.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7788705d2ca707cbbd2b5ff1645866ddac63ebf122ee6a449e18cc4eb54983d +size 368795 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498944.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498944.0 new file mode 100644 index 0000000000000000000000000000000000000000..595ff9ca6391cb84e2718435c370812e02706063 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498944.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34a26980b52523672ca790ef1d4aa6d2f2a9718cfd2764aacb44a47eeb891c8c +size 2170 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498945.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498945.0 new file mode 100644 index 0000000000000000000000000000000000000000..f04db87a1ae4f4bed990c7121c8dbac6a3eec8df --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498945.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b68c141fba1b5301e8a56b4b075d598bb61910225e149e39870d88cbec923b4e +size 2170 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498946.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498946.0 new file mode 100644 index 0000000000000000000000000000000000000000..0e49c2b6f1f21f02b7d66bad65c5ea2f91f6b1de --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705450617.gina1.1498946.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d56a85c87367dba2524854103cb0fa917e5a87b3dc9bfce8b167701fc1d23c +size 2170 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610519.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610519.0 new file mode 100644 index 0000000000000000000000000000000000000000..1c9c7f42225eb2ef410a590ae2a1f33edba57ec0 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610519.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7edc77f10fdec037eaf9b43db84eb42657bbffa59d6250094ba607ffeea0e9a +size 276026 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610520.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610520.0 new file mode 100644 index 0000000000000000000000000000000000000000..f1666c0c5d68eb3f3d894a6d4d9506405f9c9927 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610520.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0bdcf98a8c49f14de2ea16c197cc1bcc5a1a9c5d32f629ed617d4ca4e537ca6 +size 2170 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610521.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610521.0 new file mode 100644 index 0000000000000000000000000000000000000000..1474e8ee4ba5a1d4944dfeb96c79b230e8dae9d0 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610521.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0a55d85d50d9778f17cf404b752dc21123dbed4e1185acb5160704e2a7b7b5f +size 2170 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610522.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610522.0 new file mode 100644 index 0000000000000000000000000000000000000000..b41379468c97f2109231cd24ba9937d2c3c2e856 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705454078.gina301.610522.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fbf5ffdcceff08c5a8b229bf89f47ef363ea6906f06a3b3a180b03a2f96d261 +size 2170 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576669.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576669.0 new file mode 100644 index 0000000000000000000000000000000000000000..e5f62e9525a611da37e8f05d5401a70e3e22f0ce --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576669.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae26f96b329133f7afaa11b57d23661d4ccac43a2904a37ca5ff92189d11143 +size 620690 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576670.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576670.0 new file mode 100644 index 0000000000000000000000000000000000000000..8c39abddfc3ced125796bd03f03dbb862f4d2ee8 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576670.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af00695e81116da6aa753e531f6fefec091fdefc20ba5e0ad103bbd53362cb35 +size 2170 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576671.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576671.0 new file mode 100644 index 0000000000000000000000000000000000000000..de3576c9320edf24610db4504b6697ad3b9dbfff --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576671.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c494d29f5efe6acd4c545e367b5ff638ecf6d1b05344a5459db6b1ad76a7bf0f +size 2170 diff --git a/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576672.0 b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576672.0 new file mode 100644 index 0000000000000000000000000000000000000000..936d99955106415efaf6d07f3919f391bad1f623 --- /dev/null +++ b/swin_default_LR1e-2_addEOSToken/tb_log/events.out.tfevents.1705472223.gina1.1576672.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4ec1e9390f233230307f57f7ec4d8cb489d80f77a7642c0d0b7cfba2143cca +size 2170 diff --git a/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_22_05.toml b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_22_05.toml new file mode 100644 index 0000000000000000000000000000000000000000..3700274c47624ecb2dfa544537baffd48ce51978 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_22_05.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-2_AR-NAR" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-2_AR-NAR.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model_ar_nar.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_23_20.toml b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_23_20.toml new file mode 100644 index 0000000000000000000000000000000000000000..3700274c47624ecb2dfa544537baffd48ce51978 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_23_20.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-2_AR-NAR" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-2_AR-NAR.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model_ar_nar.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_33_25.toml b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_33_25.toml new file mode 100644 index 0000000000000000000000000000000000000000..3700274c47624ecb2dfa544537baffd48ce51978 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--00_33_25.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-2_AR-NAR" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-2_AR-NAR.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model_ar_nar.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-3_AR-NAR/config__2024_01_18--13_13_46.toml b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--13_13_46.toml new file mode 100644 index 0000000000000000000000000000000000000000..374ad3fefe6b17412a113349a867fda59d590aea --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--13_13_46.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-3_AR-NAR" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-3_AR-NAR.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model_ar_nar.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-3_AR-NAR/config__2024_01_18--13_14_43.toml b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--13_14_43.toml new file mode 100644 index 0000000000000000000000000000000000000000..374ad3fefe6b17412a113349a867fda59d590aea --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/config__2024_01_18--13_14_43.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-3_AR-NAR" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-3_AR-NAR.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model_ar_nar.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_21_17.log b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_21_17.log new file mode 100644 index 0000000000000000000000000000000000000000..cf958d5166d0093bde8ee2a430f637eefbddc554 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_21_17.log @@ -0,0 +1,85 @@ +01-18 00:21:17 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:21:17 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:21:17 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:21:17 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:22:18 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/config__2024_01_18--00_22_05.toml. +01-18 00:22:18 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 503.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-18 00:22:18 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─TokenEmbedding: 2-2 -- +│ │ └─Dropout: 3-4 -- +│ │ └─Embedding: 3-5 1,049,600 +│ └─Identity: 2-3 -- +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-6 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-7 151,154,688 +│ │ └─LayerNorm: 3-8 2,048 +│ └─Linear: 2-6 1,049,600 +│ └─MulticlassAccuracy: 2-7 -- +│ └─TokenEmbedding: 2-8 -- +│ │ └─Dropout: 3-9 -- +│ │ └─Embedding: 3-10 1,048,576 +│ └─ModuleList: 2-9 -- +│ │ └─TokenEmbedding: 3-11 1,049,600 +│ │ └─TokenEmbedding: 3-12 1,048,576 +│ │ └─TokenEmbedding: 3-13 1,048,576 +│ │ └─TokenEmbedding: 3-14 1,048,576 +│ │ └─TokenEmbedding: 3-15 1,048,576 +│ │ └─TokenEmbedding: 3-16 1,048,576 +│ │ └─TokenEmbedding: 3-17 1,048,576 +│ │ └─TokenEmbedding: 3-18 1,048,576 +│ └─Identity: 2-10 -- +│ └─SinePositionalEmbedding: 2-11 1 +│ │ └─Dropout: 3-19 -- +│ └─TransformerEncoder: 2-12 -- +│ │ └─ModuleList: 3-20 201,535,488 +│ │ └─AdaptiveLayerNorm: 3-21 2,101,248 +│ └─ModuleList: 2-13 -- +│ │ └─Linear: 3-22 1,048,576 +│ │ └─Linear: 3-23 1,048,576 +│ │ └─Linear: 3-24 1,048,576 +│ │ └─Linear: 3-25 1,048,576 +│ │ └─Linear: 3-26 1,048,576 +│ │ └─Linear: 3-27 1,048,576 +│ │ └─Linear: 3-28 1,048,576 +│ └─ModuleList: 2-14 -- +│ │ └─TokenEmbedding: 3-29 1,024 +│ │ └─TokenEmbedding: 3-30 1,024 +│ │ └─TokenEmbedding: 3-31 1,024 +│ │ └─TokenEmbedding: 3-32 1,024 +│ │ └─TokenEmbedding: 3-33 1,024 +│ │ └─TokenEmbedding: 3-34 1,024 +│ │ └─TokenEmbedding: 3-35 1,024 +│ └─MulticlassAccuracy: 2-15 -- +=============================================================================================== +Total params: 388,529,892 +Trainable params: 373,678,081 +Non-trainable params: 14,851,811 +=============================================================================================== +01-18 00:22:18 INFO [logging.py:61]: Training control variables: +01-18 00:22:18 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-18 00:22:18 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-18 00:22:18 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-18 00:22:18 INFO [logging.py:61]: `max_steps`: 500000 +01-18 00:22:18 INFO [logging.py:61]: `max_epochs`: 1000 +01-18 00:22:18 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-18 00:22:18 INFO [logging.py:61]: ========= Epoch 1 out of 1000 ========= +01-18 00:22:18 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_23_16.log b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_23_16.log new file mode 100644 index 0000000000000000000000000000000000000000..a8cbd6f3ab667ad4572fd0f71032e596d5f81eaf --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_23_16.log @@ -0,0 +1,85 @@ +01-18 00:23:16 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:23:16 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:23:16 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:23:16 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:23:22 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/config__2024_01_18--00_23_20.toml. +01-18 00:23:22 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 503.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-18 00:23:22 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─TokenEmbedding: 2-2 -- +│ │ └─Dropout: 3-4 -- +│ │ └─Embedding: 3-5 524,800 +│ └─Identity: 2-3 -- +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-6 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-7 37,828,608 +│ │ └─LayerNorm: 3-8 1,024 +│ └─Linear: 2-6 524,800 +│ └─MulticlassAccuracy: 2-7 -- +│ └─TokenEmbedding: 2-8 -- +│ │ └─Dropout: 3-9 -- +│ │ └─Embedding: 3-10 524,288 +│ └─ModuleList: 2-9 -- +│ │ └─TokenEmbedding: 3-11 524,800 +│ │ └─TokenEmbedding: 3-12 524,288 +│ │ └─TokenEmbedding: 3-13 524,288 +│ │ └─TokenEmbedding: 3-14 524,288 +│ │ └─TokenEmbedding: 3-15 524,288 +│ │ └─TokenEmbedding: 3-16 524,288 +│ │ └─TokenEmbedding: 3-17 524,288 +│ │ └─TokenEmbedding: 3-18 524,288 +│ └─Identity: 2-10 -- +│ └─SinePositionalEmbedding: 2-11 1 +│ │ └─Dropout: 3-19 -- +│ └─TransformerEncoder: 2-12 -- +│ │ └─ModuleList: 3-20 50,436,096 +│ │ └─AdaptiveLayerNorm: 3-21 526,336 +│ └─ModuleList: 2-13 -- +│ │ └─Linear: 3-22 524,288 +│ │ └─Linear: 3-23 524,288 +│ │ └─Linear: 3-24 524,288 +│ │ └─Linear: 3-25 524,288 +│ │ └─Linear: 3-26 524,288 +│ │ └─Linear: 3-27 524,288 +│ │ └─Linear: 3-28 524,288 +│ └─ModuleList: 2-14 -- +│ │ └─TokenEmbedding: 3-29 512 +│ │ └─TokenEmbedding: 3-30 512 +│ │ └─TokenEmbedding: 3-31 512 +│ │ └─TokenEmbedding: 3-32 512 +│ │ └─TokenEmbedding: 3-33 512 +│ │ └─TokenEmbedding: 3-34 512 +│ │ └─TokenEmbedding: 3-35 512 +│ └─MulticlassAccuracy: 2-15 -- +=============================================================================================== +Total params: 113,086,180 +Trainable params: 98,234,369 +Non-trainable params: 14,851,811 +=============================================================================================== +01-18 00:23:22 INFO [logging.py:61]: Training control variables: +01-18 00:23:22 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-18 00:23:22 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-18 00:23:22 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-18 00:23:22 INFO [logging.py:61]: `max_steps`: 500000 +01-18 00:23:22 INFO [logging.py:61]: `max_epochs`: 1000 +01-18 00:23:22 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-18 00:23:22 INFO [logging.py:61]: ========= Epoch 1 out of 1000 ========= +01-18 00:23:22 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_33_02.log b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_33_02.log new file mode 100644 index 0000000000000000000000000000000000000000..7eb68d132a656b2d549f8c8c8f9e348c1d73f097 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-2_AR-NAR_2024_01_18--00_33_02.log @@ -0,0 +1,515 @@ +01-18 00:33:02 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:33:02 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:33:02 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:33:02 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR. +01-18 00:33:32 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/config__2024_01_18--00_33_25.toml. +01-18 00:33:32 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 503.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-18 00:33:32 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─TokenEmbedding: 2-2 -- +│ │ └─Dropout: 3-4 -- +│ │ └─Embedding: 3-5 524,800 +│ └─Identity: 2-3 -- +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-6 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-7 37,828,608 +│ │ └─LayerNorm: 3-8 1,024 +│ └─Linear: 2-6 524,800 +│ └─MulticlassAccuracy: 2-7 -- +│ └─TokenEmbedding: 2-8 -- +│ │ └─Dropout: 3-9 -- +│ │ └─Embedding: 3-10 524,288 +│ └─ModuleList: 2-9 -- +│ │ └─TokenEmbedding: 3-11 524,800 +│ │ └─TokenEmbedding: 3-12 524,288 +│ │ └─TokenEmbedding: 3-13 524,288 +│ │ └─TokenEmbedding: 3-14 524,288 +│ │ └─TokenEmbedding: 3-15 524,288 +│ │ └─TokenEmbedding: 3-16 524,288 +│ │ └─TokenEmbedding: 3-17 524,288 +│ │ └─TokenEmbedding: 3-18 524,288 +│ └─Identity: 2-10 -- +│ └─SinePositionalEmbedding: 2-11 1 +│ │ └─Dropout: 3-19 -- +│ └─TransformerEncoder: 2-12 -- +│ │ └─ModuleList: 3-20 50,436,096 +│ │ └─AdaptiveLayerNorm: 3-21 526,336 +│ └─ModuleList: 2-13 -- +│ │ └─Linear: 3-22 524,288 +│ │ └─Linear: 3-23 524,288 +│ │ └─Linear: 3-24 524,288 +│ │ └─Linear: 3-25 524,288 +│ │ └─Linear: 3-26 524,288 +│ │ └─Linear: 3-27 524,288 +│ │ └─Linear: 3-28 524,288 +│ └─ModuleList: 2-14 -- +│ │ └─TokenEmbedding: 3-29 512 +│ │ └─TokenEmbedding: 3-30 512 +│ │ └─TokenEmbedding: 3-31 512 +│ │ └─TokenEmbedding: 3-32 512 +│ │ └─TokenEmbedding: 3-33 512 +│ │ └─TokenEmbedding: 3-34 512 +│ │ └─TokenEmbedding: 3-35 512 +│ └─MulticlassAccuracy: 2-15 -- +=============================================================================================== +Total params: 113,086,180 +Trainable params: 98,234,369 +Non-trainable params: 14,851,811 +=============================================================================================== +01-18 00:33:32 INFO [logging.py:61]: Training control variables: +01-18 00:33:32 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-18 00:33:32 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-18 00:33:32 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-18 00:33:32 INFO [logging.py:61]: `max_steps`: 500000 +01-18 00:33:32 INFO [logging.py:61]: `max_epochs`: 1000 +01-18 00:33:32 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-18 00:33:32 INFO [logging.py:61]: ========= Epoch 1 out of 1000 ========= +01-18 00:33:32 INFO [logging.py:61]: Begin training... +01-18 00:48:19 INFO [logging.py:61]: Loss 'loss' on epoch 1: 41.1013298034668 +01-18 00:48:19 INFO [logging.py:61]: Loss 'ar_loss' on epoch 1: 3.8041999340057373 +01-18 00:48:19 INFO [logging.py:61]: Loss 'nar_loss' on epoch 1: 37.2971305847168 +01-18 00:48:19 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 1: 0.5283006429672241 +01-18 00:48:19 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 1: 0.14399056136608124 +01-18 00:48:19 INFO [logging.py:61]: ========= Epoch 2 out of 1000 ========= +01-18 00:48:19 INFO [logging.py:61]: Begin training... +01-18 01:03:02 INFO [logging.py:61]: Loss 'loss' on epoch 2: 71.26728057861328 +01-18 01:03:02 INFO [logging.py:61]: Loss 'ar_loss' on epoch 2: 3.0147104263305664 +01-18 01:03:02 INFO [logging.py:61]: Loss 'nar_loss' on epoch 2: 68.25257110595703 +01-18 01:03:02 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 2: 0.6516079306602478 +01-18 01:03:02 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 2: 0.14630988240242004 +01-18 01:03:02 INFO [logging.py:61]: ========= Epoch 3 out of 1000 ========= +01-18 01:03:02 INFO [logging.py:61]: Begin training... +01-18 01:17:44 INFO [logging.py:61]: Loss 'loss' on epoch 3: 80.09443664550781 +01-18 01:17:44 INFO [logging.py:61]: Loss 'ar_loss' on epoch 3: 2.8742563724517822 +01-18 01:17:44 INFO [logging.py:61]: Loss 'nar_loss' on epoch 3: 77.22017669677734 +01-18 01:17:44 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 3: 0.6769989132881165 +01-18 01:17:44 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 3: 0.1532195657491684 +01-18 01:17:44 INFO [logging.py:61]: ========= Epoch 4 out of 1000 ========= +01-18 01:17:44 INFO [logging.py:61]: Begin training... +01-18 01:32:28 INFO [logging.py:61]: Loss 'loss' on epoch 4: 64.45104217529297 +01-18 01:32:28 INFO [logging.py:61]: Loss 'ar_loss' on epoch 4: 2.756040096282959 +01-18 01:32:28 INFO [logging.py:61]: Loss 'nar_loss' on epoch 4: 61.69499969482422 +01-18 01:32:28 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 4: 0.6978757381439209 +01-18 01:32:28 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 4: 0.1739574521780014 +01-18 01:32:28 INFO [logging.py:61]: ========= Epoch 5 out of 1000 ========= +01-18 01:32:28 INFO [logging.py:61]: Begin training... +01-18 01:47:11 INFO [logging.py:61]: Loss 'loss' on epoch 5: 50.534027099609375 +01-18 01:47:11 INFO [logging.py:61]: Loss 'ar_loss' on epoch 5: 2.4202704429626465 +01-18 01:47:11 INFO [logging.py:61]: Loss 'nar_loss' on epoch 5: 48.1137580871582 +01-18 01:47:11 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 5: 0.7520415186882019 +01-18 01:47:11 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 5: 0.2114027887582779 +01-18 01:47:11 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0005 +01-18 01:47:12 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0005/pytorch_model.bin +01-18 01:47:13 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0005/optimizer.bin +01-18 01:47:13 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0005/scheduler.bin +01-18 01:47:13 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0005/sampler.bin +01-18 01:47:13 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0005/sampler_1.bin +01-18 01:47:13 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0005/random_states_0.pkl +01-18 01:47:13 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0005/custom_checkpoint_0.pkl +01-18 01:47:14 INFO [logging.py:61]: ========= Epoch 6 out of 1000 ========= +01-18 01:47:14 INFO [logging.py:61]: Begin training... +01-18 02:02:24 INFO [logging.py:61]: Loss 'loss' on epoch 6: 37.49298858642578 +01-18 02:02:24 INFO [logging.py:61]: Loss 'ar_loss' on epoch 6: 1.8649616241455078 +01-18 02:02:24 INFO [logging.py:61]: Loss 'nar_loss' on epoch 6: 35.628028869628906 +01-18 02:02:24 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 6: 0.8278111219406128 +01-18 02:02:24 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 6: 0.22959543764591217 +01-18 02:02:24 INFO [logging.py:61]: ========= Epoch 7 out of 1000 ========= +01-18 02:02:24 INFO [logging.py:61]: Begin training... +01-18 02:17:03 INFO [logging.py:61]: Loss 'loss' on epoch 7: 28.96417236328125 +01-18 02:17:03 INFO [logging.py:61]: Loss 'ar_loss' on epoch 7: 1.677205204963684 +01-18 02:17:03 INFO [logging.py:61]: Loss 'nar_loss' on epoch 7: 27.286962509155273 +01-18 02:17:03 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 7: 0.8506313562393188 +01-18 02:17:03 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 7: 0.2341448813676834 +01-18 02:17:03 INFO [logging.py:61]: ========= Epoch 8 out of 1000 ========= +01-18 02:17:03 INFO [logging.py:61]: Begin training... +01-18 02:31:48 INFO [logging.py:61]: Loss 'loss' on epoch 8: 28.844099044799805 +01-18 02:31:48 INFO [logging.py:61]: Loss 'ar_loss' on epoch 8: 1.6051831245422363 +01-18 02:31:48 INFO [logging.py:61]: Loss 'nar_loss' on epoch 8: 27.238914489746094 +01-18 02:31:48 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 8: 0.85962975025177 +01-18 02:31:48 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 8: 0.2343209981918335 +01-18 02:31:48 INFO [logging.py:61]: ========= Epoch 9 out of 1000 ========= +01-18 02:31:48 INFO [logging.py:61]: Begin training... +01-18 02:46:32 INFO [logging.py:61]: Loss 'loss' on epoch 9: 18.423540115356445 +01-18 02:46:32 INFO [logging.py:61]: Loss 'ar_loss' on epoch 9: 1.5607959032058716 +01-18 02:46:32 INFO [logging.py:61]: Loss 'nar_loss' on epoch 9: 16.862743377685547 +01-18 02:46:32 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 9: 0.8656294345855713 +01-18 02:46:32 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 9: 0.268953800201416 +01-18 02:46:32 INFO [logging.py:61]: ========= Epoch 10 out of 1000 ========= +01-18 02:46:32 INFO [logging.py:61]: Begin training... +01-18 03:01:15 INFO [logging.py:61]: Loss 'loss' on epoch 10: 17.509979248046875 +01-18 03:01:15 INFO [logging.py:61]: Loss 'ar_loss' on epoch 10: 1.516535997390747 +01-18 03:01:15 INFO [logging.py:61]: Loss 'nar_loss' on epoch 10: 15.993441581726074 +01-18 03:01:15 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 10: 0.8709884881973267 +01-18 03:01:15 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 10: 0.27383264899253845 +01-18 03:01:15 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0010 +01-18 03:01:16 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0010/pytorch_model.bin +01-18 03:01:18 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0010/optimizer.bin +01-18 03:01:18 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0010/scheduler.bin +01-18 03:01:18 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0010/sampler.bin +01-18 03:01:18 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0010/sampler_1.bin +01-18 03:01:18 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0010/random_states_0.pkl +01-18 03:01:18 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0010/custom_checkpoint_0.pkl +01-18 03:01:18 INFO [logging.py:61]: ========= Epoch 11 out of 1000 ========= +01-18 03:01:18 INFO [logging.py:61]: Begin training... +01-18 03:16:19 INFO [logging.py:61]: Loss 'loss' on epoch 11: 20.488168716430664 +01-18 03:16:19 INFO [logging.py:61]: Loss 'ar_loss' on epoch 11: 1.492824673652649 +01-18 03:16:19 INFO [logging.py:61]: Loss 'nar_loss' on epoch 11: 18.995346069335938 +01-18 03:16:19 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 11: 0.8740840554237366 +01-18 03:16:19 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 11: 0.2761929929256439 +01-18 03:16:19 INFO [logging.py:61]: ========= Epoch 12 out of 1000 ========= +01-18 03:16:19 INFO [logging.py:61]: Begin training... +01-18 03:31:06 INFO [logging.py:61]: Loss 'loss' on epoch 12: 12.799497604370117 +01-18 03:31:06 INFO [logging.py:61]: Loss 'ar_loss' on epoch 12: 1.4752053022384644 +01-18 03:31:06 INFO [logging.py:61]: Loss 'nar_loss' on epoch 12: 11.324292182922363 +01-18 03:31:06 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 12: 0.8769260048866272 +01-18 03:31:06 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 12: 0.29501354694366455 +01-18 03:31:06 INFO [logging.py:61]: ========= Epoch 13 out of 1000 ========= +01-18 03:31:06 INFO [logging.py:61]: Begin training... +01-18 03:45:51 INFO [logging.py:61]: Loss 'loss' on epoch 13: 9.103281021118164 +01-18 03:45:51 INFO [logging.py:61]: Loss 'ar_loss' on epoch 13: 1.456277847290039 +01-18 03:45:51 INFO [logging.py:61]: Loss 'nar_loss' on epoch 13: 7.647003650665283 +01-18 03:45:51 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 13: 0.8797268867492676 +01-18 03:45:51 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 13: 0.3158681094646454 +01-18 03:45:51 INFO [logging.py:61]: ========= Epoch 14 out of 1000 ========= +01-18 03:45:51 INFO [logging.py:61]: Begin training... +01-18 04:00:36 INFO [logging.py:61]: Loss 'loss' on epoch 14: 13.451669692993164 +01-18 04:00:36 INFO [logging.py:61]: Loss 'ar_loss' on epoch 14: 1.4416464567184448 +01-18 04:00:36 INFO [logging.py:61]: Loss 'nar_loss' on epoch 14: 12.01002311706543 +01-18 04:00:36 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 14: 0.8816895484924316 +01-18 04:00:36 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 14: 0.2968295216560364 +01-18 04:00:36 INFO [logging.py:61]: ========= Epoch 15 out of 1000 ========= +01-18 04:00:36 INFO [logging.py:61]: Begin training... +01-18 04:15:15 INFO [logging.py:61]: Loss 'loss' on epoch 15: 10.637478828430176 +01-18 04:15:15 INFO [logging.py:61]: Loss 'ar_loss' on epoch 15: 1.4250620603561401 +01-18 04:15:15 INFO [logging.py:61]: Loss 'nar_loss' on epoch 15: 9.21241569519043 +01-18 04:15:15 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 15: 0.8841312527656555 +01-18 04:15:15 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 15: 0.3035244345664978 +01-18 04:15:15 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0015 +01-18 04:15:16 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0015/pytorch_model.bin +01-18 04:15:17 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0015/optimizer.bin +01-18 04:15:17 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0015/scheduler.bin +01-18 04:15:17 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0015/sampler.bin +01-18 04:15:17 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0015/sampler_1.bin +01-18 04:15:17 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0015/random_states_0.pkl +01-18 04:15:17 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0015/custom_checkpoint_0.pkl +01-18 04:15:17 INFO [logging.py:61]: ========= Epoch 16 out of 1000 ========= +01-18 04:15:17 INFO [logging.py:61]: Begin training... +01-18 04:30:27 INFO [logging.py:61]: Loss 'loss' on epoch 16: 7.80450439453125 +01-18 04:30:27 INFO [logging.py:61]: Loss 'ar_loss' on epoch 16: 1.4189400672912598 +01-18 04:30:27 INFO [logging.py:61]: Loss 'nar_loss' on epoch 16: 6.385564804077148 +01-18 04:30:27 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 16: 0.8853126764297485 +01-18 04:30:27 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 16: 0.3228910267353058 +01-18 04:30:27 INFO [logging.py:61]: ========= Epoch 17 out of 1000 ========= +01-18 04:30:27 INFO [logging.py:61]: Begin training... +01-18 04:45:12 INFO [logging.py:61]: Loss 'loss' on epoch 17: 7.98758602142334 +01-18 04:45:12 INFO [logging.py:61]: Loss 'ar_loss' on epoch 17: 1.402923583984375 +01-18 04:45:12 INFO [logging.py:61]: Loss 'nar_loss' on epoch 17: 6.584662914276123 +01-18 04:45:12 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 17: 0.8873386383056641 +01-18 04:45:12 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 17: 0.3134208917617798 +01-18 04:45:12 INFO [logging.py:61]: ========= Epoch 18 out of 1000 ========= +01-18 04:45:12 INFO [logging.py:61]: Begin training... +01-18 04:59:55 INFO [logging.py:61]: Loss 'loss' on epoch 18: 8.954412460327148 +01-18 04:59:55 INFO [logging.py:61]: Loss 'ar_loss' on epoch 18: 1.3915390968322754 +01-18 04:59:55 INFO [logging.py:61]: Loss 'nar_loss' on epoch 18: 7.562872886657715 +01-18 04:59:55 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 18: 0.8887759447097778 +01-18 04:59:55 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 18: 0.3137945532798767 +01-18 04:59:55 INFO [logging.py:61]: ========= Epoch 19 out of 1000 ========= +01-18 04:59:55 INFO [logging.py:61]: Begin training... +01-18 05:14:30 INFO [logging.py:61]: Loss 'loss' on epoch 19: 6.779003620147705 +01-18 05:14:30 INFO [logging.py:61]: Loss 'ar_loss' on epoch 19: 1.3836216926574707 +01-18 05:14:30 INFO [logging.py:61]: Loss 'nar_loss' on epoch 19: 5.395382404327393 +01-18 05:14:30 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 19: 0.8901618719100952 +01-18 05:14:30 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 19: 0.3347846567630768 +01-18 05:14:30 INFO [logging.py:61]: ========= Epoch 20 out of 1000 ========= +01-18 05:14:30 INFO [logging.py:61]: Begin training... +01-18 05:29:19 INFO [logging.py:61]: Loss 'loss' on epoch 20: 8.779007911682129 +01-18 05:29:19 INFO [logging.py:61]: Loss 'ar_loss' on epoch 20: 1.3702212572097778 +01-18 05:29:19 INFO [logging.py:61]: Loss 'nar_loss' on epoch 20: 7.408785343170166 +01-18 05:29:19 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 20: 0.8919013738632202 +01-18 05:29:19 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 20: 0.3212190866470337 +01-18 05:29:19 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0020 +01-18 05:29:19 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0020/pytorch_model.bin +01-18 05:29:21 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0020/optimizer.bin +01-18 05:29:21 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0020/scheduler.bin +01-18 05:29:21 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0020/sampler.bin +01-18 05:29:21 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0020/sampler_1.bin +01-18 05:29:21 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0020/random_states_0.pkl +01-18 05:29:21 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0020/custom_checkpoint_0.pkl +01-18 05:29:21 INFO [logging.py:61]: ========= Epoch 21 out of 1000 ========= +01-18 05:29:21 INFO [logging.py:61]: Begin training... +01-18 05:44:34 INFO [logging.py:61]: Loss 'loss' on epoch 21: 6.515992164611816 +01-18 05:44:34 INFO [logging.py:61]: Loss 'ar_loss' on epoch 21: 1.363883376121521 +01-18 05:44:34 INFO [logging.py:61]: Loss 'nar_loss' on epoch 21: 5.152108669281006 +01-18 05:44:34 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 21: 0.8928155899047852 +01-18 05:44:34 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 21: 0.3437054753303528 +01-18 05:44:34 INFO [logging.py:61]: ========= Epoch 22 out of 1000 ========= +01-18 05:44:34 INFO [logging.py:61]: Begin training... +01-18 05:59:19 INFO [logging.py:61]: Loss 'loss' on epoch 22: 6.411158084869385 +01-18 05:59:19 INFO [logging.py:61]: Loss 'ar_loss' on epoch 22: 1.3568339347839355 +01-18 05:59:19 INFO [logging.py:61]: Loss 'nar_loss' on epoch 22: 5.054324150085449 +01-18 05:59:19 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 22: 0.8938457369804382 +01-18 05:59:19 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 22: 0.3443514406681061 +01-18 05:59:19 INFO [logging.py:61]: ========= Epoch 23 out of 1000 ========= +01-18 05:59:19 INFO [logging.py:61]: Begin training... +01-18 06:14:01 INFO [logging.py:61]: Loss 'loss' on epoch 23: 6.536004066467285 +01-18 06:14:01 INFO [logging.py:61]: Loss 'ar_loss' on epoch 23: 1.3483623266220093 +01-18 06:14:01 INFO [logging.py:61]: Loss 'nar_loss' on epoch 23: 5.187641620635986 +01-18 06:14:01 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 23: 0.8951892256736755 +01-18 06:14:01 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 23: 0.3462466299533844 +01-18 06:14:01 INFO [logging.py:61]: ========= Epoch 24 out of 1000 ========= +01-18 06:14:01 INFO [logging.py:61]: Begin training... +01-18 06:28:46 INFO [logging.py:61]: Loss 'loss' on epoch 24: 6.32097053527832 +01-18 06:28:46 INFO [logging.py:61]: Loss 'ar_loss' on epoch 24: 1.3559930324554443 +01-18 06:28:46 INFO [logging.py:61]: Loss 'nar_loss' on epoch 24: 4.9649786949157715 +01-18 06:28:46 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 24: 0.8945856094360352 +01-18 06:28:46 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 24: 0.3438716530799866 +01-18 06:28:46 INFO [logging.py:61]: ========= Epoch 25 out of 1000 ========= +01-18 06:28:46 INFO [logging.py:61]: Begin training... +01-18 06:43:30 INFO [logging.py:61]: Loss 'loss' on epoch 25: 6.4575934410095215 +01-18 06:43:30 INFO [logging.py:61]: Loss 'ar_loss' on epoch 25: 1.3440008163452148 +01-18 06:43:30 INFO [logging.py:61]: Loss 'nar_loss' on epoch 25: 5.113592147827148 +01-18 06:43:30 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 25: 0.8960744142532349 +01-18 06:43:30 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 25: 0.3424939811229706 +01-18 06:43:30 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0025 +01-18 06:43:31 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0025/pytorch_model.bin +01-18 06:43:32 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0025/optimizer.bin +01-18 06:43:32 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0025/scheduler.bin +01-18 06:43:32 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0025/sampler.bin +01-18 06:43:32 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0025/sampler_1.bin +01-18 06:43:32 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0025/random_states_0.pkl +01-18 06:43:32 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0025/custom_checkpoint_0.pkl +01-18 06:43:32 INFO [logging.py:61]: ========= Epoch 26 out of 1000 ========= +01-18 06:43:32 INFO [logging.py:61]: Begin training... +01-18 06:59:08 INFO [logging.py:61]: Loss 'loss' on epoch 26: 5.9814558029174805 +01-18 06:59:08 INFO [logging.py:61]: Loss 'ar_loss' on epoch 26: 1.3335331678390503 +01-18 06:59:08 INFO [logging.py:61]: Loss 'nar_loss' on epoch 26: 4.647922992706299 +01-18 06:59:08 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 26: 0.8972698450088501 +01-18 06:59:08 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 26: 0.3597102165222168 +01-18 06:59:08 INFO [logging.py:61]: ========= Epoch 27 out of 1000 ========= +01-18 06:59:08 INFO [logging.py:61]: Begin training... +01-18 07:13:48 INFO [logging.py:61]: Loss 'loss' on epoch 27: 5.9211649894714355 +01-18 07:13:48 INFO [logging.py:61]: Loss 'ar_loss' on epoch 27: 1.3315753936767578 +01-18 07:13:48 INFO [logging.py:61]: Loss 'nar_loss' on epoch 27: 4.589588642120361 +01-18 07:13:48 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 27: 0.8978341817855835 +01-18 07:13:48 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 27: 0.3617432415485382 +01-18 07:13:48 INFO [logging.py:61]: ========= Epoch 28 out of 1000 ========= +01-18 07:13:48 INFO [logging.py:61]: Begin training... +01-18 07:28:33 INFO [logging.py:61]: Loss 'loss' on epoch 28: 5.8742170333862305 +01-18 07:28:33 INFO [logging.py:61]: Loss 'ar_loss' on epoch 28: 1.3296784162521362 +01-18 07:28:33 INFO [logging.py:61]: Loss 'nar_loss' on epoch 28: 4.544538497924805 +01-18 07:28:33 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 28: 0.8983495831489563 +01-18 07:28:33 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 28: 0.366417795419693 +01-18 07:28:33 INFO [logging.py:61]: ========= Epoch 29 out of 1000 ========= +01-18 07:28:33 INFO [logging.py:61]: Begin training... +01-18 07:43:17 INFO [logging.py:61]: Loss 'loss' on epoch 29: 5.860039710998535 +01-18 07:43:17 INFO [logging.py:61]: Loss 'ar_loss' on epoch 29: 1.3166179656982422 +01-18 07:43:17 INFO [logging.py:61]: Loss 'nar_loss' on epoch 29: 4.543421268463135 +01-18 07:43:17 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 29: 0.8997495174407959 +01-18 07:43:17 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 29: 0.3649654686450958 +01-18 07:43:17 INFO [logging.py:61]: ========= Epoch 30 out of 1000 ========= +01-18 07:43:17 INFO [logging.py:61]: Begin training... +01-18 07:58:02 INFO [logging.py:61]: Loss 'loss' on epoch 30: 5.796247959136963 +01-18 07:58:02 INFO [logging.py:61]: Loss 'ar_loss' on epoch 30: 1.3036959171295166 +01-18 07:58:02 INFO [logging.py:61]: Loss 'nar_loss' on epoch 30: 4.492552280426025 +01-18 07:58:02 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 30: 0.90115886926651 +01-18 07:58:02 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 30: 0.37101566791534424 +01-18 07:58:02 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0030 +01-18 07:58:03 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0030/pytorch_model.bin +01-18 07:58:04 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0030/optimizer.bin +01-18 07:58:04 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0030/scheduler.bin +01-18 07:58:04 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0030/sampler.bin +01-18 07:58:04 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0030/sampler_1.bin +01-18 07:58:04 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0030/random_states_0.pkl +01-18 07:58:04 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0030/custom_checkpoint_0.pkl +01-18 07:58:04 INFO [logging.py:61]: ========= Epoch 31 out of 1000 ========= +01-18 07:58:04 INFO [logging.py:61]: Begin training... +01-18 08:12:58 INFO [logging.py:61]: Loss 'loss' on epoch 31: 5.9744038581848145 +01-18 08:12:58 INFO [logging.py:61]: Loss 'ar_loss' on epoch 31: 1.311854362487793 +01-18 08:12:58 INFO [logging.py:61]: Loss 'nar_loss' on epoch 31: 4.662549018859863 +01-18 08:12:58 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 31: 0.9008342623710632 +01-18 08:12:58 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 31: 0.366784006357193 +01-18 08:12:58 INFO [logging.py:61]: ========= Epoch 32 out of 1000 ========= +01-18 08:12:58 INFO [logging.py:61]: Begin training... +01-18 08:27:46 INFO [logging.py:61]: Loss 'loss' on epoch 32: 5.735446453094482 +01-18 08:27:46 INFO [logging.py:61]: Loss 'ar_loss' on epoch 32: 1.3046653270721436 +01-18 08:27:46 INFO [logging.py:61]: Loss 'nar_loss' on epoch 32: 4.43078088760376 +01-18 08:27:46 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 32: 0.9014154076576233 +01-18 08:27:46 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 32: 0.37591466307640076 +01-18 08:27:46 INFO [logging.py:61]: ========= Epoch 33 out of 1000 ========= +01-18 08:27:46 INFO [logging.py:61]: Begin training... +01-18 08:42:30 INFO [logging.py:61]: Loss 'loss' on epoch 33: 5.872527122497559 +01-18 08:42:30 INFO [logging.py:61]: Loss 'ar_loss' on epoch 33: 1.2990329265594482 +01-18 08:42:30 INFO [logging.py:61]: Loss 'nar_loss' on epoch 33: 4.5734944343566895 +01-18 08:42:30 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 33: 0.9022935032844543 +01-18 08:42:30 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 33: 0.37081286311149597 +01-18 08:42:30 INFO [logging.py:61]: ========= Epoch 34 out of 1000 ========= +01-18 08:42:30 INFO [logging.py:61]: Begin training... +01-18 08:57:15 INFO [logging.py:61]: Loss 'loss' on epoch 34: 5.6954474449157715 +01-18 08:57:15 INFO [logging.py:61]: Loss 'ar_loss' on epoch 34: 1.3018174171447754 +01-18 08:57:15 INFO [logging.py:61]: Loss 'nar_loss' on epoch 34: 4.393630504608154 +01-18 08:57:15 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 34: 0.9023982882499695 +01-18 08:57:15 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 34: 0.38097846508026123 +01-18 08:57:15 INFO [logging.py:61]: ========= Epoch 35 out of 1000 ========= +01-18 08:57:15 INFO [logging.py:61]: Begin training... +01-18 09:11:52 INFO [logging.py:61]: Loss 'loss' on epoch 35: 5.682955741882324 +01-18 09:11:52 INFO [logging.py:61]: Loss 'ar_loss' on epoch 35: 1.2978262901306152 +01-18 09:11:52 INFO [logging.py:61]: Loss 'nar_loss' on epoch 35: 4.385129451751709 +01-18 09:11:52 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 35: 0.9027104377746582 +01-18 09:11:52 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 35: 0.3820667564868927 +01-18 09:11:52 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0035 +01-18 09:11:53 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0035/pytorch_model.bin +01-18 09:11:54 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0035/optimizer.bin +01-18 09:11:54 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0035/scheduler.bin +01-18 09:11:54 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0035/sampler.bin +01-18 09:11:54 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0035/sampler_1.bin +01-18 09:11:54 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0035/random_states_0.pkl +01-18 09:11:54 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0035/custom_checkpoint_0.pkl +01-18 09:11:54 INFO [logging.py:61]: ========= Epoch 36 out of 1000 ========= +01-18 09:11:54 INFO [logging.py:61]: Begin training... +01-18 09:27:23 INFO [logging.py:61]: Loss 'loss' on epoch 36: 5.689979553222656 +01-18 09:27:23 INFO [logging.py:61]: Loss 'ar_loss' on epoch 36: 1.2983156442642212 +01-18 09:27:23 INFO [logging.py:61]: Loss 'nar_loss' on epoch 36: 4.391663551330566 +01-18 09:27:23 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 36: 0.90284264087677 +01-18 09:27:23 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 36: 0.3806702196598053 +01-18 09:27:23 INFO [logging.py:61]: ========= Epoch 37 out of 1000 ========= +01-18 09:27:23 INFO [logging.py:61]: Begin training... +01-18 09:42:07 INFO [logging.py:61]: Loss 'loss' on epoch 37: 5.661297798156738 +01-18 09:42:07 INFO [logging.py:61]: Loss 'ar_loss' on epoch 37: 1.2922143936157227 +01-18 09:42:07 INFO [logging.py:61]: Loss 'nar_loss' on epoch 37: 4.369083404541016 +01-18 09:42:07 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 37: 0.9038951396942139 +01-18 09:42:07 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 37: 0.3834979236125946 +01-18 09:42:07 INFO [logging.py:61]: ========= Epoch 38 out of 1000 ========= +01-18 09:42:07 INFO [logging.py:61]: Begin training... +01-18 09:57:28 INFO [logging.py:61]: Loss 'loss' on epoch 38: 5.638166427612305 +01-18 09:57:28 INFO [logging.py:61]: Loss 'ar_loss' on epoch 38: 1.2806264162063599 +01-18 09:57:28 INFO [logging.py:61]: Loss 'nar_loss' on epoch 38: 4.357540130615234 +01-18 09:57:28 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 38: 0.9050056338310242 +01-18 09:57:28 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 38: 0.38519880175590515 +01-18 09:57:28 INFO [logging.py:61]: ========= Epoch 39 out of 1000 ========= +01-18 09:57:28 INFO [logging.py:61]: Begin training... +01-18 10:12:01 INFO [logging.py:61]: Loss 'loss' on epoch 39: 5.627877712249756 +01-18 10:12:01 INFO [logging.py:61]: Loss 'ar_loss' on epoch 39: 1.2748196125030518 +01-18 10:12:01 INFO [logging.py:61]: Loss 'nar_loss' on epoch 39: 4.353058338165283 +01-18 10:12:01 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 39: 0.9056810140609741 +01-18 10:12:01 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 39: 0.3871024549007416 +01-18 10:12:01 INFO [logging.py:61]: ========= Epoch 40 out of 1000 ========= +01-18 10:12:01 INFO [logging.py:61]: Begin training... +01-18 10:26:42 INFO [logging.py:61]: Loss 'loss' on epoch 40: 5.619797229766846 +01-18 10:26:42 INFO [logging.py:61]: Loss 'ar_loss' on epoch 40: 1.2779885530471802 +01-18 10:26:42 INFO [logging.py:61]: Loss 'nar_loss' on epoch 40: 4.341809272766113 +01-18 10:26:42 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 40: 0.9054821729660034 +01-18 10:26:42 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 40: 0.38915959000587463 +01-18 10:26:42 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0040 +01-18 10:26:43 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0040/pytorch_model.bin +01-18 10:26:45 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0040/optimizer.bin +01-18 10:26:45 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0040/scheduler.bin +01-18 10:26:45 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0040/sampler.bin +01-18 10:26:45 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0040/sampler_1.bin +01-18 10:26:45 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0040/random_states_0.pkl +01-18 10:26:45 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0040/custom_checkpoint_0.pkl +01-18 10:26:45 INFO [logging.py:61]: ========= Epoch 41 out of 1000 ========= +01-18 10:26:45 INFO [logging.py:61]: Begin training... +01-18 10:42:20 INFO [logging.py:61]: Loss 'loss' on epoch 41: 5.582706928253174 +01-18 10:42:20 INFO [logging.py:61]: Loss 'ar_loss' on epoch 41: 1.2725111246109009 +01-18 10:42:20 INFO [logging.py:61]: Loss 'nar_loss' on epoch 41: 4.3101959228515625 +01-18 10:42:20 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 41: 0.9060384035110474 +01-18 10:42:20 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 41: 0.39349165558815 +01-18 10:42:20 INFO [logging.py:61]: ========= Epoch 42 out of 1000 ========= +01-18 10:42:20 INFO [logging.py:61]: Begin training... +01-18 10:57:24 INFO [logging.py:61]: Loss 'loss' on epoch 42: 5.580837726593018 +01-18 10:57:24 INFO [logging.py:61]: Loss 'ar_loss' on epoch 42: 1.2710316181182861 +01-18 10:57:24 INFO [logging.py:61]: Loss 'nar_loss' on epoch 42: 4.309806823730469 +01-18 10:57:24 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 42: 0.9061719179153442 +01-18 10:57:24 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 42: 0.39457419514656067 +01-18 10:57:24 INFO [logging.py:61]: ========= Epoch 43 out of 1000 ========= +01-18 10:57:24 INFO [logging.py:61]: Begin training... +01-18 11:13:05 INFO [logging.py:61]: Loss 'loss' on epoch 43: 5.569582939147949 +01-18 11:13:05 INFO [logging.py:61]: Loss 'ar_loss' on epoch 43: 1.2762430906295776 +01-18 11:13:05 INFO [logging.py:61]: Loss 'nar_loss' on epoch 43: 4.293339252471924 +01-18 11:13:05 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 43: 0.9061148166656494 +01-18 11:13:05 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 43: 0.39825910329818726 +01-18 11:13:05 INFO [logging.py:61]: ========= Epoch 44 out of 1000 ========= +01-18 11:13:05 INFO [logging.py:61]: Begin training... +01-18 11:27:38 INFO [logging.py:61]: Loss 'loss' on epoch 44: 5.717195987701416 +01-18 11:27:38 INFO [logging.py:61]: Loss 'ar_loss' on epoch 44: 1.2707628011703491 +01-18 11:27:38 INFO [logging.py:61]: Loss 'nar_loss' on epoch 44: 4.446432590484619 +01-18 11:27:38 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 44: 0.9066540598869324 +01-18 11:27:38 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 44: 0.3859277665615082 +01-18 11:27:38 INFO [logging.py:61]: ========= Epoch 45 out of 1000 ========= +01-18 11:27:38 INFO [logging.py:61]: Begin training... +01-18 11:42:11 INFO [logging.py:61]: Loss 'loss' on epoch 45: 5.571887969970703 +01-18 11:42:11 INFO [logging.py:61]: Loss 'ar_loss' on epoch 45: 1.2563157081604004 +01-18 11:42:11 INFO [logging.py:61]: Loss 'nar_loss' on epoch 45: 4.315572261810303 +01-18 11:42:11 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 45: 0.9081159830093384 +01-18 11:42:11 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 45: 0.39590778946876526 +01-18 11:42:11 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0045 +01-18 11:42:12 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0045/pytorch_model.bin +01-18 11:42:13 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0045/optimizer.bin +01-18 11:42:13 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0045/scheduler.bin +01-18 11:42:13 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0045/sampler.bin +01-18 11:42:13 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0045/sampler_1.bin +01-18 11:42:13 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0045/random_states_0.pkl +01-18 11:42:13 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0045/custom_checkpoint_0.pkl +01-18 11:42:13 INFO [logging.py:61]: ========= Epoch 46 out of 1000 ========= +01-18 11:42:13 INFO [logging.py:61]: Begin training... +01-18 11:56:47 INFO [logging.py:61]: Loss 'loss' on epoch 46: 5.5488386154174805 +01-18 11:56:47 INFO [logging.py:61]: Loss 'ar_loss' on epoch 46: 1.270042061805725 +01-18 11:56:47 INFO [logging.py:61]: Loss 'nar_loss' on epoch 46: 4.278796195983887 +01-18 11:56:47 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 46: 0.9069695472717285 +01-18 11:56:47 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 46: 0.40045443177223206 +01-18 11:56:47 INFO [logging.py:61]: ========= Epoch 47 out of 1000 ========= +01-18 11:56:47 INFO [logging.py:61]: Begin training... +01-18 12:11:19 INFO [logging.py:61]: Loss 'loss' on epoch 47: 5.518341541290283 +01-18 12:11:19 INFO [logging.py:61]: Loss 'ar_loss' on epoch 47: 1.2568832635879517 +01-18 12:11:19 INFO [logging.py:61]: Loss 'nar_loss' on epoch 47: 4.261458396911621 +01-18 12:11:19 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 47: 0.9085185527801514 +01-18 12:11:19 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 47: 0.4032013416290283 +01-18 12:11:19 INFO [logging.py:61]: ========= Epoch 48 out of 1000 ========= +01-18 12:11:19 INFO [logging.py:61]: Begin training... +01-18 12:25:53 INFO [logging.py:61]: Loss 'loss' on epoch 48: 5.5060343742370605 +01-18 12:25:54 INFO [logging.py:61]: Loss 'ar_loss' on epoch 48: 1.258938193321228 +01-18 12:25:54 INFO [logging.py:61]: Loss 'nar_loss' on epoch 48: 4.247096538543701 +01-18 12:25:54 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 48: 0.9083271622657776 +01-18 12:25:54 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 48: 0.40573224425315857 +01-18 12:25:54 INFO [logging.py:61]: ========= Epoch 49 out of 1000 ========= +01-18 12:25:54 INFO [logging.py:61]: Begin training... +01-18 12:40:27 INFO [logging.py:61]: Loss 'loss' on epoch 49: 5.473796844482422 +01-18 12:40:27 INFO [logging.py:61]: Loss 'ar_loss' on epoch 49: 1.2464219331741333 +01-18 12:40:27 INFO [logging.py:61]: Loss 'nar_loss' on epoch 49: 4.22737455368042 +01-18 12:40:27 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 49: 0.9096682071685791 +01-18 12:40:27 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 49: 0.40877988934516907 +01-18 12:40:27 INFO [logging.py:61]: ========= Epoch 50 out of 1000 ========= +01-18 12:40:27 INFO [logging.py:61]: Begin training... +01-18 12:54:59 INFO [logging.py:61]: Loss 'loss' on epoch 50: 5.809966564178467 +01-18 12:54:59 INFO [logging.py:61]: Loss 'ar_loss' on epoch 50: 1.2504818439483643 +01-18 12:54:59 INFO [logging.py:61]: Loss 'nar_loss' on epoch 50: 4.559484958648682 +01-18 12:54:59 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 50: 0.9094463586807251 +01-18 12:54:59 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 50: 0.38511520624160767 +01-18 12:54:59 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0050 +01-18 12:55:00 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0050/pytorch_model.bin +01-18 12:55:01 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0050/optimizer.bin +01-18 12:55:01 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0050/scheduler.bin +01-18 12:55:01 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0050/sampler.bin +01-18 12:55:01 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0050/sampler_1.bin +01-18 12:55:01 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0050/random_states_0.pkl +01-18 12:55:01 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-2_AR-NAR/checkpoints/epoch_0050/custom_checkpoint_0.pkl +01-18 12:55:01 INFO [logging.py:61]: ========= Epoch 51 out of 1000 ========= +01-18 12:55:01 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_13_05.log b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_13_05.log new file mode 100644 index 0000000000000000000000000000000000000000..669f36e6c2821273c155b6b1239608f705a1e165 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_13_05.log @@ -0,0 +1,94 @@ +01-18 13:13:05 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR. +01-18 13:13:05 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR. +01-18 13:13:05 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR. +01-18 13:13:05 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR. +01-18 13:13:58 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR/config__2024_01_18--13_13_46.toml. +01-18 13:13:58 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 503.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-18 13:13:58 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─TokenEmbedding: 2-2 -- +│ │ └─Dropout: 3-4 -- +│ │ └─Embedding: 3-5 524,800 +│ └─Identity: 2-3 -- +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-6 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-7 37,828,608 +│ │ └─LayerNorm: 3-8 1,024 +│ └─Linear: 2-6 524,800 +│ └─MulticlassAccuracy: 2-7 -- +│ └─TokenEmbedding: 2-8 -- +│ │ └─Dropout: 3-9 -- +│ │ └─Embedding: 3-10 524,288 +│ └─ModuleList: 2-9 -- +│ │ └─TokenEmbedding: 3-11 524,800 +│ │ └─TokenEmbedding: 3-12 524,288 +│ │ └─TokenEmbedding: 3-13 524,288 +│ │ └─TokenEmbedding: 3-14 524,288 +│ │ └─TokenEmbedding: 3-15 524,288 +│ │ └─TokenEmbedding: 3-16 524,288 +│ │ └─TokenEmbedding: 3-17 524,288 +│ │ └─TokenEmbedding: 3-18 524,288 +│ └─Identity: 2-10 -- +│ └─SinePositionalEmbedding: 2-11 1 +│ │ └─Dropout: 3-19 -- +│ └─TransformerEncoder: 2-12 -- +│ │ └─ModuleList: 3-20 50,436,096 +│ │ └─AdaptiveLayerNorm: 3-21 526,336 +│ └─ModuleList: 2-13 -- +│ │ └─Linear: 3-22 524,288 +│ │ └─Linear: 3-23 524,288 +│ │ └─Linear: 3-24 524,288 +│ │ └─Linear: 3-25 524,288 +│ │ └─Linear: 3-26 524,288 +│ │ └─Linear: 3-27 524,288 +│ │ └─Linear: 3-28 524,288 +│ └─ModuleList: 2-14 -- +│ │ └─TokenEmbedding: 3-29 512 +│ │ └─TokenEmbedding: 3-30 512 +│ │ └─TokenEmbedding: 3-31 512 +│ │ └─TokenEmbedding: 3-32 512 +│ │ └─TokenEmbedding: 3-33 512 +│ │ └─TokenEmbedding: 3-34 512 +│ │ └─TokenEmbedding: 3-35 512 +│ └─MulticlassAccuracy: 2-15 -- +=============================================================================================== +Total params: 113,086,180 +Trainable params: 98,234,369 +Non-trainable params: 14,851,811 +=============================================================================================== +01-18 13:13:58 INFO [logging.py:61]: Training control variables: +01-18 13:13:58 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-18 13:13:58 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-18 13:13:58 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-18 13:13:58 INFO [logging.py:61]: `max_steps`: 500000 +01-18 13:13:58 INFO [logging.py:61]: `max_epochs`: 1000 +01-18 13:13:58 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-18 13:13:58 INFO [logging.py:61]: Loading states from /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR/checkpoints/epoch_0050 +01-18 13:13:58 INFO [logging.py:61]: All model weights loaded successfully +01-18 13:13:59 INFO [logging.py:61]: All optimizer states loaded successfully +01-18 13:13:59 INFO [logging.py:61]: All scheduler states loaded successfully +01-18 13:13:59 INFO [logging.py:61]: All dataloader sampler states loaded successfully +01-18 13:13:59 INFO [logging.py:61]: All random states loaded successfully +01-18 13:14:00 INFO [logging.py:61]: Loading in 1 custom states +01-18 13:14:00 INFO [logging.py:61]: Loading the state of TrainerState from /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR/checkpoints/epoch_0050/custom_checkpoint_0.pkl +01-18 13:14:00 INFO [logging.py:61]: Checkpoint on epoch 50 is loaded. +01-18 13:14:00 INFO [logging.py:61]: ========= Epoch 51 out of 1000 ========= +01-18 13:14:00 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_14_39.log b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_14_39.log new file mode 100644 index 0000000000000000000000000000000000000000..0adf7be9766ce22a343ffba27915256de1990db0 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_14_39.log @@ -0,0 +1,121 @@ +01-18 13:14:39 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR. +01-18 13:14:39 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR. +01-18 13:14:39 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR. +01-18 13:14:45 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR/config__2024_01_18--13_14_43.toml. +01-18 13:14:45 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 503.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-18 13:14:45 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─TokenEmbedding: 2-2 -- +│ │ └─Dropout: 3-4 -- +│ │ └─Embedding: 3-5 524,800 +│ └─Identity: 2-3 -- +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-6 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-7 37,828,608 +│ │ └─LayerNorm: 3-8 1,024 +│ └─Linear: 2-6 524,800 +│ └─MulticlassAccuracy: 2-7 -- +│ └─TokenEmbedding: 2-8 -- +│ │ └─Dropout: 3-9 -- +│ │ └─Embedding: 3-10 524,288 +│ └─ModuleList: 2-9 -- +│ │ └─TokenEmbedding: 3-11 524,800 +│ │ └─TokenEmbedding: 3-12 524,288 +│ │ └─TokenEmbedding: 3-13 524,288 +│ │ └─TokenEmbedding: 3-14 524,288 +│ │ └─TokenEmbedding: 3-15 524,288 +│ │ └─TokenEmbedding: 3-16 524,288 +│ │ └─TokenEmbedding: 3-17 524,288 +│ │ └─TokenEmbedding: 3-18 524,288 +│ └─Identity: 2-10 -- +│ └─SinePositionalEmbedding: 2-11 1 +│ │ └─Dropout: 3-19 -- +│ └─TransformerEncoder: 2-12 -- +│ │ └─ModuleList: 3-20 50,436,096 +│ │ └─AdaptiveLayerNorm: 3-21 526,336 +│ └─ModuleList: 2-13 -- +│ │ └─Linear: 3-22 524,288 +│ │ └─Linear: 3-23 524,288 +│ │ └─Linear: 3-24 524,288 +│ │ └─Linear: 3-25 524,288 +│ │ └─Linear: 3-26 524,288 +│ │ └─Linear: 3-27 524,288 +│ │ └─Linear: 3-28 524,288 +│ └─ModuleList: 2-14 -- +│ │ └─TokenEmbedding: 3-29 512 +│ │ └─TokenEmbedding: 3-30 512 +│ │ └─TokenEmbedding: 3-31 512 +│ │ └─TokenEmbedding: 3-32 512 +│ │ └─TokenEmbedding: 3-33 512 +│ │ └─TokenEmbedding: 3-34 512 +│ │ └─TokenEmbedding: 3-35 512 +│ └─MulticlassAccuracy: 2-15 -- +=============================================================================================== +Total params: 113,086,180 +Trainable params: 98,234,369 +Non-trainable params: 14,851,811 +=============================================================================================== +01-18 13:14:45 INFO [logging.py:61]: Training control variables: +01-18 13:14:45 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-18 13:14:45 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-18 13:14:45 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-18 13:14:45 INFO [logging.py:61]: `max_steps`: 500000 +01-18 13:14:45 INFO [logging.py:61]: `max_epochs`: 1000 +01-18 13:14:45 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-18 13:14:45 INFO [logging.py:61]: Loading states from /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR/checkpoints/epoch_0050 +01-18 13:14:45 INFO [logging.py:61]: All model weights loaded successfully +01-18 13:14:46 INFO [logging.py:61]: All optimizer states loaded successfully +01-18 13:14:46 INFO [logging.py:61]: All scheduler states loaded successfully +01-18 13:14:46 INFO [logging.py:61]: All dataloader sampler states loaded successfully +01-18 13:14:46 INFO [logging.py:61]: All random states loaded successfully +01-18 13:14:46 INFO [logging.py:61]: Loading in 1 custom states +01-18 13:14:46 INFO [logging.py:61]: Loading the state of TrainerState from /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR/checkpoints/epoch_0050/custom_checkpoint_0.pkl +01-18 13:14:46 INFO [logging.py:61]: Checkpoint on epoch 50 is loaded. +01-18 13:14:46 INFO [logging.py:61]: ========= Epoch 51 out of 1000 ========= +01-18 13:14:46 INFO [logging.py:61]: Begin training... +01-18 13:29:20 INFO [logging.py:61]: Loss 'loss' on epoch 51: 5.497116565704346 +01-18 13:29:20 INFO [logging.py:61]: Loss 'ar_loss' on epoch 51: 1.253153920173645 +01-18 13:29:20 INFO [logging.py:61]: Loss 'nar_loss' on epoch 51: 4.243962287902832 +01-18 13:29:20 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 51: 0.9091926217079163 +01-18 13:29:20 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 51: 0.40614935755729675 +01-18 13:29:20 INFO [logging.py:61]: ========= Epoch 52 out of 1000 ========= +01-18 13:29:20 INFO [logging.py:61]: Begin training... +01-18 13:43:53 INFO [logging.py:61]: Loss 'loss' on epoch 52: 5.452788829803467 +01-18 13:43:53 INFO [logging.py:61]: Loss 'ar_loss' on epoch 52: 1.245349645614624 +01-18 13:43:53 INFO [logging.py:61]: Loss 'nar_loss' on epoch 52: 4.207438945770264 +01-18 13:43:53 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 52: 0.9099061489105225 +01-18 13:43:53 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 52: 0.41280823945999146 +01-18 13:43:53 INFO [logging.py:61]: ========= Epoch 53 out of 1000 ========= +01-18 13:43:53 INFO [logging.py:61]: Begin training... +01-18 13:58:30 INFO [logging.py:61]: Loss 'loss' on epoch 53: 5.5037336349487305 +01-18 13:58:30 INFO [logging.py:61]: Loss 'ar_loss' on epoch 53: 1.2453527450561523 +01-18 13:58:30 INFO [logging.py:61]: Loss 'nar_loss' on epoch 53: 4.25838041305542 +01-18 13:58:30 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 53: 0.9102101922035217 +01-18 13:58:30 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 53: 0.40620309114456177 +01-18 13:58:30 INFO [logging.py:61]: ========= Epoch 54 out of 1000 ========= +01-18 13:58:30 INFO [logging.py:61]: Begin training... +01-18 14:13:06 INFO [logging.py:61]: Loss 'loss' on epoch 54: 5.462146282196045 +01-18 14:13:06 INFO [logging.py:61]: Loss 'ar_loss' on epoch 54: 1.2472673654556274 +01-18 14:13:06 INFO [logging.py:61]: Loss 'nar_loss' on epoch 54: 4.214879512786865 +01-18 14:13:06 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 54: 0.9099826216697693 +01-18 14:13:06 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 54: 0.4112021327018738 +01-18 14:13:06 INFO [logging.py:61]: ========= Epoch 55 out of 1000 ========= +01-18 14:13:06 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_14_40.log b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_14_40.log new file mode 100644 index 0000000000000000000000000000000000000000..1b0f777fcd3efccd9140744e74dc35480c2af200 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/swin_default_LR1e-3_AR-NAR_2024_01_18--13_14_40.log @@ -0,0 +1 @@ +01-18 13:14:40 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-3_AR-NAR. diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685133.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685133.0 new file mode 100644 index 0000000000000000000000000000000000000000..eb98f18e0093cb8794e889265e5f6527c509e90c --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685133.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0db7b9d9f6a021d3820b3843c34b06bc51d57a21200540d977ae644206471754 +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685134.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685134.0 new file mode 100644 index 0000000000000000000000000000000000000000..b489ab7cb48be4b7128386b5aad406a545797aff --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685134.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c455eb1988f60f1059a5d69e417cc6f3bfd31662340202195fcfd908dd2b4517 +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685135.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685135.0 new file mode 100644 index 0000000000000000000000000000000000000000..ebf03227ebe90ab2e7e98c4ea09c93417bc8e97d --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685135.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5589a30e959b9cc127b91c8fd0778037d7efcd6c7d84e5de3df1c9de60d79f6a +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685136.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685136.0 new file mode 100644 index 0000000000000000000000000000000000000000..51afd7eb5bc32ede88a709f5b519e402ad335762 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497738.gina1.1685136.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc2cf6cf8a87551e6cdaa13aaccd57b997c5523d5fcf74189141c71117e8a2b2 +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688371.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688371.0 new file mode 100644 index 0000000000000000000000000000000000000000..23fb4fabfd3b99d428800a61a2a83f9cc57d3e14 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688371.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad2246b8f994474bf05d7f34f921f5191105657fff5ab6f1c1ea723c76b824d7 +size 16886 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688372.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688372.0 new file mode 100644 index 0000000000000000000000000000000000000000..c55d3ddacb35d4a0114b20f35cb025bf60b3add3 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688372.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76f9a5e4dfe677d7a51b782cfcf642d1119c3d9ddd55f61a7d4a0fb71dc6e09e +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688373.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688373.0 new file mode 100644 index 0000000000000000000000000000000000000000..24f1b618999f5ed572c200b5227a7d35024cb6fe --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688373.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f5f21e4c295d18007c964da32d7b1a8ca5173b09af0ce62d0032c2d204c5390 +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688374.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688374.0 new file mode 100644 index 0000000000000000000000000000000000000000..a4c7164f00041ac689fcbae8610142dc77e6bb34 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705497802.gina1.1688374.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa97cba9197532f36bf3d5c158492e5793eac1bc64d0d3f4efbcecafd97c6854 +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693065.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693065.0 new file mode 100644 index 0000000000000000000000000000000000000000..d11920f1cca10e23f1d893ed9a6a7da0105d95e1 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693065.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8f451a4e9894799899b82c2ba35bb8330502198e1ad3cdd9e6d6e0990667497 +size 1392147 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693066.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693066.0 new file mode 100644 index 0000000000000000000000000000000000000000..8247a75e20e425876881419fd411a37d36c9032d --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693066.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d9c9627dc4c32e365485f25015c3e55e198e23cd38ae374c22c50bbd71f2fe2 +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693067.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693067.0 new file mode 100644 index 0000000000000000000000000000000000000000..b6ff9b27fefdfdd1d27bda3d7fbd6da83d2afff2 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693067.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45b2fa15d31fe66ac3e0ddb66e2b906fb54281a5dd9aa5f4f8fdd2631668ec0c +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693068.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693068.0 new file mode 100644 index 0000000000000000000000000000000000000000..3f3b01bb60fbe64135e14e3431ce0843f7740008 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705498412.gina1.1693068.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae622a26435c40efe71de00c971eb5022a6d469d2187073028ef5d338368ae9 +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902904.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902904.0 new file mode 100644 index 0000000000000000000000000000000000000000..0a3f95fc3540b66feb98fc71c681b5aa09c63cdc --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902904.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78c031ae9f3d415f23c57dbcf2f45c8a2971a83dad389a2c3c1bcbf750c29eda +size 3266 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902905.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902905.0 new file mode 100644 index 0000000000000000000000000000000000000000..680a6bcecad97e50c7bfb8b32b845f0a01c67897 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902905.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:731d7d744187d5c858d68909f304294a1215bf76660c0a33bf0c481bbae2a6b8 +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902906.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902906.0 new file mode 100644 index 0000000000000000000000000000000000000000..42e195bce2188eab9e0b0afcfb784c046342fa4d --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902906.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d84abd50de308ddc97b1d78aaf91cec87d6d38e3b771b6aeb05c922a61fd56b +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902907.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902907.0 new file mode 100644 index 0000000000000000000000000000000000000000..750bab9fd84da6cebf0d7f760a56fd22fca3b243 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544038.gina1.1902907.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d207613129eb1b3e74702a3e039ed92d6a2af1d52c45653aff2c278cb14d0a +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544084.gina1.1906280.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544084.gina1.1906280.0 new file mode 100644 index 0000000000000000000000000000000000000000..6ca8fac780e473fd7a4750cd9a491d7ce8e39ce1 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544084.gina1.1906280.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c2fd6ce73bd363b25ad60f086960873063a6fc723aeb8443f56ddbaf34caade +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906278.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906278.0 new file mode 100644 index 0000000000000000000000000000000000000000..12688a7e71f05a1e4e15d35d4efc6b6452ac377e --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906278.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6014bdba79c64effbc71783539675bb5cffdf946d22ecbad3082683ae55c32bc +size 137240 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906279.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906279.0 new file mode 100644 index 0000000000000000000000000000000000000000..51b10ff017c77092b5968eb70f8ff97fa46d38fe --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906279.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6a1da535825cde6ff0bd5b73f77dbc9daba972a296229ec8caf86877bbe1ccb +size 2166 diff --git a/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906281.0 b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906281.0 new file mode 100644 index 0000000000000000000000000000000000000000..0c0497fb225bd45985063c55628f4b4cb43f5597 --- /dev/null +++ b/swin_default_LR1e-3_AR-NAR/tb_log/events.out.tfevents.1705544085.gina1.1906281.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e2cf4c3e804063dc307695f33ebb3484035892070eb43b4cfd9026081fa18d0 +size 2166 diff --git a/swin_default_LR1e-4_AR-NAR/config__2024_01_18--13_07_07.toml b/swin_default_LR1e-4_AR-NAR/config__2024_01_18--13_07_07.toml new file mode 100644 index 0000000000000000000000000000000000000000..eea3bdbe634a10e63f5960b32cb6226f9a994eb0 --- /dev/null +++ b/swin_default_LR1e-4_AR-NAR/config__2024_01_18--13_07_07.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-4_AR-NAR" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-4_AR-NAR.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model_ar_nar.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 0.0001 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-4_AR-NAR/swin_default_LR1e-4_AR-NAR_2024_01_18--13_05_10.log b/swin_default_LR1e-4_AR-NAR/swin_default_LR1e-4_AR-NAR_2024_01_18--13_05_10.log new file mode 100644 index 0000000000000000000000000000000000000000..20598d8c7410b1453b59dbe718cddc60e901eb68 --- /dev/null +++ b/swin_default_LR1e-4_AR-NAR/swin_default_LR1e-4_AR-NAR_2024_01_18--13_05_10.log @@ -0,0 +1,149 @@ +01-18 13:05:10 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR. +01-18 13:05:10 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR. +01-18 13:05:10 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR. +01-18 13:05:10 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR. +01-18 13:07:34 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/config__2024_01_18--13_07_07.toml. +01-18 13:07:34 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 503.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-18 13:07:34 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─TokenEmbedding: 2-2 -- +│ │ └─Dropout: 3-4 -- +│ │ └─Embedding: 3-5 524,800 +│ └─Identity: 2-3 -- +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-6 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-7 37,828,608 +│ │ └─LayerNorm: 3-8 1,024 +│ └─Linear: 2-6 524,800 +│ └─MulticlassAccuracy: 2-7 -- +│ └─TokenEmbedding: 2-8 -- +│ │ └─Dropout: 3-9 -- +│ │ └─Embedding: 3-10 524,288 +│ └─ModuleList: 2-9 -- +│ │ └─TokenEmbedding: 3-11 524,800 +│ │ └─TokenEmbedding: 3-12 524,288 +│ │ └─TokenEmbedding: 3-13 524,288 +│ │ └─TokenEmbedding: 3-14 524,288 +│ │ └─TokenEmbedding: 3-15 524,288 +│ │ └─TokenEmbedding: 3-16 524,288 +│ │ └─TokenEmbedding: 3-17 524,288 +│ │ └─TokenEmbedding: 3-18 524,288 +│ └─Identity: 2-10 -- +│ └─SinePositionalEmbedding: 2-11 1 +│ │ └─Dropout: 3-19 -- +│ └─TransformerEncoder: 2-12 -- +│ │ └─ModuleList: 3-20 50,436,096 +│ │ └─AdaptiveLayerNorm: 3-21 526,336 +│ └─ModuleList: 2-13 -- +│ │ └─Linear: 3-22 524,288 +│ │ └─Linear: 3-23 524,288 +│ │ └─Linear: 3-24 524,288 +│ │ └─Linear: 3-25 524,288 +│ │ └─Linear: 3-26 524,288 +│ │ └─Linear: 3-27 524,288 +│ │ └─Linear: 3-28 524,288 +│ └─ModuleList: 2-14 -- +│ │ └─TokenEmbedding: 3-29 512 +│ │ └─TokenEmbedding: 3-30 512 +│ │ └─TokenEmbedding: 3-31 512 +│ │ └─TokenEmbedding: 3-32 512 +│ │ └─TokenEmbedding: 3-33 512 +│ │ └─TokenEmbedding: 3-34 512 +│ │ └─TokenEmbedding: 3-35 512 +│ └─MulticlassAccuracy: 2-15 -- +=============================================================================================== +Total params: 113,086,180 +Trainable params: 98,234,369 +Non-trainable params: 14,851,811 +=============================================================================================== +01-18 13:07:34 INFO [logging.py:61]: Training control variables: +01-18 13:07:34 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-18 13:07:34 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-18 13:07:34 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-18 13:07:34 INFO [logging.py:61]: `max_steps`: 500000 +01-18 13:07:34 INFO [logging.py:61]: `max_epochs`: 1000 +01-18 13:07:34 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-18 13:07:34 INFO [logging.py:61]: ========= Epoch 1 out of 1000 ========= +01-18 13:07:34 INFO [logging.py:61]: Begin training... +01-18 13:17:30 INFO [logging.py:61]: Loss 'loss' on epoch 1: 4.6260857582092285 +01-18 13:17:31 INFO [logging.py:61]: Loss 'ar_loss' on epoch 1: 4.6260857582092285 +01-18 13:17:31 INFO [logging.py:61]: Loss 'nar_loss' on epoch 1: 0.0 +01-18 13:17:31 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 1: 0.4006486237049103 +01-18 13:17:31 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 1: 0.0 +01-18 13:17:31 INFO [logging.py:61]: ========= Epoch 2 out of 1000 ========= +01-18 13:17:31 INFO [logging.py:61]: Begin training... +01-18 13:27:26 INFO [logging.py:61]: Loss 'loss' on epoch 2: 3.4351987838745117 +01-18 13:27:26 INFO [logging.py:61]: Loss 'ar_loss' on epoch 2: 3.4351987838745117 +01-18 13:27:26 INFO [logging.py:61]: Loss 'nar_loss' on epoch 2: 0.0 +01-18 13:27:26 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 2: 0.5857024192810059 +01-18 13:27:26 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 2: 0.0 +01-18 13:27:26 INFO [logging.py:61]: ========= Epoch 3 out of 1000 ========= +01-18 13:27:26 INFO [logging.py:61]: Begin training... +01-18 13:37:46 INFO [logging.py:61]: Loss 'loss' on epoch 3: 3.175524950027466 +01-18 13:37:46 INFO [logging.py:61]: Loss 'ar_loss' on epoch 3: 3.175524950027466 +01-18 13:37:46 INFO [logging.py:61]: Loss 'nar_loss' on epoch 3: 0.0 +01-18 13:37:46 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 3: 0.6268473267555237 +01-18 13:37:46 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 3: 0.0 +01-18 13:37:46 INFO [logging.py:61]: ========= Epoch 4 out of 1000 ========= +01-18 13:37:46 INFO [logging.py:61]: Begin training... +01-18 13:47:20 INFO [logging.py:61]: Loss 'loss' on epoch 4: 3.0606117248535156 +01-18 13:47:20 INFO [logging.py:61]: Loss 'ar_loss' on epoch 4: 3.0606117248535156 +01-18 13:47:20 INFO [logging.py:61]: Loss 'nar_loss' on epoch 4: 0.0 +01-18 13:47:20 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 4: 0.6437094211578369 +01-18 13:47:20 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 4: 0.0 +01-18 13:47:20 INFO [logging.py:61]: ========= Epoch 5 out of 1000 ========= +01-18 13:47:20 INFO [logging.py:61]: Begin training... +01-18 13:56:19 INFO [logging.py:61]: Loss 'loss' on epoch 5: 2.9828383922576904 +01-18 13:56:19 INFO [logging.py:61]: Loss 'ar_loss' on epoch 5: 2.9828383922576904 +01-18 13:56:19 INFO [logging.py:61]: Loss 'nar_loss' on epoch 5: 0.0 +01-18 13:56:19 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 5: 0.6574126482009888 +01-18 13:56:19 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 5: 0.0 +01-18 13:56:19 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/checkpoints/epoch_0005 +01-18 13:56:21 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/checkpoints/epoch_0005/pytorch_model.bin +01-18 13:56:23 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/checkpoints/epoch_0005/optimizer.bin +01-18 13:56:23 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/checkpoints/epoch_0005/scheduler.bin +01-18 13:56:23 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/checkpoints/epoch_0005/sampler.bin +01-18 13:56:23 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/checkpoints/epoch_0005/sampler_1.bin +01-18 13:56:23 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/checkpoints/epoch_0005/random_states_0.pkl +01-18 13:56:23 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-4_AR-NAR/checkpoints/epoch_0005/custom_checkpoint_0.pkl +01-18 13:56:23 INFO [logging.py:61]: ========= Epoch 6 out of 1000 ========= +01-18 13:56:23 INFO [logging.py:61]: Begin training... +01-18 14:06:06 INFO [logging.py:61]: Loss 'loss' on epoch 6: 2.9489591121673584 +01-18 14:06:06 INFO [logging.py:61]: Loss 'ar_loss' on epoch 6: 2.9489591121673584 +01-18 14:06:06 INFO [logging.py:61]: Loss 'nar_loss' on epoch 6: 0.0 +01-18 14:06:06 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 6: 0.6638808846473694 +01-18 14:06:06 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 6: 0.0 +01-18 14:06:06 INFO [logging.py:61]: ========= Epoch 7 out of 1000 ========= +01-18 14:06:06 INFO [logging.py:61]: Begin training... +01-18 14:15:31 INFO [logging.py:61]: Loss 'loss' on epoch 7: 2.9095468521118164 +01-18 14:15:31 INFO [logging.py:61]: Loss 'ar_loss' on epoch 7: 2.9095468521118164 +01-18 14:15:31 INFO [logging.py:61]: Loss 'nar_loss' on epoch 7: 0.0 +01-18 14:15:31 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 7: 0.670335590839386 +01-18 14:15:31 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 7: 0.0 +01-18 14:15:31 INFO [logging.py:61]: ========= Epoch 8 out of 1000 ========= +01-18 14:15:31 INFO [logging.py:61]: Begin training... +01-18 14:25:02 INFO [logging.py:61]: Loss 'loss' on epoch 8: 2.8823063373565674 +01-18 14:25:02 INFO [logging.py:61]: Loss 'ar_loss' on epoch 8: 2.8823063373565674 +01-18 14:25:02 INFO [logging.py:61]: Loss 'nar_loss' on epoch 8: 0.0 +01-18 14:25:02 INFO [logging.py:61]: Loss 'ar_accuracy_metric' on epoch 8: 0.6748366355895996 +01-18 14:25:02 INFO [logging.py:61]: Loss 'nar_acc_metric' on epoch 8: 0.0 +01-18 14:25:02 INFO [logging.py:61]: ========= Epoch 9 out of 1000 ========= +01-18 14:25:02 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461757.0 b/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461757.0 new file mode 100644 index 0000000000000000000000000000000000000000..b7e19352beabf6335f06aa5e6fd0afd45446d243 --- /dev/null +++ b/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461757.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4c8710483a799071219c907f61abf1dd81cb78dae380cfcc17d5774cc080489 +size 222659 diff --git a/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461758.0 b/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461758.0 new file mode 100644 index 0000000000000000000000000000000000000000..4efd758db6cc45209821755dec3365aa519c2455 --- /dev/null +++ b/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461758.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b54021e2437c13b84d594f6f40b1c33d422ba9505c999e7405225f2eb8e014ca +size 2167 diff --git a/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461761.0 b/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461761.0 new file mode 100644 index 0000000000000000000000000000000000000000..4fa77a69e064b3bd561034b9aa942f69330cc07f --- /dev/null +++ b/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461761.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ce73355cdb8c88f62c2d6794c8241a21e85be852ebbe5b45cb92b6dea627058 +size 2167 diff --git a/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461762.0 b/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461762.0 new file mode 100644 index 0000000000000000000000000000000000000000..4e460610c71883ac36b7c8dfeaa92b40d5c53406 --- /dev/null +++ b/swin_default_LR1e-4_AR-NAR/tb_log/events.out.tfevents.1705543653.gina2.461762.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7659e30f1fd7715fed5e08ea84dc853f3c17b6c866dc19dac44fb982727868e +size 2167 diff --git a/swin_default_LR1e-5_addEOSToken/config__2024_01_17--11_57_13.toml b/swin_default_LR1e-5_addEOSToken/config__2024_01_17--11_57_13.toml new file mode 100644 index 0000000000000000000000000000000000000000..a5033449cb122e865aedd5dbeb74a4f6bc9a1f86 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/config__2024_01_17--11_57_13.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-5_addEOSToken" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-5_addEOSToken.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 1e-5 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_16_44.toml b/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_16_44.toml new file mode 100644 index 0000000000000000000000000000000000000000..a5033449cb122e865aedd5dbeb74a4f6bc9a1f86 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_16_44.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-5_addEOSToken" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-5_addEOSToken.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 1e-5 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_35_24.toml b/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_35_24.toml new file mode 100644 index 0000000000000000000000000000000000000000..a5033449cb122e865aedd5dbeb74a4f6bc9a1f86 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_35_24.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-5_addEOSToken" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-5_addEOSToken.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 1e-5 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_36_04.toml b/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_36_04.toml new file mode 100644 index 0000000000000000000000000000000000000000..a5033449cb122e865aedd5dbeb74a4f6bc9a1f86 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_36_04.toml @@ -0,0 +1,89 @@ +[[validate_dataset]] +path = "dataloader.Dataset" + +[validate_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 +[validate_dataset.dataloader] +batch_size = 1 +num_workers = 1 + +[meta] +save_dir = "exp" +description = "Train a model using Generative Adversarial Networks (GANs)" +seed = 20220815 +exp_id = "swin_default_LR1e-5_addEOSToken" +config_path = "/fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/conf/swin_default_LR1e-5_addEOSToken.toml" + +[trainer] +path = "trainer.Trainer" + +[loss_function] +path = "torch.nn.MSELoss" + +[optimizer] +path = "torch.optim.AdamW" + +[model] +path = "model.Model" + +[acoustics] +n_fft = 512 +hop_length = 128 +win_length = 512 +sr = 24000 + +[train_dataset] +path = "dataloader.Dataset" + +[test_dataset] +path = "dataloader.Dataset" + +[trainer.args] +debug = false +max_steps = 0 +max_epochs = 1000 +max_grad_norm = 1.0 +save_max_score = true +save_ckpt_interval = 5 +max_patience = 200 +plot_norm = true +validation_interval = 200 +max_num_checkpoints = 100 +scheduler_name = "constant_schedule_with_warmup" +warmup_steps = 1000 +warmup_ratio = 0.0 +gradient_accumulation_steps = 1 + +[loss_function.args] + +[optimizer.args] +lr = 1e-5 + +[model.args] + +[train_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 + +[train_dataset.dataloader] +batch_size = 20 +num_workers = 10 +drop_last = true +pin_memory = true + +[test_dataset.args] +librispeech_dir = "~/data/LibriSpeech/LibriSpeech" +librispeech_metadata_fpath = "/home/xhao/proj/audiozen/recipes/librimix_sot/local/metadata/LibriSpeech/train-clean-100-24K.csv" +duration = 6.0 +sr = 24000 +num_samples = 10 + +[test_dataset.dataloader] +batch_size = 1 +num_workers = 1 diff --git a/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--11_56_49.log b/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--11_56_49.log new file mode 100644 index 0000000000000000000000000000000000000000..6a0ac7be4ca01e84f5d60ae22af56c3383a3ee3e --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--11_56_49.log @@ -0,0 +1,74 @@ +01-17 11:56:49 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 11:56:49 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 11:56:49 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 11:56:49 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 11:57:18 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/config__2024_01_17--11_57_13.toml. +01-17 11:57:18 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 1007.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-17 11:57:18 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─ModuleList: 2-2 -- +│ │ └─TokenEmbedding: 3-4 1,049,600 +│ │ └─TokenEmbedding: 3-5 1,049,600 +│ │ └─TokenEmbedding: 3-6 1,049,600 +│ │ └─TokenEmbedding: 3-7 1,049,600 +│ │ └─TokenEmbedding: 3-8 1,049,600 +│ │ └─TokenEmbedding: 3-9 1,049,600 +│ │ └─TokenEmbedding: 3-10 1,049,600 +│ │ └─TokenEmbedding: 3-11 1,049,600 +│ └─PreNet: 2-3 -- +│ │ └─Sequential: 3-12 591,360 +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-13 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-14 201,535,488 +│ │ └─AdaptiveLayerNorm: 3-15 2,101,248 +│ └─ModuleList: 2-6 -- +│ │ └─Linear: 3-16 1,049,600 +│ │ └─Linear: 3-17 1,049,600 +│ │ └─Linear: 3-18 1,049,600 +│ │ └─Linear: 3-19 1,049,600 +│ │ └─Linear: 3-20 1,049,600 +│ │ └─Linear: 3-21 1,049,600 +│ │ └─Linear: 3-22 1,049,600 +│ │ └─Linear: 3-23 1,049,600 +│ └─ModuleList: 2-7 -- +│ │ └─TokenEmbedding: 3-24 1,024 +│ │ └─TokenEmbedding: 3-25 1,024 +│ │ └─TokenEmbedding: 3-26 1,024 +│ │ └─TokenEmbedding: 3-27 1,024 +│ │ └─TokenEmbedding: 3-28 1,024 +│ │ └─TokenEmbedding: 3-29 1,024 +│ │ └─TokenEmbedding: 3-30 1,024 +│ │ └─TokenEmbedding: 3-31 1,024 +│ └─MulticlassAccuracy: 2-8 -- +=============================================================================================== +Total params: 235,881,699 +Trainable params: 221,029,888 +Non-trainable params: 14,851,811 +=============================================================================================== +01-17 11:57:18 INFO [logging.py:61]: Training control variables: +01-17 11:57:18 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-17 11:57:18 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-17 11:57:18 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-17 11:57:18 INFO [logging.py:61]: `max_steps`: 500000 +01-17 11:57:18 INFO [logging.py:61]: `max_epochs`: 1000 +01-17 11:57:18 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-17 11:57:18 INFO [logging.py:61]: ========= Epoch 1 out of 1000 ========= +01-17 11:57:18 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_15_22.log b/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_15_22.log new file mode 100644 index 0000000000000000000000000000000000000000..549323767b9cd9a5847e0d9e27ba19a39673647a --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_15_22.log @@ -0,0 +1,72 @@ +01-17 17:15:22 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:15:22 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:15:22 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:15:22 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:17:03 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_16_44.toml. +01-17 17:17:03 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 1007.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-17 17:17:03 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─ModuleList: 2-2 -- +│ │ └─TokenEmbedding: 3-4 1,049,600 +│ │ └─TokenEmbedding: 3-5 1,049,600 +│ │ └─TokenEmbedding: 3-6 1,049,600 +│ │ └─TokenEmbedding: 3-7 1,049,600 +│ │ └─TokenEmbedding: 3-8 1,049,600 +│ │ └─TokenEmbedding: 3-9 1,049,600 +│ │ └─TokenEmbedding: 3-10 1,049,600 +│ │ └─TokenEmbedding: 3-11 1,049,600 +│ └─PreNet: 2-3 -- +│ │ └─Sequential: 3-12 591,360 +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-13 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-14 201,535,488 +│ │ └─AdaptiveLayerNorm: 3-15 2,101,248 +│ └─ModuleList: 2-6 -- +│ │ └─Linear: 3-16 1,049,600 +│ │ └─Linear: 3-17 1,049,600 +│ │ └─Linear: 3-18 1,049,600 +│ │ └─Linear: 3-19 1,049,600 +│ │ └─Linear: 3-20 1,049,600 +│ │ └─Linear: 3-21 1,049,600 +│ │ └─Linear: 3-22 1,049,600 +│ │ └─Linear: 3-23 1,049,600 +│ └─ModuleList: 2-7 -- +│ │ └─TokenEmbedding: 3-24 1,024 +│ │ └─TokenEmbedding: 3-25 1,024 +│ │ └─TokenEmbedding: 3-26 1,024 +│ │ └─TokenEmbedding: 3-27 1,024 +│ │ └─TokenEmbedding: 3-28 1,024 +│ │ └─TokenEmbedding: 3-29 1,024 +│ │ └─TokenEmbedding: 3-30 1,024 +│ │ └─TokenEmbedding: 3-31 1,024 +│ └─MulticlassAccuracy: 2-8 -- +=============================================================================================== +Total params: 235,881,699 +Trainable params: 221,029,888 +Non-trainable params: 14,851,811 +=============================================================================================== +01-17 17:17:03 INFO [logging.py:61]: Training control variables: +01-17 17:17:03 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-17 17:17:03 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-17 17:17:03 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-17 17:17:03 INFO [logging.py:61]: `max_steps`: 500000 +01-17 17:17:03 INFO [logging.py:61]: `max_epochs`: 1000 +01-17 17:17:03 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. diff --git a/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_35_21.log b/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_35_21.log new file mode 100644 index 0000000000000000000000000000000000000000..3aff85a7f6def5df8c2bfd4862449424a940cab8 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_35_21.log @@ -0,0 +1,72 @@ +01-17 17:35:21 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:35:21 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:35:21 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:35:21 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:35:26 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_35_24.toml. +01-17 17:35:26 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 1007.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-17 17:35:26 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─ModuleList: 2-2 -- +│ │ └─TokenEmbedding: 3-4 1,049,600 +│ │ └─TokenEmbedding: 3-5 1,049,600 +│ │ └─TokenEmbedding: 3-6 1,049,600 +│ │ └─TokenEmbedding: 3-7 1,049,600 +│ │ └─TokenEmbedding: 3-8 1,049,600 +│ │ └─TokenEmbedding: 3-9 1,049,600 +│ │ └─TokenEmbedding: 3-10 1,049,600 +│ │ └─TokenEmbedding: 3-11 1,049,600 +│ └─PreNet: 2-3 -- +│ │ └─Sequential: 3-12 591,360 +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-13 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-14 201,535,488 +│ │ └─AdaptiveLayerNorm: 3-15 2,101,248 +│ └─ModuleList: 2-6 -- +│ │ └─Linear: 3-16 1,049,600 +│ │ └─Linear: 3-17 1,049,600 +│ │ └─Linear: 3-18 1,049,600 +│ │ └─Linear: 3-19 1,049,600 +│ │ └─Linear: 3-20 1,049,600 +│ │ └─Linear: 3-21 1,049,600 +│ │ └─Linear: 3-22 1,049,600 +│ │ └─Linear: 3-23 1,049,600 +│ └─ModuleList: 2-7 -- +│ │ └─TokenEmbedding: 3-24 1,024 +│ │ └─TokenEmbedding: 3-25 1,024 +│ │ └─TokenEmbedding: 3-26 1,024 +│ │ └─TokenEmbedding: 3-27 1,024 +│ │ └─TokenEmbedding: 3-28 1,024 +│ │ └─TokenEmbedding: 3-29 1,024 +│ │ └─TokenEmbedding: 3-30 1,024 +│ │ └─TokenEmbedding: 3-31 1,024 +│ └─MulticlassAccuracy: 2-8 -- +=============================================================================================== +Total params: 235,881,699 +Trainable params: 221,029,888 +Non-trainable params: 14,851,811 +=============================================================================================== +01-17 17:35:26 INFO [logging.py:61]: Training control variables: +01-17 17:35:26 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-17 17:35:26 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-17 17:35:26 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-17 17:35:26 INFO [logging.py:61]: `max_steps`: 500000 +01-17 17:35:26 INFO [logging.py:61]: `max_epochs`: 1000 +01-17 17:35:26 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. diff --git a/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_36_00.log b/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_36_00.log new file mode 100644 index 0000000000000000000000000000000000000000..6b0bc4539f1e3557a7e4ab1b3cbfbc83ddd654d0 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/swin_default_LR1e-5_addEOSToken_2024_01_17--17_36_00.log @@ -0,0 +1,454 @@ +01-17 17:36:00 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:36:00 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:36:00 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:36:00 INFO [logger.py:80]: Initialized logger with log file in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken. +01-17 17:36:05 INFO [logging.py:61]: Configuration file is saved to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/config__2024_01_17--17_36_04.toml. +01-17 17:36:05 INFO [logging.py:61]: Environment information: +- `Accelerate` version: 0.26.1 +- Platform: Linux-5.14.0-362.13.1.el9_3.x86_64-x86_64-with-glibc2.34 +- Python version: 3.10.13 +- Numpy version: 1.26.3 +- PyTorch version (GPU?): 2.1.2 (True) +- System RAM: 1007.48 GB +- GPU Available: True +- GPU IDs: 4 +- GPU type: NVIDIA A100-SXM4-80GB +01-17 17:36:05 INFO [logging.py:61]: + =============================================================================================== +Layer (type:depth-idx) Param # +=============================================================================================== +DistributedDataParallel -- +├─Model: 1-1 -- +│ └─EncodecModel: 2-1 -- +│ │ └─EncodecEncoder: 3-1 (7,425,792) +│ │ └─EncodecDecoder: 3-2 (7,426,018) +│ │ └─EncodecResidualVectorQuantizer: 3-3 -- +│ └─ModuleList: 2-2 -- +│ │ └─TokenEmbedding: 3-4 1,049,600 +│ │ └─TokenEmbedding: 3-5 1,049,600 +│ │ └─TokenEmbedding: 3-6 1,049,600 +│ │ └─TokenEmbedding: 3-7 1,049,600 +│ │ └─TokenEmbedding: 3-8 1,049,600 +│ │ └─TokenEmbedding: 3-9 1,049,600 +│ │ └─TokenEmbedding: 3-10 1,049,600 +│ │ └─TokenEmbedding: 3-11 1,049,600 +│ └─PreNet: 2-3 -- +│ │ └─Sequential: 3-12 591,360 +│ └─SinePositionalEmbedding: 2-4 1 +│ │ └─Dropout: 3-13 -- +│ └─TransformerEncoder: 2-5 -- +│ │ └─ModuleList: 3-14 201,535,488 +│ │ └─AdaptiveLayerNorm: 3-15 2,101,248 +│ └─ModuleList: 2-6 -- +│ │ └─Linear: 3-16 1,049,600 +│ │ └─Linear: 3-17 1,049,600 +│ │ └─Linear: 3-18 1,049,600 +│ │ └─Linear: 3-19 1,049,600 +│ │ └─Linear: 3-20 1,049,600 +│ │ └─Linear: 3-21 1,049,600 +│ │ └─Linear: 3-22 1,049,600 +│ │ └─Linear: 3-23 1,049,600 +│ └─ModuleList: 2-7 -- +│ │ └─TokenEmbedding: 3-24 1,024 +│ │ └─TokenEmbedding: 3-25 1,024 +│ │ └─TokenEmbedding: 3-26 1,024 +│ │ └─TokenEmbedding: 3-27 1,024 +│ │ └─TokenEmbedding: 3-28 1,024 +│ │ └─TokenEmbedding: 3-29 1,024 +│ │ └─TokenEmbedding: 3-30 1,024 +│ │ └─TokenEmbedding: 3-31 1,024 +│ └─MulticlassAccuracy: 2-8 -- +=============================================================================================== +Total params: 235,881,699 +Trainable params: 221,029,888 +Non-trainable params: 14,851,811 +=============================================================================================== +01-17 17:36:05 INFO [logging.py:61]: Training control variables: +01-17 17:36:05 INFO [logging.py:61]: `steps_per_epoch`: 500 +01-17 17:36:05 INFO [logging.py:61]: Gradient accumulation steps: 1 +01-17 17:36:05 INFO [logging.py:61]: `update_steps_per_epoch`: 500 +01-17 17:36:05 INFO [logging.py:61]: `max_steps`: 500000 +01-17 17:36:05 INFO [logging.py:61]: `max_epochs`: 1000 +01-17 17:36:05 INFO [logging.py:61]: warmup_steps=1000. warmup_ratio will be ignored. +01-17 17:36:05 INFO [logging.py:61]: ========= Epoch 1 out of 1000 ========= +01-17 17:36:05 INFO [logging.py:61]: Begin training... +01-17 17:53:49 INFO [logging.py:61]: Loss 'loss' on epoch 1: 6.125411510467529 +01-17 17:53:49 INFO [logging.py:61]: Loss 'acc' on epoch 1: 0.21388652920722961 +01-17 17:53:49 INFO [logging.py:61]: ========= Epoch 2 out of 1000 ========= +01-17 17:53:49 INFO [logging.py:61]: Begin training... +01-17 18:11:29 INFO [logging.py:61]: Loss 'loss' on epoch 2: 4.932284355163574 +01-17 18:11:29 INFO [logging.py:61]: Loss 'acc' on epoch 2: 0.34072327613830566 +01-17 18:11:29 INFO [logging.py:61]: ========= Epoch 3 out of 1000 ========= +01-17 18:11:29 INFO [logging.py:61]: Begin training... +01-17 18:29:09 INFO [logging.py:61]: Loss 'loss' on epoch 3: 4.676321983337402 +01-17 18:29:09 INFO [logging.py:61]: Loss 'acc' on epoch 3: 0.3615228831768036 +01-17 18:29:09 INFO [logging.py:61]: ========= Epoch 4 out of 1000 ========= +01-17 18:29:09 INFO [logging.py:61]: Begin training... +01-17 18:46:49 INFO [logging.py:61]: Loss 'loss' on epoch 4: 4.5471696853637695 +01-17 18:46:49 INFO [logging.py:61]: Loss 'acc' on epoch 4: 0.371623694896698 +01-17 18:46:49 INFO [logging.py:61]: ========= Epoch 5 out of 1000 ========= +01-17 18:46:49 INFO [logging.py:61]: Begin training... +01-17 19:04:28 INFO [logging.py:61]: Loss 'loss' on epoch 5: 4.502699851989746 +01-17 19:04:28 INFO [logging.py:61]: Loss 'acc' on epoch 5: 0.37536850571632385 +01-17 19:04:28 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0005 +01-17 19:04:30 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0005/pytorch_model.bin +01-17 19:04:32 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0005/optimizer.bin +01-17 19:04:32 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0005/scheduler.bin +01-17 19:04:32 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0005/sampler.bin +01-17 19:04:32 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0005/sampler_1.bin +01-17 19:04:32 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0005/random_states_0.pkl +01-17 19:04:32 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0005/custom_checkpoint_0.pkl +01-17 19:04:32 INFO [logging.py:61]: ========= Epoch 6 out of 1000 ========= +01-17 19:04:32 INFO [logging.py:61]: Begin training... +01-17 19:22:12 INFO [logging.py:61]: Loss 'loss' on epoch 6: 4.46772575378418 +01-17 19:22:12 INFO [logging.py:61]: Loss 'acc' on epoch 6: 0.37959420680999756 +01-17 19:22:12 INFO [logging.py:61]: ========= Epoch 7 out of 1000 ========= +01-17 19:22:12 INFO [logging.py:61]: Begin training... +01-17 19:49:38 INFO [logging.py:61]: Loss 'loss' on epoch 7: 4.449397563934326 +01-17 19:49:38 INFO [logging.py:61]: Loss 'acc' on epoch 7: 0.3816448450088501 +01-17 19:49:38 INFO [logging.py:61]: ========= Epoch 8 out of 1000 ========= +01-17 19:49:38 INFO [logging.py:61]: Begin training... +01-17 20:07:18 INFO [logging.py:61]: Loss 'loss' on epoch 8: 4.426702499389648 +01-17 20:07:18 INFO [logging.py:61]: Loss 'acc' on epoch 8: 0.3839639723300934 +01-17 20:07:18 INFO [logging.py:61]: ========= Epoch 9 out of 1000 ========= +01-17 20:07:18 INFO [logging.py:61]: Begin training... +01-17 20:24:58 INFO [logging.py:61]: Loss 'loss' on epoch 9: 4.411308765411377 +01-17 20:24:58 INFO [logging.py:61]: Loss 'acc' on epoch 9: 0.38625749945640564 +01-17 20:24:58 INFO [logging.py:61]: ========= Epoch 10 out of 1000 ========= +01-17 20:24:58 INFO [logging.py:61]: Begin training... +01-17 20:42:38 INFO [logging.py:61]: Loss 'loss' on epoch 10: 4.382351875305176 +01-17 20:42:38 INFO [logging.py:61]: Loss 'acc' on epoch 10: 0.3900945484638214 +01-17 20:42:38 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0010 +01-17 20:42:39 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0010/pytorch_model.bin +01-17 20:42:42 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0010/optimizer.bin +01-17 20:42:42 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0010/scheduler.bin +01-17 20:42:42 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0010/sampler.bin +01-17 20:42:42 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0010/sampler_1.bin +01-17 20:42:42 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0010/random_states_0.pkl +01-17 20:42:42 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0010/custom_checkpoint_0.pkl +01-17 20:42:42 INFO [logging.py:61]: ========= Epoch 11 out of 1000 ========= +01-17 20:42:42 INFO [logging.py:61]: Begin training... +01-17 21:00:22 INFO [logging.py:61]: Loss 'loss' on epoch 11: 4.384711742401123 +01-17 21:00:22 INFO [logging.py:61]: Loss 'acc' on epoch 11: 0.39045605063438416 +01-17 21:00:22 INFO [logging.py:61]: ========= Epoch 12 out of 1000 ========= +01-17 21:00:22 INFO [logging.py:61]: Begin training... +01-17 21:18:02 INFO [logging.py:61]: Loss 'loss' on epoch 12: 4.341005802154541 +01-17 21:18:02 INFO [logging.py:61]: Loss 'acc' on epoch 12: 0.397234171628952 +01-17 21:18:02 INFO [logging.py:61]: ========= Epoch 13 out of 1000 ========= +01-17 21:18:02 INFO [logging.py:61]: Begin training... +01-17 21:35:42 INFO [logging.py:61]: Loss 'loss' on epoch 13: 4.359378814697266 +01-17 21:35:42 INFO [logging.py:61]: Loss 'acc' on epoch 13: 0.39445215463638306 +01-17 21:35:42 INFO [logging.py:61]: ========= Epoch 14 out of 1000 ========= +01-17 21:35:42 INFO [logging.py:61]: Begin training... +01-17 21:53:22 INFO [logging.py:61]: Loss 'loss' on epoch 14: 4.334432601928711 +01-17 21:53:22 INFO [logging.py:61]: Loss 'acc' on epoch 14: 0.39845597743988037 +01-17 21:53:22 INFO [logging.py:61]: ========= Epoch 15 out of 1000 ========= +01-17 21:53:22 INFO [logging.py:61]: Begin training... +01-17 22:11:02 INFO [logging.py:61]: Loss 'loss' on epoch 15: 4.341060161590576 +01-17 22:11:02 INFO [logging.py:61]: Loss 'acc' on epoch 15: 0.39680665731430054 +01-17 22:11:02 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0015 +01-17 22:11:04 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0015/pytorch_model.bin +01-17 22:11:06 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0015/optimizer.bin +01-17 22:11:06 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0015/scheduler.bin +01-17 22:11:06 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0015/sampler.bin +01-17 22:11:06 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0015/sampler_1.bin +01-17 22:11:06 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0015/random_states_0.pkl +01-17 22:11:06 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0015/custom_checkpoint_0.pkl +01-17 22:11:06 INFO [logging.py:61]: ========= Epoch 16 out of 1000 ========= +01-17 22:11:06 INFO [logging.py:61]: Begin training... +01-17 22:28:46 INFO [logging.py:61]: Loss 'loss' on epoch 16: 4.32090950012207 +01-17 22:28:46 INFO [logging.py:61]: Loss 'acc' on epoch 16: 0.4006631672382355 +01-17 22:28:46 INFO [logging.py:61]: ========= Epoch 17 out of 1000 ========= +01-17 22:28:46 INFO [logging.py:61]: Begin training... +01-17 22:46:26 INFO [logging.py:61]: Loss 'loss' on epoch 17: 4.299465656280518 +01-17 22:46:26 INFO [logging.py:61]: Loss 'acc' on epoch 17: 0.4042968451976776 +01-17 22:46:26 INFO [logging.py:61]: ========= Epoch 18 out of 1000 ========= +01-17 22:46:26 INFO [logging.py:61]: Begin training... +01-17 23:04:07 INFO [logging.py:61]: Loss 'loss' on epoch 18: 4.289377212524414 +01-17 23:04:07 INFO [logging.py:61]: Loss 'acc' on epoch 18: 0.40627750754356384 +01-17 23:04:07 INFO [logging.py:61]: ========= Epoch 19 out of 1000 ========= +01-17 23:04:07 INFO [logging.py:61]: Begin training... +01-17 23:21:47 INFO [logging.py:61]: Loss 'loss' on epoch 19: 4.295515537261963 +01-17 23:21:47 INFO [logging.py:61]: Loss 'acc' on epoch 19: 0.4052572548389435 +01-17 23:21:47 INFO [logging.py:61]: ========= Epoch 20 out of 1000 ========= +01-17 23:21:47 INFO [logging.py:61]: Begin training... +01-17 23:39:27 INFO [logging.py:61]: Loss 'loss' on epoch 20: 4.267850875854492 +01-17 23:39:27 INFO [logging.py:61]: Loss 'acc' on epoch 20: 0.4097782075405121 +01-17 23:39:27 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0020 +01-17 23:39:28 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0020/pytorch_model.bin +01-17 23:39:30 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0020/optimizer.bin +01-17 23:39:30 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0020/scheduler.bin +01-17 23:39:30 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0020/sampler.bin +01-17 23:39:30 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0020/sampler_1.bin +01-17 23:39:30 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0020/random_states_0.pkl +01-17 23:39:30 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0020/custom_checkpoint_0.pkl +01-17 23:39:30 INFO [logging.py:61]: ========= Epoch 21 out of 1000 ========= +01-17 23:39:30 INFO [logging.py:61]: Begin training... +01-17 23:57:10 INFO [logging.py:61]: Loss 'loss' on epoch 21: 4.292447566986084 +01-17 23:57:10 INFO [logging.py:61]: Loss 'acc' on epoch 21: 0.4059085249900818 +01-17 23:57:10 INFO [logging.py:61]: ========= Epoch 22 out of 1000 ========= +01-17 23:57:10 INFO [logging.py:61]: Begin training... +01-18 00:14:50 INFO [logging.py:61]: Loss 'loss' on epoch 22: 4.236167907714844 +01-18 00:14:50 INFO [logging.py:61]: Loss 'acc' on epoch 22: 0.4158206880092621 +01-18 00:14:50 INFO [logging.py:61]: ========= Epoch 23 out of 1000 ========= +01-18 00:14:50 INFO [logging.py:61]: Begin training... +01-18 00:32:30 INFO [logging.py:61]: Loss 'loss' on epoch 23: 4.27931547164917 +01-18 00:32:30 INFO [logging.py:61]: Loss 'acc' on epoch 23: 0.4083727300167084 +01-18 00:32:30 INFO [logging.py:61]: ========= Epoch 24 out of 1000 ========= +01-18 00:32:30 INFO [logging.py:61]: Begin training... +01-18 00:50:10 INFO [logging.py:61]: Loss 'loss' on epoch 24: 4.229855537414551 +01-18 00:50:10 INFO [logging.py:61]: Loss 'acc' on epoch 24: 0.4171912670135498 +01-18 00:50:10 INFO [logging.py:61]: ========= Epoch 25 out of 1000 ========= +01-18 00:50:10 INFO [logging.py:61]: Begin training... +01-18 01:07:51 INFO [logging.py:61]: Loss 'loss' on epoch 25: 4.222106456756592 +01-18 01:07:51 INFO [logging.py:61]: Loss 'acc' on epoch 25: 0.4182579219341278 +01-18 01:07:51 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0025 +01-18 01:07:52 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0025/pytorch_model.bin +01-18 01:07:54 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0025/optimizer.bin +01-18 01:07:54 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0025/scheduler.bin +01-18 01:07:54 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0025/sampler.bin +01-18 01:07:54 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0025/sampler_1.bin +01-18 01:07:54 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0025/random_states_0.pkl +01-18 01:07:54 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0025/custom_checkpoint_0.pkl +01-18 01:07:54 INFO [logging.py:61]: ========= Epoch 26 out of 1000 ========= +01-18 01:07:54 INFO [logging.py:61]: Begin training... +01-18 01:25:35 INFO [logging.py:61]: Loss 'loss' on epoch 26: 4.217103004455566 +01-18 01:25:35 INFO [logging.py:61]: Loss 'acc' on epoch 26: 0.4189724922180176 +01-18 01:25:35 INFO [logging.py:61]: ========= Epoch 27 out of 1000 ========= +01-18 01:25:35 INFO [logging.py:61]: Begin training... +01-18 01:43:15 INFO [logging.py:61]: Loss 'loss' on epoch 27: 4.228879928588867 +01-18 01:43:15 INFO [logging.py:61]: Loss 'acc' on epoch 27: 0.41666004061698914 +01-18 01:43:15 INFO [logging.py:61]: ========= Epoch 28 out of 1000 ========= +01-18 01:43:15 INFO [logging.py:61]: Begin training... +01-18 02:00:55 INFO [logging.py:61]: Loss 'loss' on epoch 28: 4.248851776123047 +01-18 02:00:55 INFO [logging.py:61]: Loss 'acc' on epoch 28: 0.4130958616733551 +01-18 02:00:55 INFO [logging.py:61]: ========= Epoch 29 out of 1000 ========= +01-18 02:00:55 INFO [logging.py:61]: Begin training... +01-18 02:18:35 INFO [logging.py:61]: Loss 'loss' on epoch 29: 4.208758354187012 +01-18 02:18:35 INFO [logging.py:61]: Loss 'acc' on epoch 29: 0.4202231764793396 +01-18 02:18:35 INFO [logging.py:61]: ========= Epoch 30 out of 1000 ========= +01-18 02:18:35 INFO [logging.py:61]: Begin training... +01-18 02:36:15 INFO [logging.py:61]: Loss 'loss' on epoch 30: 4.185746669769287 +01-18 02:36:15 INFO [logging.py:61]: Loss 'acc' on epoch 30: 0.4239596128463745 +01-18 02:36:15 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0030 +01-18 02:36:17 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0030/pytorch_model.bin +01-18 02:36:19 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0030/optimizer.bin +01-18 02:36:19 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0030/scheduler.bin +01-18 02:36:19 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0030/sampler.bin +01-18 02:36:19 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0030/sampler_1.bin +01-18 02:36:19 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0030/random_states_0.pkl +01-18 02:36:19 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0030/custom_checkpoint_0.pkl +01-18 02:36:19 INFO [logging.py:61]: ========= Epoch 31 out of 1000 ========= +01-18 02:36:19 INFO [logging.py:61]: Begin training... +01-18 02:53:59 INFO [logging.py:61]: Loss 'loss' on epoch 31: 4.190670490264893 +01-18 02:53:59 INFO [logging.py:61]: Loss 'acc' on epoch 31: 0.42290133237838745 +01-18 02:53:59 INFO [logging.py:61]: ========= Epoch 32 out of 1000 ========= +01-18 02:53:59 INFO [logging.py:61]: Begin training... +01-18 03:11:39 INFO [logging.py:61]: Loss 'loss' on epoch 32: 4.168140888214111 +01-18 03:11:39 INFO [logging.py:61]: Loss 'acc' on epoch 32: 0.426896870136261 +01-18 03:11:39 INFO [logging.py:61]: ========= Epoch 33 out of 1000 ========= +01-18 03:11:39 INFO [logging.py:61]: Begin training... +01-18 03:29:19 INFO [logging.py:61]: Loss 'loss' on epoch 33: 4.166285991668701 +01-18 03:29:19 INFO [logging.py:61]: Loss 'acc' on epoch 33: 0.4269416928291321 +01-18 03:29:19 INFO [logging.py:61]: ========= Epoch 34 out of 1000 ========= +01-18 03:29:19 INFO [logging.py:61]: Begin training... +01-18 03:46:59 INFO [logging.py:61]: Loss 'loss' on epoch 34: 4.118114471435547 +01-18 03:46:59 INFO [logging.py:61]: Loss 'acc' on epoch 34: 0.4366379678249359 +01-18 03:46:59 INFO [logging.py:61]: ========= Epoch 35 out of 1000 ========= +01-18 03:46:59 INFO [logging.py:61]: Begin training... +01-18 04:04:39 INFO [logging.py:61]: Loss 'loss' on epoch 35: 4.166625022888184 +01-18 04:04:39 INFO [logging.py:61]: Loss 'acc' on epoch 35: 0.42648646235466003 +01-18 04:04:39 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0035 +01-18 04:04:40 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0035/pytorch_model.bin +01-18 04:04:43 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0035/optimizer.bin +01-18 04:04:43 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0035/scheduler.bin +01-18 04:04:43 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0035/sampler.bin +01-18 04:04:43 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0035/sampler_1.bin +01-18 04:04:43 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0035/random_states_0.pkl +01-18 04:04:43 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0035/custom_checkpoint_0.pkl +01-18 04:04:43 INFO [logging.py:61]: ========= Epoch 36 out of 1000 ========= +01-18 04:04:43 INFO [logging.py:61]: Begin training... +01-18 04:22:23 INFO [logging.py:61]: Loss 'loss' on epoch 36: 4.153348922729492 +01-18 04:22:23 INFO [logging.py:61]: Loss 'acc' on epoch 36: 0.429107129573822 +01-18 04:22:23 INFO [logging.py:61]: ========= Epoch 37 out of 1000 ========= +01-18 04:22:23 INFO [logging.py:61]: Begin training... +01-18 04:40:03 INFO [logging.py:61]: Loss 'loss' on epoch 37: 4.1347737312316895 +01-18 04:40:03 INFO [logging.py:61]: Loss 'acc' on epoch 37: 0.43268778920173645 +01-18 04:40:03 INFO [logging.py:61]: ========= Epoch 38 out of 1000 ========= +01-18 04:40:03 INFO [logging.py:61]: Begin training... +01-18 04:57:43 INFO [logging.py:61]: Loss 'loss' on epoch 38: 4.128436088562012 +01-18 04:57:43 INFO [logging.py:61]: Loss 'acc' on epoch 38: 0.4337690770626068 +01-18 04:57:43 INFO [logging.py:61]: ========= Epoch 39 out of 1000 ========= +01-18 04:57:43 INFO [logging.py:61]: Begin training... +01-18 05:15:24 INFO [logging.py:61]: Loss 'loss' on epoch 39: 4.147870063781738 +01-18 05:15:24 INFO [logging.py:61]: Loss 'acc' on epoch 39: 0.4301242232322693 +01-18 05:15:24 INFO [logging.py:61]: ========= Epoch 40 out of 1000 ========= +01-18 05:15:24 INFO [logging.py:61]: Begin training... +01-18 05:33:04 INFO [logging.py:61]: Loss 'loss' on epoch 40: 4.123072147369385 +01-18 05:33:04 INFO [logging.py:61]: Loss 'acc' on epoch 40: 0.434524804353714 +01-18 05:33:04 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0040 +01-18 05:33:05 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0040/pytorch_model.bin +01-18 05:33:08 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0040/optimizer.bin +01-18 05:33:08 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0040/scheduler.bin +01-18 05:33:08 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0040/sampler.bin +01-18 05:33:08 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0040/sampler_1.bin +01-18 05:33:08 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0040/random_states_0.pkl +01-18 05:33:08 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0040/custom_checkpoint_0.pkl +01-18 05:33:08 INFO [logging.py:61]: ========= Epoch 41 out of 1000 ========= +01-18 05:33:08 INFO [logging.py:61]: Begin training... +01-18 05:50:48 INFO [logging.py:61]: Loss 'loss' on epoch 41: 4.0986762046813965 +01-18 05:50:48 INFO [logging.py:61]: Loss 'acc' on epoch 41: 0.43930166959762573 +01-18 05:50:48 INFO [logging.py:61]: ========= Epoch 42 out of 1000 ========= +01-18 05:50:48 INFO [logging.py:61]: Begin training... +01-18 06:08:28 INFO [logging.py:61]: Loss 'loss' on epoch 42: 4.073613166809082 +01-18 06:08:28 INFO [logging.py:61]: Loss 'acc' on epoch 42: 0.44332027435302734 +01-18 06:08:28 INFO [logging.py:61]: ========= Epoch 43 out of 1000 ========= +01-18 06:08:28 INFO [logging.py:61]: Begin training... +01-18 06:26:08 INFO [logging.py:61]: Loss 'loss' on epoch 43: 4.130711555480957 +01-18 06:26:08 INFO [logging.py:61]: Loss 'acc' on epoch 43: 0.4322890043258667 +01-18 06:26:08 INFO [logging.py:61]: ========= Epoch 44 out of 1000 ========= +01-18 06:26:08 INFO [logging.py:61]: Begin training... +01-18 06:43:48 INFO [logging.py:61]: Loss 'loss' on epoch 44: 4.093181610107422 +01-18 06:43:48 INFO [logging.py:61]: Loss 'acc' on epoch 44: 0.43985316157341003 +01-18 06:43:48 INFO [logging.py:61]: ========= Epoch 45 out of 1000 ========= +01-18 06:43:48 INFO [logging.py:61]: Begin training... +01-18 07:01:29 INFO [logging.py:61]: Loss 'loss' on epoch 45: 4.079413414001465 +01-18 07:01:29 INFO [logging.py:61]: Loss 'acc' on epoch 45: 0.44259190559387207 +01-18 07:01:29 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0045 +01-18 07:01:30 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0045/pytorch_model.bin +01-18 07:01:32 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0045/optimizer.bin +01-18 07:01:32 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0045/scheduler.bin +01-18 07:01:32 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0045/sampler.bin +01-18 07:01:32 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0045/sampler_1.bin +01-18 07:01:32 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0045/random_states_0.pkl +01-18 07:01:32 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0045/custom_checkpoint_0.pkl +01-18 07:01:32 INFO [logging.py:61]: ========= Epoch 46 out of 1000 ========= +01-18 07:01:32 INFO [logging.py:61]: Begin training... +01-18 07:19:12 INFO [logging.py:61]: Loss 'loss' on epoch 46: 4.089560508728027 +01-18 07:19:12 INFO [logging.py:61]: Loss 'acc' on epoch 46: 0.4402327835559845 +01-18 07:19:12 INFO [logging.py:61]: ========= Epoch 47 out of 1000 ========= +01-18 07:19:12 INFO [logging.py:61]: Begin training... +01-18 07:36:52 INFO [logging.py:61]: Loss 'loss' on epoch 47: 4.093177318572998 +01-18 07:36:52 INFO [logging.py:61]: Loss 'acc' on epoch 47: 0.4402003288269043 +01-18 07:36:52 INFO [logging.py:61]: ========= Epoch 48 out of 1000 ========= +01-18 07:36:52 INFO [logging.py:61]: Begin training... +01-18 07:54:33 INFO [logging.py:61]: Loss 'loss' on epoch 48: 4.06325626373291 +01-18 07:54:33 INFO [logging.py:61]: Loss 'acc' on epoch 48: 0.44534069299697876 +01-18 07:54:33 INFO [logging.py:61]: ========= Epoch 49 out of 1000 ========= +01-18 07:54:33 INFO [logging.py:61]: Begin training... +01-18 08:12:13 INFO [logging.py:61]: Loss 'loss' on epoch 49: 4.018030643463135 +01-18 08:12:13 INFO [logging.py:61]: Loss 'acc' on epoch 49: 0.4532480537891388 +01-18 08:12:13 INFO [logging.py:61]: ========= Epoch 50 out of 1000 ========= +01-18 08:12:13 INFO [logging.py:61]: Begin training... +01-18 08:29:54 INFO [logging.py:61]: Loss 'loss' on epoch 50: 4.036403179168701 +01-18 08:29:54 INFO [logging.py:61]: Loss 'acc' on epoch 50: 0.44995203614234924 +01-18 08:29:54 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0050 +01-18 08:29:55 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0050/pytorch_model.bin +01-18 08:29:57 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0050/optimizer.bin +01-18 08:29:57 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0050/scheduler.bin +01-18 08:29:57 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0050/sampler.bin +01-18 08:29:57 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0050/sampler_1.bin +01-18 08:29:57 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0050/random_states_0.pkl +01-18 08:29:57 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0050/custom_checkpoint_0.pkl +01-18 08:29:57 INFO [logging.py:61]: ========= Epoch 51 out of 1000 ========= +01-18 08:29:57 INFO [logging.py:61]: Begin training... +01-18 08:47:38 INFO [logging.py:61]: Loss 'loss' on epoch 51: 3.9912354946136475 +01-18 08:47:38 INFO [logging.py:61]: Loss 'acc' on epoch 51: 0.4573386311531067 +01-18 08:47:38 INFO [logging.py:61]: ========= Epoch 52 out of 1000 ========= +01-18 08:47:38 INFO [logging.py:61]: Begin training... +01-18 09:05:19 INFO [logging.py:61]: Loss 'loss' on epoch 52: 3.988816976547241 +01-18 09:05:19 INFO [logging.py:61]: Loss 'acc' on epoch 52: 0.45804962515830994 +01-18 09:05:19 INFO [logging.py:61]: ========= Epoch 53 out of 1000 ========= +01-18 09:05:19 INFO [logging.py:61]: Begin training... +01-18 09:23:01 INFO [logging.py:61]: Loss 'loss' on epoch 53: 4.000645160675049 +01-18 09:23:01 INFO [logging.py:61]: Loss 'acc' on epoch 53: 0.45582982897758484 +01-18 09:23:01 INFO [logging.py:61]: ========= Epoch 54 out of 1000 ========= +01-18 09:23:01 INFO [logging.py:61]: Begin training... +01-18 09:41:44 INFO [logging.py:61]: Loss 'loss' on epoch 54: 4.001478672027588 +01-18 09:41:44 INFO [logging.py:61]: Loss 'acc' on epoch 54: 0.45487162470817566 +01-18 09:41:44 INFO [logging.py:61]: ========= Epoch 55 out of 1000 ========= +01-18 09:41:44 INFO [logging.py:61]: Begin training... +01-18 10:00:41 INFO [logging.py:61]: Loss 'loss' on epoch 55: 3.906637191772461 +01-18 10:00:41 INFO [logging.py:61]: Loss 'acc' on epoch 55: 0.47240233421325684 +01-18 10:00:41 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0055 +01-18 10:00:42 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0055/pytorch_model.bin +01-18 10:00:44 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0055/optimizer.bin +01-18 10:00:44 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0055/scheduler.bin +01-18 10:00:44 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0055/sampler.bin +01-18 10:00:44 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0055/sampler_1.bin +01-18 10:00:44 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0055/random_states_0.pkl +01-18 10:00:44 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0055/custom_checkpoint_0.pkl +01-18 10:00:44 INFO [logging.py:61]: ========= Epoch 56 out of 1000 ========= +01-18 10:00:44 INFO [logging.py:61]: Begin training... +01-18 10:18:30 INFO [logging.py:61]: Loss 'loss' on epoch 56: 3.9935741424560547 +01-18 10:18:30 INFO [logging.py:61]: Loss 'acc' on epoch 56: 0.45595356822013855 +01-18 10:18:30 INFO [logging.py:61]: ========= Epoch 57 out of 1000 ========= +01-18 10:18:30 INFO [logging.py:61]: Begin training... +01-18 10:36:13 INFO [logging.py:61]: Loss 'loss' on epoch 57: 3.983603000640869 +01-18 10:36:13 INFO [logging.py:61]: Loss 'acc' on epoch 57: 0.45785531401634216 +01-18 10:36:13 INFO [logging.py:61]: ========= Epoch 58 out of 1000 ========= +01-18 10:36:13 INFO [logging.py:61]: Begin training... +01-18 10:53:57 INFO [logging.py:61]: Loss 'loss' on epoch 58: 3.938398838043213 +01-18 10:53:57 INFO [logging.py:61]: Loss 'acc' on epoch 58: 0.4651034474372864 +01-18 10:53:57 INFO [logging.py:61]: ========= Epoch 59 out of 1000 ========= +01-18 10:53:57 INFO [logging.py:61]: Begin training... +01-18 11:11:41 INFO [logging.py:61]: Loss 'loss' on epoch 59: 3.8969380855560303 +01-18 11:11:41 INFO [logging.py:61]: Loss 'acc' on epoch 59: 0.4726952612400055 +01-18 11:11:41 INFO [logging.py:61]: ========= Epoch 60 out of 1000 ========= +01-18 11:11:41 INFO [logging.py:61]: Begin training... +01-18 11:29:24 INFO [logging.py:61]: Loss 'loss' on epoch 60: 3.906097888946533 +01-18 11:29:24 INFO [logging.py:61]: Loss 'acc' on epoch 60: 0.4704975485801697 +01-18 11:29:24 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0060 +01-18 11:29:25 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0060/pytorch_model.bin +01-18 11:29:28 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0060/optimizer.bin +01-18 11:29:28 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0060/scheduler.bin +01-18 11:29:28 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0060/sampler.bin +01-18 11:29:28 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0060/sampler_1.bin +01-18 11:29:28 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0060/random_states_0.pkl +01-18 11:29:28 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0060/custom_checkpoint_0.pkl +01-18 11:29:28 INFO [logging.py:61]: ========= Epoch 61 out of 1000 ========= +01-18 11:29:28 INFO [logging.py:61]: Begin training... +01-18 11:47:10 INFO [logging.py:61]: Loss 'loss' on epoch 61: 3.8588194847106934 +01-18 11:47:10 INFO [logging.py:61]: Loss 'acc' on epoch 61: 0.47811901569366455 +01-18 11:47:10 INFO [logging.py:61]: ========= Epoch 62 out of 1000 ========= +01-18 11:47:10 INFO [logging.py:61]: Begin training... +01-18 12:04:53 INFO [logging.py:61]: Loss 'loss' on epoch 62: 3.9264626502990723 +01-18 12:04:53 INFO [logging.py:61]: Loss 'acc' on epoch 62: 0.46704503893852234 +01-18 12:04:53 INFO [logging.py:61]: ========= Epoch 63 out of 1000 ========= +01-18 12:04:53 INFO [logging.py:61]: Begin training... +01-18 12:22:37 INFO [logging.py:61]: Loss 'loss' on epoch 63: 3.8577334880828857 +01-18 12:22:37 INFO [logging.py:61]: Loss 'acc' on epoch 63: 0.47825607657432556 +01-18 12:22:37 INFO [logging.py:61]: ========= Epoch 64 out of 1000 ========= +01-18 12:22:37 INFO [logging.py:61]: Begin training... +01-18 12:40:28 INFO [logging.py:61]: Loss 'loss' on epoch 64: 3.9257748126983643 +01-18 12:40:28 INFO [logging.py:61]: Loss 'acc' on epoch 64: 0.4667837619781494 +01-18 12:40:28 INFO [logging.py:61]: ========= Epoch 65 out of 1000 ========= +01-18 12:40:28 INFO [logging.py:61]: Begin training... +01-18 12:58:33 INFO [logging.py:61]: Loss 'loss' on epoch 65: 3.896535873413086 +01-18 12:58:33 INFO [logging.py:61]: Loss 'acc' on epoch 65: 0.47135165333747864 +01-18 12:58:33 INFO [logging.py:61]: Saving current state to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0065 +01-18 12:58:35 INFO [logging.py:61]: Model weights saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0065/pytorch_model.bin +01-18 12:58:37 INFO [logging.py:61]: Optimizer state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0065/optimizer.bin +01-18 12:58:37 INFO [logging.py:61]: Scheduler state saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0065/scheduler.bin +01-18 12:58:37 INFO [logging.py:61]: Sampler state for dataloader 0 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0065/sampler.bin +01-18 12:58:37 INFO [logging.py:61]: Sampler state for dataloader 1 saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0065/sampler_1.bin +01-18 12:58:37 INFO [logging.py:61]: Random states saved in /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0065/random_states_0.pkl +01-18 12:58:37 INFO [logging.py:61]: Saving the state of TrainerState to /fred/oz325/xhao/proj/audiozen/recipes/librimix_sot/tokenizer_separation/exp/swin_default_LR1e-5_addEOSToken/checkpoints/epoch_0065/custom_checkpoint_0.pkl +01-18 12:58:37 INFO [logging.py:61]: ========= Epoch 66 out of 1000 ========= +01-18 12:58:37 INFO [logging.py:61]: Begin training... +01-18 13:18:05 INFO [logging.py:61]: Loss 'loss' on epoch 66: 3.849663257598877 +01-18 13:18:05 INFO [logging.py:61]: Loss 'acc' on epoch 66: 0.4791968762874603 +01-18 13:18:05 INFO [logging.py:61]: ========= Epoch 67 out of 1000 ========= +01-18 13:18:05 INFO [logging.py:61]: Begin training... +01-18 13:35:49 INFO [logging.py:61]: Loss 'loss' on epoch 67: 3.918121099472046 +01-18 13:35:49 INFO [logging.py:61]: Loss 'acc' on epoch 67: 0.46780335903167725 +01-18 13:35:49 INFO [logging.py:61]: ========= Epoch 68 out of 1000 ========= +01-18 13:35:49 INFO [logging.py:61]: Begin training... +01-18 13:53:32 INFO [logging.py:61]: Loss 'loss' on epoch 68: 3.7919538021087646 +01-18 13:53:32 INFO [logging.py:61]: Loss 'acc' on epoch 68: 0.4886992275714874 +01-18 13:53:32 INFO [logging.py:61]: ========= Epoch 69 out of 1000 ========= +01-18 13:53:32 INFO [logging.py:61]: Begin training... +01-18 14:11:16 INFO [logging.py:61]: Loss 'loss' on epoch 69: 3.7983150482177734 +01-18 14:11:16 INFO [logging.py:61]: Loss 'acc' on epoch 69: 0.4886568486690521 +01-18 14:11:16 INFO [logging.py:61]: ========= Epoch 70 out of 1000 ========= +01-18 14:11:16 INFO [logging.py:61]: Begin training... diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601821.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601821.0 new file mode 100644 index 0000000000000000000000000000000000000000..e9fcbc0adcdeb1333ba79f366f507621441747b0 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601821.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70d24324393a4775d5a9276d1b89017677cbf9773410a22871bddeebe2db8bce +size 27796 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601822.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601822.0 new file mode 100644 index 0000000000000000000000000000000000000000..997b1f4db9476b956897ed6f157d585af68fb924 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601822.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:739fe5c57dcc14ce4d1f86ae39b20a583c647350fe3681973798b1e0ed2afbff +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601823.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601823.0 new file mode 100644 index 0000000000000000000000000000000000000000..70e345f0f9fa8b2e1457fade6ef10425b21338d6 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601823.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fe179b2f97f708a607a3536d83a92f7f8e3872a8454b7aafe4f47b810297504 +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601824.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601824.0 new file mode 100644 index 0000000000000000000000000000000000000000..1e4df9b9ec064a18d91e417d55739396a1c1b9ac --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705453038.gina301.601824.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cae8bc21afc1145386ad62bfb4ca61f65d43f488e90700a718cf2774e2e9a47b +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743187.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743187.0 new file mode 100644 index 0000000000000000000000000000000000000000..1c045827ae7996ae01b13fdbff25a9f746cf12c9 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743187.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:446777fdc9344254a80b786c60dd3d84a3c94e832753bb1f8c2924acc0782c67 +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743188.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743188.0 new file mode 100644 index 0000000000000000000000000000000000000000..93bf5ae39228b917b8e2d282d8365966e7f11356 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743188.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9cb89d43025032e39d7d3bbfe2479cac51ed80e28ca8666447ff6792b4b2889 +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743189.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743189.0 new file mode 100644 index 0000000000000000000000000000000000000000..f92883744ecd24b86d86f0f927a0f473b2a5363a --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743189.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb65a6bfca21a842db5fffdae9b775634e7635f14000d7d1aa1e95cced16070f +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743190.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743190.0 new file mode 100644 index 0000000000000000000000000000000000000000..3ffc9c97a4a43582e16004f4f7f9103ff02906e6 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705472223.gina301.743190.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba5ff377fd716ef4e463ed0664a260fdb31a36b723b665b07163c18ff0f8ca45 +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743763.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743763.0 new file mode 100644 index 0000000000000000000000000000000000000000..459f267136df711b74c347e3c672949fc5df9cf2 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743763.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ef26291fa83389e72c9dd1738eb12bc850d26ccb7e16a6531d4ddbdeb36e0db +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743764.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743764.0 new file mode 100644 index 0000000000000000000000000000000000000000..c45b45925d2e8d4a8654d0b9451b23f883fc80b4 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743764.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05575c42448223290ef0946ca031b8f92b9c8d12ce158160d984e2c761c8d3e6 +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743765.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743765.0 new file mode 100644 index 0000000000000000000000000000000000000000..0f77bb3fcf50fb21e6a19ca900e889198bb1f477 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743765.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08fa7aedb60fc43205432825fcd2fef01eb926a1878ca92581c381a3a7f3e87d +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743766.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743766.0 new file mode 100644 index 0000000000000000000000000000000000000000..2f5b862f1a85242869483a92b5ac64f64194f006 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473326.gina301.743766.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6cd84b3c9a668592d795025522bfccf67b13d1f71ccbddb339cb1ca8d182f37 +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743836.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743836.0 new file mode 100644 index 0000000000000000000000000000000000000000..a19f35a313f82ca4e72ef76e150875be9fd7aca5 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743836.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a98133ea72656d9c056ab9dd7881f50e510e6c65f8ef5e108665411f67461eb +size 1913362 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743837.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743837.0 new file mode 100644 index 0000000000000000000000000000000000000000..3ca1795801e96f47dd180efa54b277d641739a17 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743837.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5a47795ee3ce60a51f6a3853de979488bb1d996812b73d20be52b792d5696ae +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743838.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743838.0 new file mode 100644 index 0000000000000000000000000000000000000000..452171c58bbd7d146cd4f414d95a01463778b029 --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743838.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28bee077c3cc27d36520d762d159919572bca4614532b0f0e4be6a5188bf27e0 +size 2168 diff --git a/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743839.0 b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743839.0 new file mode 100644 index 0000000000000000000000000000000000000000..e1ca864a2e3a0a50f049112e377db744b56c370b --- /dev/null +++ b/swin_default_LR1e-5_addEOSToken/tb_log/events.out.tfevents.1705473365.gina301.743839.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaba4a9815c964dac69d8f32ec1422260615c0a82774790f90528fa438c21a46 +size 2168