Logs step 5000

logs/train.log

@@ -1,13 +1,13 @@
-26-01-31 02:58:51.045 - INFO:   name: DEBUG_TEST_RUN
+26-01-31 03:15:09.770 - INFO:   name: ICPR_LPDiff
   phase: train
   gpu_ids: [0]
   path:[
-    log: experiments/DEBUG_TEST_RUN_260131_025851/logs
-    tb_logger: experiments/DEBUG_TEST_RUN_260131_025851/tb_logger
-    results: experiments/DEBUG_TEST_RUN_260131_025851/results
-    checkpoint: experiments/DEBUG_TEST_RUN_260131_025851/checkpoint
+    log: experiments/ICPR_LPDiff_260131_031509/logs
+    tb_logger: experiments/ICPR_LPDiff_260131_031509/tb_logger
+    results: experiments/ICPR_LPDiff_260131_031509/results
+    checkpoint: experiments/ICPR_LPDiff_260131_031509/checkpoint
     resume_state: None
-    experiments_root: experiments/DEBUG_TEST_RUN_260131_025851
+    experiments_root: experiments/ICPR_LPDiff_260131_031509
   ]
   datasets:[
     train:[
@@ -16,8 +16,8 @@
       dataroot: ./dataset/train
       width: 128
       height: 64
-      batch_size: 2
-      num_workers: 0
+      batch_size: 64
+      num_workers: 4
       use_shuffle: True
     ]
     val:[
@@ -67,10 +67,10 @@
     use_prerain_MTA: False
     resume_training: False
     MTA: ./best_377.pt
-    n_iter: 20
-    val_freq: 10
-    save_checkpoint_freq: 10
-    print_freq: 2
+    n_iter: 200000
+    val_freq: 5000
+    save_checkpoint_freq: 5000
+    print_freq: 100
     optimizer:[
       type: adam
       lr: 0.0001
@@ -84,32 +84,65 @@
   log_eval: False
   enable_wandb: False
 
-26-01-31 02:59:02.275 - INFO: Dataset [ICPRDataset - ICPR_Train] is created.
-26-01-31 02:59:07.513 - INFO: Dataset [ICPRDataset - ICPR_Val] is created.
-26-01-31 02:59:07.513 - INFO: Initial Dataset Finished
-26-01-31 02:59:08.877 - INFO: Initialization method [orthogonal]
-26-01-31 02:59:13.684 - INFO: Model [DDPM] is created.
-26-01-31 02:59:13.684 - INFO: Initial Model Finished
-26-01-31 02:59:27.083 - INFO: <epoch:  1, iter:       2> l_pix: 8.9346e-01 
-26-01-31 02:59:27.440 - INFO: <epoch:  1, iter:       4> l_pix: 7.9007e-01 
-26-01-31 02:59:27.682 - INFO: <epoch:  1, iter:       6> l_pix: 8.2409e-01 
-26-01-31 02:59:27.992 - INFO: <epoch:  1, iter:       8> l_pix: 7.5086e-01 
-26-01-31 02:59:28.262 - INFO: <epoch:  1, iter:      10> l_pix: 7.5090e-01 
-26-01-31 03:00:06.459 - INFO: Saved model in [experiments/DEBUG_TEST_RUN_260131_025851/checkpoint/I10_E1_gen_best.pth] ...
-26-01-31 03:00:06.461 - INFO: # Validation # PSNR: 7.0245e+00
-26-01-31 03:00:06.461 - INFO: <epoch:  1, iter:      10> psnr: 7.0245e+00 loss: 9.6489e-01
-26-01-31 03:00:06.462 - INFO: Saving models and training states.
-26-01-31 03:00:08.018 - INFO: Saved model in [experiments/DEBUG_TEST_RUN_260131_025851/checkpoint/I10_E1_gen.pth] ...
-26-01-31 03:00:08.019 - INFO: ☁️ Syncing to Hugging Face: PuxAI/ICPR...
-26-01-31 03:00:25.148 - INFO: ✅ Upload ALL (ckpt, results, logs) successfully!
-26-01-31 03:00:25.531 - INFO: <epoch:  1, iter:      12> l_pix: 7.4212e-01 
-26-01-31 03:00:25.771 - INFO: <epoch:  1, iter:      14> l_pix: 7.5824e-01 
-26-01-31 03:00:25.986 - INFO: <epoch:  1, iter:      16> l_pix: 6.9023e-01 
-26-01-31 03:00:26.214 - INFO: <epoch:  1, iter:      18> l_pix: 8.4582e-01 
-26-01-31 03:00:26.451 - INFO: <epoch:  1, iter:      20> l_pix: 6.1488e-01 
-26-01-31 03:01:02.890 - INFO: Saved model in [experiments/DEBUG_TEST_RUN_260131_025851/checkpoint/I20_E1_gen_best_psnr.pth] ...
-26-01-31 03:01:02.896 - INFO: # Validation # PSNR: 7.1945e+00
-26-01-31 03:01:02.897 - INFO: <epoch:  1, iter:      20> psnr: 7.1945e+00 loss: 1.0166e+00
-26-01-31 03:01:02.898 - INFO: Saving models and training states.
-26-01-31 03:01:04.507 - INFO: Saved model in [experiments/DEBUG_TEST_RUN_260131_025851/checkpoint/I20_E1_gen.pth] ...
-26-01-31 03:01:04.509 - INFO: ☁️ Syncing to Hugging Face: PuxAI/ICPR...
+26-01-31 03:15:16.252 - INFO: Dataset [ICPRDataset - ICPR_Train] is created.
+26-01-31 03:15:20.408 - INFO: Dataset [ICPRDataset - ICPR_Val] is created.
+26-01-31 03:15:20.409 - INFO: Initial Dataset Finished
+26-01-31 03:15:21.210 - INFO: Initialization method [orthogonal]
+26-01-31 03:15:24.323 - INFO: Model [DDPM] is created.
+26-01-31 03:15:24.323 - INFO: Initial Model Finished
+26-01-31 03:16:41.666 - INFO: <epoch:  1, iter:     100> l_pix: 2.2826e-01 
+26-01-31 03:17:48.179 - INFO: <epoch:  1, iter:     200> l_pix: 5.3603e-01 
+26-01-31 03:18:54.678 - INFO: <epoch:  1, iter:     300> l_pix: 1.5491e-01 
+26-01-31 03:20:07.721 - INFO: <epoch:  2, iter:     400> l_pix: 3.1760e-01 
+26-01-31 03:21:14.231 - INFO: <epoch:  2, iter:     500> l_pix: 1.2511e-01 
+26-01-31 03:22:20.735 - INFO: <epoch:  2, iter:     600> l_pix: 1.2457e-01 
+26-01-31 03:23:28.032 - INFO: <epoch:  3, iter:     700> l_pix: 1.2696e-01 
+26-01-31 03:24:34.528 - INFO: <epoch:  3, iter:     800> l_pix: 1.0346e-01 
+26-01-31 03:25:41.033 - INFO: <epoch:  3, iter:     900> l_pix: 2.0017e-01 
+26-01-31 03:26:48.427 - INFO: <epoch:  4, iter:   1,000> l_pix: 1.1922e-01 
+26-01-31 03:27:54.969 - INFO: <epoch:  4, iter:   1,100> l_pix: 1.1436e-01 
+26-01-31 03:29:01.514 - INFO: <epoch:  4, iter:   1,200> l_pix: 1.0180e-01 
+26-01-31 03:30:09.371 - INFO: <epoch:  5, iter:   1,300> l_pix: 1.2339e-01 
+26-01-31 03:31:15.905 - INFO: <epoch:  5, iter:   1,400> l_pix: 1.0645e-01 
+26-01-31 03:32:22.408 - INFO: <epoch:  5, iter:   1,500> l_pix: 7.4137e-02 
+26-01-31 03:33:29.789 - INFO: <epoch:  6, iter:   1,600> l_pix: 8.0976e-02 
+26-01-31 03:34:36.324 - INFO: <epoch:  6, iter:   1,700> l_pix: 7.3406e-02 
+26-01-31 03:35:42.857 - INFO: <epoch:  6, iter:   1,800> l_pix: 2.5007e-01 
+26-01-31 03:36:50.158 - INFO: <epoch:  7, iter:   1,900> l_pix: 1.5067e-01 
+26-01-31 03:37:56.813 - INFO: <epoch:  7, iter:   2,000> l_pix: 7.6723e-02 
+26-01-31 03:39:03.362 - INFO: <epoch:  7, iter:   2,100> l_pix: 2.0787e-01 
+26-01-31 03:40:10.669 - INFO: <epoch:  8, iter:   2,200> l_pix: 7.4226e-02 
+26-01-31 03:41:17.231 - INFO: <epoch:  8, iter:   2,300> l_pix: 1.7326e-01 
+26-01-31 03:42:23.768 - INFO: <epoch:  8, iter:   2,400> l_pix: 4.5413e-01 
+26-01-31 03:43:30.352 - INFO: <epoch:  8, iter:   2,500> l_pix: 1.0588e-01 
+26-01-31 03:44:37.726 - INFO: <epoch:  9, iter:   2,600> l_pix: 1.0587e-01 
+26-01-31 03:45:44.255 - INFO: <epoch:  9, iter:   2,700> l_pix: 9.5614e-02 
+26-01-31 03:46:50.780 - INFO: <epoch:  9, iter:   2,800> l_pix: 9.8116e-02 
+26-01-31 03:47:58.156 - INFO: <epoch: 10, iter:   2,900> l_pix: 1.0461e-01 
+26-01-31 03:49:04.682 - INFO: <epoch: 10, iter:   3,000> l_pix: 1.7086e-01 
+26-01-31 03:50:11.204 - INFO: <epoch: 10, iter:   3,100> l_pix: 5.3971e-01 
+26-01-31 03:51:18.530 - INFO: <epoch: 11, iter:   3,200> l_pix: 9.3860e-02 
+26-01-31 03:52:25.054 - INFO: <epoch: 11, iter:   3,300> l_pix: 1.2320e-01 
+26-01-31 03:53:31.554 - INFO: <epoch: 11, iter:   3,400> l_pix: 4.6803e-02 
+26-01-31 03:54:38.969 - INFO: <epoch: 12, iter:   3,500> l_pix: 1.8803e-01 
+26-01-31 03:55:45.490 - INFO: <epoch: 12, iter:   3,600> l_pix: 5.8979e-02 
+26-01-31 03:56:52.023 - INFO: <epoch: 12, iter:   3,700> l_pix: 6.7843e-02 
+26-01-31 03:57:59.312 - INFO: <epoch: 13, iter:   3,800> l_pix: 4.9542e-02 
+26-01-31 03:59:05.828 - INFO: <epoch: 13, iter:   3,900> l_pix: 4.5841e-02 
+26-01-31 04:00:12.363 - INFO: <epoch: 13, iter:   4,000> l_pix: 6.6576e-02 
+26-01-31 04:01:19.674 - INFO: <epoch: 14, iter:   4,100> l_pix: 1.3931e-01 
+26-01-31 04:02:26.199 - INFO: <epoch: 14, iter:   4,200> l_pix: 8.2890e-02 
+26-01-31 04:03:32.711 - INFO: <epoch: 14, iter:   4,300> l_pix: 6.3951e-02 
+26-01-31 04:04:40.038 - INFO: <epoch: 15, iter:   4,400> l_pix: 1.7432e-01 
+26-01-31 04:05:46.572 - INFO: <epoch: 15, iter:   4,500> l_pix: 5.3195e-02 
+26-01-31 04:06:53.146 - INFO: <epoch: 15, iter:   4,600> l_pix: 6.9828e-02 
+26-01-31 04:08:00.672 - INFO: <epoch: 16, iter:   4,700> l_pix: 1.2374e-01 
+26-01-31 04:09:07.293 - INFO: <epoch: 16, iter:   4,800> l_pix: 9.9227e-02 
+26-01-31 04:10:13.823 - INFO: <epoch: 16, iter:   4,900> l_pix: 3.9694e-02 
+26-01-31 04:11:20.342 - INFO: <epoch: 16, iter:   5,000> l_pix: 5.2206e-02 
+26-01-31 04:11:50.978 - INFO: Saved model in [experiments/ICPR_LPDiff_260131_031509/checkpoint/I5000_E16_gen_best.pth] ...
+26-01-31 04:11:50.980 - INFO: # Validation # PSNR: 1.0103e+01
+26-01-31 04:11:50.980 - INFO: <epoch: 16, iter:   5,000> psnr: 1.0103e+01 loss: 4.5005e-01
+26-01-31 04:11:50.981 - INFO: Saving models and training states.
+26-01-31 04:11:51.936 - INFO: Saved model in [experiments/ICPR_LPDiff_260131_031509/checkpoint/I5000_E16_gen.pth] ...
+26-01-31 04:11:51.936 - INFO: ☁️ Syncing to Hugging Face: PuxAI/ICPR...

logs/val.log

@@ -1,2 +1 @@
-26-01-31 03:00:06.461 - INFO: <epoch:  1, iter:      10> psnr: 7.0245e+00 loss: 9.6489e-01
-26-01-31 03:01:02.897 - INFO: <epoch:  1, iter:      20> psnr: 7.1945e+00 loss: 1.0166e+00
+26-01-31 04:11:50.980 - INFO: <epoch: 16, iter:   5,000> psnr: 1.0103e+01 loss: 4.5005e-01