Logs step 1000

logs/train.log

@@ -1,13 +1,13 @@
-26-01-31 03:15:09.770 - INFO:   name: ICPR_LPDiff
+26-01-31 05:13:06.366 - INFO:   name: ICPR_LPDiff
   phase: train
   gpu_ids: [0]
   path:[
-    log: experiments/ICPR_LPDiff_260131_031509/logs
-    tb_logger: experiments/ICPR_LPDiff_260131_031509/tb_logger
-    results: experiments/ICPR_LPDiff_260131_031509/results
-    checkpoint: experiments/ICPR_LPDiff_260131_031509/checkpoint
+    log: experiments/ICPR_LPDiff_260131_051306/logs
+    tb_logger: experiments/ICPR_LPDiff_260131_051306/tb_logger
+    results: experiments/ICPR_LPDiff_260131_051306/results
+    checkpoint: experiments/ICPR_LPDiff_260131_051306/checkpoint
     resume_state: None
-    experiments_root: experiments/ICPR_LPDiff_260131_031509
+    experiments_root: experiments/ICPR_LPDiff_260131_051306
   ]
   datasets:[
     train:[
@@ -16,8 +16,8 @@
       dataroot: ./dataset/train
       width: 128
       height: 64
-      batch_size: 64
-      num_workers: 4
+      batch_size: 512
+      num_workers: 16
       use_shuffle: True
     ]
     val:[
@@ -67,13 +67,13 @@
     use_prerain_MTA: False
     resume_training: False
     MTA: ./best_377.pt
-    n_iter: 200000
-    val_freq: 5000
-    save_checkpoint_freq: 5000
-    print_freq: 100
+    n_iter: 25000
+    val_freq: 1000
+    save_checkpoint_freq: 1000
+    print_freq: 10
     optimizer:[
       type: adam
-      lr: 0.0001
+      lr: 0.0003
     ]
   ]
   wandb:[
@@ -84,65 +84,116 @@
   log_eval: False
   enable_wandb: False
 
-26-01-31 03:15:16.252 - INFO: Dataset [ICPRDataset - ICPR_Train] is created.
-26-01-31 03:15:20.408 - INFO: Dataset [ICPRDataset - ICPR_Val] is created.
-26-01-31 03:15:20.409 - INFO: Initial Dataset Finished
-26-01-31 03:15:21.210 - INFO: Initialization method [orthogonal]
-26-01-31 03:15:24.323 - INFO: Model [DDPM] is created.
-26-01-31 03:15:24.323 - INFO: Initial Model Finished
-26-01-31 03:16:41.666 - INFO: <epoch:  1, iter:     100> l_pix: 2.2826e-01 
-26-01-31 03:17:48.179 - INFO: <epoch:  1, iter:     200> l_pix: 5.3603e-01 
-26-01-31 03:18:54.678 - INFO: <epoch:  1, iter:     300> l_pix: 1.5491e-01 
-26-01-31 03:20:07.721 - INFO: <epoch:  2, iter:     400> l_pix: 3.1760e-01 
-26-01-31 03:21:14.231 - INFO: <epoch:  2, iter:     500> l_pix: 1.2511e-01 
-26-01-31 03:22:20.735 - INFO: <epoch:  2, iter:     600> l_pix: 1.2457e-01 
-26-01-31 03:23:28.032 - INFO: <epoch:  3, iter:     700> l_pix: 1.2696e-01 
-26-01-31 03:24:34.528 - INFO: <epoch:  3, iter:     800> l_pix: 1.0346e-01 
-26-01-31 03:25:41.033 - INFO: <epoch:  3, iter:     900> l_pix: 2.0017e-01 
-26-01-31 03:26:48.427 - INFO: <epoch:  4, iter:   1,000> l_pix: 1.1922e-01 
-26-01-31 03:27:54.969 - INFO: <epoch:  4, iter:   1,100> l_pix: 1.1436e-01 
-26-01-31 03:29:01.514 - INFO: <epoch:  4, iter:   1,200> l_pix: 1.0180e-01 
-26-01-31 03:30:09.371 - INFO: <epoch:  5, iter:   1,300> l_pix: 1.2339e-01 
-26-01-31 03:31:15.905 - INFO: <epoch:  5, iter:   1,400> l_pix: 1.0645e-01 
-26-01-31 03:32:22.408 - INFO: <epoch:  5, iter:   1,500> l_pix: 7.4137e-02 
-26-01-31 03:33:29.789 - INFO: <epoch:  6, iter:   1,600> l_pix: 8.0976e-02 
-26-01-31 03:34:36.324 - INFO: <epoch:  6, iter:   1,700> l_pix: 7.3406e-02 
-26-01-31 03:35:42.857 - INFO: <epoch:  6, iter:   1,800> l_pix: 2.5007e-01 
-26-01-31 03:36:50.158 - INFO: <epoch:  7, iter:   1,900> l_pix: 1.5067e-01 
-26-01-31 03:37:56.813 - INFO: <epoch:  7, iter:   2,000> l_pix: 7.6723e-02 
-26-01-31 03:39:03.362 - INFO: <epoch:  7, iter:   2,100> l_pix: 2.0787e-01 
-26-01-31 03:40:10.669 - INFO: <epoch:  8, iter:   2,200> l_pix: 7.4226e-02 
-26-01-31 03:41:17.231 - INFO: <epoch:  8, iter:   2,300> l_pix: 1.7326e-01 
-26-01-31 03:42:23.768 - INFO: <epoch:  8, iter:   2,400> l_pix: 4.5413e-01 
-26-01-31 03:43:30.352 - INFO: <epoch:  8, iter:   2,500> l_pix: 1.0588e-01 
-26-01-31 03:44:37.726 - INFO: <epoch:  9, iter:   2,600> l_pix: 1.0587e-01 
-26-01-31 03:45:44.255 - INFO: <epoch:  9, iter:   2,700> l_pix: 9.5614e-02 
-26-01-31 03:46:50.780 - INFO: <epoch:  9, iter:   2,800> l_pix: 9.8116e-02 
-26-01-31 03:47:58.156 - INFO: <epoch: 10, iter:   2,900> l_pix: 1.0461e-01 
-26-01-31 03:49:04.682 - INFO: <epoch: 10, iter:   3,000> l_pix: 1.7086e-01 
-26-01-31 03:50:11.204 - INFO: <epoch: 10, iter:   3,100> l_pix: 5.3971e-01 
-26-01-31 03:51:18.530 - INFO: <epoch: 11, iter:   3,200> l_pix: 9.3860e-02 
-26-01-31 03:52:25.054 - INFO: <epoch: 11, iter:   3,300> l_pix: 1.2320e-01 
-26-01-31 03:53:31.554 - INFO: <epoch: 11, iter:   3,400> l_pix: 4.6803e-02 
-26-01-31 03:54:38.969 - INFO: <epoch: 12, iter:   3,500> l_pix: 1.8803e-01 
-26-01-31 03:55:45.490 - INFO: <epoch: 12, iter:   3,600> l_pix: 5.8979e-02 
-26-01-31 03:56:52.023 - INFO: <epoch: 12, iter:   3,700> l_pix: 6.7843e-02 
-26-01-31 03:57:59.312 - INFO: <epoch: 13, iter:   3,800> l_pix: 4.9542e-02 
-26-01-31 03:59:05.828 - INFO: <epoch: 13, iter:   3,900> l_pix: 4.5841e-02 
-26-01-31 04:00:12.363 - INFO: <epoch: 13, iter:   4,000> l_pix: 6.6576e-02 
-26-01-31 04:01:19.674 - INFO: <epoch: 14, iter:   4,100> l_pix: 1.3931e-01 
-26-01-31 04:02:26.199 - INFO: <epoch: 14, iter:   4,200> l_pix: 8.2890e-02 
-26-01-31 04:03:32.711 - INFO: <epoch: 14, iter:   4,300> l_pix: 6.3951e-02 
-26-01-31 04:04:40.038 - INFO: <epoch: 15, iter:   4,400> l_pix: 1.7432e-01 
-26-01-31 04:05:46.572 - INFO: <epoch: 15, iter:   4,500> l_pix: 5.3195e-02 
-26-01-31 04:06:53.146 - INFO: <epoch: 15, iter:   4,600> l_pix: 6.9828e-02 
-26-01-31 04:08:00.672 - INFO: <epoch: 16, iter:   4,700> l_pix: 1.2374e-01 
-26-01-31 04:09:07.293 - INFO: <epoch: 16, iter:   4,800> l_pix: 9.9227e-02 
-26-01-31 04:10:13.823 - INFO: <epoch: 16, iter:   4,900> l_pix: 3.9694e-02 
-26-01-31 04:11:20.342 - INFO: <epoch: 16, iter:   5,000> l_pix: 5.2206e-02 
-26-01-31 04:11:50.978 - INFO: Saved model in [experiments/ICPR_LPDiff_260131_031509/checkpoint/I5000_E16_gen_best.pth] ...
-26-01-31 04:11:50.980 - INFO: # Validation # PSNR: 1.0103e+01
-26-01-31 04:11:50.980 - INFO: <epoch: 16, iter:   5,000> psnr: 1.0103e+01 loss: 4.5005e-01
-26-01-31 04:11:50.981 - INFO: Saving models and training states.
-26-01-31 04:11:51.936 - INFO: Saved model in [experiments/ICPR_LPDiff_260131_031509/checkpoint/I5000_E16_gen.pth] ...
-26-01-31 04:11:51.936 - INFO: ☁️ Syncing to Hugging Face: PuxAI/ICPR...
+26-01-31 05:13:10.880 - INFO: Dataset [ICPRDataset - ICPR_Train] is created.
+26-01-31 05:13:12.637 - INFO: Dataset [ICPRDataset - ICPR_Val] is created.
+26-01-31 05:13:12.637 - INFO: Initial Dataset Finished
+26-01-31 05:13:13.149 - INFO: generated new fontManager
+26-01-31 05:13:13.904 - INFO: Initialization method [orthogonal]
+26-01-31 05:13:16.054 - INFO: Model [DDPM] is created.
+26-01-31 05:13:16.054 - INFO: Initial Model Finished
+26-01-31 05:13:38.746 - INFO: <epoch:  1, iter:      10> l_pix: 6.7545e-01 
+26-01-31 05:13:43.620 - INFO: <epoch:  1, iter:      20> l_pix: 6.4135e-01 
+26-01-31 05:13:48.493 - INFO: <epoch:  1, iter:      30> l_pix: 3.0415e-01 
+26-01-31 05:13:54.327 - INFO: <epoch:  1, iter:      40> l_pix: 3.0359e-01 
+26-01-31 05:14:07.364 - INFO: <epoch:  2, iter:      50> l_pix: 2.6254e-01 
+26-01-31 05:14:12.243 - INFO: <epoch:  2, iter:      60> l_pix: 7.8696e-01 
+26-01-31 05:14:17.120 - INFO: <epoch:  2, iter:      70> l_pix: 2.7833e-01 
+26-01-31 05:14:21.595 - INFO: <epoch:  2, iter:      80> l_pix: 3.5149e-01 
+26-01-31 05:14:34.241 - INFO: <epoch:  3, iter:      90> l_pix: 2.2338e-01 
+26-01-31 05:14:39.118 - INFO: <epoch:  3, iter:     100> l_pix: 1.9077e-01 
+26-01-31 05:14:43.994 - INFO: <epoch:  3, iter:     110> l_pix: 2.2088e-01 
+26-01-31 05:14:48.438 - INFO: <epoch:  3, iter:     120> l_pix: 1.7783e-01 
+26-01-31 05:14:59.862 - INFO: <epoch:  4, iter:     130> l_pix: 4.1824e-01 
+26-01-31 05:15:04.743 - INFO: <epoch:  4, iter:     140> l_pix: 1.8410e-01 
+26-01-31 05:15:09.613 - INFO: <epoch:  4, iter:     150> l_pix: 2.4388e-01 
+26-01-31 05:15:14.050 - INFO: <epoch:  4, iter:     160> l_pix: 2.2145e-01 
+26-01-31 05:15:26.347 - INFO: <epoch:  5, iter:     170> l_pix: 1.5214e-01 
+26-01-31 05:15:31.223 - INFO: <epoch:  5, iter:     180> l_pix: 1.3835e-01 
+26-01-31 05:15:36.092 - INFO: <epoch:  5, iter:     190> l_pix: 1.3899e-01 
+26-01-31 05:15:40.535 - INFO: <epoch:  5, iter:     200> l_pix: 1.5627e-01 
+26-01-31 05:15:53.390 - INFO: <epoch:  6, iter:     210> l_pix: 1.3777e-01 
+26-01-31 05:15:58.264 - INFO: <epoch:  6, iter:     220> l_pix: 1.4471e-01 
+26-01-31 05:16:03.135 - INFO: <epoch:  6, iter:     230> l_pix: 1.4115e-01 
+26-01-31 05:16:07.573 - INFO: <epoch:  6, iter:     240> l_pix: 5.5478e-01 
+26-01-31 05:16:20.869 - INFO: <epoch:  7, iter:     250> l_pix: 1.8693e-01 
+26-01-31 05:16:25.743 - INFO: <epoch:  7, iter:     260> l_pix: 2.9627e-01 
+26-01-31 05:16:30.613 - INFO: <epoch:  7, iter:     270> l_pix: 3.2996e-01 
+26-01-31 05:16:35.055 - INFO: <epoch:  7, iter:     280> l_pix: 1.1577e-01 
+26-01-31 05:16:48.827 - INFO: <epoch:  8, iter:     290> l_pix: 1.6427e-01 
+26-01-31 05:16:53.700 - INFO: <epoch:  8, iter:     300> l_pix: 1.4078e-01 
+26-01-31 05:16:58.573 - INFO: <epoch:  8, iter:     310> l_pix: 1.4264e-01 
+26-01-31 05:17:03.018 - INFO: <epoch:  8, iter:     320> l_pix: 1.3056e-01 
+26-01-31 05:17:15.294 - INFO: <epoch:  9, iter:     330> l_pix: 1.2294e-01 
+26-01-31 05:17:20.173 - INFO: <epoch:  9, iter:     340> l_pix: 1.3783e-01 
+26-01-31 05:17:25.042 - INFO: <epoch:  9, iter:     350> l_pix: 1.6669e-01 
+26-01-31 05:17:29.483 - INFO: <epoch:  9, iter:     360> l_pix: 1.0752e-01 
+26-01-31 05:17:41.939 - INFO: <epoch: 10, iter:     370> l_pix: 2.3283e-01 
+26-01-31 05:17:46.814 - INFO: <epoch: 10, iter:     380> l_pix: 1.3487e-01 
+26-01-31 05:17:51.688 - INFO: <epoch: 10, iter:     390> l_pix: 1.2503e-01 
+26-01-31 05:17:56.127 - INFO: <epoch: 10, iter:     400> l_pix: 1.3095e-01 
+26-01-31 05:18:07.571 - INFO: <epoch: 11, iter:     410> l_pix: 1.4160e-01 
+26-01-31 05:18:12.448 - INFO: <epoch: 11, iter:     420> l_pix: 1.0003e-01 
+26-01-31 05:18:17.321 - INFO: <epoch: 11, iter:     430> l_pix: 1.4267e-01 
+26-01-31 05:18:21.758 - INFO: <epoch: 11, iter:     440> l_pix: 1.1571e-01 
+26-01-31 05:18:33.722 - INFO: <epoch: 12, iter:     450> l_pix: 1.9631e-01 
+26-01-31 05:18:38.598 - INFO: <epoch: 12, iter:     460> l_pix: 1.1798e-01 
+26-01-31 05:18:43.467 - INFO: <epoch: 12, iter:     470> l_pix: 1.2607e-01 
+26-01-31 05:18:47.906 - INFO: <epoch: 12, iter:     480> l_pix: 1.3157e-01 
+26-01-31 05:19:01.007 - INFO: <epoch: 13, iter:     490> l_pix: 1.1176e-01 
+26-01-31 05:19:05.880 - INFO: <epoch: 13, iter:     500> l_pix: 1.1281e-01 
+26-01-31 05:19:10.751 - INFO: <epoch: 13, iter:     510> l_pix: 1.3244e-01 
+26-01-31 05:19:15.196 - INFO: <epoch: 13, iter:     520> l_pix: 9.9414e-02 
+26-01-31 05:19:27.527 - INFO: <epoch: 14, iter:     530> l_pix: 1.0011e-01 
+26-01-31 05:19:32.406 - INFO: <epoch: 14, iter:     540> l_pix: 1.1723e-01 
+26-01-31 05:19:37.276 - INFO: <epoch: 14, iter:     550> l_pix: 1.5789e-01 
+26-01-31 05:19:41.717 - INFO: <epoch: 14, iter:     560> l_pix: 9.9377e-02 
+26-01-31 05:19:54.601 - INFO: <epoch: 15, iter:     570> l_pix: 1.0983e-01 
+26-01-31 05:19:59.476 - INFO: <epoch: 15, iter:     580> l_pix: 8.6299e-02 
+26-01-31 05:20:04.345 - INFO: <epoch: 15, iter:     590> l_pix: 1.0604e-01 
+26-01-31 05:20:08.786 - INFO: <epoch: 15, iter:     600> l_pix: 1.1187e-01 
+26-01-31 05:20:22.249 - INFO: <epoch: 16, iter:     610> l_pix: 1.3244e-01 
+26-01-31 05:20:27.123 - INFO: <epoch: 16, iter:     620> l_pix: 2.9328e-01 
+26-01-31 05:20:31.994 - INFO: <epoch: 16, iter:     630> l_pix: 6.4232e-01 
+26-01-31 05:20:36.433 - INFO: <epoch: 16, iter:     640> l_pix: 1.7017e-01 
+26-01-31 05:20:49.409 - INFO: <epoch: 17, iter:     650> l_pix: 9.3089e-02 
+26-01-31 05:20:54.283 - INFO: <epoch: 17, iter:     660> l_pix: 1.6027e-01 
+26-01-31 05:20:59.153 - INFO: <epoch: 17, iter:     670> l_pix: 8.7231e-02 
+26-01-31 05:21:03.594 - INFO: <epoch: 17, iter:     680> l_pix: 9.9304e-02 
+26-01-31 05:21:15.913 - INFO: <epoch: 18, iter:     690> l_pix: 1.2937e-01 
+26-01-31 05:21:20.788 - INFO: <epoch: 18, iter:     700> l_pix: 7.9223e-01 
+26-01-31 05:21:25.657 - INFO: <epoch: 18, iter:     710> l_pix: 2.5656e-01 
+26-01-31 05:21:30.093 - INFO: <epoch: 18, iter:     720> l_pix: 8.5992e-02 
+26-01-31 05:21:41.606 - INFO: <epoch: 19, iter:     730> l_pix: 1.2337e-01 
+26-01-31 05:21:46.481 - INFO: <epoch: 19, iter:     740> l_pix: 1.8112e-01 
+26-01-31 05:21:51.351 - INFO: <epoch: 19, iter:     750> l_pix: 1.1578e-01 
+26-01-31 05:21:55.789 - INFO: <epoch: 19, iter:     760> l_pix: 7.5780e-02 
+26-01-31 05:22:09.881 - INFO: <epoch: 20, iter:     770> l_pix: 1.3280e-01 
+26-01-31 05:22:14.754 - INFO: <epoch: 20, iter:     780> l_pix: 8.9268e-02 
+26-01-31 05:22:19.625 - INFO: <epoch: 20, iter:     790> l_pix: 1.6558e-01 
+26-01-31 05:22:24.062 - INFO: <epoch: 20, iter:     800> l_pix: 1.0358e-01 
+26-01-31 05:22:35.435 - INFO: <epoch: 21, iter:     810> l_pix: 1.2296e-01 
+26-01-31 05:22:40.312 - INFO: <epoch: 21, iter:     820> l_pix: 2.1908e-01 
+26-01-31 05:22:45.183 - INFO: <epoch: 21, iter:     830> l_pix: 1.3198e-01 
+26-01-31 05:22:49.619 - INFO: <epoch: 21, iter:     840> l_pix: 9.9279e-02 
+26-01-31 05:23:01.602 - INFO: <epoch: 22, iter:     850> l_pix: 1.2608e-01 
+26-01-31 05:23:06.480 - INFO: <epoch: 22, iter:     860> l_pix: 6.7757e-02 
+26-01-31 05:23:11.352 - INFO: <epoch: 22, iter:     870> l_pix: 1.0374e-01 
+26-01-31 05:23:15.790 - INFO: <epoch: 22, iter:     880> l_pix: 1.6280e-01 
+26-01-31 05:23:29.696 - INFO: <epoch: 23, iter:     890> l_pix: 9.8035e-02 
+26-01-31 05:23:34.564 - INFO: <epoch: 23, iter:     900> l_pix: 3.2271e-01 
+26-01-31 05:23:39.431 - INFO: <epoch: 23, iter:     910> l_pix: 1.1042e-01 
+26-01-31 05:23:43.866 - INFO: <epoch: 23, iter:     920> l_pix: 8.9595e-02 
+26-01-31 05:23:56.170 - INFO: <epoch: 24, iter:     930> l_pix: 7.1240e-02 
+26-01-31 05:24:01.041 - INFO: <epoch: 24, iter:     940> l_pix: 7.1903e-02 
+26-01-31 05:24:05.901 - INFO: <epoch: 24, iter:     950> l_pix: 1.0359e-01 
+26-01-31 05:24:10.330 - INFO: <epoch: 24, iter:     960> l_pix: 6.6844e-02 
+26-01-31 05:24:21.781 - INFO: <epoch: 25, iter:     970> l_pix: 8.6319e-02 
+26-01-31 05:24:26.657 - INFO: <epoch: 25, iter:     980> l_pix: 1.1493e-01 
+26-01-31 05:24:31.517 - INFO: <epoch: 25, iter:     990> l_pix: 9.4695e-02 
+26-01-31 05:24:35.946 - INFO: <epoch: 25, iter:   1,000> l_pix: 1.4095e-01 
+26-01-31 05:24:53.258 - INFO: Saved model in [experiments/ICPR_LPDiff_260131_051306/checkpoint/I1000_E25_gen_best.pth] ...
+26-01-31 05:24:53.260 - INFO: # Validation # PSNR: 1.0301e+01
+26-01-31 05:24:53.260 - INFO: <epoch: 25, iter:   1,000> psnr: 1.0301e+01 loss: 3.9501e-01
+26-01-31 05:24:53.260 - INFO: Saving models and training states.
+26-01-31 05:24:53.937 - INFO: Saved model in [experiments/ICPR_LPDiff_260131_051306/checkpoint/I1000_E25_gen.pth] ...
+26-01-31 05:24:53.938 - INFO: ☁️ Syncing to Hugging Face: PuxAI/ICPR...

logs/val.log

@@ -1 +1 @@
-26-01-31 04:11:50.980 - INFO: <epoch: 16, iter:   5,000> psnr: 1.0103e+01 loss: 4.5005e-01
+26-01-31 05:24:53.260 - INFO: <epoch: 25, iter:   1,000> psnr: 1.0301e+01 loss: 3.9501e-01