diff --git a/.gitattributes b/.gitattributes index a93f0b8132b2cb2e6bb634ac2e19116083bf3032..44836adc8c81cd6173000eed19d98e60f1b7a5cf 100644 --- a/.gitattributes +++ b/.gitattributes @@ -914,3 +914,63 @@ output/lora_rank_128_mlp_H200/step_vis/val_1_step_019999_sample01.png filter=lfs output/lora_rank_128_mlp_H200/step_vis/val_1_step_019999_sample02.png filter=lfs diff=lfs merge=lfs -text output/lora_rank_128_mlp_H200/step_vis/val_1_step_019999_sample03.png filter=lfs diff=lfs merge=lfs -text output/lora_rank_128_mlp_H200/step_vis/val_1_step_019999_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_0_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_1_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_2_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_3_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_4_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_5_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_0_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_1_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_0_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_1_step_006650.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample00.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample01.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample02.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample03.png filter=lfs diff=lfs merge=lfs -text +output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample04.png filter=lfs diff=lfs merge=lfs -text diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_0.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_0.png index db337aa34ae96a6d91d99972250a9026892b5815..ece14526843e6268e672c9e29579ced889bce2ed 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_0.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1e0fcba2045fd9db23f9f93eabe4d8084b68af06fddbf94e580162dcaccf6bf6 -size 2021377 +oid sha256:dbe45473ba97ffb4507d2a145f679cbfc8dd37513c623c38c375f4697476623c +size 2738677 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_1.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_1.png index b5fb3999631adb97406a894e8447b6f01103d44e..7bdfd8c981b4beab8722ae6cf0fa8737d891c969 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_1.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:06b90e0ed0b46af2b503ba6f89eed098e6b6811791db63ebcd1910e4d0eda53f -size 1963823 +oid sha256:9b377b6e64223be6384391831c32cb16d5a71d79b51638a5558df18107b1d292 +size 2678967 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_2.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_2.png index 074bfb77586a90d806e5cb0bbab47eb6821e7487..c61f6ca3e5bc01b662527c7aabb128277dcc0163 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_2.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_2.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e02e4b550984d33593cbb2e4807bca25f82fbabdc20f2b66ff3181470f4ed354 -size 2107509 +oid sha256:0f34966d594514cd9bc1998a6ce6139ba8e3b32cbd947207414aa30675ea4ec0 +size 2851509 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_3.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_3.png index b12c680cd15c43bad2e0600141a6bc88cfbbda93..c451646a0eb519f01676a9cba2bf4e75fd22f44d 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_3.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_3.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:48bf32bd0e44ad19b080e236e120b1ddf5d4f901535a643502f3110ab78e5c16 -size 2108829 +oid sha256:bea0197e6fa19bb0588ab2b7601568311bb183f80d1b5d0f29b12a7a236d4785 +size 2873195 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_4.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_4.png index ff1bf91635e729c1ba45b25b5c82c27f8cdb79d9..b20dc1b16f8681008050018ef6c72d7dfd54c432 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_4.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_4.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1dd243fcb0a55172ddc5f75a83d8f78296638ad7405d818671bddd4031ec69bf -size 1996244 +oid sha256:0287b57428d691affa561aa8d472df8a300cbcb62722dd4412b2130f5965d327 +size 2746053 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_5.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_5.png index e2bee937d783d3b9cbd607459910176d6c8b92d8..15492eba93193d52af12511033cb1bfd1130a818 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_5.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_5.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6c89926f2228d9c2a6afd875b4ce56e4ef7158cca881adaeca64c1300a677f40 -size 2073925 +oid sha256:035da9abdd2627a1f1a676ab9c5af3d43dca489f730b11647cf385aba0567e19 +size 2821637 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_0.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_0.png index 1655571dc063c9936fa8c5fb359594e7e42be0c5..8ecca3975940cf9a77c5085843bd65b9a8215dfe 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_0.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a67aa51e08fc491590fb2c6d32d9343fc97d7b39e92e2fdd6cf90fd4049af706 -size 1949193 +oid sha256:d4ad88ac9cbe85c3b6ae1079c314cd1b2a9231a653979af0a1cb2a23177d8235 +size 2701376 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_1.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_1.png index 387311d5bae9769347e19a0d02a0b690fa9f15c7..63b2d0410e6371fb8251b0d4e51fc7145371e6a5 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_1.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:01dff6bf97a0fae4057de303da6dc3266d7a20b0be55a8d45bd2b1cbeaccdc9e -size 2044213 +oid sha256:13479930d51eca3f7a1d6e1137e875871a88f8270c0f1760c58c993110475088 +size 2791413 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_0.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_0.png index da426eca0c739f69ad014c06a78952765cc3fc41..fc03010c80e7453dac0513ba17a9aabe2394cc3a 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_0.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ad0a12747d5be117a2579da8e4bab1c7872ec2c9dc7755511e114a0596103027 -size 1919415 +oid sha256:ba4b77877c0f96ca6fbfd2a4b8ce9bd5871ed1d068d26e7c4ce33f51dd9e8262 +size 2642236 diff --git a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_1.png b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_1.png index 68d3ba633d2ff690594b45b5789806011f2f6afe..094cd2440ece397a807cad141df3815e0711fba1 100644 --- a/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_1.png +++ b/output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:961767e52532b5c5531750484470f0e3a60e1782e8f5178490765acdd833b8df -size 1949221 +oid sha256:3e421753098e87e171c6a94afd2dd98f5c2faee91829c5c7d84508cac6b71c30 +size 2673125 diff --git a/output/lora_128_dim_384_A100/step_vis/test_0_step_006650.png b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..8a18bfd88342b0aab72afd4e38903b74472399d3 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8bd351a33686292220afcf8d4416102aac1ba689481f7c00645d26a8728a63b +size 1776211 diff --git a/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..7b3cce6c6cd4240c285ada5b9cbe1a8c359ccfc9 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62545f6beb67a735816222c301b850c59e4c63243df68452681c55fa9e603b6b +size 1683752 diff --git a/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..6c586d3bd51b3339963d46554663bae3b49221c1 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5826042175683341c63c39b07553fdf707d473a446cd606d7fb161e2f116a58c +size 1572503 diff --git a/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..7c7ecf2b565d62db6647e42d4a63dc5e20e44ac8 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e743aa7db7011015ecf955168ef5c42f8ef4bbb28243a2b2e76d9497abbafa64 +size 1807546 diff --git a/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..5d99108ea6f24a463f00839a8fac92778d83647b --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d0a5ea60c29445f31a33b0bebd196569b69c07b5d603ab0d43abacb46c5cd36 +size 1729186 diff --git a/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..1704ae656dd387b1b8e8c74cc2492b5d3a4689be --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_0_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ec5c2a78bde1b8be627460b25aabb27850de8ab7732ccfcb3c1df2034fae1c3 +size 1876842 diff --git a/output/lora_128_dim_384_A100/step_vis/test_1_step_006650.png b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..7a6f77733488143eba76a0094941da88598c8bc7 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfcce06893c544e200cc09ea4c5c4e1c1bf0b333054e984e3a048a291fe45477 +size 1702896 diff --git a/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..85dbcddd0bda94931d34e99eb5095de7a470400f --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46ca151918ce1e260ae96b56b205199ad2a280d3725f21d4be6a110340c5a549 +size 1763921 diff --git a/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..4089f90f6624cfc8b3e4f41e39eab99f08549490 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e8ac5db3de045b3617d8d54c08808ef7026b3e857162921d9447aa409e97c6c +size 1672377 diff --git a/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..b547eae1c6b3c4d418311b8a9eb849d8efdd570f --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbbcd8c835d1038bf8d293f9853aa74ce67f7a8d9625ae6d2cc3203c48b176da +size 1778826 diff --git a/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..dd609c37fbcfa3755ab29a4e926d876aabd52ed7 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec46d790a3bb7ce0064d2b4b7e67abc66752fa5de48f1016b29c8a79bbbd4b0 +size 1822829 diff --git a/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..1b81dab6514bc3c74588cdbc8c456d38b7ec29b9 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_1_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cf04a8bb1ebb3a6e469d31e11905f310f1de6d60469bb3d16c87918b785eda2 +size 1592570 diff --git a/output/lora_128_dim_384_A100/step_vis/test_2_step_006650.png b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..cabdf8d8658b3c8bb6167e7034a7eae8e56113bb --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d282ef9b706cb9d6a0bd12768bb9418ace67da1b4663cf562b65316ea93fff73 +size 1881815 diff --git a/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..0985598ed25e2ef304e11e0596febf01b868cf9d --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b179962bb6b0dd2ba5e6fe59a4b5a43913288b18e11f95212f5f164705c25d4 +size 1797117 diff --git a/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..ac57b78ee30d53c577dfb66df7b55bd17c2e0edf --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fa80919197f194fd6f7c329aaf97345aa4613cdd1ce2c60b8214c782cf31d6c +size 1648194 diff --git a/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..56768c83239aecfbcc235eb0a98d4a4f3646caa9 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2fa6737c08b0c9b4544860aed346fd6650e936fe851f0c8bccfbe8b6e435027 +size 2037702 diff --git a/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..7eb4951f465159cb4560b1ea38dfc59beae1f8b0 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6bf7c7748a7027d1aa78fa32a3f1e4c9a4dca425fe2f198d733cffac8a0f333 +size 1956447 diff --git a/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..c81cfdb40ece4444aafbde7afafac1f4ce8801fe --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_2_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1364a2070f32b607c80cf6b5e586d396dc396fc9d932b5b9ac45d3dd1cf3cea +size 1804297 diff --git a/output/lora_128_dim_384_A100/step_vis/test_3_step_006650.png b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..2034f3c3944ac10681cb809d25a1e313038bf1a7 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffa1176033dc11163d9b22d5a817a647b1e8501dd5168ddf380bcb9b5507e7c8 +size 1908235 diff --git a/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..c82252a603c0a9b5e835dbc0d6f00b8a30901a56 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e587852fecdedaed5654054e732739f74b7d10ac6c9f85cda958c23778cf58 +size 1733042 diff --git a/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..832a08097192372c888c787d4e940477728648a0 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba2c7e9cfa559136c1fe1fccdb7175ff9bd90386e6b0e8368c6f9e7fe265c5dc +size 1869638 diff --git a/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..812756079c241a9535978a044ca70212aa946ca4 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc48de80bc65701354c18bbe864327bda968f6a01a31f2403782a6c0ecec17f9 +size 1773861 diff --git a/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..b3eda582062253378ff49ffff3be6ec95e1899c3 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d8b82e4d19630714d573ee925a0066c61a329f44790ad5f13f239c9b3eee710 +size 2194919 diff --git a/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..b5c43a672c22d168c582f6ecb839ee8bf8e1c920 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_3_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d00d7c85ce83e9a67c9e577375d8317d271ee302d8ee2d0b5275a5d3316c3a0 +size 1984062 diff --git a/output/lora_128_dim_384_A100/step_vis/test_4_step_006650.png b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..462a823c12d98f59b2e2611878e427c5ce068941 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35a0bc921e61e7e75500d848883b59a367deab7e4cd25e21daa55a72a7ccf592 +size 1784402 diff --git a/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..0adf72f8423b7c912c00bd90e6816164da8bef26 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a3da4379667f502a5f401e73ae05f556f0cc260d801441c0f87bfd49d174732 +size 1889570 diff --git a/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..d26ee1a689d5e3b45119a9e2f6bc4f429eca32ea --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60899e5a2578519cfdecec941ef93f210a0b0c3e4304de56de8f0600e7ec649f +size 1966043 diff --git a/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..98e598d1174a203aa4650c55e9f0e6709bd9c09c --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:544cc63d2fdb50cdc1c1a8a8ad5013a1aaeca7f2b882e5fb29b3ffa80c8d3625 +size 1886146 diff --git a/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..e9a0be5915fd325217c31d2a02fd88f04f9b4cd8 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7a58a731ce99957a1e5e956e44df1e5119655b05be0be1e579de9346026351f +size 1670357 diff --git a/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..9c94240ea0b6024ec52d55aece4a36da57109692 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_4_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0fe3bf64a4222c2748387c2f6fc652562053b1fd813e24e67de3ecd89f9cd03 +size 1842832 diff --git a/output/lora_128_dim_384_A100/step_vis/test_5_step_006650.png b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..08fa69b35aeefff6ae52d15162f36e47840ba71e --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:058cc2670a482c1e58bbb20392de57f7e7b0d891459fd5ff57b9ded7d58b2e71 +size 1871852 diff --git a/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..b0a193a3ff843e2ad7810aebc31156ea0ae19381 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dc31f18d7ab83742b09397287ce842fe2672e7d3b8a2d551038b04f6f386405 +size 1786109 diff --git a/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..3ef28dd0eb89434d59891c23adf55d5db0f63307 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccb24e4b30282f0bca1f74198cfceee99d6d2dc0bb32148460899d0a096f09f0 +size 1719972 diff --git a/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..1da65516470a211e2e90c250749149eaf6d26417 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b2e148cdc95d9e1a6c9133ce69b726b1f1173866654c6df9c7f0df24a1068d +size 1551716 diff --git a/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..a73322573557927685f15445a79b3ef765748dac --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f996555e827895da52126f346230aafc76be9a476ed194e386cc5797cc9c46a5 +size 1863786 diff --git a/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..2388005cad5e87250226457ce8fc7d17bfb84dcc --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/test_5_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebdec0416bc313f701afe3643fd9915fccb3293a19dae16e30b11b86c65c147f +size 2027935 diff --git a/output/lora_128_dim_384_A100/step_vis/train_0_step_006650.png b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..46238f1f77043defd01b912c8403508664c0d89a --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:477e796f591685dbc4f29d3a939a1336f30105dc246be67859e36bb291805472 +size 1746817 diff --git a/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..fa263423d5d4ec4d5b6dc69adf114f49073201c4 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e42571c55feed5352e9562b18502543bdf00620876ddc032952107a757e7e08b +size 1741980 diff --git a/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..5f21eb15a2c4a9ad679ba17807a0a8d21649272f --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68da05ba10eb1a70324e171e4b4a00ff2a88550c97bdd7ed3a560962527cb1ae +size 1864709 diff --git a/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..578c18e34827c17586ef2bfcf6f07104f96c94c2 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97bb3c04779187f88631e6ad321a6f94d5e975995ae188925ae21c04136b13fb +size 1984659 diff --git a/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..9196ef03751d6cb8d952554ac8aa30246c78872e --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af9edd5a851e035563a3e419902db3f64894bc38e011e4400f105b56eb14c261 +size 1665292 diff --git a/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..b2ec7ba907f0470d56aeb9b3a7e76800e673647c --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_0_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d6c4aa576a22a2f1cd7bccfa0e205ded43a3f164fe4e9176db8dcbe6aeffb1b +size 2067301 diff --git a/output/lora_128_dim_384_A100/step_vis/train_1_step_006650.png b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..d49d50b1b8647f89eaa58f2e5983db00638afa5f --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90a00bbd009b2eab4f9c3a64799eca38bc6b475b744fbb0d5748a257dfdb7694 +size 1829878 diff --git a/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..3da660399caf2eac32dbba3174d1c0038200bcee --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052899bacc436210be317f19509ad8c2d99ede4152129a73dae1def08887b141 +size 1786362 diff --git a/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..7513c79d21f5d64d00defba127b7c5f47856b436 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc32b979979cb8f1ad2bcc72152371def49ab38a0eb878c5af06e5e810eabd2b +size 1906774 diff --git a/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..69a5d461532a331d8462a8749b5f452e1c5e82eb --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35cc92c7563d8a86a6b457e024f4420d16b35e747164d545342d33788fcc95b2 +size 1636086 diff --git a/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..41e30c6e7d7aedc625a8fef8bc90987835ff869c --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2872566e2cb28d657ce13df2a080ebbd7419d625c28ba3ca7c972aed09abe2b +size 2067392 diff --git a/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..221cb15388964ed7b66c17dd5a48da59189a4cfc --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/train_1_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6acfad0d42d82ba2e41884a0779357af025568320dc26bd2200331fc6dfd5a0f +size 1786901 diff --git a/output/lora_128_dim_384_A100/step_vis/val_0_step_006650.png b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..1be7b1dde55eb1e09a2c7455856b43868608b92a --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bc4383341ef9d195bacd60b5bbfda9b970046cfc7051f53a49e7eaa041b1114 +size 1671394 diff --git a/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..97546b3da0cde789616cc2d8318c5afb0861df48 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de62449786d331c621cb6f644e03cca5a3f1b76a6f1873a71e4d9aef8be57487 +size 1774285 diff --git a/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..a7752abc5b79d8e3a23366cc3e5afafb97b3c425 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ee795a19191aa455287db4e10f8275673aa785ae0a5fa6899af86952ecc3f19 +size 1941775 diff --git a/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..f5ec1643a53442a505b3b78910ba1762a8b60108 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:543a8c19f26a245072396f257611d4d20d53ac6821cf77365a7e7675c707942f +size 1584639 diff --git a/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..e34d2617853996b0bc7bb454ae586f6046ab1d7d --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d80cb38c869b080308370988246171b67fd6812e2cd2359d72f2497f26f1e6e +size 1699472 diff --git a/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..348e913594b267502a5bbcd7ada712be4b3aa642 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_0_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50c1c91c513b0b3c719a0d0e7ccf35d50df8e70c83f3392d79b44d71667c5aca +size 1790054 diff --git a/output/lora_128_dim_384_A100/step_vis/val_1_step_006650.png b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650.png new file mode 100644 index 0000000000000000000000000000000000000000..fe06b8818a15d795e11f6c06b37694db1ef6c061 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bceb4a68498afd43b5ca4a7999dc7ea152586e99bf0a8578cfa80a03b726225 +size 1704281 diff --git a/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample00.png b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample00.png new file mode 100644 index 0000000000000000000000000000000000000000..94c2f6630e9fcdccdff41dc1fb554918679de37b --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample00.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4011c3e3b2f5fc8e3dba0b7a432fd5cb3754d3e1f44ab7c34cabfb9382b77bc +size 1783838 diff --git a/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample01.png b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample01.png new file mode 100644 index 0000000000000000000000000000000000000000..391a78306cde724791b2e15678153172f3e75b5e --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample01.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bac00b141357b2cf198619e33cff3d40a7eb91f151131ee6fed0becbcad7722 +size 1867370 diff --git a/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample02.png b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample02.png new file mode 100644 index 0000000000000000000000000000000000000000..6d8cf15a58705e7182266e1e942a139412f6e524 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample02.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05d527f3747a2bf6dec84cbef409cb0c2c6fed2335a635fbbbb7a37b5de23aae +size 1735941 diff --git a/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample03.png b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample03.png new file mode 100644 index 0000000000000000000000000000000000000000..dd02af87745c264ff698ca28fdd2e7e3550fe1a8 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample03.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:482c4647da8cfbf954681656a1b8e35975e72fce14d07eb35583545211edbcbf +size 1639586 diff --git a/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample04.png b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample04.png new file mode 100644 index 0000000000000000000000000000000000000000..d3f07a27808c9f90d5e132dc309aa59daf5ab4f3 --- /dev/null +++ b/output/lora_128_dim_384_A100/step_vis/val_1_step_006650_sample04.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31e5ed2171106441f33d62a45a294f3dba4075ba8670e0b5dc3ade8613644689 +size 1618540 diff --git a/output/lora_128_dim_384_A100/train.log b/output/lora_128_dim_384_A100/train.log index 6a7ef44ea1a0aa3139f9b1039d389aaa22839d43..50e526a524cbdadc87a2166196902a753afb2c75 100644 --- a/output/lora_128_dim_384_A100/train.log +++ b/output/lora_128_dim_384_A100/train.log @@ -5922,3 +5922,6699 @@ Starting training: 500 epochs × 133 steps = 66500 total steps ..._step_000000_sample04.png: 100%|#################################################################################################| 2.70MB / 2.70MB ...s/train_1_step_000000.png: 100%|#################################################################################################| 2.04MB / 2.04MB ..._step_000000_sample03.png: 100%|#################################################################################################| 2.67MB / 2.67MB + [HF Upload] ✓ https://huggingface.co/JasonXF/SynthUrbanSAT-Output/tree/main/output/lora_128_dim_384_A100 + [Epoch 0][10/133] loss=0.937363 avg=0.902560 VRAM=38.4GiB | 0.0% done | ETA(epoch): 24304s + [Epoch 0][20/133] loss=0.792233 avg=0.904441 VRAM=38.3GiB | 0.0% done | ETA(epoch): 12182s + [Epoch 0][30/133] loss=0.840362 avg=0.896203 VRAM=38.4GiB | 0.0% done | ETA(epoch): 8023s + [Epoch 0][40/133] loss=1.026177 avg=0.896547 VRAM=38.3GiB | 0.1% done | ETA(epoch): 5851s + [Epoch 0][50/133] loss=0.882616 avg=0.903844 VRAM=38.4GiB | 0.1% done | ETA(epoch): 4479s + [Epoch 0][60/133] loss=0.985102 avg=0.906280 VRAM=38.3GiB | 0.1% done | ETA(epoch): 3502s + [Epoch 0][70/133] loss=0.913582 avg=0.904951 VRAM=38.4GiB | 0.1% done | ETA(epoch): 2752s + [Epoch 0][80/133] loss=0.869109 avg=0.905451 VRAM=38.3GiB | 0.1% done | ETA(epoch): 2145s + [Epoch 0][90/133] loss=0.872389 avg=0.906280 VRAM=38.4GiB | 0.1% done | ETA(epoch): 1633s + [Epoch 0][100/133] loss=0.831392 avg=0.903249 VRAM=38.3GiB | 0.2% done | ETA(epoch): 1187s + [Epoch 0][110/133] loss=0.919739 avg=0.906070 VRAM=38.4GiB | 0.2% done | ETA(epoch): 790s + [Epoch 0][120/133] loss=0.921420 avg=0.905642 VRAM=38.3GiB | 0.2% done | ETA(epoch): 429s + [Epoch 0][130/133] loss=0.866190 avg=0.905786 VRAM=38.4GiB | 0.2% done | ETA(epoch): 95s + Train loss: 0.906772 (4191.4s) ETA: 34947min + Val loss: 0.989512 [t_0.0-0.2=1.0932 t_0.2-0.4=1.0331 t_0.4-0.6=0.9222 t_0.6-0.8=0.7556 t_0.8-1.0=1.0491] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0000 (BEST) +[MEM @ epoch 0 end] RAM: 44.5/1007.0 GiB (4.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 1/499 (0% done) --- + [Epoch 1][10/133] loss=0.839493 avg=0.919614 VRAM=38.4GiB | 0.2% done | ETA(epoch): 2216s + [Epoch 1][20/133] loss=0.872305 avg=0.943195 VRAM=38.3GiB | 0.2% done | ETA(epoch): 2029s + [Epoch 1][30/133] loss=0.960927 avg=0.943920 VRAM=38.4GiB | 0.2% done | ETA(epoch): 1848s + [Epoch 1][40/133] loss=0.946095 avg=0.931753 VRAM=38.3GiB | 0.3% done | ETA(epoch): 1668s + [Epoch 1][50/133] loss=0.756278 avg=0.918593 VRAM=38.4GiB | 0.3% done | ETA(epoch): 1489s + [Epoch 1][60/133] loss=0.880924 avg=0.918641 VRAM=38.3GiB | 0.3% done | ETA(epoch): 1310s + [Epoch 1][70/133] loss=0.776703 avg=0.914670 VRAM=38.4GiB | 0.3% done | ETA(epoch): 1130s + [Epoch 1][80/133] loss=0.765338 avg=0.913329 VRAM=38.3GiB | 0.3% done | ETA(epoch): 951s + [Epoch 1][90/133] loss=0.915909 avg=0.913537 VRAM=38.4GiB | 0.3% done | ETA(epoch): 771s + [Epoch 1][100/133] loss=0.906444 avg=0.910423 VRAM=38.3GiB | 0.4% done | ETA(epoch): 592s + [Epoch 1][110/133] loss=1.052891 avg=0.911504 VRAM=38.4GiB | 0.4% done | ETA(epoch): 412s + [Epoch 1][120/133] loss=1.030833 avg=0.912994 VRAM=38.3GiB | 0.4% done | ETA(epoch): 233s + [Epoch 1][130/133] loss=0.846860 avg=0.911235 VRAM=38.4GiB | 0.4% done | ETA(epoch): 54s + Train loss: 0.911914 (2383.3s) ETA: 28046min + Val loss: 0.927958 [t_0.0-0.2=1.0927 t_0.2-0.4=1.0007 t_0.4-0.6=0.8703 t_0.6-0.8=0.7718 t_0.8-1.0=0.9617] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0001 (BEST) +[MEM @ epoch 1 end] RAM: 48.1/1007.0 GiB (4.8%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 2/499 (0% done) --- + [Epoch 2][10/133] loss=0.863856 avg=0.911735 VRAM=38.4GiB | 0.4% done | ETA(epoch): 2202s + [Epoch 2][20/133] loss=0.895166 avg=0.903446 VRAM=38.3GiB | 0.4% done | ETA(epoch): 2023s + [Epoch 2][30/133] loss=0.740886 avg=0.883981 VRAM=38.4GiB | 0.4% done | ETA(epoch): 1843s + [Epoch 2][40/133] loss=1.018766 avg=0.892999 VRAM=38.3GiB | 0.5% done | ETA(epoch): 1663s + [Epoch 2][50/133] loss=0.969655 avg=0.898535 VRAM=38.4GiB | 0.5% done | ETA(epoch): 1485s + [Epoch 2][60/133] loss=0.993872 avg=0.898333 VRAM=38.3GiB | 0.5% done | ETA(epoch): 1306s + [Epoch 2][70/133] loss=0.913542 avg=0.898410 VRAM=38.4GiB | 0.5% done | ETA(epoch): 1127s + [Epoch 2][80/133] loss=0.876821 avg=0.901515 VRAM=38.3GiB | 0.5% done | ETA(epoch): 949s + [Epoch 2][90/133] loss=0.774090 avg=0.898591 VRAM=38.4GiB | 0.5% done | ETA(epoch): 770s + [Epoch 2][100/133] loss=0.931752 avg=0.898310 VRAM=38.3GiB | 0.6% done | ETA(epoch): 591s + [Epoch 2][110/133] loss=1.016933 avg=0.902365 VRAM=38.4GiB | 0.6% done | ETA(epoch): 412s + [Epoch 2][120/133] loss=0.912723 avg=0.901777 VRAM=38.3GiB | 0.6% done | ETA(epoch): 233s + [Epoch 2][130/133] loss=1.047471 avg=0.900042 VRAM=38.4GiB | 0.6% done | ETA(epoch): 54s + Train loss: 0.898834 (2382.1s) ETA: 25710min + Val loss: 0.942709 [t_0.0-0.2=1.0729 t_0.2-0.4=1.0249 t_0.4-0.6=0.8091 t_0.6-0.8=0.7774 t_0.8-1.0=1.0433] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0002 +[MEM @ epoch 2 end] RAM: 60.1/1007.0 GiB (6.0%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 3/499 (1% done) --- + [Epoch 3][10/133] loss=0.841994 avg=0.900071 VRAM=38.4GiB | 0.6% done | ETA(epoch): 2200s + [Epoch 3][20/133] loss=0.964176 avg=0.893224 VRAM=38.3GiB | 0.6% done | ETA(epoch): 2028s + [Epoch 3][30/133] loss=1.026013 avg=0.879517 VRAM=38.4GiB | 0.6% done | ETA(epoch): 1849s + [Epoch 3][40/133] loss=0.923857 avg=0.887802 VRAM=38.3GiB | 0.7% done | ETA(epoch): 1669s + [Epoch 3][50/133] loss=0.803731 avg=0.887804 VRAM=38.4GiB | 0.7% done | ETA(epoch): 1489s + [Epoch 3][60/133] loss=0.869277 avg=0.888120 VRAM=38.3GiB | 0.7% done | ETA(epoch): 1309s + [Epoch 3][70/133] loss=0.879003 avg=0.891276 VRAM=38.4GiB | 0.7% done | ETA(epoch): 1130s + [Epoch 3][80/133] loss=0.850176 avg=0.888092 VRAM=38.3GiB | 0.7% done | ETA(epoch): 950s + [Epoch 3][90/133] loss=0.753254 avg=0.885211 VRAM=38.4GiB | 0.7% done | ETA(epoch): 771s + [Epoch 3][100/133] loss=0.786793 avg=0.885453 VRAM=38.3GiB | 0.8% done | ETA(epoch): 592s + [Epoch 3][110/133] loss=1.002169 avg=0.886139 VRAM=38.4GiB | 0.8% done | ETA(epoch): 412s + [Epoch 3][120/133] loss=0.929148 avg=0.885497 VRAM=38.3GiB | 0.8% done | ETA(epoch): 233s + [Epoch 3][130/133] loss=0.829764 avg=0.883915 VRAM=38.4GiB | 0.8% done | ETA(epoch): 54s + Train loss: 0.884268 (2385.6s) ETA: 24528min + Val loss: 0.912448 [t_0.0-0.2=1.0909 t_0.2-0.4=1.0090 t_0.4-0.6=0.8533 t_0.6-0.8=0.7626 t_0.8-1.0=0.8628] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0003 (BEST) + Deleted old checkpoint: checkpoint_epoch_0000 +[MEM @ epoch 3 end] RAM: 62.8/1007.0 GiB (6.2%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 4/499 (1% done) --- + [Epoch 4][10/133] loss=0.872064 avg=0.920418 VRAM=38.4GiB | 0.8% done | ETA(epoch): 2230s + [Epoch 4][20/133] loss=0.805010 avg=0.892022 VRAM=38.3GiB | 0.8% done | ETA(epoch): 2039s + [Epoch 4][30/133] loss=0.898158 avg=0.889846 VRAM=38.4GiB | 0.8% done | ETA(epoch): 1856s + [Epoch 4][40/133] loss=0.933690 avg=0.886762 VRAM=38.3GiB | 0.9% done | ETA(epoch): 1673s + [Epoch 4][50/133] loss=0.790216 avg=0.891210 VRAM=38.4GiB | 0.9% done | ETA(epoch): 1493s + [Epoch 4][60/133] loss=0.804030 avg=0.891523 VRAM=38.3GiB | 0.9% done | ETA(epoch): 1312s + [Epoch 4][70/133] loss=0.929425 avg=0.897882 VRAM=38.4GiB | 0.9% done | ETA(epoch): 1133s + [Epoch 4][80/133] loss=0.821988 avg=0.900008 VRAM=38.3GiB | 0.9% done | ETA(epoch): 953s + [Epoch 4][90/133] loss=1.036605 avg=0.895613 VRAM=38.4GiB | 0.9% done | ETA(epoch): 773s + [Epoch 4][100/133] loss=0.795134 avg=0.894741 VRAM=38.3GiB | 1.0% done | ETA(epoch): 593s + [Epoch 4][110/133] loss=0.827026 avg=0.892866 VRAM=38.4GiB | 1.0% done | ETA(epoch): 413s + [Epoch 4][120/133] loss=0.911934 avg=0.891557 VRAM=38.3GiB | 1.0% done | ETA(epoch): 234s + [Epoch 4][130/133] loss=0.836589 avg=0.892149 VRAM=38.4GiB | 1.0% done | ETA(epoch): 54s + Train loss: 0.892294 (2390.3s) ETA: 23810min + Val loss: 0.927830 [t_0.0-0.2=1.0769 t_0.2-0.4=1.0105 t_0.4-0.6=0.8817 t_0.6-0.8=0.7199 t_0.8-1.0=0.9356] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0004 + Deleted old checkpoint: checkpoint_epoch_0001 +[MEM @ epoch 4 end] RAM: 67.0/1007.0 GiB (6.7%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 5/499 (1% done) --- + [Epoch 5][10/133] loss=0.960754 avg=0.914703 VRAM=38.4GiB | 1.0% done | ETA(epoch): 2204s + [Epoch 5][20/133] loss=0.731850 avg=0.900644 VRAM=38.3GiB | 1.0% done | ETA(epoch): 2024s + [Epoch 5][30/133] loss=0.839370 avg=0.895473 VRAM=38.4GiB | 1.0% done | ETA(epoch): 1844s + [Epoch 5][40/133] loss=0.907067 avg=0.888754 VRAM=38.3GiB | 1.1% done | ETA(epoch): 1665s + [Epoch 5][50/133] loss=1.019039 avg=0.887576 VRAM=38.4GiB | 1.1% done | ETA(epoch): 1486s + [Epoch 5][60/133] loss=0.905228 avg=0.880834 VRAM=38.3GiB | 1.1% done | ETA(epoch): 1307s + [Epoch 5][70/133] loss=0.784577 avg=0.881728 VRAM=38.4GiB | 1.1% done | ETA(epoch): 1128s + [Epoch 5][80/133] loss=0.928128 avg=0.881208 VRAM=38.3GiB | 1.1% done | ETA(epoch): 949s + [Epoch 5][90/133] loss=1.006608 avg=0.879864 VRAM=38.4GiB | 1.1% done | ETA(epoch): 770s + [Epoch 5][100/133] loss=0.842781 avg=0.880333 VRAM=38.3GiB | 1.2% done | ETA(epoch): 591s + [Epoch 5][110/133] loss=0.923272 avg=0.881443 VRAM=38.4GiB | 1.2% done | ETA(epoch): 412s + [Epoch 5][120/133] loss=0.980575 avg=0.884864 VRAM=38.3GiB | 1.2% done | ETA(epoch): 233s + [Epoch 5][130/133] loss=0.758079 avg=0.881862 VRAM=38.4GiB | 1.2% done | ETA(epoch): 54s + Train loss: 0.883253 (2382.8s) ETA: 23312min + Val loss: 0.896555 [t_0.0-0.2=1.0660 t_0.2-0.4=1.0177 t_0.4-0.6=0.8705 t_0.6-0.8=0.7367 t_0.8-1.0=0.8176] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0005 (BEST) + Deleted old checkpoint: checkpoint_epoch_0002 +[MEM @ epoch 5 end] RAM: 52.6/1007.0 GiB (5.2%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 6/499 (1% done) --- + [Epoch 6][10/133] loss=0.819137 avg=0.832533 VRAM=38.4GiB | 1.2% done | ETA(epoch): 2203s + [Epoch 6][20/133] loss=0.856562 avg=0.827326 VRAM=38.3GiB | 1.2% done | ETA(epoch): 2025s + [Epoch 6][30/133] loss=0.836367 avg=0.836934 VRAM=38.4GiB | 1.2% done | ETA(epoch): 1846s + [Epoch 6][40/133] loss=0.974980 avg=0.833976 VRAM=38.3GiB | 1.3% done | ETA(epoch): 1667s + [Epoch 6][50/133] loss=0.742404 avg=0.845714 VRAM=38.4GiB | 1.3% done | ETA(epoch): 1488s + [Epoch 6][60/133] loss=0.797904 avg=0.843314 VRAM=38.3GiB | 1.3% done | ETA(epoch): 1309s + [Epoch 6][70/133] loss=0.955947 avg=0.846686 VRAM=38.4GiB | 1.3% done | ETA(epoch): 1129s + [Epoch 6][80/133] loss=1.018990 avg=0.851498 VRAM=38.3GiB | 1.3% done | ETA(epoch): 950s + [Epoch 6][90/133] loss=0.917380 avg=0.855755 VRAM=38.4GiB | 1.3% done | ETA(epoch): 771s + [Epoch 6][100/133] loss=0.922153 avg=0.860317 VRAM=38.3GiB | 1.4% done | ETA(epoch): 592s + [Epoch 6][110/133] loss=0.861571 avg=0.863169 VRAM=38.4GiB | 1.4% done | ETA(epoch): 412s + [Epoch 6][120/133] loss=0.919967 avg=0.863321 VRAM=38.3GiB | 1.4% done | ETA(epoch): 233s + [Epoch 6][130/133] loss=0.972445 avg=0.862479 VRAM=38.4GiB | 1.4% done | ETA(epoch): 54s + Train loss: 0.862987 (2384.3s) ETA: 22939min + Val loss: 0.891180 [t_0.0-0.2=1.0898 t_0.2-0.4=0.9526 t_0.4-0.6=0.8342 t_0.6-0.8=0.7535 t_0.8-1.0=0.7325] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0006 (BEST) + Deleted old checkpoint: checkpoint_epoch_0003 +[MEM @ epoch 6 end] RAM: 51.9/1007.0 GiB (5.2%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 7/499 (1% done) --- + [Epoch 7][10/133] loss=0.753163 avg=0.837751 VRAM=38.4GiB | 1.4% done | ETA(epoch): 2202s + [Epoch 7][20/133] loss=0.933751 avg=0.854740 VRAM=38.3GiB | 1.4% done | ETA(epoch): 2022s + [Epoch 7][30/133] loss=0.898100 avg=0.865183 VRAM=38.4GiB | 1.4% done | ETA(epoch): 1844s + [Epoch 7][40/133] loss=0.959123 avg=0.877441 VRAM=38.3GiB | 1.5% done | ETA(epoch): 1665s + [Epoch 7][50/133] loss=0.922296 avg=0.881913 VRAM=38.4GiB | 1.5% done | ETA(epoch): 1486s + [Epoch 7][60/133] loss=0.795820 avg=0.882128 VRAM=38.3GiB | 1.5% done | ETA(epoch): 1307s + [Epoch 7][70/133] loss=0.825931 avg=0.883445 VRAM=38.4GiB | 1.5% done | ETA(epoch): 1128s + [Epoch 7][80/133] loss=0.981761 avg=0.887221 VRAM=38.3GiB | 1.5% done | ETA(epoch): 949s + [Epoch 7][90/133] loss=0.885804 avg=0.883606 VRAM=38.4GiB | 1.5% done | ETA(epoch): 770s + [Epoch 7][100/133] loss=0.954298 avg=0.879701 VRAM=38.3GiB | 1.6% done | ETA(epoch): 591s + [Epoch 7][110/133] loss=0.943999 avg=0.880310 VRAM=38.4GiB | 1.6% done | ETA(epoch): 412s + [Epoch 7][120/133] loss=0.922131 avg=0.878148 VRAM=38.3GiB | 1.6% done | ETA(epoch): 233s + [Epoch 7][130/133] loss=0.887675 avg=0.880644 VRAM=38.4GiB | 1.6% done | ETA(epoch): 54s + Train loss: 0.882259 (2380.9s) ETA: 22647min + Val loss: 0.878770 [t_0.0-0.2=1.0854 t_0.2-0.4=1.0026 t_0.4-0.6=0.8427 t_0.6-0.8=0.7302 t_0.8-1.0=0.7996] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0007 (BEST) + Deleted old checkpoint: checkpoint_epoch_0004 +[MEM @ epoch 7 end] RAM: 51.6/1007.0 GiB (5.1%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 8/499 (2% done) --- + [Epoch 8][10/133] loss=0.828246 avg=0.927995 VRAM=38.4GiB | 1.6% done | ETA(epoch): 2196s + [Epoch 8][20/133] loss=0.776031 avg=0.895309 VRAM=38.3GiB | 1.6% done | ETA(epoch): 2019s + [Epoch 8][30/133] loss=0.821663 avg=0.877674 VRAM=38.4GiB | 1.6% done | ETA(epoch): 1840s + [Epoch 8][40/133] loss=0.840940 avg=0.870839 VRAM=38.3GiB | 1.7% done | ETA(epoch): 1662s + [Epoch 8][50/133] loss=0.805177 avg=0.878252 VRAM=38.4GiB | 1.7% done | ETA(epoch): 1482s + [Epoch 8][60/133] loss=0.782290 avg=0.875233 VRAM=38.3GiB | 1.7% done | ETA(epoch): 1304s + [Epoch 8][70/133] loss=0.910781 avg=0.872572 VRAM=38.4GiB | 1.7% done | ETA(epoch): 1126s + [Epoch 8][80/133] loss=0.818519 avg=0.867869 VRAM=38.3GiB | 1.7% done | ETA(epoch): 947s + [Epoch 8][90/133] loss=0.914914 avg=0.870060 VRAM=38.4GiB | 1.7% done | ETA(epoch): 769s + [Epoch 8][100/133] loss=0.866912 avg=0.870843 VRAM=38.3GiB | 1.8% done | ETA(epoch): 590s + [Epoch 8][110/133] loss=1.097188 avg=0.869642 VRAM=38.4GiB | 1.8% done | ETA(epoch): 412s + [Epoch 8][120/133] loss=0.923699 avg=0.873562 VRAM=38.3GiB | 1.8% done | ETA(epoch): 233s + [Epoch 8][130/133] loss=0.805690 avg=0.873807 VRAM=38.4GiB | 1.8% done | ETA(epoch): 54s + Train loss: 0.874730 (2381.4s) ETA: 22411min + Val loss: 0.895715 [t_0.0-0.2=1.0672 t_0.2-0.4=1.0517 t_0.4-0.6=0.8403 t_0.6-0.8=0.7435 t_0.8-1.0=0.7320] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0008 + Deleted old checkpoint: checkpoint_epoch_0005 +[MEM @ epoch 8 end] RAM: 41.2/1007.0 GiB (4.1%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 9/499 (2% done) --- + [Epoch 9][10/133] loss=0.906177 avg=0.899119 VRAM=38.4GiB | 1.8% done | ETA(epoch): 2195s + [Epoch 9][20/133] loss=1.013546 avg=0.884039 VRAM=38.3GiB | 1.8% done | ETA(epoch): 2017s + [Epoch 9][30/133] loss=0.982018 avg=0.885578 VRAM=38.4GiB | 1.8% done | ETA(epoch): 1839s + [Epoch 9][40/133] loss=0.862772 avg=0.879784 VRAM=38.3GiB | 1.9% done | ETA(epoch): 1661s + [Epoch 9][50/133] loss=0.865373 avg=0.886176 VRAM=38.4GiB | 1.9% done | ETA(epoch): 1482s + [Epoch 9][60/133] loss=0.840828 avg=0.884414 VRAM=38.3GiB | 1.9% done | ETA(epoch): 1304s + [Epoch 9][70/133] loss=0.935121 avg=0.886067 VRAM=38.4GiB | 1.9% done | ETA(epoch): 1125s + [Epoch 9][80/133] loss=0.828527 avg=0.889757 VRAM=38.3GiB | 1.9% done | ETA(epoch): 947s + [Epoch 9][90/133] loss=0.848532 avg=0.887131 VRAM=38.4GiB | 1.9% done | ETA(epoch): 768s + [Epoch 9][100/133] loss=1.010312 avg=0.886971 VRAM=38.3GiB | 2.0% done | ETA(epoch): 590s + [Epoch 9][110/133] loss=0.884129 avg=0.883990 VRAM=38.4GiB | 2.0% done | ETA(epoch): 412s + [Epoch 9][120/133] loss=0.758193 avg=0.883808 VRAM=38.3GiB | 2.0% done | ETA(epoch): 233s + [Epoch 9][130/133] loss=0.698337 avg=0.879812 VRAM=38.4GiB | 2.0% done | ETA(epoch): 54s + Train loss: 0.880697 (2381.2s) ETA: 22213min + Val loss: 0.916252 [t_0.0-0.2=1.0965 t_0.2-0.4=0.9969 t_0.4-0.6=0.8749 t_0.6-0.8=0.7575 t_0.8-1.0=0.7308] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0009 + Deleted old checkpoint: checkpoint_epoch_0006 +[MEM @ epoch 9 end] RAM: 41.3/1007.0 GiB (4.1%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 10/499 (2% done) --- + [Epoch 10][10/133] loss=0.708718 avg=0.827795 VRAM=38.4GiB | 2.0% done | ETA(epoch): 2194s + [Epoch 10][20/133] loss=1.083581 avg=0.872049 VRAM=38.3GiB | 2.0% done | ETA(epoch): 2019s + [Epoch 10][30/133] loss=0.962710 avg=0.882280 VRAM=38.4GiB | 2.0% done | ETA(epoch): 1841s + [Epoch 10][40/133] loss=0.931366 avg=0.879538 VRAM=38.3GiB | 2.1% done | ETA(epoch): 1663s + [Epoch 10][50/133] loss=0.985583 avg=0.877996 VRAM=38.4GiB | 2.1% done | ETA(epoch): 1484s + [Epoch 10][60/133] loss=0.837644 avg=0.872836 VRAM=38.3GiB | 2.1% done | ETA(epoch): 1306s + [Epoch 10][70/133] loss=1.058156 avg=0.877936 VRAM=38.4GiB | 2.1% done | ETA(epoch): 1127s + [Epoch 10][80/133] loss=0.740740 avg=0.876437 VRAM=38.3GiB | 2.1% done | ETA(epoch): 948s + [Epoch 10][90/133] loss=0.963482 avg=0.877942 VRAM=38.4GiB | 2.1% done | ETA(epoch): 769s + [Epoch 10][100/133] loss=0.911760 avg=0.874902 VRAM=38.3GiB | 2.2% done | ETA(epoch): 590s + [Epoch 10][110/133] loss=0.870662 avg=0.880404 VRAM=38.4GiB | 2.2% done | ETA(epoch): 411s + [Epoch 10][120/133] loss=0.791958 avg=0.879677 VRAM=38.3GiB | 2.2% done | ETA(epoch): 233s + [Epoch 10][130/133] loss=0.781462 avg=0.880438 VRAM=38.4GiB | 2.2% done | ETA(epoch): 54s + Train loss: 0.880769 (2380.2s) ETA: 22043min + Val loss: 0.890132 [t_0.0-0.2=1.0991 t_0.2-0.4=0.9861 t_0.4-0.6=0.8487 t_0.6-0.8=0.7304 t_0.8-1.0=0.7313] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0010 +[MEM @ epoch 10 end] RAM: 42.4/1007.0 GiB (4.2%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 11/499 (2% done) --- + [Epoch 11][10/133] loss=0.843086 avg=0.867335 VRAM=38.4GiB | 2.2% done | ETA(epoch): 2201s + [Epoch 11][20/133] loss=0.975943 avg=0.875194 VRAM=38.3GiB | 2.2% done | ETA(epoch): 2022s + [Epoch 11][30/133] loss=0.795215 avg=0.862231 VRAM=38.4GiB | 2.2% done | ETA(epoch): 1843s + [Epoch 11][40/133] loss=0.837438 avg=0.863409 VRAM=38.3GiB | 2.3% done | ETA(epoch): 1664s + [Epoch 11][50/133] loss=1.021595 avg=0.865995 VRAM=38.4GiB | 2.3% done | ETA(epoch): 1485s + [Epoch 11][60/133] loss=0.964752 avg=0.859866 VRAM=38.3GiB | 2.3% done | ETA(epoch): 1307s + [Epoch 11][70/133] loss=0.799993 avg=0.860234 VRAM=38.4GiB | 2.3% done | ETA(epoch): 1128s + [Epoch 11][80/133] loss=0.979786 avg=0.864353 VRAM=38.3GiB | 2.3% done | ETA(epoch): 949s + [Epoch 11][90/133] loss=0.963019 avg=0.866261 VRAM=38.4GiB | 2.3% done | ETA(epoch): 770s + [Epoch 11][100/133] loss=0.863559 avg=0.873328 VRAM=38.3GiB | 2.4% done | ETA(epoch): 591s + [Epoch 11][110/133] loss=0.720007 avg=0.872358 VRAM=38.4GiB | 2.4% done | ETA(epoch): 412s + [Epoch 11][120/133] loss=0.974573 avg=0.872201 VRAM=38.3GiB | 2.4% done | ETA(epoch): 233s + [Epoch 11][130/133] loss=0.952214 avg=0.871690 VRAM=38.4GiB | 2.4% done | ETA(epoch): 54s + Train loss: 0.872838 (2381.5s) ETA: 21894min + Val loss: 0.868290 [t_0.0-0.2=1.0784 t_0.2-0.4=1.0037 t_0.4-0.6=0.8410 t_0.6-0.8=0.7411 t_0.8-1.0=0.7309] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0011 (BEST) + Deleted old checkpoint: checkpoint_epoch_0007 + Deleted old checkpoint: checkpoint_epoch_0008 +[MEM @ epoch 11 end] RAM: 42.6/1007.0 GiB (4.2%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 12/499 (2% done) --- + [Epoch 12][10/133] loss=0.850487 avg=0.844228 VRAM=38.4GiB | 2.4% done | ETA(epoch): 2194s + [Epoch 12][20/133] loss=0.948374 avg=0.844470 VRAM=38.3GiB | 2.4% done | ETA(epoch): 2020s + [Epoch 12][30/133] loss=0.872223 avg=0.866204 VRAM=38.4GiB | 2.4% done | ETA(epoch): 1842s + [Epoch 12][40/133] loss=0.808613 avg=0.876705 VRAM=38.3GiB | 2.5% done | ETA(epoch): 1663s + [Epoch 12][50/133] loss=0.738890 avg=0.874905 VRAM=38.4GiB | 2.5% done | ETA(epoch): 1485s + [Epoch 12][60/133] loss=1.051799 avg=0.882883 VRAM=38.3GiB | 2.5% done | ETA(epoch): 1306s + [Epoch 12][70/133] loss=0.897342 avg=0.877327 VRAM=38.4GiB | 2.5% done | ETA(epoch): 1127s + [Epoch 12][80/133] loss=0.793001 avg=0.877771 VRAM=38.3GiB | 2.5% done | ETA(epoch): 948s + [Epoch 12][90/133] loss=0.888010 avg=0.880759 VRAM=38.4GiB | 2.5% done | ETA(epoch): 769s + [Epoch 12][100/133] loss=0.946425 avg=0.881932 VRAM=38.3GiB | 2.6% done | ETA(epoch): 590s + [Epoch 12][110/133] loss=0.787864 avg=0.878454 VRAM=38.4GiB | 2.6% done | ETA(epoch): 412s + [Epoch 12][120/133] loss=0.870160 avg=0.876307 VRAM=38.3GiB | 2.6% done | ETA(epoch): 233s + [Epoch 12][130/133] loss=0.818155 avg=0.873995 VRAM=38.4GiB | 2.6% done | ETA(epoch): 54s + Train loss: 0.872995 (2379.8s) ETA: 21761min + Val loss: 0.879780 [t_0.0-0.2=1.0761 t_0.2-0.4=0.9862 t_0.4-0.6=0.8674 t_0.6-0.8=0.7524 t_0.8-1.0=0.7381] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0012 + Deleted old checkpoint: checkpoint_epoch_0009 +[MEM @ epoch 12 end] RAM: 43.9/1007.0 GiB (4.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 13/499 (3% done) --- + [Epoch 13][10/133] loss=1.001422 avg=0.936081 VRAM=38.4GiB | 2.6% done | ETA(epoch): 2199s + [Epoch 13][20/133] loss=0.854630 avg=0.890782 VRAM=38.3GiB | 2.6% done | ETA(epoch): 2021s + [Epoch 13][30/133] loss=0.770016 avg=0.865370 VRAM=38.4GiB | 2.6% done | ETA(epoch): 1841s + [Epoch 13][40/133] loss=0.795019 avg=0.877043 VRAM=38.3GiB | 2.7% done | ETA(epoch): 1663s + [Epoch 13][50/133] loss=0.967319 avg=0.873123 VRAM=38.4GiB | 2.7% done | ETA(epoch): 1485s + [Epoch 13][60/133] loss=1.059752 avg=0.875095 VRAM=38.3GiB | 2.7% done | ETA(epoch): 1306s + [Epoch 13][70/133] loss=0.778554 avg=0.866852 VRAM=38.4GiB | 2.7% done | ETA(epoch): 1127s + [Epoch 13][80/133] loss=0.752861 avg=0.862563 VRAM=38.3GiB | 2.7% done | ETA(epoch): 948s + [Epoch 13][90/133] loss=0.778992 avg=0.868901 VRAM=38.4GiB | 2.7% done | ETA(epoch): 769s + [Epoch 13][100/133] loss=0.864326 avg=0.866768 VRAM=38.3GiB | 2.8% done | ETA(epoch): 590s + [Epoch 13][110/133] loss=0.856763 avg=0.868427 VRAM=38.4GiB | 2.8% done | ETA(epoch): 412s + [Epoch 13][120/133] loss=0.870209 avg=0.871048 VRAM=38.3GiB | 2.8% done | ETA(epoch): 233s + [Epoch 13][130/133] loss=0.895121 avg=0.869933 VRAM=38.4GiB | 2.8% done | ETA(epoch): 54s + Train loss: 0.869065 (2385.1s) ETA: 21644min + Val loss: 0.871217 [t_0.0-0.2=1.0766 t_0.2-0.4=0.9913 t_0.4-0.6=0.8730 t_0.6-0.8=0.7293 t_0.8-1.0=0.7319] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0013 + Deleted old checkpoint: checkpoint_epoch_0010 +[MEM @ epoch 13 end] RAM: 42.6/1007.0 GiB (4.2%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 14/499 (3% done) --- + [Epoch 14][10/133] loss=0.878983 avg=0.874122 VRAM=38.4GiB | 2.8% done | ETA(epoch): 2196s + [Epoch 14][20/133] loss=0.912322 avg=0.857057 VRAM=38.3GiB | 2.8% done | ETA(epoch): 2042s + [Epoch 14][30/133] loss=0.830843 avg=0.858372 VRAM=38.4GiB | 2.8% done | ETA(epoch): 1855s + [Epoch 14][40/133] loss=0.932959 avg=0.863574 VRAM=38.3GiB | 2.9% done | ETA(epoch): 1671s + [Epoch 14][50/133] loss=0.796364 avg=0.871697 VRAM=38.4GiB | 2.9% done | ETA(epoch): 1490s + [Epoch 14][60/133] loss=0.883935 avg=0.871457 VRAM=38.3GiB | 2.9% done | ETA(epoch): 1309s + [Epoch 14][70/133] loss=0.987267 avg=0.869903 VRAM=38.4GiB | 2.9% done | ETA(epoch): 1129s + [Epoch 14][80/133] loss=0.871323 avg=0.877258 VRAM=38.3GiB | 2.9% done | ETA(epoch): 950s + [Epoch 14][90/133] loss=0.974360 avg=0.876280 VRAM=38.4GiB | 2.9% done | ETA(epoch): 771s + [Epoch 14][100/133] loss=0.848575 avg=0.876606 VRAM=38.3GiB | 3.0% done | ETA(epoch): 591s + [Epoch 14][110/133] loss=0.771036 avg=0.875346 VRAM=38.4GiB | 3.0% done | ETA(epoch): 412s + [Epoch 14][120/133] loss=0.968287 avg=0.872967 VRAM=38.3GiB | 3.0% done | ETA(epoch): 233s + [Epoch 14][130/133] loss=0.805763 avg=0.871667 VRAM=38.4GiB | 3.0% done | ETA(epoch): 54s + Train loss: 0.872823 (2384.0s) ETA: 21537min + Val loss: 0.876365 [t_0.0-0.2=1.0896 t_0.2-0.4=1.0113 t_0.4-0.6=0.8672 t_0.6-0.8=0.6986 t_0.8-1.0=0.7683] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0014 +[MEM @ epoch 14 end] RAM: 56.4/1007.0 GiB (5.6%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 15/499 (3% done) --- + [Epoch 15][10/133] loss=0.872184 avg=0.863602 VRAM=38.4GiB | 3.0% done | ETA(epoch): 2199s + [Epoch 15][20/133] loss=0.961843 avg=0.847901 VRAM=38.3GiB | 3.0% done | ETA(epoch): 2019s + [Epoch 15][30/133] loss=1.020192 avg=0.857268 VRAM=38.4GiB | 3.0% done | ETA(epoch): 1842s + [Epoch 15][40/133] loss=1.008827 avg=0.865385 VRAM=38.3GiB | 3.1% done | ETA(epoch): 1663s + [Epoch 15][50/133] loss=0.781387 avg=0.856517 VRAM=38.4GiB | 3.1% done | ETA(epoch): 1484s + [Epoch 15][60/133] loss=0.888807 avg=0.863936 VRAM=38.3GiB | 3.1% done | ETA(epoch): 1306s + [Epoch 15][70/133] loss=0.908181 avg=0.865972 VRAM=38.4GiB | 3.1% done | ETA(epoch): 1127s + [Epoch 15][80/133] loss=1.042755 avg=0.869684 VRAM=38.3GiB | 3.1% done | ETA(epoch): 948s + [Epoch 15][90/133] loss=0.875968 avg=0.868339 VRAM=38.4GiB | 3.1% done | ETA(epoch): 770s + [Epoch 15][100/133] loss=0.901767 avg=0.862872 VRAM=38.3GiB | 3.2% done | ETA(epoch): 591s + [Epoch 15][110/133] loss=0.944795 avg=0.863501 VRAM=38.4GiB | 3.2% done | ETA(epoch): 412s + [Epoch 15][120/133] loss=1.000549 avg=0.860586 VRAM=38.3GiB | 3.2% done | ETA(epoch): 233s + [Epoch 15][130/133] loss=0.917080 avg=0.862393 VRAM=38.4GiB | 3.2% done | ETA(epoch): 54s + Train loss: 0.861751 (2382.7s) ETA: 21437min + Val loss: 0.894497 [t_0.0-0.2=1.0851 t_0.2-0.4=1.0093 t_0.4-0.6=0.8574 t_0.6-0.8=0.7279 t_0.8-1.0=0.7456] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0015 + Deleted old checkpoint: checkpoint_epoch_0012 +[MEM @ epoch 15 end] RAM: 54.5/1007.0 GiB (5.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 16/499 (3% done) --- + [Epoch 16][10/133] loss=0.710673 avg=0.821987 VRAM=38.4GiB | 3.2% done | ETA(epoch): 2198s + [Epoch 16][20/133] loss=1.025571 avg=0.853537 VRAM=38.3GiB | 3.2% done | ETA(epoch): 2021s + [Epoch 16][30/133] loss=0.783307 avg=0.864766 VRAM=38.4GiB | 3.2% done | ETA(epoch): 1843s + [Epoch 16][40/133] loss=0.987077 avg=0.872564 VRAM=38.3GiB | 3.3% done | ETA(epoch): 1663s + [Epoch 16][50/133] loss=0.816814 avg=0.859977 VRAM=38.4GiB | 3.3% done | ETA(epoch): 1485s + [Epoch 16][60/133] loss=0.820111 avg=0.865422 VRAM=38.3GiB | 3.3% done | ETA(epoch): 1306s + [Epoch 16][70/133] loss=1.035239 avg=0.867806 VRAM=38.4GiB | 3.3% done | ETA(epoch): 1127s + [Epoch 16][80/133] loss=0.871212 avg=0.871737 VRAM=38.3GiB | 3.3% done | ETA(epoch): 950s + [Epoch 16][90/133] loss=0.808437 avg=0.871199 VRAM=38.4GiB | 3.3% done | ETA(epoch): 771s + [Epoch 16][100/133] loss=0.832055 avg=0.871531 VRAM=38.3GiB | 3.4% done | ETA(epoch): 591s + [Epoch 16][110/133] loss=0.858387 avg=0.872552 VRAM=38.4GiB | 3.4% done | ETA(epoch): 412s + [Epoch 16][120/133] loss=0.797624 avg=0.874240 VRAM=38.3GiB | 3.4% done | ETA(epoch): 233s + [Epoch 16][130/133] loss=1.004551 avg=0.874156 VRAM=38.4GiB | 3.4% done | ETA(epoch): 54s + Train loss: 0.875320 (2384.7s) ETA: 21344min + Val loss: 0.900710 [t_0.0-0.2=1.0680 t_0.2-0.4=0.9941 t_0.4-0.6=0.8692 t_0.6-0.8=0.7295 t_0.8-1.0=0.7251] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0016 + Deleted old checkpoint: checkpoint_epoch_0013 +[MEM @ epoch 16 end] RAM: 55.4/1007.0 GiB (5.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 17/499 (3% done) --- + [Epoch 17][10/133] loss=0.879025 avg=0.881089 VRAM=38.4GiB | 3.4% done | ETA(epoch): 2192s + [Epoch 17][20/133] loss=0.696983 avg=0.858507 VRAM=38.3GiB | 3.4% done | ETA(epoch): 2016s + [Epoch 17][30/133] loss=0.829745 avg=0.861871 VRAM=38.4GiB | 3.4% done | ETA(epoch): 1839s + [Epoch 17][40/133] loss=0.807958 avg=0.861848 VRAM=38.3GiB | 3.5% done | ETA(epoch): 1661s + [Epoch 17][50/133] loss=0.773826 avg=0.862051 VRAM=38.4GiB | 3.5% done | ETA(epoch): 1484s + [Epoch 17][60/133] loss=1.001316 avg=0.864447 VRAM=38.3GiB | 3.5% done | ETA(epoch): 1305s + [Epoch 17][70/133] loss=0.935773 avg=0.867336 VRAM=38.4GiB | 3.5% done | ETA(epoch): 1126s + [Epoch 17][80/133] loss=0.889991 avg=0.871175 VRAM=38.3GiB | 3.5% done | ETA(epoch): 948s + [Epoch 17][90/133] loss=0.739277 avg=0.870548 VRAM=38.4GiB | 3.5% done | ETA(epoch): 770s + [Epoch 17][100/133] loss=0.918908 avg=0.864107 VRAM=38.3GiB | 3.6% done | ETA(epoch): 592s + [Epoch 17][110/133] loss=0.839466 avg=0.863434 VRAM=38.4GiB | 3.6% done | ETA(epoch): 412s + [Epoch 17][120/133] loss=0.835388 avg=0.863943 VRAM=38.3GiB | 3.6% done | ETA(epoch): 233s + [Epoch 17][130/133] loss=0.854827 avg=0.865325 VRAM=38.4GiB | 3.6% done | ETA(epoch): 54s + Train loss: 0.865089 (2382.8s) ETA: 21257min + Val loss: 0.867439 [t_0.0-0.2=1.0948 t_0.2-0.4=1.0237 t_0.4-0.6=0.8162 t_0.6-0.8=0.7096 t_0.8-1.0=0.7097] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0017 (BEST) + Deleted old checkpoint: checkpoint_epoch_0011 + Deleted old checkpoint: checkpoint_epoch_0014 +[MEM @ epoch 17 end] RAM: 53.8/1007.0 GiB (5.3%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 18/499 (4% done) --- + [Epoch 18][10/133] loss=0.844173 avg=0.873051 VRAM=38.4GiB | 3.6% done | ETA(epoch): 2199s + [Epoch 18][20/133] loss=0.831395 avg=0.858221 VRAM=38.3GiB | 3.6% done | ETA(epoch): 2019s + [Epoch 18][30/133] loss=0.880785 avg=0.860729 VRAM=38.4GiB | 3.6% done | ETA(epoch): 1842s + [Epoch 18][40/133] loss=0.837552 avg=0.865912 VRAM=38.3GiB | 3.7% done | ETA(epoch): 1663s + [Epoch 18][50/133] loss=0.851577 avg=0.861131 VRAM=38.4GiB | 3.7% done | ETA(epoch): 1485s + [Epoch 18][60/133] loss=0.782547 avg=0.865655 VRAM=38.3GiB | 3.7% done | ETA(epoch): 1305s + [Epoch 18][70/133] loss=0.974058 avg=0.866731 VRAM=38.4GiB | 3.7% done | ETA(epoch): 1127s + [Epoch 18][80/133] loss=0.947856 avg=0.870901 VRAM=38.3GiB | 3.7% done | ETA(epoch): 948s + [Epoch 18][90/133] loss=0.795135 avg=0.869010 VRAM=38.4GiB | 3.7% done | ETA(epoch): 769s + [Epoch 18][100/133] loss=0.836942 avg=0.873832 VRAM=38.3GiB | 3.8% done | ETA(epoch): 590s + [Epoch 18][110/133] loss=0.888694 avg=0.872040 VRAM=38.4GiB | 3.8% done | ETA(epoch): 411s + [Epoch 18][120/133] loss=0.968306 avg=0.870556 VRAM=38.3GiB | 3.8% done | ETA(epoch): 232s + [Epoch 18][130/133] loss=0.892266 avg=0.872599 VRAM=38.4GiB | 3.8% done | ETA(epoch): 54s + Train loss: 0.872229 (2377.7s) ETA: 21172min + Val loss: 0.880152 [t_0.0-0.2=1.0906 t_0.2-0.4=1.0016 t_0.4-0.6=0.8567 t_0.6-0.8=0.7431 t_0.8-1.0=0.7024] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0018 + Deleted old checkpoint: checkpoint_epoch_0015 +[MEM @ epoch 18 end] RAM: 54.2/1007.0 GiB (5.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 19/499 (4% done) --- + [Epoch 19][10/133] loss=0.808655 avg=0.891813 VRAM=38.4GiB | 3.8% done | ETA(epoch): 2202s + [Epoch 19][20/133] loss=0.888531 avg=0.891690 VRAM=38.3GiB | 3.8% done | ETA(epoch): 2020s + [Epoch 19][30/133] loss=0.790091 avg=0.897754 VRAM=38.4GiB | 3.8% done | ETA(epoch): 1842s + [Epoch 19][40/133] loss=0.881384 avg=0.893600 VRAM=38.3GiB | 3.9% done | ETA(epoch): 1663s + [Epoch 19][50/133] loss=1.059092 avg=0.892649 VRAM=38.4GiB | 3.9% done | ETA(epoch): 1484s + [Epoch 19][60/133] loss=0.989594 avg=0.883197 VRAM=38.3GiB | 3.9% done | ETA(epoch): 1306s + [Epoch 19][70/133] loss=0.910562 avg=0.886417 VRAM=38.4GiB | 3.9% done | ETA(epoch): 1127s + [Epoch 19][80/133] loss=0.923303 avg=0.888877 VRAM=38.3GiB | 3.9% done | ETA(epoch): 948s + [Epoch 19][90/133] loss=0.757954 avg=0.886573 VRAM=38.4GiB | 3.9% done | ETA(epoch): 769s + [Epoch 19][100/133] loss=0.785163 avg=0.886846 VRAM=38.3GiB | 4.0% done | ETA(epoch): 590s + [Epoch 19][110/133] loss=0.925548 avg=0.888917 VRAM=38.4GiB | 4.0% done | ETA(epoch): 412s + [Epoch 19][120/133] loss=0.964721 avg=0.884146 VRAM=38.3GiB | 4.0% done | ETA(epoch): 233s + [Epoch 19][130/133] loss=0.958596 avg=0.880259 VRAM=38.4GiB | 4.0% done | ETA(epoch): 54s + Train loss: 0.879872 (2381.6s) ETA: 21093min + Val loss: 0.878398 [t_0.0-0.2=1.0925 t_0.2-0.4=0.9996 t_0.4-0.6=0.8610 t_0.6-0.8=0.7381 t_0.8-1.0=0.7227] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0019 + Deleted old checkpoint: checkpoint_epoch_0016 +[MEM @ epoch 19 end] RAM: 45.0/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 20/499 (4% done) --- + [Epoch 20][10/133] loss=0.877217 avg=0.915603 VRAM=38.4GiB | 4.0% done | ETA(epoch): 2192s + [Epoch 20][20/133] loss=0.885897 avg=0.911068 VRAM=38.3GiB | 4.0% done | ETA(epoch): 2016s + [Epoch 20][30/133] loss=0.822979 avg=0.885451 VRAM=38.4GiB | 4.0% done | ETA(epoch): 1853s + [Epoch 20][40/133] loss=0.806842 avg=0.874111 VRAM=38.3GiB | 4.1% done | ETA(epoch): 1671s + [Epoch 20][50/133] loss=0.948496 avg=0.881645 VRAM=38.4GiB | 4.1% done | ETA(epoch): 1491s + [Epoch 20][60/133] loss=0.792279 avg=0.873281 VRAM=38.3GiB | 4.1% done | ETA(epoch): 1310s + [Epoch 20][70/133] loss=0.860882 avg=0.869839 VRAM=38.4GiB | 4.1% done | ETA(epoch): 1133s + [Epoch 20][80/133] loss=0.782328 avg=0.866788 VRAM=38.3GiB | 4.1% done | ETA(epoch): 953s + [Epoch 20][90/133] loss=0.750024 avg=0.866368 VRAM=38.4GiB | 4.1% done | ETA(epoch): 773s + [Epoch 20][100/133] loss=0.914553 avg=0.859974 VRAM=38.3GiB | 4.2% done | ETA(epoch): 593s + [Epoch 20][110/133] loss=0.769923 avg=0.860683 VRAM=38.4GiB | 4.2% done | ETA(epoch): 413s + [Epoch 20][120/133] loss=0.782516 avg=0.855628 VRAM=38.3GiB | 4.2% done | ETA(epoch): 233s + [Epoch 20][130/133] loss=1.078539 avg=0.861317 VRAM=38.4GiB | 4.2% done | ETA(epoch): 54s + Train loss: 0.861304 (2388.1s) ETA: 21020min + Val loss: 0.877006 [t_0.0-0.2=1.0503 t_0.2-0.4=1.0155 t_0.4-0.6=0.8349 t_0.6-0.8=0.7543 t_0.8-1.0=0.7006] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0020 +[MEM @ epoch 20 end] RAM: 45.6/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 21/499 (4% done) --- + [Epoch 21][10/133] loss=0.871239 avg=0.795870 VRAM=38.4GiB | 4.2% done | ETA(epoch): 2198s + [Epoch 21][20/133] loss=0.716387 avg=0.835481 VRAM=38.3GiB | 4.2% done | ETA(epoch): 2023s + [Epoch 21][30/133] loss=0.747229 avg=0.836141 VRAM=38.4GiB | 4.2% done | ETA(epoch): 1848s + [Epoch 21][40/133] loss=0.890088 avg=0.839567 VRAM=38.3GiB | 4.3% done | ETA(epoch): 1672s + [Epoch 21][50/133] loss=0.967192 avg=0.846712 VRAM=38.4GiB | 4.3% done | ETA(epoch): 1491s + [Epoch 21][60/133] loss=0.983116 avg=0.852029 VRAM=38.3GiB | 4.3% done | ETA(epoch): 1310s + [Epoch 21][70/133] loss=0.882070 avg=0.852566 VRAM=38.4GiB | 4.3% done | ETA(epoch): 1130s + [Epoch 21][80/133] loss=0.932539 avg=0.860229 VRAM=38.3GiB | 4.3% done | ETA(epoch): 951s + [Epoch 21][90/133] loss=0.868704 avg=0.863946 VRAM=38.4GiB | 4.3% done | ETA(epoch): 771s + [Epoch 21][100/133] loss=0.975596 avg=0.861929 VRAM=38.3GiB | 4.4% done | ETA(epoch): 592s + [Epoch 21][110/133] loss=0.894147 avg=0.863954 VRAM=38.4GiB | 4.4% done | ETA(epoch): 412s + [Epoch 21][120/133] loss=0.823404 avg=0.864612 VRAM=38.3GiB | 4.4% done | ETA(epoch): 233s + [Epoch 21][130/133] loss=0.895014 avg=0.865720 VRAM=38.4GiB | 4.4% done | ETA(epoch): 54s + Train loss: 0.866004 (2383.2s) ETA: 20947min + Val loss: 0.868534 [t_0.0-0.2=1.0796 t_0.2-0.4=1.0185 t_0.4-0.6=0.8770 t_0.6-0.8=0.7214 t_0.8-1.0=0.7111] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0021 + Deleted old checkpoint: checkpoint_epoch_0018 +[MEM @ epoch 21 end] RAM: 44.2/1007.0 GiB (4.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 22/499 (4% done) --- + [Epoch 22][10/133] loss=1.002735 avg=0.891096 VRAM=38.4GiB | 4.4% done | ETA(epoch): 2196s + [Epoch 22][20/133] loss=0.887124 avg=0.895535 VRAM=38.3GiB | 4.4% done | ETA(epoch): 2018s + [Epoch 22][30/133] loss=0.848973 avg=0.891593 VRAM=38.4GiB | 4.4% done | ETA(epoch): 1841s + [Epoch 22][40/133] loss=1.048726 avg=0.895761 VRAM=38.3GiB | 4.5% done | ETA(epoch): 1664s + [Epoch 22][50/133] loss=0.861461 avg=0.892943 VRAM=38.4GiB | 4.5% done | ETA(epoch): 1484s + [Epoch 22][60/133] loss=0.970960 avg=0.889408 VRAM=38.3GiB | 4.5% done | ETA(epoch): 1305s + [Epoch 22][70/133] loss=0.915942 avg=0.884289 VRAM=38.4GiB | 4.5% done | ETA(epoch): 1126s + [Epoch 22][80/133] loss=0.812526 avg=0.880950 VRAM=38.3GiB | 4.5% done | ETA(epoch): 948s + [Epoch 22][90/133] loss=0.738770 avg=0.877603 VRAM=38.4GiB | 4.5% done | ETA(epoch): 769s + [Epoch 22][100/133] loss=0.921073 avg=0.874923 VRAM=38.3GiB | 4.6% done | ETA(epoch): 590s + [Epoch 22][110/133] loss=0.977864 avg=0.874456 VRAM=38.4GiB | 4.6% done | ETA(epoch): 411s + [Epoch 22][120/133] loss=0.964275 avg=0.874878 VRAM=38.3GiB | 4.6% done | ETA(epoch): 232s + [Epoch 22][130/133] loss=0.897280 avg=0.873978 VRAM=38.4GiB | 4.6% done | ETA(epoch): 54s + Train loss: 0.875381 (2378.8s) ETA: 20876min + Val loss: 0.862790 [t_0.0-0.2=1.0802 t_0.2-0.4=1.0088 t_0.4-0.6=0.8751 t_0.6-0.8=0.7178 t_0.8-1.0=0.7053] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0022 (BEST) + Deleted old checkpoint: checkpoint_epoch_0017 + Deleted old checkpoint: checkpoint_epoch_0019 +[MEM @ epoch 22 end] RAM: 45.0/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 23/499 (5% done) --- + [Epoch 23][10/133] loss=0.909763 avg=0.921693 VRAM=38.4GiB | 4.6% done | ETA(epoch): 2208s + [Epoch 23][20/133] loss=0.905639 avg=0.897073 VRAM=38.3GiB | 4.6% done | ETA(epoch): 2034s + [Epoch 23][30/133] loss=0.855933 avg=0.892916 VRAM=38.4GiB | 4.6% done | ETA(epoch): 1850s + [Epoch 23][40/133] loss=0.975256 avg=0.889881 VRAM=38.3GiB | 4.7% done | ETA(epoch): 1669s + [Epoch 23][50/133] loss=1.028978 avg=0.891130 VRAM=38.4GiB | 4.7% done | ETA(epoch): 1489s + [Epoch 23][60/133] loss=0.865014 avg=0.887174 VRAM=38.3GiB | 4.7% done | ETA(epoch): 1309s + [Epoch 23][70/133] loss=0.973312 avg=0.883696 VRAM=38.4GiB | 4.7% done | ETA(epoch): 1129s + [Epoch 23][80/133] loss=1.063625 avg=0.886303 VRAM=38.3GiB | 4.7% done | ETA(epoch): 950s + [Epoch 23][90/133] loss=0.866590 avg=0.881840 VRAM=38.4GiB | 4.7% done | ETA(epoch): 771s + [Epoch 23][100/133] loss=0.716303 avg=0.878499 VRAM=38.3GiB | 4.8% done | ETA(epoch): 591s + [Epoch 23][110/133] loss=0.938200 avg=0.878587 VRAM=38.4GiB | 4.8% done | ETA(epoch): 412s + [Epoch 23][120/133] loss=0.907645 avg=0.879459 VRAM=38.3GiB | 4.8% done | ETA(epoch): 233s + [Epoch 23][130/133] loss=0.782872 avg=0.877925 VRAM=38.4GiB | 4.8% done | ETA(epoch): 54s + Train loss: 0.876561 (2382.1s) ETA: 20808min + Val loss: 0.857622 [t_0.0-0.2=1.0776 t_0.2-0.4=1.0145 t_0.4-0.6=0.8153 t_0.6-0.8=0.7365 t_0.8-1.0=0.7053] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0023 (BEST) + Deleted old checkpoint: checkpoint_epoch_0020 +[MEM @ epoch 23 end] RAM: 44.7/1007.0 GiB (4.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 24/499 (5% done) --- + [Epoch 24][10/133] loss=0.985291 avg=0.936922 VRAM=38.4GiB | 4.8% done | ETA(epoch): 2205s + [Epoch 24][20/133] loss=0.933445 avg=0.899341 VRAM=38.3GiB | 4.8% done | ETA(epoch): 2023s + [Epoch 24][30/133] loss=0.787979 avg=0.893779 VRAM=38.4GiB | 4.8% done | ETA(epoch): 1844s + [Epoch 24][40/133] loss=0.733892 avg=0.874669 VRAM=38.3GiB | 4.9% done | ETA(epoch): 1665s + [Epoch 24][50/133] loss=0.947768 avg=0.878443 VRAM=38.4GiB | 4.9% done | ETA(epoch): 1486s + [Epoch 24][60/133] loss=0.954784 avg=0.874547 VRAM=38.3GiB | 4.9% done | ETA(epoch): 1309s + [Epoch 24][70/133] loss=0.808263 avg=0.873165 VRAM=38.4GiB | 4.9% done | ETA(epoch): 1130s + [Epoch 24][80/133] loss=0.778960 avg=0.868263 VRAM=38.3GiB | 4.9% done | ETA(epoch): 950s + [Epoch 24][90/133] loss=0.932655 avg=0.863128 VRAM=38.4GiB | 4.9% done | ETA(epoch): 772s + [Epoch 24][100/133] loss=0.910070 avg=0.862678 VRAM=38.3GiB | 5.0% done | ETA(epoch): 592s + [Epoch 24][110/133] loss=1.031656 avg=0.864694 VRAM=38.4GiB | 5.0% done | ETA(epoch): 413s + [Epoch 24][120/133] loss=1.045520 avg=0.868150 VRAM=38.3GiB | 5.0% done | ETA(epoch): 233s + [Epoch 24][130/133] loss=0.848071 avg=0.871181 VRAM=38.4GiB | 5.0% done | ETA(epoch): 54s + Train loss: 0.870749 (2386.3s) ETA: 20743min + Val loss: 0.879571 [t_0.0-0.2=1.0696 t_0.2-0.4=1.0033 t_0.4-0.6=0.8359 t_0.6-0.8=0.7437 t_0.8-1.0=0.6969] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0024 + Deleted old checkpoint: checkpoint_epoch_0021 +[MEM @ epoch 24 end] RAM: 45.0/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 25/499 (5% done) --- + [Epoch 25][10/133] loss=0.778661 avg=0.824825 VRAM=38.4GiB | 5.0% done | ETA(epoch): 2239s + [Epoch 25][20/133] loss=0.943010 avg=0.857515 VRAM=38.3GiB | 5.0% done | ETA(epoch): 2040s + [Epoch 25][30/133] loss=0.756169 avg=0.866476 VRAM=38.4GiB | 5.0% done | ETA(epoch): 1855s + [Epoch 25][40/133] loss=0.758743 avg=0.861918 VRAM=38.3GiB | 5.1% done | ETA(epoch): 1673s + [Epoch 25][50/133] loss=0.838171 avg=0.868094 VRAM=38.4GiB | 5.1% done | ETA(epoch): 1490s + [Epoch 25][60/133] loss=0.803491 avg=0.865196 VRAM=38.3GiB | 5.1% done | ETA(epoch): 1310s + [Epoch 25][70/133] loss=0.888001 avg=0.864575 VRAM=38.4GiB | 5.1% done | ETA(epoch): 1130s + [Epoch 25][80/133] loss=0.857309 avg=0.868389 VRAM=38.3GiB | 5.1% done | ETA(epoch): 950s + [Epoch 25][90/133] loss=0.770293 avg=0.872982 VRAM=38.4GiB | 5.1% done | ETA(epoch): 771s + [Epoch 25][100/133] loss=0.857863 avg=0.874479 VRAM=38.3GiB | 5.2% done | ETA(epoch): 591s + [Epoch 25][110/133] loss=0.822387 avg=0.872921 VRAM=38.4GiB | 5.2% done | ETA(epoch): 412s + [Epoch 25][120/133] loss=0.917062 avg=0.868396 VRAM=38.3GiB | 5.2% done | ETA(epoch): 233s + [Epoch 25][130/133] loss=0.913956 avg=0.863948 VRAM=38.4GiB | 5.2% done | ETA(epoch): 54s + Train loss: 0.864028 (2381.3s) ETA: 20679min + Val loss: 0.886741 [t_0.0-0.2=1.0566 t_0.2-0.4=1.0327 t_0.4-0.6=0.8594 t_0.6-0.8=0.7212 t_0.8-1.0=0.6946] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0025 + Deleted old checkpoint: checkpoint_epoch_0022 +[MEM @ epoch 25 end] RAM: 45.5/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 26/499 (5% done) --- + [Epoch 26][10/133] loss=0.780628 avg=0.863271 VRAM=38.4GiB | 5.2% done | ETA(epoch): 2196s + [Epoch 26][20/133] loss=0.707116 avg=0.869076 VRAM=38.3GiB | 5.2% done | ETA(epoch): 2017s + [Epoch 26][30/133] loss=0.874112 avg=0.868117 VRAM=38.4GiB | 5.2% done | ETA(epoch): 1839s + [Epoch 26][40/133] loss=0.862996 avg=0.861300 VRAM=38.3GiB | 5.3% done | ETA(epoch): 1661s + [Epoch 26][50/133] loss=0.825098 avg=0.865502 VRAM=38.4GiB | 5.3% done | ETA(epoch): 1485s + [Epoch 26][60/133] loss=0.846348 avg=0.866196 VRAM=38.3GiB | 5.3% done | ETA(epoch): 1306s + [Epoch 26][70/133] loss=0.915973 avg=0.866144 VRAM=38.4GiB | 5.3% done | ETA(epoch): 1129s + [Epoch 26][80/133] loss=0.743140 avg=0.860602 VRAM=38.3GiB | 5.3% done | ETA(epoch): 950s + [Epoch 26][90/133] loss=0.771098 avg=0.862675 VRAM=38.4GiB | 5.3% done | ETA(epoch): 770s + [Epoch 26][100/133] loss=0.948258 avg=0.861118 VRAM=38.3GiB | 5.4% done | ETA(epoch): 592s + [Epoch 26][110/133] loss=0.806764 avg=0.861934 VRAM=38.4GiB | 5.4% done | ETA(epoch): 412s + [Epoch 26][120/133] loss=0.799396 avg=0.863527 VRAM=38.3GiB | 5.4% done | ETA(epoch): 233s + [Epoch 26][130/133] loss=0.820932 avg=0.864831 VRAM=38.4GiB | 5.4% done | ETA(epoch): 54s + Train loss: 0.864397 (2385.3s) ETA: 20618min + Val loss: 0.853731 [t_0.0-0.2=1.0592 t_0.2-0.4=0.9907 t_0.4-0.6=0.8403 t_0.6-0.8=0.7427 t_0.8-1.0=0.6916] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0026 (BEST) + Deleted old checkpoint: checkpoint_epoch_0023 +[MEM @ epoch 26 end] RAM: 44.5/1007.0 GiB (4.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 27/499 (5% done) --- + [Epoch 27][10/133] loss=0.830352 avg=0.863677 VRAM=38.4GiB | 5.4% done | ETA(epoch): 2197s + [Epoch 27][20/133] loss=0.824267 avg=0.888559 VRAM=38.3GiB | 5.4% done | ETA(epoch): 2019s + [Epoch 27][30/133] loss=0.797465 avg=0.888951 VRAM=38.4GiB | 5.4% done | ETA(epoch): 1841s + [Epoch 27][40/133] loss=0.734370 avg=0.893740 VRAM=38.3GiB | 5.5% done | ETA(epoch): 1667s + [Epoch 27][50/133] loss=0.944886 avg=0.893025 VRAM=38.4GiB | 5.5% done | ETA(epoch): 1487s + [Epoch 27][60/133] loss=0.732928 avg=0.894192 VRAM=38.3GiB | 5.5% done | ETA(epoch): 1310s + [Epoch 27][70/133] loss=0.733780 avg=0.883719 VRAM=38.4GiB | 5.5% done | ETA(epoch): 1130s + [Epoch 27][80/133] loss=0.920868 avg=0.879049 VRAM=38.3GiB | 5.5% done | ETA(epoch): 950s + [Epoch 27][90/133] loss=0.888179 avg=0.874712 VRAM=38.4GiB | 5.5% done | ETA(epoch): 773s + [Epoch 27][100/133] loss=0.855338 avg=0.872912 VRAM=38.3GiB | 5.6% done | ETA(epoch): 594s + [Epoch 27][110/133] loss=0.812454 avg=0.876801 VRAM=38.4GiB | 5.6% done | ETA(epoch): 413s + [Epoch 27][120/133] loss=0.988481 avg=0.874514 VRAM=38.3GiB | 5.6% done | ETA(epoch): 234s + [Epoch 27][130/133] loss=0.805650 avg=0.871981 VRAM=38.4GiB | 5.6% done | ETA(epoch): 54s + Train loss: 0.871477 (2388.1s) ETA: 20560min + Val loss: 0.889808 [t_0.0-0.2=1.0846 t_0.2-0.4=0.9979 t_0.4-0.6=0.8550 t_0.6-0.8=0.7237 t_0.8-1.0=0.6754] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0027 + Deleted old checkpoint: checkpoint_epoch_0024 +[MEM @ epoch 27 end] RAM: 44.4/1007.0 GiB (4.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 28/499 (6% done) --- + [Epoch 28][10/133] loss=0.886866 avg=0.809424 VRAM=38.4GiB | 5.6% done | ETA(epoch): 2192s + [Epoch 28][20/133] loss=0.920214 avg=0.841656 VRAM=38.3GiB | 5.6% done | ETA(epoch): 2014s + [Epoch 28][30/133] loss=0.722535 avg=0.854411 VRAM=38.4GiB | 5.6% done | ETA(epoch): 1838s + [Epoch 28][40/133] loss=0.823925 avg=0.863451 VRAM=38.3GiB | 5.7% done | ETA(epoch): 1660s + [Epoch 28][50/133] loss=0.931469 avg=0.871872 VRAM=38.4GiB | 5.7% done | ETA(epoch): 1484s + [Epoch 28][60/133] loss=0.711443 avg=0.871225 VRAM=38.3GiB | 5.7% done | ETA(epoch): 1305s + [Epoch 28][70/133] loss=0.745433 avg=0.870389 VRAM=38.4GiB | 5.7% done | ETA(epoch): 1127s + [Epoch 28][80/133] loss=0.920458 avg=0.873943 VRAM=38.3GiB | 5.7% done | ETA(epoch): 951s + [Epoch 28][90/133] loss=1.050516 avg=0.872235 VRAM=38.4GiB | 5.7% done | ETA(epoch): 771s + [Epoch 28][100/133] loss=0.874887 avg=0.869492 VRAM=38.3GiB | 5.8% done | ETA(epoch): 592s + [Epoch 28][110/133] loss=0.992868 avg=0.872154 VRAM=38.4GiB | 5.8% done | ETA(epoch): 412s + [Epoch 28][120/133] loss=1.008923 avg=0.872409 VRAM=38.3GiB | 5.8% done | ETA(epoch): 233s + [Epoch 28][130/133] loss=0.927058 avg=0.872112 VRAM=38.4GiB | 5.8% done | ETA(epoch): 54s + Train loss: 0.871712 (2382.4s) ETA: 20501min + Val loss: 0.865230 [t_0.0-0.2=1.0795 t_0.2-0.4=1.0189 t_0.4-0.6=0.8832 t_0.6-0.8=0.7386 t_0.8-1.0=0.6909] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0028 + Deleted old checkpoint: checkpoint_epoch_0025 +[MEM @ epoch 28 end] RAM: 46.1/1007.0 GiB (4.6%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 29/499 (6% done) --- + [Epoch 29][10/133] loss=0.876414 avg=0.871864 VRAM=38.4GiB | 5.8% done | ETA(epoch): 2208s + [Epoch 29][20/133] loss=0.851677 avg=0.861879 VRAM=38.3GiB | 5.8% done | ETA(epoch): 2022s + [Epoch 29][30/133] loss=0.734665 avg=0.844880 VRAM=38.4GiB | 5.8% done | ETA(epoch): 1841s + [Epoch 29][40/133] loss=0.782955 avg=0.855472 VRAM=38.3GiB | 5.9% done | ETA(epoch): 1662s + [Epoch 29][50/133] loss=0.910833 avg=0.861243 VRAM=38.4GiB | 5.9% done | ETA(epoch): 1483s + [Epoch 29][60/133] loss=0.800480 avg=0.857932 VRAM=38.3GiB | 5.9% done | ETA(epoch): 1304s + [Epoch 29][70/133] loss=0.850203 avg=0.857191 VRAM=38.4GiB | 5.9% done | ETA(epoch): 1126s + [Epoch 29][80/133] loss=0.968786 avg=0.854903 VRAM=38.3GiB | 5.9% done | ETA(epoch): 947s + [Epoch 29][90/133] loss=0.849451 avg=0.857064 VRAM=38.4GiB | 5.9% done | ETA(epoch): 769s + [Epoch 29][100/133] loss=0.917943 avg=0.862400 VRAM=38.3GiB | 6.0% done | ETA(epoch): 590s + [Epoch 29][110/133] loss=0.836530 avg=0.864344 VRAM=38.4GiB | 6.0% done | ETA(epoch): 411s + [Epoch 29][120/133] loss=0.919419 avg=0.865544 VRAM=38.3GiB | 6.0% done | ETA(epoch): 232s + [Epoch 29][130/133] loss=0.990676 avg=0.867406 VRAM=38.4GiB | 6.0% done | ETA(epoch): 54s + Train loss: 0.868135 (2377.0s) ETA: 20441min + Val loss: 0.876191 [t_0.0-0.2=1.0632 t_0.2-0.4=1.0124 t_0.4-0.6=0.8646 t_0.6-0.8=0.7149 t_0.8-1.0=0.6819] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0029 +[MEM @ epoch 29 end] RAM: 45.7/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 30/499 (6% done) --- + [Epoch 30][10/133] loss=0.898193 avg=0.849703 VRAM=38.4GiB | 6.0% done | ETA(epoch): 2201s + [Epoch 30][20/133] loss=0.752623 avg=0.823661 VRAM=38.3GiB | 6.0% done | ETA(epoch): 2021s + [Epoch 30][30/133] loss=0.881966 avg=0.834648 VRAM=38.4GiB | 6.0% done | ETA(epoch): 1848s + [Epoch 30][40/133] loss=0.906873 avg=0.845626 VRAM=38.3GiB | 6.1% done | ETA(epoch): 1671s + [Epoch 30][50/133] loss=0.822889 avg=0.850537 VRAM=38.4GiB | 6.1% done | ETA(epoch): 1490s + [Epoch 30][60/133] loss=0.778993 avg=0.855467 VRAM=38.3GiB | 6.1% done | ETA(epoch): 1310s + [Epoch 30][70/133] loss=0.839459 avg=0.855306 VRAM=38.4GiB | 6.1% done | ETA(epoch): 1130s + [Epoch 30][80/133] loss=0.877520 avg=0.860872 VRAM=38.3GiB | 6.1% done | ETA(epoch): 950s + [Epoch 30][90/133] loss=0.829199 avg=0.860941 VRAM=38.4GiB | 6.1% done | ETA(epoch): 770s + [Epoch 30][100/133] loss=0.891273 avg=0.863368 VRAM=38.3GiB | 6.2% done | ETA(epoch): 591s + [Epoch 30][110/133] loss=0.884669 avg=0.863326 VRAM=38.4GiB | 6.2% done | ETA(epoch): 412s + [Epoch 30][120/133] loss=0.662977 avg=0.861509 VRAM=38.3GiB | 6.2% done | ETA(epoch): 233s + [Epoch 30][130/133] loss=0.844080 avg=0.860022 VRAM=38.4GiB | 6.2% done | ETA(epoch): 54s + Train loss: 0.859505 (2379.3s) ETA: 20382min + Val loss: 0.870774 [t_0.0-0.2=1.0886 t_0.2-0.4=1.0351 t_0.4-0.6=0.8536 t_0.6-0.8=0.7359 t_0.8-1.0=0.6897] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0030 + Deleted old checkpoint: checkpoint_epoch_0027 +[MEM @ epoch 30 end] RAM: 45.8/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 31/499 (6% done) --- + [Epoch 31][10/133] loss=0.814111 avg=0.884080 VRAM=38.4GiB | 6.2% done | ETA(epoch): 2195s + [Epoch 31][20/133] loss=0.816840 avg=0.890627 VRAM=38.3GiB | 6.2% done | ETA(epoch): 2018s + [Epoch 31][30/133] loss=0.762473 avg=0.865126 VRAM=38.4GiB | 6.2% done | ETA(epoch): 1840s + [Epoch 31][40/133] loss=1.046906 avg=0.874659 VRAM=38.3GiB | 6.3% done | ETA(epoch): 1661s + [Epoch 31][50/133] loss=0.979773 avg=0.876589 VRAM=38.4GiB | 6.3% done | ETA(epoch): 1483s + [Epoch 31][60/133] loss=0.795062 avg=0.872333 VRAM=38.3GiB | 6.3% done | ETA(epoch): 1304s + [Epoch 31][70/133] loss=0.775976 avg=0.875278 VRAM=38.4GiB | 6.3% done | ETA(epoch): 1125s + [Epoch 31][80/133] loss=0.925478 avg=0.875052 VRAM=38.3GiB | 6.3% done | ETA(epoch): 947s + [Epoch 31][90/133] loss=0.981834 avg=0.872726 VRAM=38.4GiB | 6.3% done | ETA(epoch): 768s + [Epoch 31][100/133] loss=0.933854 avg=0.873548 VRAM=38.3GiB | 6.4% done | ETA(epoch): 590s + [Epoch 31][110/133] loss=0.720145 avg=0.874197 VRAM=38.4GiB | 6.4% done | ETA(epoch): 411s + [Epoch 31][120/133] loss=0.707899 avg=0.870808 VRAM=38.3GiB | 6.4% done | ETA(epoch): 232s + [Epoch 31][130/133] loss=0.742803 avg=0.868276 VRAM=38.4GiB | 6.4% done | ETA(epoch): 54s + Train loss: 0.867280 (2377.2s) ETA: 20324min + Val loss: 0.874796 [t_0.0-0.2=1.0838 t_0.2-0.4=1.0126 t_0.4-0.6=0.8402 t_0.6-0.8=0.7103 t_0.8-1.0=0.6950] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0031 + Deleted old checkpoint: checkpoint_epoch_0028 +[MEM @ epoch 31 end] RAM: 46.3/1007.0 GiB (4.6%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 32/499 (6% done) --- + [Epoch 32][10/133] loss=0.956530 avg=0.906332 VRAM=38.4GiB | 6.4% done | ETA(epoch): 2196s + [Epoch 32][20/133] loss=0.911616 avg=0.903297 VRAM=38.3GiB | 6.4% done | ETA(epoch): 2017s + [Epoch 32][30/133] loss=0.895159 avg=0.908455 VRAM=38.4GiB | 6.4% done | ETA(epoch): 1838s + [Epoch 32][40/133] loss=0.809401 avg=0.888664 VRAM=38.3GiB | 6.5% done | ETA(epoch): 1660s + [Epoch 32][50/133] loss=0.937098 avg=0.884293 VRAM=38.4GiB | 6.5% done | ETA(epoch): 1482s + [Epoch 32][60/133] loss=0.983407 avg=0.889045 VRAM=38.3GiB | 6.5% done | ETA(epoch): 1303s + [Epoch 32][70/133] loss=0.794281 avg=0.875804 VRAM=38.4GiB | 6.5% done | ETA(epoch): 1125s + [Epoch 32][80/133] loss=0.742096 avg=0.865864 VRAM=38.3GiB | 6.5% done | ETA(epoch): 947s + [Epoch 32][90/133] loss=0.914272 avg=0.866189 VRAM=38.4GiB | 6.5% done | ETA(epoch): 768s + [Epoch 32][100/133] loss=0.713033 avg=0.867278 VRAM=38.3GiB | 6.6% done | ETA(epoch): 589s + [Epoch 32][110/133] loss=0.948477 avg=0.869191 VRAM=38.4GiB | 6.6% done | ETA(epoch): 411s + [Epoch 32][120/133] loss=0.808325 avg=0.871274 VRAM=38.3GiB | 6.6% done | ETA(epoch): 232s + [Epoch 32][130/133] loss=0.918632 avg=0.872040 VRAM=38.4GiB | 6.6% done | ETA(epoch): 54s + Train loss: 0.872092 (2375.2s) ETA: 20267min + Val loss: 0.881829 [t_0.0-0.2=1.0759 t_0.2-0.4=1.0282 t_0.4-0.6=0.8377 t_0.6-0.8=0.7131 t_0.8-1.0=0.6818] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0032 + Deleted old checkpoint: checkpoint_epoch_0029 +[MEM @ epoch 32 end] RAM: 47.0/1007.0 GiB (4.7%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 33/499 (7% done) --- + [Epoch 33][10/133] loss=0.736909 avg=0.875380 VRAM=38.4GiB | 6.6% done | ETA(epoch): 2237s + [Epoch 33][20/133] loss=0.807003 avg=0.880533 VRAM=38.3GiB | 6.6% done | ETA(epoch): 2038s + [Epoch 33][30/133] loss=0.846376 avg=0.876852 VRAM=38.4GiB | 6.6% done | ETA(epoch): 1852s + [Epoch 33][40/133] loss=0.954137 avg=0.891018 VRAM=38.3GiB | 6.7% done | ETA(epoch): 1671s + [Epoch 33][50/133] loss=0.821631 avg=0.883077 VRAM=38.4GiB | 6.7% done | ETA(epoch): 1489s + [Epoch 33][60/133] loss=0.979627 avg=0.891324 VRAM=38.3GiB | 6.7% done | ETA(epoch): 1311s + [Epoch 33][70/133] loss=0.814727 avg=0.883830 VRAM=38.4GiB | 6.7% done | ETA(epoch): 1131s + [Epoch 33][80/133] loss=0.776461 avg=0.878749 VRAM=38.3GiB | 6.7% done | ETA(epoch): 951s + [Epoch 33][90/133] loss=1.017188 avg=0.884109 VRAM=38.4GiB | 6.7% done | ETA(epoch): 772s + [Epoch 33][100/133] loss=0.923592 avg=0.878103 VRAM=38.3GiB | 6.8% done | ETA(epoch): 592s + [Epoch 33][110/133] loss=0.967628 avg=0.873035 VRAM=38.4GiB | 6.8% done | ETA(epoch): 413s + [Epoch 33][120/133] loss=0.772975 avg=0.869259 VRAM=38.3GiB | 6.8% done | ETA(epoch): 233s + [Epoch 33][130/133] loss=0.880729 avg=0.869714 VRAM=38.4GiB | 6.8% done | ETA(epoch): 54s + Train loss: 0.867755 (2384.1s) ETA: 20213min + Val loss: 0.888573 [t_0.0-0.2=1.0749 t_0.2-0.4=1.0028 t_0.4-0.6=0.8480 t_0.6-0.8=0.7289 t_0.8-1.0=0.6822] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0033 + Deleted old checkpoint: checkpoint_epoch_0030 +[MEM @ epoch 33 end] RAM: 46.1/1007.0 GiB (4.6%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 34/499 (7% done) --- + [Epoch 34][10/133] loss=0.702362 avg=0.857566 VRAM=38.4GiB | 6.8% done | ETA(epoch): 2192s + [Epoch 34][20/133] loss=0.694326 avg=0.832465 VRAM=38.3GiB | 6.8% done | ETA(epoch): 2016s + [Epoch 34][30/133] loss=1.057511 avg=0.847309 VRAM=38.4GiB | 6.8% done | ETA(epoch): 1840s + [Epoch 34][40/133] loss=0.897835 avg=0.859137 VRAM=38.3GiB | 6.9% done | ETA(epoch): 1662s + [Epoch 34][50/133] loss=0.843036 avg=0.848813 VRAM=38.4GiB | 6.9% done | ETA(epoch): 1483s + [Epoch 34][60/133] loss=0.892767 avg=0.854059 VRAM=38.3GiB | 6.9% done | ETA(epoch): 1305s + [Epoch 34][70/133] loss=0.897666 avg=0.860831 VRAM=38.4GiB | 6.9% done | ETA(epoch): 1128s + [Epoch 34][80/133] loss=0.900378 avg=0.857413 VRAM=38.3GiB | 6.9% done | ETA(epoch): 949s + [Epoch 34][90/133] loss=0.813873 avg=0.861708 VRAM=38.4GiB | 6.9% done | ETA(epoch): 770s + [Epoch 34][100/133] loss=0.871118 avg=0.861800 VRAM=38.3GiB | 7.0% done | ETA(epoch): 591s + [Epoch 34][110/133] loss=0.921602 avg=0.860095 VRAM=38.4GiB | 7.0% done | ETA(epoch): 412s + [Epoch 34][120/133] loss=0.892116 avg=0.855469 VRAM=38.3GiB | 7.0% done | ETA(epoch): 233s + [Epoch 34][130/133] loss=1.007599 avg=0.854602 VRAM=38.4GiB | 7.0% done | ETA(epoch): 54s + Train loss: 0.854719 (2384.0s) ETA: 20160min + Val loss: 0.882121 [t_0.0-0.2=1.0685 t_0.2-0.4=1.0215 t_0.4-0.6=0.8311 t_0.6-0.8=0.7318 t_0.8-1.0=0.6827] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0034 + Deleted old checkpoint: checkpoint_epoch_0031 +[MEM @ epoch 34 end] RAM: 44.9/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 35/499 (7% done) --- + [Epoch 35][10/133] loss=0.997830 avg=0.863385 VRAM=38.4GiB | 7.0% done | ETA(epoch): 2198s + [Epoch 35][20/133] loss=0.757938 avg=0.859838 VRAM=38.3GiB | 7.0% done | ETA(epoch): 2017s + [Epoch 35][30/133] loss=0.826715 avg=0.856951 VRAM=38.4GiB | 7.0% done | ETA(epoch): 1840s + [Epoch 35][40/133] loss=0.929776 avg=0.844027 VRAM=38.3GiB | 7.1% done | ETA(epoch): 1661s + [Epoch 35][50/133] loss=0.982511 avg=0.833045 VRAM=38.4GiB | 7.1% done | ETA(epoch): 1485s + [Epoch 35][60/133] loss=0.864294 avg=0.839694 VRAM=38.3GiB | 7.1% done | ETA(epoch): 1309s + [Epoch 35][70/133] loss=0.979145 avg=0.843209 VRAM=38.4GiB | 7.1% done | ETA(epoch): 1129s + [Epoch 35][80/133] loss=0.828956 avg=0.842104 VRAM=38.3GiB | 7.1% done | ETA(epoch): 949s + [Epoch 35][90/133] loss=0.827720 avg=0.846025 VRAM=38.4GiB | 7.1% done | ETA(epoch): 770s + [Epoch 35][100/133] loss=1.009997 avg=0.844811 VRAM=38.3GiB | 7.2% done | ETA(epoch): 591s + [Epoch 35][110/133] loss=0.676239 avg=0.844793 VRAM=38.4GiB | 7.2% done | ETA(epoch): 412s + [Epoch 35][120/133] loss=0.967400 avg=0.844857 VRAM=38.3GiB | 7.2% done | ETA(epoch): 233s + [Epoch 35][130/133] loss=0.757333 avg=0.850472 VRAM=38.4GiB | 7.2% done | ETA(epoch): 54s + Train loss: 0.851828 (2381.7s) ETA: 20106min + Val loss: 0.858223 [t_0.0-0.2=1.0749 t_0.2-0.4=1.0231 t_0.4-0.6=0.8361 t_0.6-0.8=0.7332 t_0.8-1.0=0.6822] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0035 + Deleted old checkpoint: checkpoint_epoch_0032 +[MEM @ epoch 35 end] RAM: 45.4/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 36/499 (7% done) --- + [Epoch 36][10/133] loss=0.869637 avg=0.879626 VRAM=38.4GiB | 7.2% done | ETA(epoch): 2191s + [Epoch 36][20/133] loss=0.784220 avg=0.860562 VRAM=38.3GiB | 7.2% done | ETA(epoch): 2017s + [Epoch 36][30/133] loss=1.015686 avg=0.864018 VRAM=38.4GiB | 7.2% done | ETA(epoch): 1838s + [Epoch 36][40/133] loss=0.961907 avg=0.857245 VRAM=38.3GiB | 7.3% done | ETA(epoch): 1661s + [Epoch 36][50/133] loss=0.762576 avg=0.860380 VRAM=38.4GiB | 7.3% done | ETA(epoch): 1483s + [Epoch 36][60/133] loss=1.001557 avg=0.865506 VRAM=38.3GiB | 7.3% done | ETA(epoch): 1304s + [Epoch 36][70/133] loss=0.901584 avg=0.866854 VRAM=38.4GiB | 7.3% done | ETA(epoch): 1125s + [Epoch 36][80/133] loss=0.978259 avg=0.865257 VRAM=38.3GiB | 7.3% done | ETA(epoch): 947s + [Epoch 36][90/133] loss=1.012546 avg=0.868081 VRAM=38.4GiB | 7.3% done | ETA(epoch): 768s + [Epoch 36][100/133] loss=0.991199 avg=0.874371 VRAM=38.3GiB | 7.4% done | ETA(epoch): 590s + [Epoch 36][110/133] loss=0.956660 avg=0.871405 VRAM=38.4GiB | 7.4% done | ETA(epoch): 411s + [Epoch 36][120/133] loss=0.961753 avg=0.873255 VRAM=38.3GiB | 7.4% done | ETA(epoch): 232s + [Epoch 36][130/133] loss=0.893167 avg=0.876694 VRAM=38.4GiB | 7.4% done | ETA(epoch): 54s + Train loss: 0.877220 (2376.7s) ETA: 20052min + Val loss: 0.873434 [t_0.0-0.2=1.0724 t_0.2-0.4=0.9855 t_0.4-0.6=0.8447 t_0.6-0.8=0.7397 t_0.8-1.0=0.6684] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0036 + Deleted old checkpoint: checkpoint_epoch_0033 +[MEM @ epoch 36 end] RAM: 48.1/1007.0 GiB (4.8%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 37/499 (7% done) --- + [Epoch 37][10/133] loss=0.857382 avg=0.825247 VRAM=38.4GiB | 7.4% done | ETA(epoch): 2195s + [Epoch 37][20/133] loss=0.711670 avg=0.829663 VRAM=38.3GiB | 7.4% done | ETA(epoch): 2020s + [Epoch 37][30/133] loss=0.757527 avg=0.835941 VRAM=38.4GiB | 7.4% done | ETA(epoch): 1842s + [Epoch 37][40/133] loss=0.984616 avg=0.836884 VRAM=38.3GiB | 7.5% done | ETA(epoch): 1662s + [Epoch 37][50/133] loss=0.863423 avg=0.839530 VRAM=38.4GiB | 7.5% done | ETA(epoch): 1484s + [Epoch 37][60/133] loss=1.074039 avg=0.842256 VRAM=38.3GiB | 7.5% done | ETA(epoch): 1305s + [Epoch 37][70/133] loss=0.822100 avg=0.842679 VRAM=38.4GiB | 7.5% done | ETA(epoch): 1126s + [Epoch 37][80/133] loss=0.752087 avg=0.845896 VRAM=38.3GiB | 7.5% done | ETA(epoch): 948s + [Epoch 37][90/133] loss=0.765052 avg=0.847600 VRAM=38.4GiB | 7.5% done | ETA(epoch): 769s + [Epoch 37][100/133] loss=0.745851 avg=0.849702 VRAM=38.3GiB | 7.6% done | ETA(epoch): 591s + [Epoch 37][110/133] loss=0.937959 avg=0.850173 VRAM=38.4GiB | 7.6% done | ETA(epoch): 412s + [Epoch 37][120/133] loss=0.994871 avg=0.851541 VRAM=38.3GiB | 7.6% done | ETA(epoch): 233s + [Epoch 37][130/133] loss=0.863272 avg=0.851016 VRAM=38.4GiB | 7.6% done | ETA(epoch): 54s + Train loss: 0.851632 (2380.1s) ETA: 19999min + Val loss: 0.861795 [t_0.0-0.2=1.0827 t_0.2-0.4=0.9748 t_0.4-0.6=0.8495 t_0.6-0.8=0.7226 t_0.8-1.0=0.6870] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0037 + Deleted old checkpoint: checkpoint_epoch_0034 +[MEM @ epoch 37 end] RAM: 45.6/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 38/499 (8% done) --- + [Epoch 38][10/133] loss=0.754793 avg=0.884548 VRAM=38.4GiB | 7.6% done | ETA(epoch): 2211s + [Epoch 38][20/133] loss=0.808268 avg=0.863940 VRAM=38.3GiB | 7.6% done | ETA(epoch): 2024s + [Epoch 38][30/133] loss=0.761935 avg=0.861919 VRAM=38.4GiB | 7.6% done | ETA(epoch): 1843s + [Epoch 38][40/133] loss=0.832598 avg=0.846170 VRAM=38.3GiB | 7.7% done | ETA(epoch): 1665s + [Epoch 38][50/133] loss=1.038682 avg=0.846069 VRAM=38.4GiB | 7.7% done | ETA(epoch): 1486s + [Epoch 38][60/133] loss=1.033909 avg=0.857107 VRAM=38.3GiB | 7.7% done | ETA(epoch): 1306s + [Epoch 38][70/133] loss=0.867504 avg=0.851180 VRAM=38.4GiB | 7.7% done | ETA(epoch): 1127s + [Epoch 38][80/133] loss=0.851943 avg=0.852143 VRAM=38.3GiB | 7.7% done | ETA(epoch): 948s + [Epoch 38][90/133] loss=0.599605 avg=0.850557 VRAM=38.4GiB | 7.7% done | ETA(epoch): 769s + [Epoch 38][100/133] loss=0.901649 avg=0.851949 VRAM=38.3GiB | 7.8% done | ETA(epoch): 590s + [Epoch 38][110/133] loss=0.974281 avg=0.854884 VRAM=38.4GiB | 7.8% done | ETA(epoch): 411s + [Epoch 38][120/133] loss=0.854068 avg=0.852059 VRAM=38.3GiB | 7.8% done | ETA(epoch): 232s + [Epoch 38][130/133] loss=0.758047 avg=0.851983 VRAM=38.4GiB | 7.8% done | ETA(epoch): 54s + Train loss: 0.854391 (2376.9s) ETA: 19947min + Val loss: 0.872630 [t_0.0-0.2=1.0628 t_0.2-0.4=1.0081 t_0.4-0.6=0.8598 t_0.6-0.8=0.7205 t_0.8-1.0=0.6909] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0038 + Deleted old checkpoint: checkpoint_epoch_0035 +[MEM @ epoch 38 end] RAM: 45.4/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 39/499 (8% done) --- + [Epoch 39][10/133] loss=0.926677 avg=0.882846 VRAM=38.4GiB | 7.8% done | ETA(epoch): 2195s + [Epoch 39][20/133] loss=0.800273 avg=0.886011 VRAM=38.3GiB | 7.8% done | ETA(epoch): 2017s + [Epoch 39][30/133] loss=0.892487 avg=0.878212 VRAM=38.4GiB | 7.8% done | ETA(epoch): 1847s + [Epoch 39][40/133] loss=0.756650 avg=0.869792 VRAM=38.3GiB | 7.9% done | ETA(epoch): 1665s + [Epoch 39][50/133] loss=0.737557 avg=0.874469 VRAM=38.4GiB | 7.9% done | ETA(epoch): 1487s + [Epoch 39][60/133] loss=0.877766 avg=0.873815 VRAM=38.3GiB | 7.9% done | ETA(epoch): 1307s + [Epoch 39][70/133] loss=1.025093 avg=0.884039 VRAM=38.4GiB | 7.9% done | ETA(epoch): 1127s + [Epoch 39][80/133] loss=0.817953 avg=0.882475 VRAM=38.3GiB | 7.9% done | ETA(epoch): 950s + [Epoch 39][90/133] loss=0.854708 avg=0.881264 VRAM=38.4GiB | 7.9% done | ETA(epoch): 770s + [Epoch 39][100/133] loss=0.867334 avg=0.875757 VRAM=38.3GiB | 8.0% done | ETA(epoch): 592s + [Epoch 39][110/133] loss=0.782222 avg=0.872463 VRAM=38.4GiB | 8.0% done | ETA(epoch): 413s + [Epoch 39][120/133] loss=1.092508 avg=0.869251 VRAM=38.3GiB | 8.0% done | ETA(epoch): 233s + [Epoch 39][130/133] loss=0.913419 avg=0.869873 VRAM=38.4GiB | 8.0% done | ETA(epoch): 54s + Train loss: 0.870806 (2384.3s) ETA: 19895min + Val loss: 0.854891 [t_0.0-0.2=1.0573 t_0.2-0.4=0.9984 t_0.4-0.6=0.8486 t_0.6-0.8=0.7417 t_0.8-1.0=0.6877] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0039 + Deleted old checkpoint: checkpoint_epoch_0036 +[MEM @ epoch 39 end] RAM: 45.6/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 40/499 (8% done) --- + [Epoch 40][10/133] loss=0.918324 avg=0.854222 VRAM=38.4GiB | 8.0% done | ETA(epoch): 2193s + [Epoch 40][20/133] loss=0.865943 avg=0.857513 VRAM=38.3GiB | 8.0% done | ETA(epoch): 2016s + [Epoch 40][30/133] loss=0.739231 avg=0.851796 VRAM=38.4GiB | 8.0% done | ETA(epoch): 1838s + [Epoch 40][40/133] loss=0.728671 avg=0.846483 VRAM=38.3GiB | 8.1% done | ETA(epoch): 1664s + [Epoch 40][50/133] loss=0.966738 avg=0.853972 VRAM=38.4GiB | 8.1% done | ETA(epoch): 1484s + [Epoch 40][60/133] loss=0.769317 avg=0.848856 VRAM=38.3GiB | 8.1% done | ETA(epoch): 1307s + [Epoch 40][70/133] loss=0.969736 avg=0.855393 VRAM=38.4GiB | 8.1% done | ETA(epoch): 1128s + [Epoch 40][80/133] loss=0.887414 avg=0.863236 VRAM=38.3GiB | 8.1% done | ETA(epoch): 949s + [Epoch 40][90/133] loss=0.743930 avg=0.859645 VRAM=38.4GiB | 8.1% done | ETA(epoch): 770s + [Epoch 40][100/133] loss=0.753780 avg=0.857554 VRAM=38.3GiB | 8.2% done | ETA(epoch): 590s + [Epoch 40][110/133] loss=0.938396 avg=0.857148 VRAM=38.4GiB | 8.2% done | ETA(epoch): 411s + [Epoch 40][120/133] loss=0.971210 avg=0.856753 VRAM=38.3GiB | 8.2% done | ETA(epoch): 233s + [Epoch 40][130/133] loss=0.855161 avg=0.859490 VRAM=38.4GiB | 8.2% done | ETA(epoch): 54s + Train loss: 0.860891 (2379.2s) ETA: 19844min + Val loss: 0.872935 [t_0.0-0.2=1.0831 t_0.2-0.4=0.9723 t_0.4-0.6=0.8511 t_0.6-0.8=0.7268 t_0.8-1.0=0.6654] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0040 + Deleted old checkpoint: checkpoint_epoch_0037 +[MEM @ epoch 40 end] RAM: 45.8/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 41/499 (8% done) --- + [Epoch 41][10/133] loss=0.815489 avg=0.892119 VRAM=38.4GiB | 8.2% done | ETA(epoch): 2222s + [Epoch 41][20/133] loss=0.841629 avg=0.870620 VRAM=38.3GiB | 8.2% done | ETA(epoch): 2027s + [Epoch 41][30/133] loss=0.872856 avg=0.863243 VRAM=38.4GiB | 8.2% done | ETA(epoch): 1844s + [Epoch 41][40/133] loss=0.933296 avg=0.867635 VRAM=38.3GiB | 8.3% done | ETA(epoch): 1664s + [Epoch 41][50/133] loss=0.824280 avg=0.864209 VRAM=38.4GiB | 8.3% done | ETA(epoch): 1485s + [Epoch 41][60/133] loss=0.818009 avg=0.866685 VRAM=38.3GiB | 8.3% done | ETA(epoch): 1306s + [Epoch 41][70/133] loss=0.734521 avg=0.856224 VRAM=38.4GiB | 8.3% done | ETA(epoch): 1127s + [Epoch 41][80/133] loss=1.034929 avg=0.857372 VRAM=38.3GiB | 8.3% done | ETA(epoch): 948s + [Epoch 41][90/133] loss=0.842697 avg=0.861803 VRAM=38.4GiB | 8.3% done | ETA(epoch): 769s + [Epoch 41][100/133] loss=0.971844 avg=0.865015 VRAM=38.3GiB | 8.4% done | ETA(epoch): 590s + [Epoch 41][110/133] loss=0.853958 avg=0.867454 VRAM=38.4GiB | 8.4% done | ETA(epoch): 411s + [Epoch 41][120/133] loss=0.905122 avg=0.865830 VRAM=38.3GiB | 8.4% done | ETA(epoch): 232s + [Epoch 41][130/133] loss=0.984508 avg=0.867314 VRAM=38.4GiB | 8.4% done | ETA(epoch): 54s + Train loss: 0.866723 (2379.1s) ETA: 19792min + Val loss: 0.853423 [t_0.0-0.2=1.0731 t_0.2-0.4=1.0256 t_0.4-0.6=0.8516 t_0.6-0.8=0.7085 t_0.8-1.0=0.6785] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0041 (BEST) + Deleted old checkpoint: checkpoint_epoch_0026 + Deleted old checkpoint: checkpoint_epoch_0038 +[MEM @ epoch 41 end] RAM: 46.1/1007.0 GiB (4.6%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 42/499 (8% done) --- + [Epoch 42][10/133] loss=0.821288 avg=0.882640 VRAM=38.4GiB | 8.4% done | ETA(epoch): 2195s + [Epoch 42][20/133] loss=0.885514 avg=0.871098 VRAM=38.3GiB | 8.4% done | ETA(epoch): 2018s + [Epoch 42][30/133] loss=0.835985 avg=0.882652 VRAM=38.4GiB | 8.4% done | ETA(epoch): 1840s + [Epoch 42][40/133] loss=0.812877 avg=0.876484 VRAM=38.3GiB | 8.5% done | ETA(epoch): 1661s + [Epoch 42][50/133] loss=0.989257 avg=0.866912 VRAM=38.4GiB | 8.5% done | ETA(epoch): 1483s + [Epoch 42][60/133] loss=1.037834 avg=0.873141 VRAM=38.3GiB | 8.5% done | ETA(epoch): 1304s + [Epoch 42][70/133] loss=0.757398 avg=0.865682 VRAM=38.4GiB | 8.5% done | ETA(epoch): 1125s + [Epoch 42][80/133] loss=0.746917 avg=0.864665 VRAM=38.3GiB | 8.5% done | ETA(epoch): 947s + [Epoch 42][90/133] loss=0.860942 avg=0.864824 VRAM=38.4GiB | 8.5% done | ETA(epoch): 768s + [Epoch 42][100/133] loss=0.995956 avg=0.868911 VRAM=38.3GiB | 8.6% done | ETA(epoch): 590s + [Epoch 42][110/133] loss=0.872620 avg=0.869297 VRAM=38.4GiB | 8.6% done | ETA(epoch): 411s + [Epoch 42][120/133] loss=0.925454 avg=0.866585 VRAM=38.3GiB | 8.6% done | ETA(epoch): 232s + [Epoch 42][130/133] loss=0.782117 avg=0.866792 VRAM=38.4GiB | 8.6% done | ETA(epoch): 54s + Train loss: 0.868776 (2375.8s) ETA: 19741min + Val loss: 0.875517 [t_0.0-0.2=1.0727 t_0.2-0.4=1.0039 t_0.4-0.6=0.8528 t_0.6-0.8=0.7370 t_0.8-1.0=0.6728] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0042 + Deleted old checkpoint: checkpoint_epoch_0039 +[MEM @ epoch 42 end] RAM: 47.0/1007.0 GiB (4.7%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 43/499 (9% done) --- + [Epoch 43][10/133] loss=0.880199 avg=0.917287 VRAM=38.4GiB | 8.6% done | ETA(epoch): 2192s + [Epoch 43][20/133] loss=0.720508 avg=0.892494 VRAM=38.3GiB | 8.6% done | ETA(epoch): 2016s + [Epoch 43][30/133] loss=0.832846 avg=0.889259 VRAM=38.4GiB | 8.6% done | ETA(epoch): 1838s + [Epoch 43][40/133] loss=0.821186 avg=0.887352 VRAM=38.3GiB | 8.7% done | ETA(epoch): 1660s + [Epoch 43][50/133] loss=0.864779 avg=0.885032 VRAM=38.4GiB | 8.7% done | ETA(epoch): 1482s + [Epoch 43][60/133] loss=0.766657 avg=0.878806 VRAM=38.3GiB | 8.7% done | ETA(epoch): 1303s + [Epoch 43][70/133] loss=0.780493 avg=0.870307 VRAM=38.4GiB | 8.7% done | ETA(epoch): 1125s + [Epoch 43][80/133] loss=0.923550 avg=0.871647 VRAM=38.3GiB | 8.7% done | ETA(epoch): 947s + [Epoch 43][90/133] loss=0.795328 avg=0.866723 VRAM=38.4GiB | 8.7% done | ETA(epoch): 768s + [Epoch 43][100/133] loss=1.002560 avg=0.865874 VRAM=38.3GiB | 8.8% done | ETA(epoch): 590s + [Epoch 43][110/133] loss=0.752669 avg=0.864128 VRAM=38.4GiB | 8.8% done | ETA(epoch): 411s + [Epoch 43][120/133] loss=0.848897 avg=0.865327 VRAM=38.3GiB | 8.8% done | ETA(epoch): 232s + [Epoch 43][130/133] loss=0.943711 avg=0.867414 VRAM=38.4GiB | 8.8% done | ETA(epoch): 54s + Train loss: 0.866637 (2376.0s) ETA: 19690min + Val loss: 0.841129 [t_0.0-0.2=1.0707 t_0.2-0.4=0.9926 t_0.4-0.6=0.8709 t_0.6-0.8=0.7148 t_0.8-1.0=0.6655] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0043 (BEST) + Deleted old checkpoint: checkpoint_epoch_0040 +[MEM @ epoch 43 end] RAM: 47.4/1007.0 GiB (4.7%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 44/499 (9% done) --- + [Epoch 44][10/133] loss=0.835351 avg=0.837525 VRAM=38.4GiB | 8.8% done | ETA(epoch): 2194s + [Epoch 44][20/133] loss=0.888802 avg=0.857820 VRAM=38.3GiB | 8.8% done | ETA(epoch): 2018s + [Epoch 44][30/133] loss=0.780897 avg=0.855039 VRAM=38.4GiB | 8.8% done | ETA(epoch): 1839s + [Epoch 44][40/133] loss=0.831667 avg=0.858150 VRAM=38.3GiB | 8.9% done | ETA(epoch): 1661s + [Epoch 44][50/133] loss=0.907471 avg=0.852921 VRAM=38.4GiB | 8.9% done | ETA(epoch): 1482s + [Epoch 44][60/133] loss=0.928493 avg=0.859149 VRAM=38.3GiB | 8.9% done | ETA(epoch): 1304s + [Epoch 44][70/133] loss=0.903859 avg=0.858012 VRAM=38.4GiB | 8.9% done | ETA(epoch): 1125s + [Epoch 44][80/133] loss=0.769217 avg=0.856570 VRAM=38.3GiB | 8.9% done | ETA(epoch): 947s + [Epoch 44][90/133] loss=1.043346 avg=0.858648 VRAM=38.4GiB | 8.9% done | ETA(epoch): 769s + [Epoch 44][100/133] loss=0.831927 avg=0.859661 VRAM=38.3GiB | 9.0% done | ETA(epoch): 591s + [Epoch 44][110/133] loss=0.939501 avg=0.859256 VRAM=38.4GiB | 9.0% done | ETA(epoch): 412s + [Epoch 44][120/133] loss=0.921425 avg=0.858898 VRAM=38.3GiB | 9.0% done | ETA(epoch): 233s + [Epoch 44][130/133] loss=0.837782 avg=0.858718 VRAM=38.4GiB | 9.0% done | ETA(epoch): 54s + Train loss: 0.856616 (2380.6s) ETA: 19640min + Val loss: 0.882423 [t_0.0-0.2=1.0778 t_0.2-0.4=0.9998 t_0.4-0.6=0.8339 t_0.6-0.8=0.7206 t_0.8-1.0=0.6741] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0044 + Deleted old checkpoint: checkpoint_epoch_0041 +[MEM @ epoch 44 end] RAM: 44.8/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 45/499 (9% done) --- + [Epoch 45][10/133] loss=0.838732 avg=0.886239 VRAM=38.4GiB | 9.0% done | ETA(epoch): 2201s + [Epoch 45][20/133] loss=0.851137 avg=0.861530 VRAM=38.3GiB | 9.0% done | ETA(epoch): 2021s + [Epoch 45][30/133] loss=0.906795 avg=0.873222 VRAM=38.4GiB | 9.0% done | ETA(epoch): 1848s + [Epoch 45][40/133] loss=0.805591 avg=0.858172 VRAM=38.3GiB | 9.1% done | ETA(epoch): 1666s + [Epoch 45][50/133] loss=0.718258 avg=0.862818 VRAM=38.4GiB | 9.1% done | ETA(epoch): 1486s + [Epoch 45][60/133] loss=0.769588 avg=0.857235 VRAM=38.3GiB | 9.1% done | ETA(epoch): 1307s + [Epoch 45][70/133] loss=0.972668 avg=0.863999 VRAM=38.4GiB | 9.1% done | ETA(epoch): 1127s + [Epoch 45][80/133] loss=0.884450 avg=0.862531 VRAM=38.3GiB | 9.1% done | ETA(epoch): 948s + [Epoch 45][90/133] loss=0.734400 avg=0.860824 VRAM=38.4GiB | 9.1% done | ETA(epoch): 769s + [Epoch 45][100/133] loss=1.075095 avg=0.864638 VRAM=38.3GiB | 9.2% done | ETA(epoch): 590s + [Epoch 45][110/133] loss=0.705822 avg=0.862164 VRAM=38.4GiB | 9.2% done | ETA(epoch): 411s + [Epoch 45][120/133] loss=1.084314 avg=0.865976 VRAM=38.3GiB | 9.2% done | ETA(epoch): 232s + [Epoch 45][130/133] loss=0.913205 avg=0.868438 VRAM=38.4GiB | 9.2% done | ETA(epoch): 54s + Train loss: 0.867957 (2377.8s) ETA: 19591min + Val loss: 0.877517 [t_0.0-0.2=1.0561 t_0.2-0.4=0.9743 t_0.4-0.6=0.8823 t_0.6-0.8=0.7450 t_0.8-1.0=0.6659] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0045 + Deleted old checkpoint: checkpoint_epoch_0042 +[MEM @ epoch 45 end] RAM: 44.8/1007.0 GiB (4.4%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 46/499 (9% done) --- + [Epoch 46][10/133] loss=0.776327 avg=0.851932 VRAM=38.4GiB | 9.2% done | ETA(epoch): 2192s + [Epoch 46][20/133] loss=0.973841 avg=0.855038 VRAM=38.3GiB | 9.2% done | ETA(epoch): 2019s + [Epoch 46][30/133] loss=0.915657 avg=0.845135 VRAM=38.4GiB | 9.2% done | ETA(epoch): 1840s + [Epoch 46][40/133] loss=0.943386 avg=0.849785 VRAM=38.3GiB | 9.3% done | ETA(epoch): 1662s + [Epoch 46][50/133] loss=0.787840 avg=0.851818 VRAM=38.4GiB | 9.3% done | ETA(epoch): 1483s + [Epoch 46][60/133] loss=0.824876 avg=0.853923 VRAM=38.3GiB | 9.3% done | ETA(epoch): 1305s + [Epoch 46][70/133] loss=0.917679 avg=0.850542 VRAM=38.4GiB | 9.3% done | ETA(epoch): 1126s + [Epoch 46][80/133] loss=0.984812 avg=0.853556 VRAM=38.3GiB | 9.3% done | ETA(epoch): 949s + [Epoch 46][90/133] loss=0.785761 avg=0.854125 VRAM=38.4GiB | 9.3% done | ETA(epoch): 770s + [Epoch 46][100/133] loss=0.821992 avg=0.855801 VRAM=38.3GiB | 9.4% done | ETA(epoch): 591s + [Epoch 46][110/133] loss=0.973889 avg=0.859306 VRAM=38.4GiB | 9.4% done | ETA(epoch): 412s + [Epoch 46][120/133] loss=1.027300 avg=0.859212 VRAM=38.3GiB | 9.4% done | ETA(epoch): 233s + [Epoch 46][130/133] loss=0.833799 avg=0.859365 VRAM=38.4GiB | 9.4% done | ETA(epoch): 54s + Train loss: 0.859897 (2382.6s) ETA: 19542min + Val loss: 0.860380 [t_0.0-0.2=1.0611 t_0.2-0.4=0.9569 t_0.4-0.6=0.8588 t_0.6-0.8=0.7454 t_0.8-1.0=0.6568] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0046 +[MEM @ epoch 46 end] RAM: 45.1/1007.0 GiB (4.5%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 47/499 (9% done) --- + [Epoch 47][10/133] loss=0.865754 avg=0.875508 VRAM=38.4GiB | 9.4% done | ETA(epoch): 2195s + [Epoch 47][20/133] loss=0.828984 avg=0.858829 VRAM=38.3GiB | 9.4% done | ETA(epoch): 2017s + [Epoch 47][30/133] loss=1.044913 avg=0.841076 VRAM=38.4GiB | 9.4% done | ETA(epoch): 1841s + [Epoch 47][40/133] loss=0.797536 avg=0.839371 VRAM=38.3GiB | 9.5% done | ETA(epoch): 1662s + [Epoch 47][50/133] loss=0.927674 avg=0.843061 VRAM=38.4GiB | 9.5% done | ETA(epoch): 1484s + [Epoch 47][60/133] loss=1.014404 avg=0.852949 VRAM=38.3GiB | 9.5% done | ETA(epoch): 1305s + [Epoch 47][70/133] loss=0.834411 avg=0.852711 VRAM=38.4GiB | 9.5% done | ETA(epoch): 1126s + [Epoch 47][80/133] loss=0.836162 avg=0.854212 VRAM=38.3GiB | 9.5% done | ETA(epoch): 948s + [Epoch 47][90/133] loss=0.795815 avg=0.860602 VRAM=38.4GiB | 9.5% done | ETA(epoch): 769s + [Epoch 47][100/133] loss=0.659238 avg=0.855151 VRAM=38.3GiB | 9.6% done | ETA(epoch): 590s + [Epoch 47][110/133] loss=0.847418 avg=0.855199 VRAM=38.4GiB | 9.6% done | ETA(epoch): 411s + [Epoch 47][120/133] loss=0.946502 avg=0.859154 VRAM=38.3GiB | 9.6% done | ETA(epoch): 232s + [Epoch 47][130/133] loss=0.960661 avg=0.857227 VRAM=38.4GiB | 9.6% done | ETA(epoch): 54s + Train loss: 0.859359 (2378.6s) ETA: 19493min + Val loss: 0.896368 [t_0.0-0.2=1.0750 t_0.2-0.4=1.0002 t_0.4-0.6=0.8202 t_0.6-0.8=0.7323 t_0.8-1.0=0.6645] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0047 + Deleted old checkpoint: checkpoint_epoch_0044 +[MEM @ epoch 47 end] RAM: 46.2/1007.0 GiB (4.6%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 48/499 (10% done) --- + [Epoch 48][10/133] loss=0.766297 avg=0.840810 VRAM=38.4GiB | 9.6% done | ETA(epoch): 2237s + [Epoch 48][20/133] loss=0.772314 avg=0.830576 VRAM=38.3GiB | 9.6% done | ETA(epoch): 2060s + [Epoch 48][30/133] loss=0.842754 avg=0.839722 VRAM=38.4GiB | 9.6% done | ETA(epoch): 1868s + [Epoch 48][40/133] loss=0.844563 avg=0.842836 VRAM=38.3GiB | 9.7% done | ETA(epoch): 1686s + [Epoch 48][50/133] loss=0.823124 avg=0.842208 VRAM=38.4GiB | 9.7% done | ETA(epoch): 1500s + [Epoch 48][60/133] loss=0.779937 avg=0.843402 VRAM=38.3GiB | 9.7% done | ETA(epoch): 1317s + [Epoch 48][70/133] loss=0.756273 avg=0.841576 VRAM=38.4GiB | 9.7% done | ETA(epoch): 1135s + [Epoch 48][80/133] loss=0.988368 avg=0.837793 VRAM=38.3GiB | 9.7% done | ETA(epoch): 954s + [Epoch 48][90/133] loss=0.889109 avg=0.843626 VRAM=38.4GiB | 9.7% done | ETA(epoch): 774s + [Epoch 48][100/133] loss=0.769374 avg=0.839660 VRAM=38.3GiB | 9.8% done | ETA(epoch): 593s + [Epoch 48][110/133] loss=1.017270 avg=0.849914 VRAM=38.4GiB | 9.8% done | ETA(epoch): 413s + [Epoch 48][120/133] loss=0.846474 avg=0.853776 VRAM=38.3GiB | 9.8% done | ETA(epoch): 234s + [Epoch 48][130/133] loss=0.809811 avg=0.854334 VRAM=38.4GiB | 9.8% done | ETA(epoch): 54s + Train loss: 0.855484 (2389.0s) ETA: 19446min + Val loss: 0.863178 [t_0.0-0.2=1.0786 t_0.2-0.4=1.0099 t_0.4-0.6=0.8422 t_0.6-0.8=0.7180 t_0.8-1.0=0.6647] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0048 + Deleted old checkpoint: checkpoint_epoch_0045 +[MEM @ epoch 48 end] RAM: 47.6/1007.0 GiB (4.7%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 49/499 (10% done) --- + [Epoch 49][10/133] loss=0.883173 avg=0.861942 VRAM=38.4GiB | 9.8% done | ETA(epoch): 2194s + [Epoch 49][20/133] loss=0.838293 avg=0.849828 VRAM=38.3GiB | 9.8% done | ETA(epoch): 2036s + [Epoch 49][30/133] loss=0.736972 avg=0.850184 VRAM=38.4GiB | 9.8% done | ETA(epoch): 1850s + [Epoch 49][40/133] loss=0.715258 avg=0.858064 VRAM=38.3GiB | 9.9% done | ETA(epoch): 1668s + [Epoch 49][50/133] loss=0.776190 avg=0.853011 VRAM=38.4GiB | 9.9% done | ETA(epoch): 1487s + [Epoch 49][60/133] loss=0.826272 avg=0.858246 VRAM=38.3GiB | 9.9% done | ETA(epoch): 1307s + [Epoch 49][70/133] loss=0.973277 avg=0.866057 VRAM=38.4GiB | 9.9% done | ETA(epoch): 1128s + [Epoch 49][80/133] loss=0.910689 avg=0.861617 VRAM=38.3GiB | 9.9% done | ETA(epoch): 949s + [Epoch 49][90/133] loss=0.783218 avg=0.857567 VRAM=38.4GiB | 9.9% done | ETA(epoch): 769s + [Epoch 49][100/133] loss=1.027619 avg=0.860529 VRAM=38.3GiB | 10.0% done | ETA(epoch): 590s + [Epoch 49][110/133] loss=0.691425 avg=0.859165 VRAM=38.4GiB | 10.0% done | ETA(epoch): 411s + [Epoch 49][120/133] loss=0.827091 avg=0.859154 VRAM=38.3GiB | 10.0% done | ETA(epoch): 232s + [Epoch 49][130/133] loss=0.866830 avg=0.861255 VRAM=38.4GiB | 10.0% done | ETA(epoch): 54s + Train loss: 0.860644 (2378.7s) ETA: 19398min + Val loss: 0.846870 [t_0.0-0.2=1.0750 t_0.2-0.4=0.9920 t_0.4-0.6=0.8225 t_0.6-0.8=0.7030 t_0.8-1.0=0.6622] + Checkpoint saved: output/lora_128_dim_384_A100/checkpoint_epoch_0049 + Deleted old checkpoint: checkpoint_epoch_0046 +[MEM @ epoch 49 end] RAM: 50.3/1007.0 GiB (5.0%) | VRAM: 38.1/79.3 GiB (48.1%) + +--- Epoch 50/499 (10% done) --- + [MilestoneVis] train_0 step 6650 ✓ + [MilestoneGrid] train_0 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_0.png + [MilestoneVis] train_1 step 6650 ✓ + [MilestoneGrid] train_1 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_train_1.png + [MilestoneVis] val_0 step 6650 ✓ + [MilestoneGrid] val_0 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_0.png + [MilestoneVis] val_1 step 6650 ✓ + [MilestoneGrid] val_1 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_val_1.png + [MilestoneVis] test_0 step 6650 ✓ + [MilestoneGrid] test_0 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_0.png + [MilestoneVis] test_1 step 6650 ✓ + [MilestoneGrid] test_1 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_1.png + [MilestoneVis] test_2 step 6650 ✓ + [MilestoneGrid] test_2 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_2.png + [MilestoneVis] test_3 step 6650 ✓ + [MilestoneGrid] test_3 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_3.png + [MilestoneVis] test_4 step 6650 ✓ + [MilestoneGrid] test_4 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_4.png + [MilestoneVis] test_5 step 6650 ✓ + [MilestoneGrid] test_5 → output/lora_128_dim_384_A100/milestone_vis/milestone_grid_test_5.png + [HF Upload] syncing to JasonXF/SynthUrbanSAT-Output/output/lora_128_dim_384_A100 ... + Processing Files (0 / 0) : | | 0.00B / 0.00B + New Data Upload : | | 0.00B / 0.00B  + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.62MB / 2.62MB  + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.66MB / 2.66MB  + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.70MB / 2.70MB  + + + + + ...is/test_0_step_000000.png: 100%|#################################################################################################| 2.02MB / 2.02MB  + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.74MB / 2.74MB  + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.62MB / 2.62MB  + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.66MB / 2.66MB  + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.70MB / 2.70MB  + + + + + ...is/test_0_step_000000.png: 100%|#################################################################################################| 2.02MB / 2.02MB  + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.74MB / 2.74MB  Processing Files (5 / 5) : 4%|####2 | 12.7MB / 291MB, 12.7MB/s + + + + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.66MB / 2.66MB  + + + + + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.70MB / 2.70MB  + + + + + + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.68MB / 2.68MB  + + + + + + + + + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.69MB / 2.69MB  + + + + + + + + + + + ...is/test_1_step_000000.png: 100%|#################################################################################################| 1.96MB / 1.96MB  + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.62MB / 2.62MB  + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.66MB / 2.66MB  + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.70MB / 2.70MB  + + + + + ...is/test_0_step_000000.png: 100%|#################################################################################################| 2.02MB / 2.02MB  + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.74MB / 2.74MB  + + + + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.66MB / 2.66MB  + + + + + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.70MB / 2.70MB  + + + + + + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.68MB / 2.68MB  + + + + + + + + + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.69MB / 2.69MB  + + + + + + + + + + + [+ 8 files] : 100%|#################################################################################################| 22.9MB / 22.9MB  Processing Files (18 / 18) : 16%|###############4 | 46.4MB / 291MB, 38.7MB/s + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.69MB / 2.69MB  + + + ...is/test_1_step_000000.png: 100%|#################################################################################################| 1.96MB / 1.96MB  + + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.64MB / 2.64MB  + + + + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.71MB / 2.71MB  + + + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.73MB / 2.73MB  + + + + + + + ...is/test_2_step_000000.png: 100%|#################################################################################################| 2.10MB / 2.10MB  + + + + + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.67MB / 2.67MB  + + + + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.66MB / 2.66MB  + + + + + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.74MB / 2.74MB  + + + + + + + + + + + [+ 4 files] : 100%|#################################################################################################| 13.0MB / 13.0MB  Processing Files (22 / 22) : 20%|##################9 | 56.7MB / 291MB, 40.5MB/s + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.73MB / 2.73MB  + + + ...is/test_2_step_000000.png: 100%|#################################################################################################| 2.10MB / 2.10MB  + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.67MB / 2.67MB  + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.66MB / 2.66MB  + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.74MB / 2.74MB  + + + + + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.71MB / 2.71MB  + + + + + + + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.73MB / 2.73MB  + + + + + + + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.72MB / 2.72MB  + + + + + + + + + + ...is/test_3_step_000000.png: 100%|#################################################################################################| 2.10MB / 2.10MB  + + + + + + + + + + + [+ 3 files] : 100%|#################################################################################################| 10.8MB / 10.8MB  Processing Files (25 / 25) : 22%|#####################6 | 64.8MB / 291MB, 40.5MB/s + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.66MB / 2.66MB  + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.74MB / 2.74MB  + + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.71MB / 2.71MB  + + + + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.73MB / 2.73MB  + + + + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.72MB / 2.72MB  + + + + + + + ...is/test_3_step_000000.png: 100%|#################################################################################################| 2.10MB / 2.10MB  + + + + + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.74MB / 2.74MB  + + + + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.71MB / 2.71MB  + + + + + + + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.69MB / 2.69MB  + + + + + + + + + + + [+ 6 files] : 100%|#################################################################################################| 18.3MB / 18.3MB  Processing Files (31 / 31) : 28%|##########################8 | 80.4MB / 291MB, 44.7MB/s + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.74MB / 2.74MB  + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.71MB / 2.71MB  + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.69MB / 2.69MB  + + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.67MB / 2.67MB  + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.72MB / 2.72MB  + + + + + + + ...is/test_4_step_000000.png: 100%|#################################################################################################| 1.99MB / 1.99MB  + + + + + + + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.75MB / 2.75MB  + + + + + + + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.73MB / 2.73MB  + + + + + + + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.71MB / 2.71MB  + + + + + + + + + + + [+ 5 files] : 100%|#################################################################################################| 15.5MB / 15.5MB  Processing Files (36 / 36) : 32%|###############################1 | 93.2MB / 291MB, 46.6MB/s + + ...is/test_4_step_000000.png: 100%|#################################################################################################| 1.99MB / 1.99MB  + + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.75MB / 2.75MB  + + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.73MB / 2.73MB  + + + + + ..._step_000000_sample04.png: 100%|#################################################################################################| 2.71MB / 2.71MB  + + + + + + ..._step_000000_sample01.png: 100%|#################################################################################################| 2.77MB / 2.77MB  + + + + + + + ..._step_000000_sample00.png: 100%|#################################################################################################| 2.70MB / 2.70MB  + + + + + + + + ...is/test_5_step_000000.png: 100%|#################################################################################################| 2.07MB / 2.07MB  + + + + + + + + + ..._step_000000_sample03.png: 100%|#################################################################################################| 2.71MB / 2.71MB  + + + + + + + + + + ..._step_000000_sample02.png: 100%|#################################################################################################| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 44 files] : 34%|################################9 | 34.1MB / 100MB  Processing Files (48 / 80) : 43%|#########################################6 | 125MB / 291MB, 56.7MB/s + New Data Upload : 1%|7 | 524kB / 66.7MB, 238kB/s  + + ...milestone_grid_test_0.png: 1%|7 | 21.5kB / 2.74MB  + + + ...milestone_grid_test_1.png: 1%|7 | 21.1kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 1%|7 | 22.4kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 1%|7 | 22.6kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 1%|7 | 21.6kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 1%|7 | 22.2kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 1%|7 | 21.2kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 1%|7 | 21.9kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 1%|7 | 20.8kB / 2.64MB  + + + + + + + + + + + [+ 28 files] : 28%|##########################7 | 15.8MB / 57.3MB  Processing Files (54 / 86) : 48%|##############################################7 | 140MB / 291MB, 58.4MB/s + + ...milestone_grid_test_0.png: 2%|#5 | 43.0kB / 2.74MB  + + + ...milestone_grid_test_1.png: 2%|#5 | 42.1kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 2%|#5 | 44.8kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 2%|#5 | 45.2kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 2%|#5 | 43.2kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 2%|#5 | 44.3kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 2%|#5 | 42.5kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 2%|#5 | 43.9kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 2%|#5 | 41.5kB / 2.64MB  + + + + + + + + + + + [+ 23 files] : 6%|#####7 | 2.60MB / 43.8MB  Processing Files (55 / 87) : 49%|###############################################5 | 143MB / 291MB, 54.9MB/s + New Data Upload : 2%|#5 | 1.05MB / 66.7MB, 403kB/s  + + ...milestone_grid_test_0.png: 3%|### | 86.1kB / 2.74MB  + + + ...milestone_grid_test_1.png: 3%|### | 84.2kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 3%|### | 89.6kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 3%|### | 90.3kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 3%|### | 86.3kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 3%|### | 88.7kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 3%|### | 84.9kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 3%|### | 87.7kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 3%|### | 83.1kB / 2.64MB  + + + + + + + + + + + [+ 27 files] : 27%|#########################8 | 14.8MB / 55.3MB  Processing Files (60 / 92) : 54%|####################################################4 | 157MB / 291MB, 56.1MB/s + New Data Upload : 3%|### | 2.10MB / 66.7MB, 749kB/s  + + ...milestone_grid_test_0.png: 6%|#####3 | 151kB / 2.74MB  + + + ...milestone_grid_test_1.png: 6%|#####3 | 147kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 6%|#####3 | 157kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 6%|#####3 | 158kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 6%|#####3 | 151kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 6%|#####3 | 155kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 6%|#####3 | 149kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 6%|#####3 | 154kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 6%|#####3 | 145kB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 6%|#####3 | 2.30MB / 41.9MB  Processing Files (60 / 92) : 55%|####################################################9 | 159MB / 291MB, 52.9MB/s + New Data Upload : 3%|##6 | 3.67MB / 132MB, 1.22MB/s  + + ...milestone_grid_test_0.png: 11%|##########6 | 301kB / 2.74MB  + + + ...milestone_grid_test_1.png: 11%|##########6 | 295kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 11%|##########6 | 314kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 11%|##########6 | 316kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 11%|##########6 | 302kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 11%|##########6 | 310kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 11%|##########6 | 297kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 11%|##########6 | 307kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 11%|##########6 | 291kB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 11%|##########6 | 4.61MB / 41.9MB  Processing Files (60 / 92) : 56%|######################################################1 | 162MB / 291MB, 50.7MB/s + New Data Upload : 6%|#####3 | 7.34MB / 132MB, 2.29MB/s  + + ...milestone_grid_test_0.png: 16%|###############2 | 430kB / 2.74MB  + + + ...milestone_grid_test_1.png: 16%|###############2 | 421kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 16%|###############2 | 448kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 16%|###############2 | 452kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 16%|###############2 | 432kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 16%|###############2 | 443kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 16%|###############2 | 425kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 16%|###############2 | 439kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 16%|###############2 | 415kB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 16%|###############2 | 6.58MB / 41.9MB  Processing Files (60 / 92) : 57%|#######################################################2 | 165MB / 291MB, 48.7MB/s + New Data Upload : 8%|#######6 | 10.5MB / 132MB, 3.08MB/s  + + ...milestone_grid_test_0.png: 20%|###################8 | 560kB / 2.74MB  + + + ...milestone_grid_test_1.png: 20%|###################8 | 547kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 20%|###################8 | 583kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 20%|###################8 | 587kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 20%|###################8 | 561kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 20%|###################8 | 577kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 20%|###################8 | 552kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 20%|###################8 | 570kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 20%|###################8 | 540kB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 20%|###################8 | 8.55MB / 41.9MB  Processing Files (60 / 92) : 58%|########################################################2 | 169MB / 291MB, 46.8MB/s + New Data Upload : 10%|#########9 | 13.6MB / 132MB, 3.79MB/s  + + ...milestone_grid_test_0.png: 23%|######################1 | 624kB / 2.74MB  + + + ...milestone_grid_test_1.png: 23%|######################1 | 611kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 23%|######################1 | 650kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 23%|######################1 | 655kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 23%|######################1 | 626kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 23%|######################1 | 643kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 23%|######################1 | 616kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 23%|######################1 | 636kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 23%|######################1 | 602kB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 23%|######################1 | 9.54MB / 41.9MB  Processing Files (60 / 92) : 59%|########################################################8 | 170MB / 291MB, 44.8MB/s + New Data Upload : 11%|###########1 | 15.2MB / 132MB, 4.00MB/s  + + ...milestone_grid_test_0.png: 28%|##########################6 | 753kB / 2.74MB  + + + ...milestone_grid_test_1.png: 28%|##########################6 | 737kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 28%|##########################6 | 784kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 28%|##########################6 | 790kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 28%|##########################6 | 755kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 28%|##########################6 | 776kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 28%|##########################6 | 743kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 28%|##########################6 | 768kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 28%|##########################6 | 727kB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 28%|##########################6 | 11.5MB / 41.9MB  Processing Files (60 / 92) : 60%|#########################################################8 | 173MB / 291MB, 43.3MB/s + New Data Upload : 14%|#############4 | 18.3MB / 132MB, 4.59MB/s  + + ...milestone_grid_test_0.png: 33%|################################ | 904kB / 2.74MB  + + + ...milestone_grid_test_1.png: 33%|################################ | 884kB / 2.68MB  + + + + ...milestone_grid_test_2.png: 33%|################################ | 941kB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 33%|################################ | 948kB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 33%|################################ | 906kB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 33%|################################ | 931kB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 33%|################################ | 892kB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 33%|################################ | 921kB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 33%|################################ | 872kB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 33%|################################ | 13.8MB / 41.9MB  Processing Files (60 / 92) : 61%|########################################################### | 177MB / 291MB, 42.2MB/s + New Data Upload : 17%|################1 | 22.0MB / 132MB, 5.24MB/s  + + ...milestone_grid_test_0.png: 38%|####################################5 | 1.03MB / 2.74MB  + + + ...milestone_grid_test_1.png: 38%|####################################5 | 1.01MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 38%|####################################5 | 1.08MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 38%|####################################5 | 1.08MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 38%|####################################5 | 1.04MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 38%|####################################5 | 1.06MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 38%|####################################5 | 1.02MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 38%|####################################5 | 1.05MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 38%|####################################5 | 997kB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 38%|####################################5 | 15.8MB / 41.9MB  Processing Files (60 / 92) : 62%|############################################################1 | 180MB / 291MB, 40.9MB/s + New Data Upload : 19%|##################4 | 25.2MB / 132MB, 5.72MB/s  + + ...milestone_grid_test_0.png: 42%|########################################4 | 1.14MB / 2.74MB  + + + ...milestone_grid_test_1.png: 42%|########################################4 | 1.12MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 42%|########################################4 | 1.19MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 42%|########################################4 | 1.20MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 42%|########################################4 | 1.14MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 42%|########################################4 | 1.18MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 42%|########################################4 | 1.13MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 42%|########################################4 | 1.16MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 42%|########################################4 | 1.10MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 42%|########################################4 | 17.4MB / 41.9MB  Processing Files (60 / 92) : 63%|############################################################# | 183MB / 291MB, 39.7MB/s + New Data Upload : 21%|####################3 | 27.8MB / 132MB, 6.04MB/s  + + ...milestone_grid_test_0.png: 46%|############################################9 | 1.27MB / 2.74MB  + + + ...milestone_grid_test_1.png: 46%|############################################9 | 1.24MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 46%|############################################9 | 1.32MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 46%|############################################9 | 1.33MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 46%|############################################9 | 1.27MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 46%|############################################9 | 1.31MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 46%|############################################9 | 1.25MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 46%|############################################9 | 1.29MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 46%|############################################9 | 1.23MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 46%|############################################9 | 19.4MB / 41.9MB  Processing Files (60 / 92) : 64%|############################################################## | 186MB / 291MB, 38.7MB/s + New Data Upload : 23%|######################6 | 30.9MB / 132MB, 6.44MB/s  + + ...milestone_grid_test_0.png: 51%|#################################################5 | 1.40MB / 2.74MB  + + + ...milestone_grid_test_1.png: 51%|#################################################5 | 1.37MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 51%|#################################################5 | 1.46MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 51%|#################################################5 | 1.47MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 51%|#################################################5 | 1.40MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 51%|#################################################5 | 1.44MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 51%|#################################################5 | 1.38MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 51%|#################################################5 | 1.43MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 51%|#################################################5 | 1.35MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 51%|#################################################5 | 21.4MB / 41.9MB  Processing Files (60 / 92) : 65%|###############################################################1 | 189MB / 291MB, 37.8MB/s + New Data Upload : 26%|########################9 | 34.1MB / 132MB, 6.82MB/s  + + ...milestone_grid_test_0.png: 55%|#####################################################3 | 1.51MB / 2.74MB  + + + ...milestone_grid_test_1.png: 55%|#####################################################3 | 1.47MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 55%|#####################################################3 | 1.57MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 55%|#####################################################3 | 1.58MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 55%|#####################################################3 | 1.51MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 55%|#####################################################3 | 1.55MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 55%|#####################################################3 | 1.49MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 55%|#####################################################3 | 1.54MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 55%|#####################################################3 | 1.45MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 55%|#####################################################3 | 23.0MB / 41.9MB  Processing Files (60 / 92) : 66%|###############################################################9 | 192MB / 291MB, 36.9MB/s + New Data Upload : 28%|##########################8 | 36.7MB / 132MB, 7.06MB/s  + + ...milestone_grid_test_0.png: 61%|##########################################################6 | 1.66MB / 2.74MB  + + + ...milestone_grid_test_1.png: 61%|##########################################################6 | 1.62MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 61%|##########################################################6 | 1.73MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 61%|##########################################################6 | 1.74MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 61%|##########################################################6 | 1.66MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 61%|##########################################################6 | 1.71MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 61%|##########################################################6 | 1.63MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 61%|##########################################################6 | 1.69MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 61%|##########################################################6 | 1.60MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 61%|##########################################################6 | 25.3MB / 41.9MB  Processing Files (60 / 92) : 67%|#################################################################2 | 195MB / 291MB, 36.2MB/s + New Data Upload : 30%|#############################5 | 40.4MB / 132MB, 7.47MB/s  + + ...milestone_grid_test_0.png: 64%|#############################################################7 | 1.74MB / 2.74MB  + + + ...milestone_grid_test_1.png: 64%|#############################################################7 | 1.71MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 64%|#############################################################7 | 1.82MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 64%|#############################################################7 | 1.83MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 64%|#############################################################7 | 1.75MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 64%|#############################################################7 | 1.80MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 64%|#############################################################7 | 1.72MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 64%|#############################################################7 | 1.78MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 64%|#############################################################7 | 1.68MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 64%|#############################################################7 | 26.6MB / 41.9MB  Processing Files (60 / 92) : 68%|#################################################################9 | 197MB / 291MB, 35.3MB/s + New Data Upload : 32%|###############################1 | 42.5MB / 132MB, 7.58MB/s  + + ...milestone_grid_test_0.png: 68%|##################################################################3 | 1.87MB / 2.74MB  + + + ...milestone_grid_test_1.png: 68%|##################################################################3 | 1.83MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 68%|##################################################################3 | 1.95MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 68%|##################################################################3 | 1.96MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 68%|##################################################################3 | 1.88MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 68%|##################################################################3 | 1.93MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 68%|##################################################################3 | 1.85MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 68%|##################################################################3 | 1.91MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 68%|##################################################################3 | 1.81MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 68%|##################################################################3 | 28.6MB / 41.9MB  Processing Files (60 / 92) : 69%|##################################################################9 | 201MB / 291MB, 34.6MB/s + New Data Upload : 34%|#################################4 | 45.6MB / 132MB, 7.86MB/s  + + ...milestone_grid_test_0.png: 72%|######################################################################1 | 1.98MB / 2.74MB  + + + ...milestone_grid_test_1.png: 72%|######################################################################1 | 1.94MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 72%|######################################################################1 | 2.06MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 72%|######################################################################1 | 2.08MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 72%|######################################################################1 | 1.99MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 72%|######################################################################1 | 2.04MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 72%|######################################################################1 | 1.95MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 72%|######################################################################1 | 2.02MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 72%|######################################################################1 | 1.91MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 72%|######################################################################1 | 30.3MB / 41.9MB  Processing Files (60 / 92) : 70%|###################################################################8 | 203MB / 291MB, 33.9MB/s + New Data Upload : 36%|###################################3 | 48.2MB / 132MB, 8.04MB/s  + + ...milestone_grid_test_0.png: 77%|##########################################################################7 | 2.11MB / 2.74MB  + + + ...milestone_grid_test_1.png: 77%|##########################################################################7 | 2.06MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 77%|##########################################################################7 | 2.20MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 77%|##########################################################################7 | 2.21MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 77%|##########################################################################7 | 2.11MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 77%|##########################################################################7 | 2.17MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 77%|##########################################################################7 | 2.08MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 77%|##########################################################################7 | 2.15MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 77%|##########################################################################7 | 2.03MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 77%|##########################################################################7 | 32.2MB / 41.9MB  Processing Files (60 / 92) : 71%|####################################################################8 | 206MB / 291MB, 33.3MB/s + New Data Upload : 39%|#####################################6 | 51.4MB / 132MB, 8.29MB/s  + + ...milestone_grid_test_0.png: 83%|################################################################################8 | 2.28MB / 2.74MB  + + + ...milestone_grid_test_1.png: 83%|################################################################################8 | 2.23MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 83%|################################################################################8 | 2.38MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 83%|################################################################################8 | 2.39MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 83%|################################################################################8 | 2.29MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 83%|################################################################################8 | 2.35MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 83%|################################################################################8 | 2.25MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 83%|################################################################################8 | 2.33MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 83%|################################################################################8 | 2.20MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 83%|################################################################################8 | 34.9MB / 41.9MB  Processing Files (60 / 92) : 72%|######################################################################2 | 211MB / 291MB, 32.9MB/s + New Data Upload : 42%|########################################7 | 55.6MB / 132MB, 8.68MB/s  + + ...milestone_grid_test_0.png: 85%|##################################################################################3 | 2.32MB / 2.74MB  + + + ...milestone_grid_test_1.png: 85%|##################################################################################3 | 2.27MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 85%|##################################################################################3 | 2.42MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 85%|##################################################################################3 | 2.44MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 85%|##################################################################################3 | 2.33MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 85%|##################################################################################3 | 2.39MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 85%|##################################################################################3 | 2.29MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 85%|##################################################################################3 | 2.37MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 85%|##################################################################################3 | 2.24MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 85%|##################################################################################3 | 35.5MB / 41.9MB  Processing Files (60 / 92) : 73%|######################################################################6 | 212MB / 291MB, 32.1MB/s + New Data Upload : 43%|#########################################4 | 56.6MB / 132MB, 8.58MB/s  + + ...milestone_grid_test_0.png: 90%|#######################################################################################6 | 2.48MB / 2.74MB  + + + ...milestone_grid_test_1.png: 90%|#######################################################################################6 | 2.42MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 90%|#######################################################################################6 | 2.58MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 90%|#######################################################################################6 | 2.60MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 90%|#######################################################################################6 | 2.48MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 90%|#######################################################################################6 | 2.55MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 90%|#######################################################################################6 | 2.44MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 90%|#######################################################################################6 | 2.52MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 90%|#######################################################################################6 | 2.39MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 90%|#######################################################################################6 | 37.8MB / 41.9MB  Processing Files (60 / 92) : 74%|#######################################################################8 | 215MB / 291MB, 31.7MB/s + New Data Upload : 46%|############################################1 | 60.3MB / 132MB, 8.87MB/s  + + ...milestone_grid_test_0.png: 95%|############################################################################################2 | 2.60MB / 2.74MB  + + + ...milestone_grid_test_1.png: 95%|############################################################################################2 | 2.55MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 95%|############################################################################################2 | 2.71MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 95%|############################################################################################2 | 2.73MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 95%|############################################################################################2 | 2.61MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 95%|############################################################################################2 | 2.68MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 95%|############################################################################################2 | 2.57MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 95%|############################################################################################2 | 2.65MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 95%|############################################################################################2 | 2.51MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 95%|############################################################################################2 | 39.8MB / 41.9MB  Processing Files (60 / 92) : 75%|########################################################################8 | 218MB / 291MB, 31.2MB/s + New Data Upload : 48%|##############################################4 | 63.4MB / 132MB, 9.06MB/s  + + ...milestone_grid_test_0.png: 100%|################################################################################################8| 2.73MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|################################################################################################8| 2.67MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|################################################################################################8| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|################################################################################################8| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|################################################################################################8| 2.74MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|################################################################################################8| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|################################################################################################8| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|################################################################################################8| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|################################################################################################8| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 100%|################################################################################################8| 41.8MB / 41.9MB  Processing Files (60 / 92) : 76%|#########################################################################9 | 222MB / 291MB, 30.8MB/s + New Data Upload : 50%|################################################7 | 66.6MB / 132MB, 9.25MB/s  + + ...milestone_grid_test_0.png: 100%|################################################################################################8| 2.73MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|################################################################################################8| 2.67MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|################################################################################################8| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|################################################################################################8| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|################################################################################################8| 2.74MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|################################################################################################8| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|################################################################################################8| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|################################################################################################8| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|################################################################################################8| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 100%|################################################################################################8| 41.8MB / 41.9MB  + + ...milestone_grid_test_0.png: 100%|################################################################################################8| 2.73MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|################################################################################################8| 2.67MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|################################################################################################8| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|################################################################################################8| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|################################################################################################8| 2.74MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|################################################################################################8| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|################################################################################################8| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|################################################################################################8| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|################################################################################################8| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 100%|################################################################################################8| 41.8MB / 41.9MB  + + ...milestone_grid_test_0.png: 100%|################################################################################################8| 2.73MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|################################################################################################8| 2.67MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|################################################################################################8| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|################################################################################################8| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|################################################################################################8| 2.74MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|################################################################################################8| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|################################################################################################8| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|################################################################################################8| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|################################################################################################8| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 100%|################################################################################################8| 41.8MB / 41.9MB  + + ...milestone_grid_test_0.png: 100%|################################################################################################8| 2.73MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|################################################################################################8| 2.67MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|################################################################################################8| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|################################################################################################8| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|################################################################################################8| 2.74MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|################################################################################################8| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|################################################################################################8| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|################################################################################################8| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|################################################################################################8| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 100%|################################################################################################8| 41.8MB / 41.9MB  + + ...milestone_grid_test_0.png: 100%|################################################################################################8| 2.73MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|################################################################################################8| 2.67MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|################################################################################################8| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|################################################################################################8| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|################################################################################################8| 2.74MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|################################################################################################8| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|################################################################################################8| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|################################################################################################8| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|################################################################################################8| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 100%|################################################################################################8| 41.8MB / 41.9MB  + + ...milestone_grid_test_0.png: 100%|################################################################################################8| 2.73MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|################################################################################################8| 2.67MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|################################################################################################8| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|################################################################################################8| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|################################################################################################8| 2.74MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|################################################################################################8| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|################################################################################################8| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|################################################################################################8| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|################################################################################################8| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 100%|################################################################################################8| 41.8MB / 41.9MB  + + ...milestone_grid_test_0.png: 100%|################################################################################################8| 2.73MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|################################################################################################8| 2.67MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|################################################################################################8| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|################################################################################################8| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|################################################################################################8| 2.74MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|################################################################################################8| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|################################################################################################8| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|################################################################################################8| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|################################################################################################8| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 22 files] : 100%|################################################################################################8| 41.8MB / 41.9MB  + + ...milestone_grid_test_0.png: 100%|#################################################################################################| 2.74MB / 2.74MB  + + + ...milestone_grid_test_1.png: 100%|#################################################################################################| 2.68MB / 2.68MB  + + + + ...milestone_grid_test_2.png: 100%|#################################################################################################| 2.85MB / 2.85MB  + + + + + ...milestone_grid_test_3.png: 100%|#################################################################################################| 2.87MB / 2.87MB  + + + + + + ...milestone_grid_test_4.png: 100%|#################################################################################################| 2.75MB / 2.75MB  + + + + + + + ...milestone_grid_test_5.png: 100%|#################################################################################################| 2.82MB / 2.82MB  + + + + + + + + ...ilestone_grid_train_0.png: 100%|#################################################################################################| 2.70MB / 2.70MB  + + + + + + + + + ...ilestone_grid_train_1.png: 100%|#################################################################################################| 2.79MB / 2.79MB  + + + + + + + + + + .../milestone_grid_val_0.png: 100%|#################################################################################################| 2.64MB / 2.64MB  + + + + + + + + + + + [+ 24 files] : 94%|###########################################################################################1 | 42.4MB / 45.1MB  Processing Files (92 / 94) : 76%|##########################################################################1 | 222MB / 291MB, 25.3MB/s + New Data Upload : 50%|################################################ | 67.2MB / 136MB, 7.64MB/s  + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.65MB / 1.65MB  + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 2.04MB / 2.04MB  + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.96MB / 1.96MB  + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.80MB / 1.80MB  + + + + + + ...is/test_3_step_006650.png: 100%|#################################################################################################| 1.91MB / 1.91MB  + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.73MB / 1.73MB  + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + + + + + + + ..._step_006650_sample03.png: 16%|###############6 | 264kB / 1.64MB  + + + + + + + + + + + [+ 36 files] : 1%|#1 | 785kB / 67.3MB  Processing Files (92 / 130) : 77%|##########################################################################3 | 223MB / 291MB, 24.7MB/s + New Data Upload : 50%|################################################4 | 67.8MB / 136MB, 7.53MB/s  + + ..._step_006650_sample03.png: 32%|###############################2 | 528kB / 1.64MB  + + + ..._step_006650_sample04.png: 32%|###############################2 | 521kB / 1.62MB  + + + + ..._step_006650_sample03.png: 1%|7 | 17.5kB / 2.19MB  + + + + + ..._step_006650_sample04.png: 1%|7 | 15.8kB / 1.98MB  + + + + + + ...is/test_4_step_006650.png: 1%|7 | 14.2kB / 1.78MB  + + + + + + + ..._step_006650_sample00.png: 1%|7 | 15.1kB / 1.89MB  + + + + + + + + ..._step_006650_sample01.png: 1%|7 | 15.7kB / 1.97MB  + + + + + + + + + ..._step_006650_sample02.png: 1%|7 | 15.1kB / 1.89MB  + + + + + + + + + + ..._step_006650_sample03.png: 1%|7 | 13.3kB / 1.67MB  + + + + + + + + + + + [+ 28 files] : 1%|7 | 417kB / 52.3MB  Processing Files (92 / 130) : 77%|##########################################################################5 | 223MB / 291MB, 24.3MB/s + New Data Upload : 50%|################################################8 | 68.3MB / 136MB, 7.42MB/s  + + ..._step_006650_sample03.png: 32%|###############################2 | 528kB / 1.64MB  + + + ..._step_006650_sample04.png: 32%|###############################2 | 521kB / 1.62MB  + + + + ..._step_006650_sample03.png: 2%|#5 | 35.0kB / 2.19MB  + + + + + ..._step_006650_sample04.png: 2%|#5 | 31.7kB / 1.98MB  + + + + + + ...is/test_4_step_006650.png: 2%|#5 | 28.5kB / 1.78MB  + + + + + + + ..._step_006650_sample00.png: 2%|#5 | 30.2kB / 1.89MB  + + + + + + + + ..._step_006650_sample01.png: 2%|#5 | 31.4kB / 1.97MB  + + + + + + + + + ..._step_006650_sample02.png: 2%|#5 | 30.1kB / 1.89MB  + + + + + + + + + + ..._step_006650_sample03.png: 2%|#5 | 26.7kB / 1.67MB  + + + + + + + + + + + [+ 28 files] : 2%|#5 | 835kB / 52.3MB  Processing Files (92 / 130) : 77%|##########################################################################6 | 224MB / 291MB, 23.8MB/s + New Data Upload : 51%|#################################################2 | 68.8MB / 136MB, 7.32MB/s  + + ..._step_006650_sample03.png: 80%|############################################################################## | 1.32MB / 1.64MB  + + + ..._step_006650_sample04.png: 80%|############################################################################## | 1.30MB / 1.62MB  + + + + ..._step_006650_sample03.png: 2%|##3 | 52.6kB / 2.19MB  + + + + + ..._step_006650_sample04.png: 2%|##3 | 47.5kB / 1.98MB  + + + + + + ...is/test_4_step_006650.png: 2%|##3 | 42.7kB / 1.78MB  + + + + + + + ..._step_006650_sample00.png: 2%|##3 | 45.2kB / 1.89MB  + + + + + + + + ..._step_006650_sample01.png: 2%|##3 | 47.1kB / 1.97MB  + + + + + + + + + ..._step_006650_sample02.png: 2%|##3 | 45.2kB / 1.89MB  + + + + + + + + + + ..._step_006650_sample03.png: 2%|##3 | 40.0kB / 1.67MB  + + + + + + + + + + + [+ 28 files] : 2%|##3 | 1.25MB / 52.3MB  Processing Files (92 / 130) : 78%|###########################################################################3 | 226MB / 291MB, 23.5MB/s + New Data Upload : 52%|##################################################7 | 70.9MB / 136MB, 7.39MB/s  + + ..._step_006650_sample03.png: 97%|#############################################################################################6 | 1.58MB / 1.64MB  + + + ..._step_006650_sample04.png: 97%|#############################################################################################6 | 1.56MB / 1.62MB  + + + + ..._step_006650_sample03.png: 6%|#####4 | 123kB / 2.19MB  + + + + + ..._step_006650_sample04.png: 6%|#####4 | 111kB / 1.98MB  + + + + + + ...is/test_4_step_006650.png: 6%|#####4 | 99.7kB / 1.78MB  + + + + + + + ..._step_006650_sample00.png: 6%|#####4 | 106kB / 1.89MB  + + + + + + + + ..._step_006650_sample01.png: 6%|#####4 | 110kB / 1.97MB  + + + + + + + + + ..._step_006650_sample02.png: 6%|#####4 | 105kB / 1.89MB  + + + + + + + + + + ..._step_006650_sample03.png: 6%|#####4 | 93.3kB / 1.67MB  + + + + + + + + + + + [+ 28 files] : 6%|#####4 | 2.92MB / 52.3MB  Processing Files (92 / 130) : 79%|############################################################################2 | 229MB / 291MB, 23.3MB/s + New Data Upload : 54%|####################################################5 | 73.5MB / 136MB, 7.50MB/s  + + ..._step_006650_sample03.png: 97%|#############################################################################################6 | 1.58MB / 1.64MB  + + + ..._step_006650_sample04.png: 97%|#############################################################################################6 | 1.56MB / 1.62MB  + + + + ..._step_006650_sample03.png: 9%|########5 | 193kB / 2.19MB  + + + + + ..._step_006650_sample04.png: 9%|########5 | 174kB / 1.98MB  + + + + + + ...is/test_4_step_006650.png: 9%|########5 | 157kB / 1.78MB  + + + + + + + ..._step_006650_sample00.png: 9%|########5 | 166kB / 1.89MB  + + + + + + + + ..._step_006650_sample01.png: 9%|########5 | 173kB / 1.97MB  + + + + + + + + + ..._step_006650_sample02.png: 9%|########5 | 166kB / 1.89MB  + + + + + + + + + + ..._step_006650_sample03.png: 9%|########5 | 147kB / 1.67MB  + + + + + + + + + + + [+ 28 files] : 9%|########5 | 4.59MB / 52.3MB  Processing Files (92 / 130) : 79%|############################################################################9 | 231MB / 291MB, 23.1MB/s + New Data Upload : 56%|###################################################### | 75.6MB / 136MB, 7.56MB/s  + + ..._step_006650_sample03.png: 97%|#############################################################################################6 | 1.58MB / 1.64MB  + + + ..._step_006650_sample04.png: 97%|#############################################################################################6 | 1.56MB / 1.62MB  + + + + ..._step_006650_sample03.png: 14%|#############9 | 315kB / 2.19MB  + + + + + ..._step_006650_sample04.png: 14%|#############9 | 285kB / 1.98MB  + + + + + + ...is/test_4_step_006650.png: 14%|#############9 | 256kB / 1.78MB  + + + + + + + ..._step_006650_sample00.png: 14%|#############9 | 271kB / 1.89MB  + + + + + + + + ..._step_006650_sample01.png: 14%|#############9 | 282kB / 1.97MB  + + + + + + + + + ..._step_006650_sample02.png: 14%|#############9 | 271kB / 1.89MB  + + + + + + + + + + ..._step_006650_sample03.png: 14%|#############9 | 240kB / 1.67MB  + + + + + + + + + + + [+ 28 files] : 14%|#############9 | 7.51MB / 52.3MB  Processing Files (92 / 130) : 81%|##############################################################################1 | 234MB / 291MB, 23.0MB/s + New Data Upload : 58%|########################################################7 | 79.3MB / 136MB, 7.77MB/s  + + ..._step_006650_sample03.png: 97%|#############################################################################################6 | 1.58MB / 1.64MB  + + + ..._step_006650_sample04.png: 97%|#############################################################################################6 | 1.56MB / 1.62MB  + + + + ..._step_006650_sample03.png: 17%|################2 | 368kB / 2.19MB  + + + + + ..._step_006650_sample04.png: 17%|################2 | 333kB / 1.98MB  + + + + + + ...is/test_4_step_006650.png: 17%|################2 | 299kB / 1.78MB  + + + + + + + ..._step_006650_sample00.png: 17%|################2 | 317kB / 1.89MB  + + + + + + + + ..._step_006650_sample01.png: 17%|################2 | 330kB / 1.97MB  + + + + + + + + + ..._step_006650_sample02.png: 17%|################2 | 316kB / 1.89MB  + + + + + + + + + + ..._step_006650_sample03.png: 17%|################2 | 280kB / 1.67MB  + + + + + + + + + + + [+ 28 files] : 17%|################2 | 8.77MB / 52.3MB  Processing Files (92 / 130) : 81%|##############################################################################7 | 236MB / 291MB, 23.1MB/s + New Data Upload : 60%|#########################################################8 | 80.9MB / 136MB, 7.93MB/s  + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.64MB / 1.64MB  + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.62MB / 1.62MB  + + + + ..._step_006650_sample03.png: 22%|####################9 | 473kB / 2.19MB  + + + + + ..._step_006650_sample04.png: 22%|####################9 | 428kB / 1.98MB  + + + + + + ...is/test_4_step_006650.png: 22%|####################9 | 385kB / 1.78MB  + + + + + + + ..._step_006650_sample00.png: 22%|####################9 | 407kB / 1.89MB  + + + + + + + + ..._step_006650_sample01.png: 22%|####################9 | 424kB / 1.97MB  + + + + + + + + + ..._step_006650_sample02.png: 22%|####################9 | 406kB / 1.89MB  + + + + + + + + + + ..._step_006650_sample03.png: 22%|####################9 | 360kB / 1.67MB  + + + + + + + + + + + [+ 28 files] : 22%|####################9 | 11.3MB / 52.3MB  Processing Files (94 / 130) : 82%|###############################################################################8 | 239MB / 291MB, 23.4MB/s + New Data Upload : 62%|############################################################1 | 84.1MB / 136MB, 8.25MB/s  + + ..._step_006650_sample03.png: 26%|#########################5 | 578kB / 2.19MB  + + + ..._step_006650_sample04.png: 26%|#########################5 | 523kB / 1.98MB  + + + + ...is/test_4_step_006650.png: 26%|#########################5 | 470kB / 1.78MB  + + + + + ..._step_006650_sample00.png: 26%|#########################5 | 498kB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 26%|#########################5 | 518kB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 26%|#########################5 | 497kB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 26%|#########################5 | 440kB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 26%|#########################5 | 485kB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 26%|#########################5 | 493kB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 26%|#########################5 | 12.8MB / 48.6MB  Processing Files (94 / 130) : 83%|################################################################################8 | 242MB / 291MB, 23.8MB/s + New Data Upload : 64%|##############################################################4 | 87.3MB / 136MB, 8.56MB/s  + + ..._step_006650_sample03.png: 31%|##############################1 | 683kB / 2.19MB  + + + ..._step_006650_sample04.png: 31%|##############################1 | 618kB / 1.98MB  + + + + ...is/test_4_step_006650.png: 31%|##############################1 | 555kB / 1.78MB  + + + + + ..._step_006650_sample00.png: 31%|##############################1 | 588kB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 31%|##############################1 | 612kB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 31%|##############################1 | 587kB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 31%|##############################1 | 520kB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 31%|##############################1 | 574kB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 31%|##############################1 | 583kB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 31%|##############################1 | 15.1MB / 48.6MB  Processing Files (94 / 130) : 84%|#################################################################################9 | 245MB / 291MB, 24.1MB/s + New Data Upload : 67%|################################################################6 | 90.4MB / 136MB, 8.86MB/s  + + ..._step_006650_sample03.png: 36%|##################################8 | 788kB / 2.19MB  + + + ..._step_006650_sample04.png: 36%|##################################8 | 713kB / 1.98MB  + + + + ...is/test_4_step_006650.png: 36%|##################################8 | 641kB / 1.78MB  + + + + + ..._step_006650_sample00.png: 36%|##################################8 | 679kB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 36%|##################################8 | 706kB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 36%|##################################8 | 677kB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 36%|##################################8 | 600kB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 36%|##################################8 | 662kB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 36%|##################################8 | 672kB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 36%|##################################8 | 17.5MB / 48.6MB  Processing Files (94 / 130) : 86%|##################################################################################9 | 249MB / 291MB, 23.1MB/s + New Data Upload : 69%|##################################################################9 | 93.6MB / 136MB, 9.17MB/s  + + ..._step_006650_sample03.png: 38%|#####################################1 | 841kB / 2.19MB  + + + ..._step_006650_sample04.png: 38%|#####################################1 | 760kB / 1.98MB  + + + + ...is/test_4_step_006650.png: 38%|#####################################1 | 684kB / 1.78MB  + + + + + ..._step_006650_sample00.png: 38%|#####################################1 | 724kB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 38%|#####################################1 | 753kB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 38%|#####################################1 | 723kB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 38%|#####################################1 | 640kB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 38%|#####################################1 | 706kB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 38%|#####################################1 | 717kB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 38%|#####################################1 | 18.6MB / 48.6MB  Processing Files (94 / 130) : 86%|###################################################################################4 | 250MB / 291MB, 20.0MB/s + New Data Upload : 70%|#################################################################### | 95.1MB / 136MB, 9.33MB/s  + + ..._step_006650_sample03.png: 44%|##########################################5 | 964kB / 2.19MB  + + + ..._step_006650_sample04.png: 44%|##########################################5 | 871kB / 1.98MB  + + + + ...is/test_4_step_006650.png: 44%|##########################################5 | 783kB / 1.78MB  + + + + + ..._step_006650_sample00.png: 44%|##########################################5 | 830kB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 44%|##########################################5 | 863kB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 44%|##########################################5 | 828kB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 44%|##########################################5 | 733kB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 44%|##########################################5 | 809kB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 44%|##########################################5 | 822kB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 44%|##########################################5 | 21.3MB / 48.6MB  Processing Files (94 / 130) : 87%|####################################################################################7 | 254MB / 291MB, 19.3MB/s + New Data Upload : 73%|######################################################################6 | 98.8MB / 136MB, 9.69MB/s  + + ..._step_006650_sample03.png: 46%|############################################9 | 1.02MB / 2.19MB  + + + ..._step_006650_sample04.png: 46%|############################################9 | 919kB / 1.98MB  + + + + ...is/test_4_step_006650.png: 46%|############################################9 | 826kB / 1.78MB  + + + + + ..._step_006650_sample00.png: 46%|############################################9 | 875kB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 46%|############################################9 | 910kB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 46%|############################################9 | 873kB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 46%|############################################9 | 773kB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 46%|############################################9 | 853kB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 46%|############################################9 | 867kB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 46%|############################################9 | 22.5MB / 48.6MB  Processing Files (94 / 130) : 88%|#####################################################################################2 | 255MB / 291MB, 18.7MB/s + New Data Upload : 74%|#######################################################################7 | 100MB / 136MB, 9.84MB/s  + + ..._step_006650_sample03.png: 52%|##################################################3 | 1.14MB / 2.19MB  + + + ..._step_006650_sample04.png: 52%|##################################################3 | 1.03MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 52%|##################################################3 | 926kB / 1.78MB  + + + + + ..._step_006650_sample00.png: 52%|##################################################3 | 980kB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 52%|##################################################3 | 1.02MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 52%|##################################################3 | 979kB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 52%|##################################################3 | 867kB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 52%|##################################################3 | 956kB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 52%|##################################################3 | 971kB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 52%|##################################################3 | 25.2MB / 48.6MB  Processing Files (94 / 130) : 89%|######################################################################################4 | 259MB / 291MB, 17.5MB/s + New Data Upload : 77%|##########################################################################4 | 104MB / 136MB, 10.2MB/s  + + ..._step_006650_sample03.png: 56%|######################################################1 | 1.23MB / 2.19MB  + + + ..._step_006650_sample04.png: 56%|######################################################1 | 1.11MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 56%|######################################################1 | 997kB / 1.78MB  + + + + + ..._step_006650_sample00.png: 56%|######################################################1 | 1.06MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 56%|######################################################1 | 1.10MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 56%|######################################################1 | 1.05MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 56%|######################################################1 | 933kB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 56%|######################################################1 | 1.03MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 56%|######################################################1 | 1.05MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 56%|######################################################1 | 27.1MB / 48.6MB  Processing Files (94 / 130) : 90%|#######################################################################################3 | 262MB / 291MB, 16.5MB/s + New Data Upload : 79%|############################################################################2 | 107MB / 136MB, 10.5MB/s  + + ..._step_006650_sample03.png: 61%|###########################################################6 | 1.35MB / 2.19MB  + + + ..._step_006650_sample04.png: 61%|###########################################################6 | 1.22MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 61%|###########################################################6 | 1.10MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 61%|###########################################################6 | 1.16MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 61%|###########################################################6 | 1.21MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 61%|###########################################################6 | 1.16MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 61%|###########################################################6 | 1.03MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 61%|###########################################################6 | 1.13MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 61%|###########################################################6 | 1.15MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 61%|###########################################################6 | 29.9MB / 48.6MB  Processing Files (94 / 130) : 91%|########################################################################################5 | 265MB / 291MB, 13.8MB/s + New Data Upload : 81%|##############################################################################8 | 110MB / 136MB, 10.8MB/s  + + ..._step_006650_sample03.png: 64%|#############################################################9 | 1.40MB / 2.19MB  + + + ..._step_006650_sample04.png: 64%|#############################################################9 | 1.27MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 64%|#############################################################9 | 1.14MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 64%|#############################################################9 | 1.21MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 64%|#############################################################9 | 1.26MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 64%|#############################################################9 | 1.20MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 64%|#############################################################9 | 1.07MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 64%|#############################################################9 | 1.18MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 64%|#############################################################9 | 1.20MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 64%|#############################################################9 | 31.0MB / 48.6MB  Processing Files (94 / 130) : 92%|######################################################################################### | 267MB / 291MB, 12.4MB/s + New Data Upload : 82%|################################################################################ | 112MB / 136MB, 10.9MB/s  + + ..._step_006650_sample03.png: 69%|###################################################################3 | 1.52MB / 2.19MB  + + + ..._step_006650_sample04.png: 69%|###################################################################3 | 1.38MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 69%|###################################################################3 | 1.24MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 69%|###################################################################3 | 1.31MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 69%|###################################################################3 | 1.37MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 69%|###################################################################3 | 1.31MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 69%|###################################################################3 | 1.16MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 69%|###################################################################3 | 1.28MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 69%|###################################################################3 | 1.30MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 69%|###################################################################3 | 33.7MB / 48.6MB  Processing Files (94 / 130) : 93%|##########################################################################################3 | 271MB / 291MB, 12.5MB/s + New Data Upload : 85%|##################################################################################6 | 116MB / 136MB, 11.2MB/s  + + ..._step_006650_sample03.png: 75%|########################################################################7 | 1.65MB / 2.19MB  + + + ..._step_006650_sample04.png: 75%|########################################################################7 | 1.49MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 75%|########################################################################7 | 1.34MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 75%|########################################################################7 | 1.42MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 75%|########################################################################7 | 1.48MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 75%|########################################################################7 | 1.42MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 75%|########################################################################7 | 1.25MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 75%|########################################################################7 | 1.38MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 75%|########################################################################7 | 1.40MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 75%|########################################################################7 | 36.5MB / 48.6MB  Processing Files (94 / 130) : 94%|###########################################################################################5 | 274MB / 291MB, 11.5MB/s + New Data Upload : 88%|#####################################################################################2 | 119MB / 136MB, 11.5MB/s  + + ..._step_006650_sample03.png: 77%|###########################################################################1 | 1.70MB / 2.19MB  + + + ..._step_006650_sample04.png: 77%|###########################################################################1 | 1.54MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 77%|###########################################################################1 | 1.38MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 77%|###########################################################################1 | 1.46MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 77%|###########################################################################1 | 1.52MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 77%|###########################################################################1 | 1.46MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 77%|###########################################################################1 | 1.29MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 77%|###########################################################################1 | 1.43MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 77%|###########################################################################1 | 1.45MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 77%|###########################################################################1 | 37.6MB / 48.6MB  Processing Files (94 / 130) : 95%|############################################################################################ | 276MB / 291MB, 11.5MB/s + New Data Upload : 89%|######################################################################################3 | 121MB / 136MB, 11.5MB/s  + + ..._step_006650_sample03.png: 82%|###############################################################################7 | 1.80MB / 2.19MB  + + + ..._step_006650_sample04.png: 82%|###############################################################################7 | 1.63MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 82%|###############################################################################7 | 1.47MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 82%|###############################################################################7 | 1.55MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 82%|###############################################################################7 | 1.62MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 82%|###############################################################################7 | 1.55MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 82%|###############################################################################7 | 1.37MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 82%|###############################################################################7 | 1.52MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 82%|###############################################################################7 | 1.54MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 82%|###############################################################################7 | 39.9MB / 48.6MB  Processing Files (94 / 130) : 96%|#############################################################################################1 | 279MB / 291MB, 11.4MB/s + New Data Upload : 91%|########################################################################################6 | 124MB / 136MB, 11.4MB/s  + + ..._step_006650_sample03.png: 87%|####################################################################################3 | 1.91MB / 2.19MB  + + + ..._step_006650_sample04.png: 87%|####################################################################################3 | 1.73MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 87%|####################################################################################3 | 1.55MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 87%|####################################################################################3 | 1.64MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 87%|####################################################################################3 | 1.71MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 87%|####################################################################################3 | 1.64MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 87%|####################################################################################3 | 1.45MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 87%|####################################################################################3 | 1.60MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 87%|####################################################################################3 | 1.63MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 87%|####################################################################################3 | 42.3MB / 48.6MB  Processing Files (94 / 130) : 97%|##############################################################################################1 | 282MB / 291MB, 11.4MB/s + New Data Upload : 94%|##########################################################################################8 | 127MB / 136MB, 11.4MB/s  + + ..._step_006650_sample03.png: 91%|########################################################################################2 | 2.00MB / 2.19MB  + + + ..._step_006650_sample04.png: 91%|########################################################################################2 | 1.81MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 91%|########################################################################################2 | 1.62MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 91%|########################################################################################2 | 1.72MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 91%|########################################################################################2 | 1.79MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 91%|########################################################################################2 | 1.72MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 91%|########################################################################################2 | 1.52MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 91%|########################################################################################2 | 1.68MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 91%|########################################################################################2 | 1.70MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 91%|########################################################################################2 | 44.2MB / 48.6MB  Processing Files (94 / 130) : 98%|############################################################################################### | 285MB / 291MB, 11.4MB/s + New Data Upload : 96%|############################################################################################7 | 130MB / 136MB, 11.4MB/s  + + ..._step_006650_sample03.png: 93%|##########################################################################################5 | 2.05MB / 2.19MB  + + + ..._step_006650_sample04.png: 93%|##########################################################################################5 | 1.85MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 93%|##########################################################################################5 | 1.67MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 93%|##########################################################################################5 | 1.76MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 93%|##########################################################################################5 | 1.84MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 93%|##########################################################################################5 | 1.76MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 93%|##########################################################################################5 | 1.56MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 93%|##########################################################################################5 | 1.72MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 93%|##########################################################################################5 | 1.75MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 93%|##########################################################################################5 | 45.4MB / 48.6MB  Processing Files (94 / 130) : 99%|###############################################################################################5 | 286MB / 291MB, 11.4MB/s + New Data Upload : 97%|#############################################################################################8 | 131MB / 136MB, 11.4MB/s  + + ..._step_006650_sample03.png: 98%|###############################################################################################2 | 2.15MB / 2.19MB  + + + ..._step_006650_sample04.png: 98%|###############################################################################################2 | 1.95MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 98%|###############################################################################################2 | 1.75MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 98%|###############################################################################################2 | 1.86MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 98%|###############################################################################################2 | 1.93MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 98%|###############################################################################################2 | 1.85MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 98%|###############################################################################################2 | 1.64MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 98%|###############################################################################################2 | 1.81MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 98%|###############################################################################################2 | 1.84MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 98%|###############################################################################################2 | 47.7MB / 48.6MB  Processing Files (94 / 130) : 100%|################################################################################################6| 289MB / 291MB, 11.4MB/s + New Data Upload : 99%|################################################################################################1| 134MB / 136MB, 11.4MB/s  + + ..._step_006650_sample03.png: 100%|################################################################################################7| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|################################################################################################7| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|################################################################################################7| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|################################################################################################7| 1.96MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|################################################################################################7| 1.88MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|################################################################################################7| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|################################################################################################7| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|################################################################################################7| 48.5MB / 48.6MB  Processing Files (94 / 130) : 100%|################################################################################################9| 290MB / 291MB, 11.1MB/s + New Data Upload : 100%|################################################################################################8| 135MB / 136MB, 11.1MB/s  + + ..._step_006650_sample03.png: 100%|################################################################################################7| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|################################################################################################7| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|################################################################################################7| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|################################################################################################7| 1.96MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|################################################################################################7| 1.88MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|################################################################################################7| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|################################################################################################7| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|################################################################################################7| 48.5MB / 48.6MB  + + ..._step_006650_sample03.png: 100%|################################################################################################7| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|################################################################################################7| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|################################################################################################7| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|################################################################################################7| 1.96MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|################################################################################################7| 1.88MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|################################################################################################7| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|################################################################################################7| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|################################################################################################7| 48.5MB / 48.6MB  + + ..._step_006650_sample03.png: 100%|################################################################################################7| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|################################################################################################7| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|################################################################################################7| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|################################################################################################7| 1.96MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|################################################################################################7| 1.88MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|################################################################################################7| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|################################################################################################7| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|################################################################################################7| 48.5MB / 48.6MB  + + ..._step_006650_sample03.png: 100%|################################################################################################7| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|################################################################################################7| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|################################################################################################7| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|################################################################################################7| 1.96MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|################################################################################################7| 1.88MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|################################################################################################7| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|################################################################################################7| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|################################################################################################7| 48.5MB / 48.6MB  + + ..._step_006650_sample03.png: 100%|################################################################################################7| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|################################################################################################7| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|################################################################################################7| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|################################################################################################7| 1.96MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|################################################################################################7| 1.88MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|################################################################################################7| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|################################################################################################7| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|################################################################################################7| 48.5MB / 48.6MB  + + ..._step_006650_sample03.png: 100%|################################################################################################7| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|################################################################################################7| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|################################################################################################7| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|################################################################################################7| 1.96MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|################################################################################################7| 1.88MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|################################################################################################7| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|################################################################################################7| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|################################################################################################7| 48.5MB / 48.6MB  + + ..._step_006650_sample03.png: 100%|################################################################################################7| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|################################################################################################7| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|################################################################################################7| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|################################################################################################7| 1.96MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|################################################################################################7| 1.88MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|################################################################################################7| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|################################################################################################7| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|################################################################################################7| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|################################################################################################7| 48.5MB / 48.6MB  + + ..._step_006650_sample03.png: 100%|#################################################################################################| 2.19MB / 2.19MB  + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.98MB / 1.98MB  + + + + ...is/test_4_step_006650.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.89MB / 1.89MB  + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.97MB / 1.97MB  + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.89MB / 1.89MB  + + + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.84MB / 1.84MB  + + + + + + + + + + ...is/test_5_step_006650.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + [+ 26 files] : 100%|#################################################################################################| 48.6MB / 48.6MB  Processing Files (130 / 130) : 100%|#################################################################################################| 291MB / 291MB, 8.83MB/s + New Data Upload : 100%|#################################################################################################| 136MB / 136MB, 8.83MB/s  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  + + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB  + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB  + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB  + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB  + + + + + + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB  + + + + + + + + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB  + + + + + + + + + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB  + + + + + + + + + + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB  + + + + + + + + + + + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB  Processing Files (130 / 130) : 100%|#################################################################################################| 291MB / 291MB, 6.91MB/s + New Data Upload : 100%|#################################################################################################| 136MB / 136MB, 6.91MB/s + ...vis/val_0_step_006650.png: 100%|#################################################################################################| 1.67MB / 1.67MB + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.77MB / 1.77MB + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.94MB / 1.94MB + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.58MB / 1.58MB + ..._step_006650_sample03.png: 100%|#################################################################################################| 1.70MB / 1.70MB + ..._step_006650_sample04.png: 100%|#################################################################################################| 1.79MB / 1.79MB + ...vis/val_1_step_006650.png: 100%|#################################################################################################| 1.70MB / 1.70MB + ..._step_006650_sample00.png: 100%|#################################################################################################| 1.78MB / 1.78MB + ..._step_006650_sample01.png: 100%|#################################################################################################| 1.87MB / 1.87MB + ..._step_006650_sample02.png: 100%|#################################################################################################| 1.74MB / 1.74MB